iPad driver: Make setting iTunes Category from series optional. News download now optimizations for iPad output. Fix #5922 (Calibre crashes on Mac OSX when fetching annotations)

resources/recipes/nytimes.recipe

@@ -17,7 +17,7 @@ class NYTimes(BasicNewsRecipe):
     title       = 'New York Times Top Stories'
     __author__  = 'GRiker'
     language = 'en'
-    requires_version = (0, 7, 3)
+    requires_version = (0, 7, 5)
     description = 'Top Stories from the New York Times'
 
     # List of sections typically included in Top Stories.  Use a keyword from the
@@ -79,6 +79,7 @@ class NYTimes(BasicNewsRecipe):
                             'doubleRule',
                             'dottedLine',
                             'entry-meta',
+                            'entry-response module',
                             'icon enlargeThis',
                             'leftNavTabs',
                             'module box nav',
@@ -110,6 +111,7 @@ class NYTimes(BasicNewsRecipe):
                             'navigation',
                             'portfolioInline',
                             'relatedArticles',
+                            'respond',
                             'side_search',
                             'side_index',
                             'side_tool',

resources/recipes/nytimes_sub.recipe

@@ -13,14 +13,14 @@ Story
 import re, string, time
 from calibre import strftime
 from calibre.web.feeds.recipes import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, NavigableString, Tag
+from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup, NavigableString, Tag
 
 class NYTimes(BasicNewsRecipe):
 
     title       = 'The New York Times'
     __author__  = 'GRiker'
     language = 'en'
-    requires_version = (0, 7, 3)
+    requires_version = (0, 7, 5)
 
     description = 'Daily news from the New York Times (subscription version)'
     allSectionKeywords = ['The Front Page', 'International','National','Obituaries','Editorials',
@@ -66,6 +66,7 @@ class NYTimes(BasicNewsRecipe):
                             'doubleRule',
                             'dottedLine',
                             'entry-meta',
+                            'entry-response module',
                             'icon enlargeThis',
                             'leftNavTabs',
                             'module box nav',
@@ -97,6 +98,7 @@ class NYTimes(BasicNewsRecipe):
                             'navigation',
                             'portfolioInline',
                             'relatedArticles',
+                            'respond',
                             'side_search',
                             'side_index',
                             'side_tool',
@@ -417,12 +419,11 @@ class NYTimes(BasicNewsRecipe):
 
         return soup
 
-    def postprocess_book(self, oeb, opts, log) :
+    def populate_article_metadata(self,article,soup,first):
-        print "\npostprocess_book()\n"
+        '''
-
+        Extract author and description from article, add to article metadata
-        def extract_byline(href) :
+        '''
-            # <meta name="byline" content=
+        def extract_author(soup):
-            soup = BeautifulSoup(str(oeb.manifest.hrefs[href]))
             byline = soup.find('meta',attrs={'name':['byl','CLMST']})
             if byline :
                 author = byline['content']
@@ -432,50 +433,32 @@ class NYTimes(BasicNewsRecipe):
                 if byline:
                     author = byline.renderContents()
                 else:
-                    print "couldn't find byline in %s" % href
                     print soup.prettify()
                     return None
-            # Kill commas - Kindle switches to '&'
+            return author
-            return re.sub(',','',author)
 
-        def extract_description(href) :
+        def extract_description(soup):
-            soup = BeautifulSoup(str(oeb.manifest.hrefs[href]))
             description = soup.find('meta',attrs={'name':['description','description ']})
             if description :
-#                print repr(description['content'])
-#                print self.massageNCXText(description['content'])
                 return self.massageNCXText(description['content'])
             else:
                 # Take first paragraph of article
-                articleBody = soup.find('div',attrs={'id':'articleBody'})
+                articlebody = soup.find('div',attrs={'id':'articlebody'})
-                if not articleBody:
+                if not articlebody:
                     # Try again with class instead of id
-                    articleBody = soup.find('div',attrs={'class':'articleBody'})
+                    articlebody = soup.find('div',attrs={'class':'articlebody'})
-                    if not articleBody:
+                    if not articlebody:
-                        print 'postprocess_book.extract_description(): Did not find <div id="articleBody">:'
+                        print 'postprocess_book.extract_description(): Did not find <div id="articlebody">:'
                         print soup.prettify()
                         return None
-                paras = articleBody.findAll('p')
+                paras = articlebody.findAll('p')
                 for p in paras:
                     if p.renderContents() > '' :
                         return self.massageNCXText(self.tag_to_string(p,use_alt=False))
                 return None
 
-        # Method entry point here
+        article.author = extract_author(soup)
-        # Single section toc looks different than multi-section tocs
+        article.summary = article.text_summary = extract_description(soup)
-        if oeb.toc.depth() == 2 :
-            for article in oeb.toc :
-                if article.author is None :
-                    article.author = extract_byline(article.href)
-                if article.description is None :
-                    article.description = extract_description(article.href).decode('utf-8')
-        elif oeb.toc.depth() == 3 :
-            for section in oeb.toc :
-                for article in section :
-                    if article.author is None :
-                        article.author = extract_byline(article.href)
-                    if article.description is None :
-                        article.description = extract_description(article.href)
 
     def strip_anchors(self,soup):
         paras = soup.findAll(True)

src/calibre/customize/profiles.py

@@ -36,7 +36,7 @@ class Plugin(_Plugin):
         self.fnames = dict((name, sz) for name, _, sz in self.fsizes if name)
         self.fnums = dict((num, sz) for _, num, sz in self.fsizes if num)
 
-
+# Input profiles {{{
 class InputProfile(Plugin):
 
     author = 'Kovid Goyal'
@@ -218,6 +218,8 @@ input_profiles = [InputProfile, SonyReaderInput, SonyReader300Input,
 
 input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower()))
 
+# }}}
+
 class OutputProfile(Plugin):
 
     author = 'Kovid Goyal'
@@ -237,11 +239,12 @@ class OutputProfile(Plugin):
     # If True the MOBI renderer on the device supports MOBI indexing
     supports_mobi_indexing = False
 
-    # Device supports displaying a nested TOC
-    supports_nested_toc = True
-
     # If True output should be optimized for a touchscreen interface
     touchscreen = False
+    touchscreen_news_css = ''
+    # A list of extra (beyond CSS 2.1) modules supported by the device
+    # Format is a cssutils profile dictionary (see iPad for example)
+    extra_css_modules = []
 
     @classmethod
     def tags_to_string(cls, tags):
@@ -256,8 +259,86 @@ class iPadOutput(OutputProfile):
     screen_size = (768, 1024)
     comic_screen_size = (768, 1024)
     dpi = 132.0
-    supports_nested_toc = False
+    extra_css_modules = [
+        {
+            'name':'webkit',
+            'props': { '-webkit-border-bottom-left-radius':'{length}',
+                '-webkit-border-bottom-right-radius':'{length}',
+                '-webkit-border-top-left-radius':'{length}',
+                '-webkit-border-top-right-radius':'{length}',
+                '-webkit-border-radius': r'{border-width}(\s+{border-width}){0,3}|inherit',
+            },
+            'macros': {'border-width': '{length}|medium|thick|thin'}
+        }
+    ]
     touchscreen = True
+    # touchscreen_news_css {{{
+    touchscreen_news_css = u'''
+			/* hr used in articles */
+            .caption_divider {
+            	border:#ccc 1px solid;
+				}
+
+            .touchscreen_navbar {
+                background:#ccc;
+                border:#ccc 1px solid;
+                border-collapse:separate;
+                border-spacing:1px;
+                margin-left: 5%;
+                margin-right: 5%;
+                width: 90%;
+                -webkit-border-radius:4px;
+                }
+            .touchscreen_navbar td {
+                background:#fff;
+                font-family:Helvetica;
+                font-size:80%;
+                padding: 5px;
+                text-align:center;
+                }
+            .touchscreen_navbar td:first-child {
+				-webkit-border-top-left-radius:4px;
+				-webkit-border-bottom-left-radius:4px;
+				}
+			.touchscreen_navbar td:last-child {
+				-webkit-border-top-right-radius:4px;
+				-webkit-border-bottom-right-radius:4px;
+				}
+
+            .feed_link {
+                font-style: italic;
+                }
+
+
+            /* Feed summary formatting */
+            .feed_title {
+                text-align: center;
+                font-size: 160%;
+                }
+
+            .summary_headline {
+                font-weight:bold;
+                text-align:left;
+				}
+
+            .summary_byline {
+                text-align:left;
+                font-family:monospace;
+				}
+
+            .summary_text {
+                text-align:left;
+				}
+
+            .feed {
+                font-family:sans-serif;
+                font-weight:bold;
+                font-size:larger;
+				}
+
+        '''
+        # }}}
+
 
 class SonyReaderOutput(OutputProfile):
 

src/calibre/devices/apple/driver.py

@@ -10,12 +10,13 @@ from calibre.constants import __appname__, __version__, DEBUG
 from calibre import fit_image
 from calibre.constants import isosx, iswindows
 from calibre.devices.errors import UserFeedback
+from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.epub import set_metadata
 from calibre.library.server.utils import strftime
-from calibre.utils.config import Config, config_dir
+from calibre.utils.config import config_dir
 from calibre.utils.date import isoformat, now, parse_date
 from calibre.utils.logging import Log
 from calibre.utils.zipfile import ZipFile
@@ -33,8 +34,15 @@ if isosx:
 if iswindows:
     import pythoncom, win32com.client
 
+class DriverBase(DeviceConfig, DevicePlugin):
+    # Needed for config_widget to work
+    FORMATS = ['epub', 'pdf']
 
-class ITUNES(DevicePlugin):
+    @classmethod
+    def _config_base_name(cls):
+        return 'iTunes'
+
+class ITUNES(DriverBase):
     '''
     Calling sequences:
     Initialization:
@@ -78,12 +86,11 @@ class ITUNES(DevicePlugin):
     supported_platforms = ['osx','windows']
     author = 'GRiker'
     #: The version of this plugin as a 3-tuple (major, minor, revision)
-    version        = (0,7,0)
+    version        = (0,8,0)
 
     OPEN_FEEDBACK_MESSAGE = _(
         'Apple device detected, launching iTunes, please wait ...')
 
-    FORMATS = ['epub']
 
     # Product IDs:
     #  0x1292:iPhone 3G
@@ -141,6 +148,10 @@ class ITUNES(DevicePlugin):
                 'SongNames',
                 ]
 
+    # Cover art size limits
+    MAX_COVER_WIDTH = 510
+    MAX_COVER_HEIGHT = 680
+
     # Properties
     cached_books = {}
     cache_dir = os.path.join(config_dir, 'caches', 'itunes')
@@ -159,7 +170,6 @@ class ITUNES(DevicePlugin):
     sources = None
     update_msg = None
     update_needed = False
-    use_series_data = True
 
     # Public methods
     def add_books_to_metadata(self, locations, metadata, booklists):
@@ -173,16 +183,17 @@ class ITUNES(DevicePlugin):
                                 (L{books}(oncard=None), L{books}(oncard='carda'),
                                 L{books}(oncard='cardb')).
         '''
+        if DEBUG:
+            self.log.info("ITUNES.add_books_to_metadata()")
 
         task_count = float(len(self.update_list))
 
         # Delete any obsolete copies of the book from the booklist
         if self.update_list:
-            if True:
+            if False:
-                self.log.info("ITUNES.add_books_to_metadata()")
+                self._dump_booklist(booklists[0], header='before',indent=2)
-                #self._dump_booklist(booklists[0], header='before',indent=2)
+                self._dump_update_list(header='before',indent=2)
-                #self._dump_update_list(header='before',indent=2)
+                self._dump_cached_books(header='before',indent=2)
-                #self._dump_cached_books(header='before',indent=2)
 
             for (j,p_book) in enumerate(self.update_list):
                 if False:
@@ -230,12 +241,12 @@ class ITUNES(DevicePlugin):
 
         # Add new books to booklists[0]
         for new_book in locations[0]:
-            if False:
+            if DEBUG:
                 self.log.info("  adding '%s' by '%s' to booklists[0]" %
                     (new_book.title, new_book.author))
             booklists[0].append(new_book)
 
-        if False:
+        if DEBUG:
             self._dump_booklist(booklists[0],header='after',indent=2)
             self._dump_cached_books(header='after',indent=2)
 
@@ -329,7 +340,8 @@ class ITUNES(DevicePlugin):
                              'title':book.Name,
                              'author':book.Artist,
                              'lib_book':library_books[this_book.path] if this_book.path in library_books else None,
-                             'uuid': book.Composer
+                             'uuid': book.Composer,
+                             'format': 'pdf' if book.KindAsString.startswith('PDF') else 'epub'
                              }
 
                             if self.report_progress is not None:
@@ -343,9 +355,9 @@ class ITUNES(DevicePlugin):
                 if self.report_progress is not None:
                     self.report_progress(1.0, _('finished'))
                 self.cached_books = cached_books
-#                 if DEBUG:
+                if DEBUG:
-#                     self._dump_booklist(booklist, 'returning from books():')
+                    self._dump_booklist(booklist, 'returning from books()', indent=2)
-#                     self._dump_cached_books('returning from books():')
+                    self._dump_cached_books('returning from books()',indent=2)
                 return booklist
         else:
             return []
@@ -506,6 +518,19 @@ class ITUNES(DevicePlugin):
         '''
         return (None,None)
 
+    @classmethod
+    def config_widget(cls):
+        '''
+        Return a QWidget with settings for the device interface
+        '''
+        cw = DriverBase.config_widget()
+        # Turn off the Save template
+        cw.opt_save_template.setVisible(False)
+        cw.label.setVisible(False)
+        # Repurpose the checkbox
+        cw.opt_read_metadata.setText(_("Use Series as Genre in iTunes/iBooks"))
+        return cw
+
     def delete_books(self, paths, end_session=True):
         '''
         Delete books at paths on device.
@@ -685,6 +710,9 @@ class ITUNES(DevicePlugin):
         @param booklists:  A tuple containing the result of calls to
                                 (L{books}(oncard=None), L{books}(oncard='carda'),
                                 L{books}(oncard='cardb')).
+
+        NB: This will not find books that were added by a different installation of calibre
+            as uuids are different
         '''
         if DEBUG:
             self.log.info("ITUNES.remove_books_from_metadata()")
@@ -732,17 +760,6 @@ class ITUNES(DevicePlugin):
         '''
         self.report_progress = report_progress
 
-    def settings(self):
-        '''
-        Should return an opts object. The opts object should have one attribute
-        `format_map` which is an ordered list of formats for the device.
-        '''
-        klass = self if isinstance(self, type) else self.__class__
-        c = Config('device_drivers_%s' % klass.__name__, _('settings for device drivers'))
-        c.add_opt('format_map', default=self.FORMATS,
-            help=_('Ordered list of formats the device will accept'))
-        return c.parse()
-
     def sync_booklists(self, booklists, end_session=True):
         '''
         Update metadata on device.
@@ -750,6 +767,10 @@ class ITUNES(DevicePlugin):
                                 (L{books}(oncard=None), L{books}(oncard='carda'),
                                 L{books}(oncard='cardb')).
         '''
+
+        if DEBUG:
+            self.log.info("ITUNES.sync_booklists()")
+
         if self.update_needed:
             if DEBUG:
                 self.log.info(' calling _update_device')
@@ -812,30 +833,33 @@ class ITUNES(DevicePlugin):
         self.problem_msg = _("Some cover art could not be converted.\n"
                                      "Click 'Show Details' for a list.")
 
-        if False:
+        if DEBUG:
             self.log.info("ITUNES.upload_books()")
             self._dump_files(files, header='upload_books()',indent=2)
             self._dump_update_list(header='upload_books()',indent=2)
 
         if isosx:
             for (i,file) in enumerate(files):
+                format = file.rpartition('.')[2].lower()
                 path = self.path_template % (metadata[i].title, metadata[i].author[0])
                 self._remove_existing_copy(path, metadata[i])
-                fpath = self._get_fpath(file, metadata[i], update_md=True)
+                fpath = self._get_fpath(file, metadata[i], format, update_md=True)
                 db_added, lb_added = self._add_new_copy(fpath, metadata[i])
-                thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added)
+                thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
-                this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb)
+                this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
                 new_booklist.append(this_book)
                 self._update_iTunes_metadata(metadata[i], db_added, lb_added, this_book)
 
                 # Add new_book to self.cached_paths
                 self.cached_books[this_book.path] = {
-                    'title': metadata[i].title,
                    'author': metadata[i].author,
-                 'lib_book': lb_added,
                  'dev_book': db_added,
+                   'format': format,
+                 'lib_book': lb_added,
+                    'title': metadata[i].title,
                      'uuid': metadata[i].uuid }
 
+
                 # Report progress
                 if self.report_progress is not None:
                     self.report_progress(i+1/file_count, _('%d of %d') % (i+1, file_count))
@@ -846,9 +870,10 @@ class ITUNES(DevicePlugin):
                 self.iTunes = win32com.client.Dispatch("iTunes.Application")
 
                 for (i,file) in enumerate(files):
+                    format = file.rpartition('.')[2].lower()
                     path = self.path_template % (metadata[i].title, metadata[i].author[0])
                     self._remove_existing_copy(path, metadata[i])
-                    fpath = self._get_fpath(file, metadata[i], update_md=True)
+                    fpath = self._get_fpath(file, metadata[i],format, update_md=True)
                     db_added, lb_added = self._add_new_copy(fpath, metadata[i])
 
                     if self.manual_sync_mode and not db_added:
@@ -857,17 +882,18 @@ class ITUNES(DevicePlugin):
                                             "Click 'Show Details...' for affected books.")
                         self.problem_titles.append("'%s' by %s" % (metadata[i].title, metadata[i].author[0]))
 
-                    thumb = self._cover_to_thumb(path, metadata[i], lb_added, db_added)
+                    thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
-                    this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb)
+                    this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
                     new_booklist.append(this_book)
                     self._update_iTunes_metadata(metadata[i], db_added, lb_added, this_book)
 
                     # Add new_book to self.cached_paths
                     self.cached_books[this_book.path] = {
-                        'title': metadata[i].title,
                        'author': metadata[i].author[0],
-                     'lib_book': lb_added,
                      'dev_book': db_added,
+                       'format': format,
+                     'lib_book': lb_added,
+                        'title': metadata[i].title,
                          'uuid': metadata[i].uuid}
 
                     # Report progress
@@ -968,7 +994,8 @@ class ITUNES(DevicePlugin):
                         db_added = self._find_device_book(
                                      {'title': metadata.title,
                                      'author': metadata.authors[0],
-                                       'uuid': metadata.uuid})
+                                       'uuid': metadata.uuid,
+                                     'format': fpath.rpartition('.')[2].lower()})
 
                     return db_added
 
@@ -1021,7 +1048,8 @@ class ITUNES(DevicePlugin):
                 added = self._find_library_book(
                     { 'title': metadata.title,
                      'author': metadata.author[0],
-                       'uuid': metadata.uuid})
+                       'uuid': metadata.uuid,
+                     'format': file.rpartition('.')[2].lower()})
         return added
 
     def _add_new_copy(self, fpath, metadata):
@@ -1047,23 +1075,50 @@ class ITUNES(DevicePlugin):
 
         return db_added, lb_added
 
-    def _cover_to_thumb(self, path, metadata, db_added, lb_added):
+    def _cover_to_thumb(self, path, metadata, db_added, lb_added, format):
         '''
         assumes pythoncom wrapper for db_added
+        as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
         '''
         self.log.info(" ITUNES._cover_to_thumb()")
+
         thumb = None
         if metadata.cover:
+
+            if (format == 'epub'):
+                # Pre-shrink cover
+                # self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT
+                try:
+                    img = PILImage.open(metadata.cover)
+                    width = img.size[0]
+                    height = img.size[1]
+                    scaled, nwidth, nheight = fit_image(width, height, self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT)
+                    if scaled:
+                        if DEBUG:
+                            self.log.info("   '%s' scaled from %sx%s to %sx%s" %
+                                          (metadata.cover,width,height,nwidth,nheight))
+                        img = img.resize((nwidth, nheight), PILImage.ANTIALIAS)
+                        cd = cStringIO.StringIO()
+                        img.convert('RGB').save(cd, 'JPEG')
+                        cover_data = cd.getvalue()
+                        cd.close()
+                    else:
+                        with open(metadata.cover,'r+b') as cd:
+                            cover_data = cd.read()
+                except:
+                    self.problem_titles.append("'%s' by %s" % (metadata.title, metadata.author[0]))
+                    self.log.error("  error scaling '%s' for '%s'" % (metadata.cover,metadata.title))
+                    return thumb
+
                 if isosx:
-                cover_data = open(metadata.cover,'rb')
                     if lb_added:
-                    lb_added.artworks[1].data_.set(cover_data.read())
+                        lb_added.artworks[1].data_.set(cover_data)
 
                     if db_added:
                         # The following command generates an error, but the artwork does in fact
                         # get sent to the device.  Seems like a bug in Apple's automation interface
                         try:
-                        db_added.artworks[1].data_.set(cover_data.read())
+                            db_added.artworks[1].data_.set(cover_data)
                         except:
                             if DEBUG:
                                 self.log.warning("  iTunes automation interface reported an error"
@@ -1076,17 +1131,26 @@ class ITUNES(DevicePlugin):
 
 
                 elif iswindows:
+                    # Write the data to a real file for Windows iTunes
+                    tc = os.path.join(tempfile.gettempdir(), "cover.jpg")
+                    with open(tc,'wb') as tmp_cover:
+                        tmp_cover.write(cover_data)
+
                     if lb_added:
                         if lb_added.Artwork.Count:
-                        lb_added.Artwork.Item(1).SetArtworkFromFile(metadata.cover)
+                            lb_added.Artwork.Item(1).SetArtworkFromFile(tc)
                         else:
-                        lb_added.AddArtworkFromFile(metadata.cover)
+                            lb_added.AddArtworkFromFile(tc)
 
                     if db_added:
                         if db_added.Artwork.Count:
-                        db_added.Artwork.Item(1).SetArtworkFromFile(metadata.cover)
+                            db_added.Artwork.Item(1).SetArtworkFromFile(tc)
                         else:
-                        db_added.AddArtworkFromFile(metadata.cover)
+                            db_added.AddArtworkFromFile(tc)
+
+            elif format == 'pdf':
+                if DEBUG:
+                    self.log.info("   unable to set PDF cover via automation interface")
 
             try:
                 # Resize for thumb
@@ -1097,6 +1161,7 @@ class ITUNES(DevicePlugin):
                 of = cStringIO.StringIO()
                 im.convert('RGB').save(of, 'JPEG')
                 thumb = of.getvalue()
+                of.close()
 
                 # Refresh the thumbnail cache
                 if DEBUG:
@@ -1105,14 +1170,15 @@ class ITUNES(DevicePlugin):
                 zfw = ZipFile(archive_path, mode='a')
                 thumb_path = path.rpartition('.')[0] + '.jpg'
                 zfw.writestr(thumb_path, thumb)
-                zfw.close()
             except:
                 self.problem_titles.append("'%s' by %s" % (metadata.title, metadata.author[0]))
                 self.log.error("  error converting '%s' to thumb for '%s'" % (metadata.cover,metadata.title))
+            finally:
+                zfw.close()
 
         return thumb
 
-    def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb):
+    def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb, format):
         '''
         '''
         if DEBUG:
@@ -1122,6 +1188,7 @@ class ITUNES(DevicePlugin):
 
         this_book.db_id = None
         this_book.device_collections = []
+        this_book.format = format
         this_book.library_id = lb_added
         this_book.path = path
         this_book.thumbnail = thumb
@@ -1319,10 +1386,11 @@ class ITUNES(DevicePlugin):
                   self.cached_books[cb]['uuid']))
         elif iswindows:
             for cb in self.cached_books.keys():
-                self.log.info("%s%-40.40s %-30.30s %s" %
+                self.log.info("%s%-40.40s %-30.30s %-4.4s %s" %
                  (' '*indent,
                   self.cached_books[cb]['title'],
                   self.cached_books[cb]['author'],
+                  self.cached_books[cb]['format'],
                   self.cached_books[cb]['uuid']))
 
         self.log.info()
@@ -1338,8 +1406,9 @@ class ITUNES(DevicePlugin):
         fnames = zf.namelist()
         opf = [x for x in fnames if '.opf' in x][0]
         if opf:
-            opf_raw = cStringIO.StringIO(zf.read(opf)).getvalue()
+            opf_raw = cStringIO.StringIO(zf.read(opf))
-            soup = BeautifulSoup(opf_raw)
+            soup = BeautifulSoup(opf_raw.getvalue())
+            opf_raw.close()
             title = soup.find('dc:title').renderContents()
             author = soup.find('dc:creator').renderContents()
             ts = soup.find('meta',attrs={'name':'calibre:timestamp'})
@@ -1440,6 +1509,22 @@ class ITUNES(DevicePlugin):
                                 self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
                             return hit
 
+                # PDF metadata was rewritten at export as 'safe(title) - safe(author)'
+                if search['format'] == 'pdf':
+                    title = re.sub(r'[^0-9a-zA-Z ]', '_', search['title'])
+                    author = re.sub(r'[^0-9a-zA-Z ]', '_', search['author'])
+                    if DEBUG:
+                        self.log.info("   searching by name: '%s - %s'" % (title,author))
+                    hits = dev_books.Search('%s - %s' % (title,author),
+                                             self.SearchField.index('All'))
+                    if hits:
+                        hit = hits[0]
+                        self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
+                        return hit
+                    else:
+                        if DEBUG:
+                            self.log.info("   no PDF hits")
+
                 attempts -= 1
                 time.sleep(0.5)
                 if DEBUG:
@@ -1509,6 +1594,22 @@ class ITUNES(DevicePlugin):
                                 self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
                             return hit
 
+                # PDF metadata was rewritten at export as 'safe(title) - safe(author)'
+                if search['format'] == 'pdf':
+                    title = re.sub(r'[^0-9a-zA-Z ]', '_', search['title'])
+                    author = re.sub(r'[^0-9a-zA-Z ]', '_', search['author'])
+                    if DEBUG:
+                        self.log.info("   searching by name: %s - %s" % (title,author))
+                    hits = lib_books.Search('%s - %s' % (title,author),
+                                             self.SearchField.index('All'))
+                    if hits:
+                        hit = hits[0]
+                        self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
+                        return hit
+                    else:
+                        if DEBUG:
+                            self.log.info("   no PDF hits")
+
                 attempts -= 1
                 time.sleep(0.5)
                 if DEBUG:
@@ -1523,10 +1624,12 @@ class ITUNES(DevicePlugin):
         Convert iTunes artwork to thumbnail
         Cache generated thumbnails
         cache_dir = os.path.join(config_dir, 'caches', 'itunes')
+        as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
         '''
 
         archive_path = os.path.join(self.cache_dir, "thumbs.zip")
         thumb_path = book_path.rpartition('.')[0] + '.jpg'
+        format = book_path.rpartition('.')[2].lower()
 
         try:
             zfr = ZipFile(archive_path)
@@ -1539,22 +1642,30 @@ class ITUNES(DevicePlugin):
 
         self.log.info(" ITUNES._generate_thumbnail()")
         if isosx:
+            if format == 'epub':
                 try:
+                    if False:
+                        self.log.info("   fetching artwork from %s\n   %s" % (book_path,book))
                     # Resize the cover
                     data = book.artworks[1].raw_data().data
                     #self._dump_hex(data[:256])
-                im = PILImage.open(cStringIO.StringIO(data))
+                    img_data = cStringIO.StringIO(data)
+                    im = PILImage.open(img_data)
                     scaled, width, height = fit_image(im.size[0],im.size[1], 60, 80)
                     im = im.resize((int(width),int(height)), PILImage.ANTIALIAS)
+                    img_data.close()
+
                     thumb = cStringIO.StringIO()
                     im.convert('RGB').save(thumb,'JPEG')
+                    thumb_data = thumb.getvalue()
+                    thumb.close()
 
                     # Cache the tagged thumb
                     if DEBUG:
                         self.log.info("  generated thumb for '%s', caching" % book.name())
-                zfw.writestr(thumb_path, thumb.getvalue())
+                    zfw.writestr(thumb_path, thumb_data)
                     zfw.close()
-                return thumb.getvalue()
+                    return thumb_data
                 except:
                     self.log.error("  error generating thumb for '%s'" % book.name())
                     try:
@@ -1562,14 +1673,19 @@ class ITUNES(DevicePlugin):
                     except:
                         pass
                     return None
+            else:
+                if DEBUG:
+                    self.log.info("  unable to generate PDF thumbs")
+                return None
 
         elif iswindows:
 
             if not book.Artwork.Count:
                 if DEBUG:
-                    self.log.info("  no artwork available")
+                    self.log.info("  no artwork available for '%s'" % book.Name)
                 return None
 
+            if format == 'epub':
                 # Save the cover from iTunes
                 try:
                     tmp_thumb = os.path.join(tempfile.gettempdir(), "thumb.%s" % self.ArtworkFormat[book.Artwork.Item(1).Format])
@@ -1580,14 +1696,16 @@ class ITUNES(DevicePlugin):
                     im = im.resize((int(width),int(height)), PILImage.ANTIALIAS)
                     thumb = cStringIO.StringIO()
                     im.convert('RGB').save(thumb,'JPEG')
+                    thumb_data = thumb.getvalue()
                     os.remove(tmp_thumb)
+                    thumb.close()
 
                     # Cache the tagged thumb
                     if DEBUG:
                         self.log.info("  generated thumb for '%s', caching" % book.Name)
-                zfw.writestr(thumb_path, thumb.getvalue())
+                    zfw.writestr(thumb_path, thumb_data)
                     zfw.close()
-                return thumb.getvalue()
+                    return thumb_data
                 except:
                     self.log.error("  error generating thumb for '%s'" % book.Name)
                     try:
@@ -1595,11 +1713,18 @@ class ITUNES(DevicePlugin):
                     except:
                         pass
                     return None
+            else:
+                if DEBUG:
+                    self.log.info("  unable to generate PDF thumbs")
+                return None
 
     def _get_device_book_size(self, file, compressed_size):
         '''
         Calculate the exploded size of file
         '''
+        exploded_file_size = compressed_size
+        format = file.rpartition('.')[2].lower()
+        if format == 'epub':
             myZip = ZipFile(file,'r')
             myZipList = myZip.infolist()
             exploded_file_size = 0
@@ -1701,7 +1826,7 @@ class ITUNES(DevicePlugin):
                         self.log.error("  no iPad|Books playlist found")
                 return pl
 
-    def _get_fpath(self,file, metadata, update_md=False):
+    def _get_fpath(self,file, metadata, format, update_md=False):
         '''
         If the database copy will be deleted after upload, we have to
         use file (the PersistentTemporaryFile), which will be around until
@@ -1725,7 +1850,7 @@ class ITUNES(DevicePlugin):
             if DEBUG:
                 self.log.info("   file will be deleted after upload")
 
-        if update_md:
+        if format == 'epub' and update_md:
             self._update_epub_metadata(fpath, metadata)
 
         return fpath
@@ -1950,10 +2075,12 @@ class ITUNES(DevicePlugin):
 
             # Read the current storage path for iTunes media from the XML file
             with open(self.iTunes.LibraryXMLPath, 'r') as xml:
-                soup = BeautifulSoup(xml.read().decode('utf-8'))
+                for line in xml:
-                mf = soup.find('key',text="Music Folder").parent
+                    if line.strip().startswith('<key>Music Folder'):
-                string = mf.findNext('string').renderContents()
+                        soup = BeautifulSoup(line)
+                        string = soup.find('string').renderContents()
                         media_dir = os.path.abspath(string[len('file://localhost/'):].replace('%20',' '))
+                        break
                 if os.path.exists(media_dir):
                     self.iTunes_media = media_dir
                 else:
@@ -2028,7 +2155,9 @@ class ITUNES(DevicePlugin):
             # Delete existing from Library|Books, add to self.update_list
             # for deletion from booklist[0] during add_books_to_metadata
             for book in self.cached_books:
-                if self.cached_books[book]['uuid'] == metadata.uuid:
+                if (self.cached_books[book]['uuid'] == metadata.uuid) or \
+                   (self.cached_books[book]['title'] == metadata.title and \
+                    self.cached_books[book]['author'] == metadata.authors[0]):
                     self.update_list.append(self.cached_books[book])
                     self._remove_from_iTunes(self.cached_books[book])
                     if DEBUG:
@@ -2036,7 +2165,7 @@ class ITUNES(DevicePlugin):
                     break
                 else:
                     if DEBUG:
-                        self.log.info("  '%s' not in cached_books" % metadata.title)
+                        self.log.info("  '%s' not found in cached_books" % metadata.title)
 
     def _remove_from_device(self, cached_book):
         '''
@@ -2158,12 +2287,14 @@ class ITUNES(DevicePlugin):
             fnames = zf_opf.namelist()
             opf = [x for x in fnames if '.opf' in x][0]
             if opf:
-                opf_raw = cStringIO.StringIO(zf_opf.read(opf)).getvalue()
+                opf_raw = cStringIO.StringIO(zf_opf.read(opf))
-                soup = BeautifulSoup(opf_raw)
+                soup = BeautifulSoup(opf_raw.getvalue())
+                opf_raw.close()
+
+                # Touch existing calibre timestamp
                 md = soup.find('metadata')
                 ts = md.find('meta',attrs={'name':'calibre:timestamp'})
                 if ts:
-                    # Touch existing calibre timestamp
                     timestamp = ts['content']
                     old_ts = parse_date(timestamp)
                     metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour,
@@ -2172,6 +2303,15 @@ class ITUNES(DevicePlugin):
                     metadata.timestamp = isoformat(now())
                     if DEBUG:
                         self.log.info("   add timestamp: %s" % metadata.timestamp)
+
+                # Fix the language declaration for iBooks 1.1
+                patched_language = 'en-US'
+                language = md.find('dc:language')
+                if language:
+                    self.log.info("   changing <dc:language> from '%s' to '%s'" %
+                                  (language.renderContents(),patched_language))
+                metadata.language = patched_language
+
             zf_opf.close()
 
             # If 'News' in tags, tweak the title/author for friendlier display in iBooks
@@ -2257,6 +2397,9 @@ class ITUNES(DevicePlugin):
                 lb_added.enabled.set(True)
                 lb_added.sort_artist.set(metadata.author_sort.title())
                 lb_added.sort_name.set(this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    lb_added.artist.set(metadata.authors[0])
+                    lb_added.name.set(metadata.title)
 
             if db_added:
                 db_added.album.set(metadata.title)
@@ -2265,6 +2408,9 @@ class ITUNES(DevicePlugin):
                 db_added.enabled.set(True)
                 db_added.sort_artist.set(metadata.author_sort.title())
                 db_added.sort_name.set(this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    db_added.artist.set(metadata.authors[0])
+                    db_added.name.set(metadata.title)
 
             if metadata.comments:
                 if lb_added:
@@ -2284,7 +2430,9 @@ class ITUNES(DevicePlugin):
 
             # Set genre from series if available, else first alpha tag
             # Otherwise iTunes grabs the first dc:subject from the opf metadata
-            if self.use_series_data and metadata.series:
+            if metadata.series and self.settings().read_metadata:
+                if DEBUG:
+                    self.log.info("   using Series name as Genre")
                 if lb_added:
                     lb_added.sort_name.set("%s %03d" % (metadata.series, metadata.series_index))
                     lb_added.genre.set(metadata.series)
@@ -2298,6 +2446,8 @@ class ITUNES(DevicePlugin):
                     db_added.episode_number.set(metadata.series_index)
 
             elif metadata.tags:
+                if DEBUG:
+                    self.log.info("   using Tag as Genre")
                 for tag in metadata.tags:
                     if self._is_alpha(tag[0]):
                         if lb_added:
@@ -2314,6 +2464,9 @@ class ITUNES(DevicePlugin):
                 lb_added.Enabled = True
                 lb_added.SortArtist = (metadata.author_sort.title())
                 lb_added.SortName = (this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    lb_added.Artist = metadata.authors[0]
+                    lb_added.Name = metadata.title
 
             if db_added:
                 db_added.Album = metadata.title
@@ -2322,6 +2475,9 @@ class ITUNES(DevicePlugin):
                 db_added.Enabled = True
                 db_added.SortArtist = (metadata.author_sort.title())
                 db_added.SortName = (this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    db_added.Artist = metadata.authors[0]
+                    db_added.Name = metadata.title
 
             if metadata.comments:
                 if lb_added:
@@ -2345,7 +2501,9 @@ class ITUNES(DevicePlugin):
             # Otherwise iBooks uses first <dc:subject> from opf
             # iTunes balks on setting EpisodeNumber, but it sticks (9.1.1.12)
 
-            if self.use_series_data and metadata.series:
+            if metadata.series and self.settings().read_metadata:
+                if DEBUG:
+                    self.log.info("   using Series name as Genre")
                 if lb_added:
                     lb_added.SortName = "%s %03d" % (metadata.series, metadata.series_index)
                     lb_added.Genre = metadata.series
@@ -2365,6 +2523,8 @@ class ITUNES(DevicePlugin):
                             self.log.warning("  iTunes automation interface reported an error"
                                              " setting EpisodeNumber on iDevice")
             elif metadata.tags:
+                if DEBUG:
+                    self.log.info("   using Tag as Genre")
                 for tag in metadata.tags:
                     if self._is_alpha(tag[0]):
                         if lb_added:

src/calibre/devices/interface.py

@@ -59,7 +59,7 @@ class DevicePlugin(Plugin):
             return cls.__name__
         return cls.name
 
-
+    # Device detection {{{
     def test_bcd_windows(self, device_id, bcd):
         if bcd is None or len(bcd) == 0:
             return True
@@ -152,6 +152,7 @@ class DevicePlugin(Plugin):
                                     return True, dev
         return False, None
 
+    # }}}
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None) :
@@ -378,8 +379,6 @@ class DevicePlugin(Plugin):
         raise NotImplementedError()
 
 
-
-
 class BookList(list):
     '''
     A list of books. Each Book object must have the fields:

src/calibre/devices/kindle/driver.py

@@ -429,6 +429,7 @@ class Bookmark():
                 entries, = unpack('>I', data[9:13])
                 current_entry = 0
                 e_base = 0x0d
+                self.pdf_page_offset = 0
                 while current_entry < entries:
                     '''
                     location, = unpack('>I', data[e_base+2:e_base+6])

src/calibre/devices/usbms/device.py

@@ -78,9 +78,6 @@ class Device(DeviceConfig, DevicePlugin):
     STORAGE_CARD_VOLUME_LABEL = ''
     STORAGE_CARD2_VOLUME_LABEL = None
 
-    SUPPORTS_SUB_DIRS = False
-    MUST_READ_METADATA = False
-    SUPPORTS_USE_AUTHOR_SORT = False
 
     EBOOK_DIR_MAIN = ''
     EBOOK_DIR_CARD_A = ''

src/calibre/devices/usbms/deviceconfig.py

@@ -13,6 +13,10 @@ class DeviceConfig(object):
     EXTRA_CUSTOMIZATION_MESSAGE = None
     EXTRA_CUSTOMIZATION_DEFAULT = None
 
+    SUPPORTS_SUB_DIRS = False
+    MUST_READ_METADATA = False
+    SUPPORTS_USE_AUTHOR_SORT = False
+
     #: If None the default is used
     SAVE_TEMPLATE = None
 
@@ -23,9 +27,14 @@ class DeviceConfig(object):
             config().parse().send_template
 
     @classmethod
-    def _config(cls):
+    def _config_base_name(cls):
         klass = cls if isinstance(cls, type) else cls.__class__
-        c = Config('device_drivers_%s' % klass.__name__, _('settings for device drivers'))
+        return klass.__name__
+
+    @classmethod
+    def _config(cls):
+        name = cls._config_base_name()
+        c = Config('device_drivers_%s' % name, _('settings for device drivers'))
         c.add_opt('format_map', default=cls.FORMATS,
                 help=_('Ordered list of formats the device will accept'))
         c.add_opt('use_subdirs', default=True,

src/calibre/ebooks/oeb/stylizer.py

@@ -126,6 +126,13 @@ class Stylizer(object):
             head = head[0]
         else:
             head = []
+
+        # Add cssutils parsing profiles from output_profile
+        for profile in self.opts.output_profile.extra_css_modules:
+            cssutils.profile.addProfile(profile['name'],
+                                        profile['props'],
+                                        profile['macros'])
+
         parser = cssutils.CSSParser(fetcher=self._fetch_css_file,
                 log=logging.getLogger('calibre.css'))
         self.font_face_rules = []

src/calibre/gui2/actions.py

@@ -176,6 +176,7 @@ class AnnotationsAction(object): # {{{
 
             def mark_book_as_read(self,id):
                 read_tag = gprefs.get('catalog_epub_mobi_read_tag')
+                if read_tag:
                     self.db.set_tags(id, [read_tag], append=True)
 
             def canceled(self):

src/calibre/web/feeds/news.py

@@ -585,6 +585,8 @@ class BasicNewsRecipe(Recipe):
         self.lrf = options.lrf
         self.output_profile = options.output_profile
         self.touchscreen = getattr(self.output_profile, 'touchscreen', False)
+        if self.touchscreen:
+            self.template_css += self.output_profile.touchscreen_news_css
 
         self.output_dir = os.path.abspath(self.output_dir)
         if options.test:
@@ -638,7 +640,8 @@ class BasicNewsRecipe(Recipe):
         if self.delay > 0:
             self.simultaneous_downloads = 1
 
-        self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else templates.NavBarTemplate()
+        self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else \
+                      templates.NavBarTemplate()
         self.failed_downloads = []
         self.partial_failures = []
 
@@ -726,7 +729,6 @@ class BasicNewsRecipe(Recipe):
         timefmt = self.timefmt
         if self.touchscreen:
             templ = templates.TouchscreenIndexTemplate()
-            timefmt = '%A, %d %b %Y'
         return templ.generate(self.title, "mastheadImage.jpg", timefmt, feeds,
                               extra_css=css).render(doctype='xhtml')
 
@@ -752,7 +754,8 @@ class BasicNewsRecipe(Recipe):
 
 
 
-    def feed2index(self, feed):
+    def feed2index(self, f, feeds):
+        feed = feeds[f]
         if feed.image_url is not None: # Download feed image
             imgdir = os.path.join(self.output_dir, 'images')
             if not os.path.isdir(imgdir):
@@ -782,33 +785,9 @@ class BasicNewsRecipe(Recipe):
         css = self.template_css + '\n\n' +(self.extra_css if self.extra_css else '')
 
         if self.touchscreen:
-            touchscreen_css = u'''
-                    .summary_headline {
-                        font-weight:bold; text-align:left;
-                    }
-
-                    .summary_byline {
-                        text-align:left;
-                        font-family:monospace;
-                    }
-
-                    .summary_text {
-                        text-align:left;
-                    }
-
-                    .feed {
-                        font-family:sans-serif; font-weight:bold; font-size:larger;
-                    }
-
-                    .calibre_navbar {
-                        font-family:monospace;
-                    }
-
-            '''
-
             templ = templates.TouchscreenFeedTemplate()
-            css = touchscreen_css + '\n\n' + (self.extra_css if self.extra_css else '')
+
-        return templ.generate(feed, self.description_limiter,
+        return templ.generate(f, feeds, self.description_limiter,
                               extra_css=css).render(doctype='xhtml')
 
 
@@ -951,7 +930,7 @@ class BasicNewsRecipe(Recipe):
         #feeds.restore_duplicates()
 
         for f, feed in enumerate(feeds):
-            html = self.feed2index(feed)
+            html = self.feed2index(f,feeds)
             feed_dir = os.path.join(self.output_dir, 'feed_%d'%f)
             with open(os.path.join(feed_dir, 'index.html'), 'wb') as fi:
                 fi.write(html)

src/calibre/web/feeds/templates.py

@@ -3,9 +3,12 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
+
+import copy
+
 from lxml import html, etree
 from lxml.html.builder import HTML, HEAD, TITLE, STYLE, DIV, BODY, \
-        STRONG, BR, SPAN, A, HR, UL, LI, H2, IMG, P as PT, \
+        STRONG, EM, BR, SPAN, A, HR, UL, LI, H2, IMG, P as PT, \
         TABLE, TD, TR
 
 from calibre import preferred_encoding, strftime, isbytestring
@@ -14,6 +17,7 @@ def CLASS(*args, **kwargs): # class is a reserved word in Python
     kwargs['class'] = ' '.join(args)
     return kwargs
 
+# Regular templates
 class Template(object):
 
     IS_HTML = True
@@ -44,105 +48,35 @@ class Template(object):
         return etree.tostring(self.root, encoding='utf-8', xml_declaration=True,
                 pretty_print=True)
 
-class NavBarTemplate(Template):
+class EmbeddedContent(Template):
 
-    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
+    def _generate(self, article, style=None, extra_css=None):
-                 two_levels, url, __appname__, prefix='', center=True,
+        content = article.content if article.content else ''
-                 extra_css=None, style=None):
+        summary = article.summary if article.summary else ''
-        head = HEAD(TITLE('navbar'))
+        text = content if len(content) > len(summary) else summary
+        head = HEAD(TITLE(article.title))
         if style:
             head.append(STYLE(style, type='text/css'))
         if extra_css:
             head.append(STYLE(extra_css, type='text/css'))
 
-        if prefix and not prefix.endswith('/'):
+        if isbytestring(text):
-            prefix += '/'
+            text = text.decode('utf-8', 'replace')
-        align = 'center' if center else 'left'
+        elements = html.fragments_fromstring(text)
-        navbar = DIV(CLASS('calibre_navbar', 'calibre_rescale_70',
+        self.root = HTML(head,
-            style='text-align:'+align))
+                BODY(H2(article.title), DIV()))
-        if bottom:
+        div = self.root.find('body').find('div')
-            navbar.append(HR())
+        if elements and isinstance(elements[0], unicode):
-            text = 'This article was downloaded by '
+            div.text = elements[0]
-            p = PT(text, STRONG(__appname__), A(url, href=url), style='text-align:left')
+            elements = list(elements)[1:]
-            p[0].tail = ' from '
+        for elem in elements:
-            navbar.append(p)
+            elem.getparent().remove(elem)
-            navbar.append(BR())
+            div.append(elem)
-            navbar.append(BR())
-        else:
-            next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
-                    else 'article_%d'%(art+1)
-            up = '../..' if art == number_of_articles_in_feed - 1 else '..'
-            href = '%s%s/%s/index.html'%(prefix, up, next)
-            navbar.text = '| '
-            navbar.append(A('Next', href=href))
-        href = '%s../index.html#article_%d'%(prefix, art)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Section Menu', href=href))
-        href = '%s../../index.html#feed_%d'%(prefix, feed)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Main Menu', href=href))
-        if art > 0 and not bottom:
-            href = '%s../article_%d/index.html'%(prefix, art-1)
-            navbar.iterchildren(reversed=True).next().tail = ' | '
-            navbar.append(A('Previous', href=href))
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        if not bottom:
-            navbar.append(HR())
-
-        self.root = HTML(head, BODY(navbar))
-
-class TouchscreenNavBarTemplate(Template):
-
-    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
-                 two_levels, url, __appname__, prefix='', center=True,
-                 extra_css=None, style=None):
-        head = HEAD(TITLE('navbar'))
-        if style:
-            head.append(STYLE(style, type='text/css'))
-        if extra_css:
-            head.append(STYLE(extra_css, type='text/css'))
-
-        if prefix and not prefix.endswith('/'):
-            prefix += '/'
-        align = 'center' if center else 'left'
-        navbar = DIV(CLASS('calibre_navbar', 'calibre_rescale_100',
-            style='text-align:'+align))
-        if bottom:
-            navbar.append(DIV(style="border-top:1px solid gray;border-bottom:1em solid white"))
-            text = 'This article was downloaded by '
-            p = PT(text, STRONG(__appname__), A(url, href=url), style='text-align:left')
-            p[0].tail = ' from '
-            navbar.append(p)
-            navbar.append(BR())
-            navbar.append(BR())
-        else:
-            next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
-                    else 'article_%d'%(art+1)
-            up = '../..' if art == number_of_articles_in_feed - 1 else '..'
-            href = '%s%s/%s/index.html'%(prefix, up, next)
-            navbar.text = '| '
-            navbar.append(A('Next', href=href))
-
-        href = '%s../index.html#article_%d'%(prefix, art)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Section Menu', href=href))
-        href = '%s../../index.html#feed_%d'%(prefix, feed)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Main Menu', href=href))
-        if art > 0 and not bottom:
-            href = '%s../article_%d/index.html'%(prefix, art-1)
-            navbar.iterchildren(reversed=True).next().tail = ' | '
-            navbar.append(A('Previous', href=href))
-
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        if not bottom:
-            navbar.append(DIV(style="border-top:1px solid gray;border-bottom:1em solid white"))
-
-        self.root = HTML(head, BODY(navbar))
 
 class IndexTemplate(Template):
 
     def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None):
+        self.IS_HTML = False
         if isinstance(datefmt, unicode):
             datefmt = datefmt.encode(preferred_encoding)
         date = strftime(datefmt)
@@ -164,43 +98,10 @@ class IndexTemplate(Template):
                 CLASS('calibre_rescale_100'))
         self.root = HTML(head, BODY(div))
 
-class TouchscreenIndexTemplate(Template):
-
-    def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None):
-        if isinstance(datefmt, unicode):
-            datefmt = datefmt.encode(preferred_encoding)
-        date = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
-        masthead_p = etree.Element("p")
-        masthead_p.set("style","text-align:center")
-        masthead_img = etree.Element("img")
-        masthead_img.set("src",masthead)
-        masthead_img.set("alt","masthead")
-        masthead_p.append(masthead_img)
-
-        head = HEAD(TITLE(title))
-        if style:
-            head.append(STYLE(style, type='text/css'))
-        if extra_css:
-            head.append(STYLE(extra_css, type='text/css'))
-
-        toc = TABLE(CLASS('toc'),width="100%",border="0",cellpadding="3px")
-        for i, feed in enumerate(feeds):
-            if feed:
-                tr = TR()
-                tr.append(TD( CLASS('calibre_rescale_120'), A(feed.title, href='feed_%d/index.html'%i)))
-                tr.append(TD( '%s' % len(feed.articles), style="text-align:right"))
-                toc.append(tr)
-        div = DIV(
-                masthead_p,
-                PT(date, style='text-align:center'),
-                #DIV(style="border-color:gray;border-top-style:solid;border-width:thin"),
-                DIV(style="border-top:1px solid gray;border-bottom:1em solid white"),
-                toc)
-        self.root = HTML(head, BODY(div))
-
 class FeedTemplate(Template):
 
-    def _generate(self, feed, cutoff, extra_css=None, style=None):
+    def _generate(self, f, feeds, cutoff, extra_css=None, style=None):
+        feed = feeds[f]
         head = HEAD(TITLE(feed.title))
         if style:
             head.append(STYLE(style, type='text/css'))
@@ -248,9 +149,147 @@ class FeedTemplate(Template):
 
         self.root = HTML(head, body)
 
+class NavBarTemplate(Template):
+
+    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
+                 two_levels, url, __appname__, prefix='', center=True,
+                 extra_css=None, style=None):
+        head = HEAD(TITLE('navbar'))
+        if style:
+            head.append(STYLE(style, type='text/css'))
+        if extra_css:
+            head.append(STYLE(extra_css, type='text/css'))
+
+        if prefix and not prefix.endswith('/'):
+            prefix += '/'
+        align = 'center' if center else 'left'
+
+        navbar = DIV(CLASS('calibre_navbar', 'calibre_rescale_70',
+            style='text-align:'+align))
+        if bottom:
+            navbar.append(HR())
+            text = 'This article was downloaded by '
+            p = PT(text, STRONG(__appname__), A(url, href=url), style='text-align:left')
+            p[0].tail = ' from '
+            navbar.append(p)
+            navbar.append(BR())
+            navbar.append(BR())
+        else:
+            next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
+                    else 'article_%d'%(art+1)
+            up = '../..' if art == number_of_articles_in_feed - 1 else '..'
+            href = '%s%s/%s/index.html'%(prefix, up, next)
+            navbar.text = '| '
+            navbar.append(A('Next', href=href))
+        href = '%s../index.html#article_%d'%(prefix, art)
+        navbar.iterchildren(reversed=True).next().tail = ' | '
+        navbar.append(A('Section Menu', href=href))
+        href = '%s../../index.html#feed_%d'%(prefix, feed)
+        navbar.iterchildren(reversed=True).next().tail = ' | '
+        navbar.append(A('Main Menu', href=href))
+        if art > 0 and not bottom:
+            href = '%s../article_%d/index.html'%(prefix, art-1)
+            navbar.iterchildren(reversed=True).next().tail = ' | '
+            navbar.append(A('Previous', href=href))
+        navbar.iterchildren(reversed=True).next().tail = ' | '
+        if not bottom:
+            navbar.append(HR())
+
+        self.root = HTML(head, BODY(navbar))
+
+
+# Touchscreen templates
+class TouchscreenIndexTemplate(Template):
+
+    def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None):
+
+        self.IS_HTML = False
+
+        if isinstance(datefmt, unicode):
+            datefmt = datefmt.encode(preferred_encoding)
+        date = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
+        masthead_p = etree.Element("p")
+        masthead_p.set("style","text-align:center")
+        masthead_img = etree.Element("img")
+        masthead_img.set("src",masthead)
+        masthead_img.set("alt","masthead")
+        masthead_p.append(masthead_img)
+
+        head = HEAD(TITLE(title))
+        if style:
+            head.append(STYLE(style, type='text/css'))
+        if extra_css:
+            head.append(STYLE(extra_css, type='text/css'))
+
+        toc = TABLE(CLASS('toc'),width="100%",border="0",cellpadding="3px")
+        for i, feed in enumerate(feeds):
+            if feed:
+                tr = TR()
+                tr.append(TD( CLASS('calibre_rescale_120'), A(feed.title, href='feed_%d/index.html'%i)))
+                tr.append(TD( '%s' % len(feed.articles), style="text-align:right"))
+                toc.append(tr)
+        div = DIV(
+                masthead_p,
+                PT(date, style='text-align:center'),
+                #DIV(style="border-color:gray;border-top-style:solid;border-width:thin"),
+                DIV(style="border-top:1px solid gray;border-bottom:1em solid white"),
+                toc)
+        self.root = HTML(head, BODY(div))
+
 class TouchscreenFeedTemplate(Template):
 
-    def _generate(self, feed, cutoff, extra_css=None, style=None):
+    def _generate(self, f, feeds, cutoff, extra_css=None, style=None):
+
+        def trim_title(title,clip=18):
+            if len(title)>clip:
+                tokens = title.split(' ')
+                new_title_tokens = []
+                new_title_len = 0
+                if len(tokens[0]) > clip:
+                    return tokens[0][:clip] + '...'
+                for token in tokens:
+                    if len(token) + new_title_len < clip:
+                        new_title_tokens.append(token)
+                        new_title_len += len(token)
+                    else:
+                        new_title_tokens.append('...')
+                        title = ' '.join(new_title_tokens)
+                        break
+            return title
+
+        self.IS_HTML = False
+        feed = feeds[f]
+
+        # Construct the navbar
+        navbar_t = TABLE(CLASS('touchscreen_navbar'))
+        navbar_tr = TR()
+
+        # Previous Section
+        link = ''
+        if f > 0:
+            link = A(CLASS('feed_link'),
+                     trim_title(feeds[f-1].title),
+                     href = '../feed_%d/index.html' % int(f-1))
+        navbar_tr.append(TD(link, width="40%", align="center"))
+
+        # Up to Sections
+        link = A(STRONG('Sections'), href="../index.html")
+        navbar_tr.append(TD(link,width="20%",align="center"))
+
+        # Next Section
+        link = ''
+        if f < len(feeds)-1:
+            link = A(CLASS('feed_link'),
+                     trim_title(feeds[f+1].title),
+                     href = '../feed_%d/index.html' % int(f+1))
+        navbar_tr.append(TD(link, width="40%", align="center", ))
+        navbar_t.append(navbar_tr)
+        top_navbar = navbar_t
+        bottom_navbar = copy.copy(navbar_t)
+        #print "\n%s\n" % etree.tostring(navbar_t, pretty_print=True)
+
+
+        # Build the page
         head = HEAD(TITLE(feed.title))
         if style:
             head.append(STYLE(style, type='text/css'))
@@ -258,10 +297,11 @@ class TouchscreenFeedTemplate(Template):
             head.append(STYLE(extra_css, type='text/css'))
         body = BODY(style='page-break-before:always')
         div = DIV(
-                H2(feed.title, CLASS('calibre_feed_title', 'calibre_rescale_160')),
+                top_navbar,
-                DIV(style="border-top:1px solid gray;border-bottom:1em solid white")
+                H2(feed.title, CLASS('feed_title'))
                 )
         body.append(div)
+
         if getattr(feed, 'image', None):
             div.append(DIV(IMG(
                 alt = feed.image_alt if feed.image_alt else '',
@@ -280,7 +320,6 @@ class TouchscreenFeedTemplate(Template):
                 continue
             tr = TR()
 
-            if True:
             div_td = DIV(
                     A(article.title, CLASS('summary_headline','calibre_rescale_120',
                                     href=article.url)),
@@ -292,53 +331,53 @@ class TouchscreenFeedTemplate(Template):
                 div_td.append(DIV(cutoff(article.text_summary),
                     CLASS('summary_text', 'calibre_rescale_100')))
             tr.append(TD(div_td))
-            else:
-                td = TD(
-                        A(article.title, CLASS('summary_headline','calibre_rescale_120',
-                                        href=article.url))
-                        )
-                if article.author:
-                    td.append(DIV(article.author,
-                        CLASS('summary_byline', 'calibre_rescale_100')))
-                if article.summary:
-                    td.append(DIV(cutoff(article.text_summary),
-                        CLASS('summary_text', 'calibre_rescale_100')))
-
-                tr.append(td)
-
             toc.append(tr)
+
         div.append(toc)
-
+        div.append(BR())
-        navbar = DIV('| ', CLASS('calibre_navbar', 'calibre_rescale_100'),style='text-align:center')
+        div.append(bottom_navbar)
-        link = A('Up one level', href="../index.html")
-        link.tail = ' |'
-        navbar.append(link)
-        div.append(navbar)
-
         self.root = HTML(head, body)
 
-class EmbeddedContent(Template):
+class TouchscreenNavBarTemplate(Template):
 
-    def _generate(self, article, style=None, extra_css=None):
+    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
-        content = article.content if article.content else ''
+                 two_levels, url, __appname__, prefix='', center=True,
-        summary = article.summary if article.summary else ''
+                 extra_css=None, style=None):
-        text = content if len(content) > len(summary) else summary
+        head = HEAD(TITLE('navbar'))
-        head = HEAD(TITLE(article.title))
         if style:
             head.append(STYLE(style, type='text/css'))
         if extra_css:
             head.append(STYLE(extra_css, type='text/css'))
 
-        if isbytestring(text):
+        navbar = DIV()
-            text = text.decode('utf-8', 'replace')
+        navbar_t = TABLE(CLASS('touchscreen_navbar'))
-        elements = html.fragments_fromstring(text)
+        navbar_tr = TR()
-        self.root = HTML(head,
-                BODY(H2(article.title), DIV()))
-        div = self.root.find('body').find('div')
-        if elements and isinstance(elements[0], unicode):
-            div.text = elements[0]
-            elements = list(elements)[1:]
-        for elem in elements:
-            elem.getparent().remove(elem)
-            div.append(elem)
 
+        # | Previous
+        if art > 0:
+            href = '%s../article_%d/index.html'%(prefix, art-1)
+            navbar_tr.append(TD(A(EM('Previous'),href=href),
+                               width="32%"))
+        else:
+            navbar_tr.append(TD('', width="32%"))
+
+        # | Articles | Sections |
+        href = '%s../index.html#article_%d'%(prefix, art)
+        navbar_tr.append(TD(A(STRONG('Articles'), href=href),width="18%"))
+
+        href = '%s../../index.html#feed_%d'%(prefix, feed)
+        navbar_tr.append(TD(A(STRONG('Sections'), href=href),width="18%"))
+
+        # | Next
+        next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
+                else 'article_%d'%(art+1)
+        up = '../..' if art == number_of_articles_in_feed - 1 else '..'
+        href = '%s%s/%s/index.html'%(prefix, up, next)
+
+        navbar_tr.append(TD(A(EM('Next'),href=href),
+                           width="32%"))
+        navbar_t.append(navbar_tr)
+        navbar.append(navbar_t)
+        #print "\n%s\n" % etree.tostring(navbar, pretty_print=True)
+
+        self.root = HTML(head, BODY(navbar))