Merge upstream changes.

installer/linux/freeze.py

@@ -16,6 +16,7 @@ def freeze():
     from calibre.linux import entry_points
     from calibre import walk
     from calibre.web.feeds.recipes import recipe_modules
+    from calibre.ebooks.lrf.fonts import FONT_MAP
     import calibre
     
 
@@ -37,6 +38,7 @@ def freeze():
                        '/usr/lib/libxml2.so.2',
                        '/usr/lib/libxslt.so.1',
                        '/usr/lib/libxslt.so.1',
+                       '/usr/lib/libexslt.so.0',
                        '/usr/lib/libMagickWand.so',
                        '/usr/lib/libMagickCore.so',
                        ]
@@ -72,6 +74,7 @@ def freeze():
         os.makedirs(DIST_DIR)
     
     includes = [x[0] for x in executables.values()]
+    includes += ['calibre.ebooks.lrf.fonts.prs500.'+x for x in FONT_MAP.values()]
     
     excludes = ['matplotlib', "Tkconstants", "Tkinter", "tcl", "_imagingtk", 
                 "ImageTk", "FixTk", 'wx', 'PyQt4.QtAssistant', 'PyQt4.QtOpenGL.so', 

installer/osx/freeze.py

@@ -326,7 +326,7 @@ def main():
                                        'genshi', 'calibre.web.feeds.recipes.*',
                                        'calibre.ebooks.lrf.any.*', 'calibre.ebooks.lrf.feeds.*',
                                        'keyword', 'codeop', 'pydoc', 'readline',
-                                       'BeautifulSoup'
+                                       'BeautifulSoup', 'calibre.ebooks.lrf.fonts.prs500.*',
                                        ],
                          'packages' : ['PIL', 'Authorization', 'lxml'],
                          'excludes' : ['IPython'],

installer/windows/freeze.py

@@ -176,6 +176,7 @@ def main(args=sys.argv):
                                              'BeautifulSoup', 'pyreadline',
                                              'pydoc', 'IPython.Extensions.*',
                                              'calibre.web.feeds.recipes.*',
+                                             'calibre.ebooks.lrf.fonts.prs500.*',
                                              'PyQt4.QtWebKit', 'PyQt4.QtNetwork',
                                              ],
                                   'packages'  : ['PIL', 'lxml', 'cherrypy'],

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.124'
+__version__   = '0.4.126'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/customize/ui.py

@@ -105,31 +105,44 @@ def reread_metadata_plugins():
     for plugin in _initialized_plugins:
         if isinstance(plugin, MetadataReaderPlugin):
             for ft in plugin.file_types:
-                _metadata_readers[ft] = plugin
+                if not _metadata_readers.has_key(ft):
+                    _metadata_readers[ft] = []
+                _metadata_readers[ft].append(plugin)
         elif isinstance(plugin, MetadataWriterPlugin):
             for ft in plugin.file_types:
-                _metadata_writers[ft] = plugin 
+                if not _metadata_writers.has_key(ft):
+                    _metadata_writers[ft] = []
+                _metadata_writers[ft].append(plugin) 
+                
+    
                 
 def get_file_type_metadata(stream, ftype):
     mi = MetaInformation(None, None)
-    try:
+    ftype = ftype.lower().strip()
-        plugin = _metadata_readers[ftype.lower().strip()]
+    if _metadata_readers.has_key(ftype):
+        for plugin in _metadata_readers[ftype]:
             if not is_disabled(plugin):
                 with plugin:
+                    try:
                         mi = plugin.get_metadata(stream, ftype.lower().strip())
+                        break
                     except:
-        pass
+                        continue
     return mi
 
 def set_file_type_metadata(stream, mi, ftype):
-    try:
+    ftype = ftype.lower().strip()
-        plugin = _metadata_writers[ftype.lower().strip()]
+    if _metadata_writers.has_key(ftype):
+        for plugin in _metadata_writers[ftype]:
             if not is_disabled(plugin):
                 with plugin:
+                    try:
                         plugin.set_metadata(stream, mi, ftype.lower().strip())
+                        break
                     except:
                         traceback.print_exc()
     
+                
 def _run_filetype_plugins(path_to_file, ft=None, occasion='preprocess'):
     occasion = {'import':_on_import, 'preprocess':_on_preprocess, 
                 'postprocess':_on_postprocess}[occasion]

src/calibre/devices/cybookg3/books.py

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2009, John Schember <john at nachtimwald.com'
+__copyright__ = '2009, John Schember <john at nachtimwald.com>'
 
 '''
 '''
@@ -20,6 +20,13 @@ class Book(object):
         self.thumbnail = None
         self.tags = []
         
+    @apply
+    def title_sorter():
+        doc = '''String to sort the title. If absent, title is returned'''
+        def fget(self):
+            return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', self.title).rstrip()
+        return property(doc=doc, fget=fget)
+
     @apply
     def thumbnail():
         return None
@@ -42,6 +49,8 @@ class BookList(_BookList):
             # Filter out anything that isn't in the list of supported ebook types
             for book_type in EBOOK_TYPES:
                 for filename in fnmatch.filter(files, '*.%s' % (book_type)):
+                    book_title = ''
+                    book_author = ''
                     # Calibre uses a specific format for file names. They take the form
                     # title_-_author_number.extention We want to see if the file name is
                     # in this format.
@@ -56,8 +65,7 @@ class BookList(_BookList):
                     else:
                         book_title = os.path.splitext(filename)[0].replace('_', ' ')
 
-                    book_path = os.path.join(path, filename)
+                    self.append(Book(os.path.join(path, filename), book_title, book_author))
-                    self.append(Book(book_path, book_title, book_author))
             
     def add_book(self, path, title):
         self.append(Book(path, title, ""))

src/calibre/devices/cybookg3/driver.py

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2009, John Schember <john at nachtimwald.com'
+__copyright__ = '2009, John Schember <john at nachtimwald.com>'
 
 '''
 Device driver for Bookeen's Cybook Gen 3
@@ -19,7 +19,9 @@ class CYBOOKG3(Device):
     VENDOR_ID   = 0x0bda
     PRODUCT_ID  = 0x0703
     BCD         = 0x110
-    #THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device
+
+    VENDOR_NAME = 'BOOKEEN'
+    PRODUCT_NAME = 'CYBOOK_GEN3'
     
     MAIN_MEMORY_VOLUME_LABEL  = 'Cybook Gen 3 Main Memory'
     STORAGE_CARD_VOLUME_LABEL = 'Cybook Gen 3 Storage Card'
@@ -220,10 +222,49 @@ class CYBOOKG3(Device):
             path = path.replace('card:', self._card_prefix[:-1])
         return path
         
+    @classmethod
+    def windows_match_device(cls, device_id):
+        device_id = device_id.upper()
+        if 'VEN_'+cls.VENDOR_NAME in device_id and \
+               'PROD_'+cls.PRODUCT_NAME in device_id:
+            return True
+                    
+        vid, pid = hex(cls.VENDOR_ID)[2:], hex(cls.PRODUCT_ID)[2:]        
+        while len(vid) < 4: vid = '0' + vid
+        while len(pid) < 4: pid = '0' + pid        
+        if 'VID_'+vid in device_id and 'PID_'+pid in device_id:
+            return True
+            
+        return False
+
+    # This only supports Windows >= 2000
     def open_windows(self):
-        raise NotImplementedError()
+        drives = []
+        wmi = __import__('wmi', globals(), locals(), [], -1) 
+        c = wmi.WMI()
+        for drive in c.Win32_DiskDrive():
+            if self.__class__.windows_match_device(str(drive.PNPDeviceID)):
+                if drive.Partitions == 0:
+                    continue
+                try:
+                    partition = drive.associators("Win32_DiskDriveToDiskPartition")[0]
+                    logical_disk = partition.associators('Win32_LogicalDiskToPartition')[0]
+                    prefix = logical_disk.DeviceID+os.sep
+                    drives.append((drive.Index, prefix))
+                except IndexError:
+                    continue
+                
+        if not drives:
+            raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.')%self.__class__.__name__)
+        
+        drives.sort(cmp=lambda a, b: cmp(a[0], b[0]))
+        self._main_prefix = drives[0][1]
+        if len(drives) > 1:
+            self._card_prefix = drives[1][1]            
+        
     def open_osx(self):
         raise NotImplementedError()
+
     def open_linux(self):
         import dbus
         bus = dbus.SystemBus() 

src/calibre/ebooks/epub/from_html.py

@@ -35,7 +35,7 @@ Conversion of HTML/OPF files follows several stages:
 import os, sys, cStringIO, logging, re, functools, shutil
 
 from lxml.etree import XPath
-from lxml import html
+from lxml import html, etree
 from PyQt4.Qt import QApplication, QPixmap
 
 from calibre.ebooks.html import Processor, merge_metadata, get_filelist,\
@@ -61,7 +61,7 @@ def remove_bad_link(element, attribute, link, pos):
             element.set(attribute, '')
             del element.attrib[attribute]
 
-def check(opf_path, pretty_print):
+def check_links(opf_path, pretty_print):
     '''
     Find and remove all invalid links in the HTML files 
     '''
@@ -284,6 +284,16 @@ def find_oeb_cover(htmlfile):
     if match:
         return match.group(1)
 
+def condense_ncx(ncx_path):
+    tree = etree.parse(ncx_path)
+    for tag in tree.getroot().iter(tag=etree.Element):
+        if tag.text:
+            tag.text = tag.text.strip()
+        if tag.tail:
+            tag.tail = tag.tail.strip()
+    compressed = etree.tostring(tree.getroot(), encoding='utf-8')
+    open(ncx_path, 'wb').write(compressed)
+
 def convert(htmlfile, opts, notification=None, create_epub=True, 
             oeb_cover=False, extract_to=None):
     htmlfile = os.path.abspath(htmlfile)
@@ -366,7 +376,8 @@ def convert(htmlfile, opts, notification=None, create_epub=True,
             if opts.show_ncx:
                 print toc
         split(opf_path, opts, stylesheet_map)
-        check(opf_path, opts.pretty_print)
+        check_links(opf_path, opts.pretty_print)
+        
         opf = OPF(opf_path, tdir)
         opf.remove_guide()
         oeb_cover_file = None
@@ -387,6 +398,13 @@ def convert(htmlfile, opts, notification=None, create_epub=True,
             if not raw.startswith('<?xml '):
                 raw = '<?xml version="1.0"  encoding="UTF-8"?>\n'+raw
             f.write(raw)
+        ncx_path = os.path.join(os.path.dirname(opf_path), 'toc.ncx')
+        if os.path.exists(ncx_path) and os.stat(ncx_path).st_size > opts.profile.flow_size:
+            logger.info('Condensing NCX from %d bytes...'%os.stat(ncx_path).st_size)
+            condense_ncx(ncx_path)
+            if os.stat(ncx_path).st_size > opts.profile.flow_size:
+                logger.warn('NCX still larger than allowed size at %d bytes. Menu based Table of Contents may not work on device.'%os.stat(ncx_path).st_size)
+            
         if create_epub:
             epub = initialize_container(opts.output)
             epub.add_dir(tdir)

src/calibre/ebooks/html.py

@@ -458,6 +458,8 @@ class Parser(PreProcessor, LoggingInterface):
     def parse_html(self):
         ''' Create lxml ElementTree from HTML '''
         self.log_info('\tParsing '+os.sep.join(self.htmlfile.path.split(os.sep)[-3:]))
+        if self.htmlfile.is_binary:
+            raise ValueError('Not a valid HTML file: '+self.htmlfile.path)
         src = open(self.htmlfile.path, 'rb').read().decode(self.htmlfile.encoding, 'replace').strip()
         src = src.replace('\x00', '')
         src = self.preprocess(src)

src/calibre/ebooks/lrf/fonts/__init__.py

@@ -50,7 +50,8 @@ def get_font_path(name):
         try:
             font_mod = __import__('calibre.ebooks.lrf.fonts.prs500', {}, {}, 
                                   [fname], -1)
-        except ImportError:
+            getattr(font_mod, fname)
+        except (ImportError, AttributeError):
             font_mod = __import__('calibre.ebooks.lrf.fonts.liberation', {}, {}, 
                                   [LIBERATION_FONT_MAP[name]], -1)
         p = PersistentTemporaryFile('.ttf', 'font_')

src/calibre/ebooks/lrf/html/convert_from.py

@@ -245,7 +245,6 @@ class HTMLConverter(object, LoggingInterface):
         
         self.override_css = {}
         self.override_pcss = {}
-        self.table_render_job_server = None
          
         if self._override_css is not None:
             if os.access(self._override_css, os.R_OK):
@@ -266,7 +265,6 @@ class HTMLConverter(object, LoggingInterface):
         paths = [os.path.abspath(path) for path in paths]
         paths = [path.decode(sys.getfilesystemencoding()) if not isinstance(path, unicode) else path for path in paths]
         
-        try:
         while len(paths) > 0 and self.link_level <= self.link_levels:
             for path in paths:
                 if path in self.processed_files:
@@ -298,9 +296,6 @@ class HTMLConverter(object, LoggingInterface):
         if self.base_font_size > 0:
             self.log_info('\tRationalizing font sizes...')
             self.book.rationalize_font_sizes(self.base_font_size)
-        finally:
-            if self.table_render_job_server is not None:
-                self.table_render_job_server.killall()
         
     def is_baen(self, soup):
         return bool(soup.find('meta', attrs={'name':'Publisher', 
@@ -1732,15 +1727,11 @@ class HTMLConverter(object, LoggingInterface):
                 self.process_children(tag, tag_css, tag_pseudo_css)
             elif tagname == 'table' and not self.ignore_tables and not self.in_table:
                 if self.render_tables_as_images:
-                    if self.table_render_job_server is None:
-                        from calibre.parallel import Server
-                        self.table_render_job_server = Server(number_of_workers=1)
                     print 'Rendering table...'
                     from calibre.ebooks.lrf.html.table_as_image import render_table
                     pheight = int(self.current_page.pageStyle.attrs['textheight'])
                     pwidth  = int(self.current_page.pageStyle.attrs['textwidth'])
-                    images = render_table(self.table_render_job_server,
+                    images = render_table(self.soup, tag, tag_css, 
-                                          self.soup, tag, tag_css, 
                                           os.path.dirname(self.target_prefix), 
                                           pwidth, pheight, self.profile.dpi, 
                                           self.text_size_multiplier_for_rendered_tables)

src/calibre/ebooks/lrf/html/table_as_image.py

@@ -6,14 +6,11 @@ __docformat__ = 'restructuredtext en'
 '''
 Render HTML tables as images.
 '''
-import os, tempfile, atexit, shutil, time
+import os, tempfile, atexit, shutil
-from PyQt4.Qt import QUrl, QApplication, QSize, \
+from PyQt4.Qt import QUrl, QApplication, QSize, QEventLoop, \
                      SIGNAL, QPainter, QImage, QObject, Qt
 from PyQt4.QtWebKit import QWebPage
 
-from calibre.parallel import ParallelJob
-
-__app = None
 
 class HTMLTableRenderer(QObject):
 
@@ -27,13 +24,15 @@ class HTMLTableRenderer(QObject):
         self.app = None
         self.width, self.height, self.dpi = width, height, dpi
         self.base_dir = base_dir
+        self.images = []
+        self.tdir = tempfile.mkdtemp(prefix='calibre_render_table')
+        self.loop = QEventLoop()
         self.page = QWebPage()
         self.connect(self.page, SIGNAL('loadFinished(bool)'), self.render_html)
         self.page.mainFrame().setTextSizeMultiplier(factor)
         self.page.mainFrame().setHtml(html, 
                                 QUrl('file:'+os.path.abspath(self.base_dir)))
-        self.images = []
+        
-        self.tdir = tempfile.mkdtemp(prefix='calibre_render_table')
         
     def render_html(self, ok):
         try:
@@ -63,7 +62,7 @@ class HTMLTableRenderer(QObject):
         finally:
             QApplication.quit()
         
-def render_table(server, soup, table, css, base_dir, width, height, dpi, factor=1.0):
+def render_table(soup, table, css, base_dir, width, height, dpi, factor=1.0):
     head = ''
     for e in soup.findAll(['link', 'style']):
         head += unicode(e)+'\n\n'
@@ -83,24 +82,13 @@ def render_table(server, soup, table, css, base_dir, width, height, dpi, factor=
     </body>
 </html>
     '''%(head, width-10, style, unicode(table))
-    job = ParallelJob('render_table',  lambda j : j, None,
+    images, tdir = do_render(html, base_dir, width, height, dpi, factor)
-                            args=[html, base_dir, width, height, dpi, factor])
-    server.add_job(job)
-    while not job.has_run:
-        time.sleep(2)
-    
-    if job.exception is not None:
-        print 'Failed to render table'
-        print job.exception
-        print job.traceback
-    images, tdir = job.result
     atexit.register(shutil.rmtree, tdir)
     return images
     
 def do_render(html, base_dir, width, height, dpi, factor):
-    app = QApplication.instance()
+    if QApplication.instance() is None:
-    if app is None:
+        QApplication([])
-        app = QApplication([])
     tr = HTMLTableRenderer(html, base_dir, width, height, dpi, factor)
-    app.exec_()
+    tr.loop.exec_()
     return tr.images, tr.tdir

src/calibre/ebooks/mobi/reader.py

@@ -257,6 +257,8 @@ class MobiReader(object):
                 pass
             try:
                 styles.append('text-indent: %s' % tag['width'])
+                if tag['width'].startswith('-'):
+                    styles.append('margin-left: %s'%(tag['width'][1:]))
                 del tag['width']
             except KeyError:
                 pass
@@ -335,9 +337,8 @@ class MobiReader(object):
             if flags & 1:
                 num += sizeof_trailing_entry(data, size - num)
             flags >>= 1
-        # Flag indicates overlapping multibyte character data
         if self.book_header.extra_flags & 1:
-            num += ord(data[size - num - 1]) + 1
+            num += (ord(data[size - num - 1]) & 0x3) + 1
         return num
 
     def text_section(self, index):

src/calibre/gui2/__init__.py

@@ -54,8 +54,12 @@ def _config():
     c.add_opt('autolaunch_server', default=False, help=_('Automatically launch content server on application startup'))
     c.add_opt('oldest_news', default=60, help=_('Oldest news kept in database'))
     c.add_opt('systray_icon', default=True, help=_('Show system tray icon'))
-    c.add_opt('upload_news_to_device', default=True, help=_('Upload downloaded news to device'))
+    c.add_opt('upload_news_to_device', default=True, 
-    c.add_opt('delete_news_from_library_on_upload', default=False, help=_('Delete books from library after uploading to device'))
+              help=_('Upload downloaded news to device'))
+    c.add_opt('delete_news_from_library_on_upload', default=False, 
+              help=_('Delete books from library after uploading to device'))
+    c.add_opt('separate_cover_flow', default=False, 
+              help=_('Show the cover flow in a separate window instead of in the main calibre window'))
     return ConfigProxy(c)
     
 config = _config()

src/calibre/gui2/cover_flow.py

@@ -69,11 +69,11 @@ if pictureflow is not None:
             
     class CoverFlow(pictureflow.PictureFlow):
         
-        def __init__(self, height=300, parent=None):
+        def __init__(self, height=300, parent=None, text_height=25):
             pictureflow.PictureFlow.__init__(self, parent, 
                                 config['cover_flow_queue_length']+1)
             self.setSlideSize(QSize(int(2/3. * height), height))
-            self.setMinimumSize(QSize(int(2.35*0.67*height), (5/3.)*height+25))
+            self.setMinimumSize(QSize(int(2.35*0.67*height), (5/3.)*height+text_height))
             self.setFocusPolicy(Qt.WheelFocus)
             self.setSizePolicy(QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum))
             

src/calibre/gui2/dialogs/config.py

@@ -244,6 +244,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
         self.connect(self.remove_plugin, SIGNAL('clicked()'), lambda : self.modify_plugin(op='remove'))
         self.connect(self.button_plugin_browse, SIGNAL('clicked()'), self.find_plugin)
         self.connect(self.button_plugin_add, SIGNAL('clicked()'), self.add_plugin)
+        self.separate_cover_flow.setChecked(config['separate_cover_flow'])
     
     def add_plugin(self):
         path = unicode(self.plugin_path.text())
@@ -392,6 +393,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
         config['column_map'] = cols
         config['toolbar_icon_size'] = self.ICON_SIZES[self.toolbar_button_size.currentIndex()]
         config['show_text_in_toolbar'] = bool(self.show_toolbar_text.isChecked())
+        config['separate_cover_flow'] = bool(self.separate_cover_flow.isChecked())
         pattern = self.filename_pattern.commit()
         prefs['filename_pattern'] = pattern
         p = {0:'normal', 1:'high', 2:'low'}[self.priority.currentIndex()]

src/calibre/gui2/dialogs/config.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>800</width>
-    <height>563</height>
+    <height>570</height>
    </rect>
   </property>
   <property name="windowTitle" >
@@ -356,7 +356,7 @@
            </item>
           </layout>
          </item>
-         <item row="5" column="0" >
+         <item row="7" column="0" >
           <widget class="QGroupBox" name="groupBox_2" >
            <property name="title" >
             <string>Toolbar</string>
@@ -404,7 +404,7 @@
            </layout>
           </widget>
          </item>
-         <item row="6" column="0" >
+         <item row="8" column="0" >
           <widget class="QGroupBox" name="groupBox" >
            <property name="title" >
             <string>Select visible &amp;columns in library view</string>
@@ -492,20 +492,27 @@
            </property>
           </widget>
          </item>
-         <item row="3" column="0" >
+         <item row="4" column="0" >
           <widget class="QCheckBox" name="sync_news" >
            <property name="text" >
             <string>Automatically send downloaded &amp;news to ebook reader</string>
            </property>
           </widget>
          </item>
-         <item row="4" column="0" >
+         <item row="5" column="0" >
           <widget class="QCheckBox" name="delete_news" >
            <property name="text" >
             <string>&amp;Delete news from library when it is sent to reader</string>
            </property>
           </widget>
          </item>
+         <item row="3" column="0" >
+          <widget class="QCheckBox" name="separate_cover_flow" >
+           <property name="text" >
+            <string>Show cover &amp;browser in a separate window (needs restart)</string>
+           </property>
+          </widget>
+         </item>
         </layout>
        </widget>
        <widget class="QWidget" name="page_2" >

src/calibre/gui2/dialogs/metadata_single.ui

@@ -600,6 +600,33 @@
    <header>widgets.h</header>
   </customwidget>
  </customwidgets>
+ <tabstops>
+  <tabstop>title</tabstop>
+  <tabstop>swap_button</tabstop>
+  <tabstop>authors</tabstop>
+  <tabstop>author_sort</tabstop>
+  <tabstop>auto_author_sort</tabstop>
+  <tabstop>rating</tabstop>
+  <tabstop>publisher</tabstop>
+  <tabstop>tags</tabstop>
+  <tabstop>tag_editor_button</tabstop>
+  <tabstop>series</tabstop>
+  <tabstop>remove_series_button</tabstop>
+  <tabstop>series_index</tabstop>
+  <tabstop>isbn</tabstop>
+  <tabstop>comments</tabstop>
+  <tabstop>fetch_metadata_button</tabstop>
+  <tabstop>fetch_cover_button</tabstop>
+  <tabstop>password_button</tabstop>
+  <tabstop>cover_button</tabstop>
+  <tabstop>reset_cover</tabstop>
+  <tabstop>cover_path</tabstop>
+  <tabstop>add_format_button</tabstop>
+  <tabstop>button_set_cover</tabstop>
+  <tabstop>remove_format_button</tabstop>
+  <tabstop>formats</tabstop>
+  <tabstop>button_box</tabstop>
+ </tabstops>
  <resources>
   <include location="../images.qrc" />
  </resources>

src/calibre/gui2/library.py

@@ -8,9 +8,9 @@ from math import cos, sin, pi
 from PyQt4.QtGui import QTableView, QAbstractItemView, QColor, \
                         QItemDelegate, QPainterPath, QLinearGradient, QBrush, \
                         QPen, QStyle, QPainter, QLineEdit, \
-                        QPalette, QImage, QApplication, QMenu
+                        QPalette, QImage, QApplication, QMenu, QStyledItemDelegate
 from PyQt4.QtCore import QAbstractTableModel, QVariant, Qt, QString, \
-                         SIGNAL, QObject, QSize, QModelIndex
+                         SIGNAL, QObject, QSize, QModelIndex, QDate
 
 from calibre import strftime
 from calibre.ptempfile import PersistentTemporaryFile
@@ -82,6 +82,17 @@ class LibraryDelegate(QItemDelegate):
         sb.setMaximum(5)
         return sb
 
+class DateDelegate(QStyledItemDelegate):
+    
+    def displayText(self, val, locale):
+        d = val.toDate()
+        return d.toString('dd MMM yyyy')
+        if d.isNull():
+            return ''
+        d = datetime(d.year(), d.month(), d.day())
+        return strftime(BooksView.TIME_FMT, d.timetuple())
+        
+
 class BooksModel(QAbstractTableModel):
     coding = zip(
     [1000,900,500,400,100,90,50,40,10,9,5,4,1],
@@ -114,7 +125,8 @@ class BooksModel(QAbstractTableModel):
         QAbstractTableModel.__init__(self, parent)
         self.db = None
         self.column_map = config['column_map']
-        self.editable_cols = ['title', 'authors', 'rating', 'publisher', 'tags', 'series']
+        self.editable_cols = ['title', 'authors', 'rating', 'publisher', 
+                              'tags', 'series', 'timestamp']
         self.default_image = QImage(':/images/book.svg')
         self.sorted_on = ('timestamp', Qt.AscendingOrder)
         self.last_search = '' # The last search performed on this model
@@ -136,7 +148,12 @@ class BooksModel(QAbstractTableModel):
                 idx = self.column_map.index('rating')
             except ValueError:
                 idx = -1
-            self.emit(SIGNAL('columns_sorted(int)'), idx)
+            try:
+                tidx = self.column_map.index('timestamp')
+            except ValueError:
+                tidx = -1
+            
+            self.emit(SIGNAL('columns_sorted(int,int)'), idx, tidx)
         
     
     def set_database(self, db):
@@ -443,7 +460,7 @@ class BooksModel(QAbstractTableModel):
             dt = self.db.data[r][tmdx]
             if dt:
                 dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight)
-                return strftime(BooksView.TIME_FMT, dt.timetuple())
+                return QDate(dt.year, dt.month, dt.day)
         
         def rating(r):
             r = self.db.data[r][ridx]
@@ -508,35 +525,40 @@ class BooksModel(QAbstractTableModel):
         return flags
 
     def setData(self, index, value, role):
-        done = False
         if role == Qt.EditRole:
             row, col = index.row(), index.column()
             column = self.column_map[col]
             if column not in self.editable_cols:
                 return False
-            val = unicode(value.toString().toUtf8(), 'utf-8').strip() if column != 'rating' else \
+            val = int(value.toInt()[0]) if column == 'rating' else \
-                  int(value.toInt()[0])
+                  value.toDate() if column == 'timestamp' else \
+                  unicode(value.toString())
+            id = self.db.id(row)  
             if column == 'rating':
                 val = 0 if val < 0 else 5 if val > 5 else val
                 val *= 2
-            if column == 'series':
+            elif column == 'series':
                 pat = re.compile(r'\[(\d+)\]')
                 match = pat.search(val)
-                id = self.db.id(row)
                 if match is not None:
                     self.db.set_series_index(id, int(match.group(1)))
                     val = pat.sub('', val)
                 val = val.strip()
                 if val:
                     self.db.set_series(id, val)
+            elif column == 'timestamp':
+                if val.isNull() or not val.isValid():
+                    return False
+                dt = datetime(val.year(), val.month(), val.day()) + timedelta(seconds=time.timezone) - timedelta(hours=time.daylight)
+                self.db.set_timestamp(id, dt)
             else:
                 self.db.set(row, column, val)
             self.emit(SIGNAL("dataChanged(QModelIndex, QModelIndex)"), \
                                 index, index)
             if column == self.sorted_on[0]:
                 self.resort()
-            done = True
+            
-        return done
+        return True
 
 class BooksView(TableView):
     TIME_FMT = '%d %b %Y'
@@ -555,25 +577,29 @@ class BooksView(TableView):
     def __init__(self, parent, modelcls=BooksModel):
         TableView.__init__(self, parent)
         self.rating_delegate = LibraryDelegate(self)
+        self.timestamp_delegate = DateDelegate(self)
         self.display_parent = parent
         self._model = modelcls(self)
         self.setModel(self._model)
         self.setSelectionBehavior(QAbstractItemView.SelectRows)
         self.setSortingEnabled(True)
         try:
-            self.columns_sorted(self._model.column_map.index('rating'))
+            self.columns_sorted(self._model.column_map.index('rating'),
+                                self._model.column_map.index('timestamp'))
         except ValueError:
             pass
         QObject.connect(self.selectionModel(), SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'),
                         self._model.current_changed)
-        self.connect(self._model, SIGNAL('columns_sorted(int)'), self.columns_sorted, Qt.QueuedConnection)
+        self.connect(self._model, SIGNAL('columns_sorted(int, int)'), self.columns_sorted, Qt.QueuedConnection)
         
-    def columns_sorted(self, col):
+    def columns_sorted(self, rating_col, timestamp_col):
         for i in range(self.model().columnCount(None)):
             if self.itemDelegateForColumn(i) == self.rating_delegate:
                 self.setItemDelegateForColumn(i, self.itemDelegate())
-        if col > -1:
+        if rating_col > -1:
-            self.setItemDelegateForColumn(col, self.rating_delegate)
+            self.setItemDelegateForColumn(rating_col, self.rating_delegate)
+        if timestamp_col > -1:
+            self.setItemDelegateForColumn(timestamp_col, self.timestamp_delegate)
             
     def set_context_menu(self, edit_metadata, send_to_device, convert, view, 
                          save, open_folder, book_details, similar_menu=None):

src/calibre/gui2/main.py

@@ -7,7 +7,7 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
                      QModelIndex, QPixmap, QColor, QPainter, QMenu, QIcon, \
                      QToolButton, QDialog, QDesktopServices, QFileDialog, \
                      QSystemTrayIcon, QApplication, QKeySequence, QAction, \
-                     QProgressDialog, QMessageBox
+                     QProgressDialog, QMessageBox, QStackedLayout
 from PyQt4.QtSvg import QSvgRenderer
 
 from calibre import __version__, __appname__, islinux, sanitize_file_name, \
@@ -22,7 +22,8 @@ from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \
                            pixmap_to_data, choose_dir, ORG_NAME, \
                            set_sidebar_directories, Dispatcher, \
                            SingleApplication, Application, available_height, \
-                           max_available_height, config, info_dialog
+                           max_available_height, config, info_dialog, \
+                           available_width
 from calibre.gui2.cover_flow import CoverFlow, DatabaseImages, pictureflowerror
 from calibre.library.database import LibraryDatabase
 from calibre.gui2.dialogs.scheduler import Scheduler
@@ -342,8 +343,15 @@ class Main(MainWindow, Ui_MainWindow):
         ########################### Cover Flow ################################
         self.cover_flow = None
         if CoverFlow is not None:
-            self.cover_flow = CoverFlow(height=220 if available_height() > 950 else 170 if available_height() > 850 else 140)
+            text_height = 40 if config['separate_cover_flow'] else 25
+            ah = available_height()
+            cfh = ah-100
+            cfh = 3./5 * cfh - text_height
+            if not config['separate_cover_flow']:
+                cfh = 220 if ah > 950 else 170 if ah > 850 else 140
+            self.cover_flow = CoverFlow(height=cfh, text_height=text_height)
             self.cover_flow.setVisible(False)
+            if not config['separate_cover_flow']:
                 self.library.layout().addWidget(self.cover_flow)
             self.connect(self.cover_flow, SIGNAL('currentChanged(int)'), self.sync_cf_to_listview)
             self.connect(self.cover_flow, SIGNAL('itemActivated(int)'), self.show_book_info)
@@ -410,6 +418,29 @@ class Main(MainWindow, Ui_MainWindow):
                 
     
     def toggle_cover_flow(self, show):
+        if config['separate_cover_flow']:
+            if show:
+                d = QDialog(self)
+                ah, aw = available_height(), available_width()
+                d.resize(int(aw/2.), ah-60)
+                d._layout = QStackedLayout()
+                d.setLayout(d._layout)
+                d.setWindowTitle(_('Browse by covers'))
+                d.layout().addWidget(self.cover_flow)
+                self.cover_flow.setVisible(True)
+                self.cover_flow.setFocus(Qt.OtherFocusReason)
+                self.library_view.scrollTo(self.library_view.currentIndex())
+                d.show()
+                self.connect(d, SIGNAL('finished(int)'), 
+                             lambda x: self.status_bar.cover_flow_button.setChecked(False))
+                self.cf_dialog = d
+            else:
+                cfd = getattr(self, 'cf_dialog', None)
+                if cfd is not None:
+                    self.cover_flow.setVisible(False)
+                    cfd.hide()
+                    self.cf_dialog = None
+        else:
             if show:
                 self.library_view.setCurrentIndex(self.library_view.currentIndex())
                 self.cover_flow.setVisible(True)
@@ -583,6 +614,7 @@ class Main(MainWindow, Ui_MainWindow):
         try:
             duplicates = self.library_view.model().db.recursive_import(root, single, callback=callback)
         finally:
+            progress.hide()
             progress.close()            
         if duplicates:
             files = _('<p>Books with the same title as the following already exist in the database. Add them anyway?<ul>')
@@ -702,7 +734,8 @@ class Main(MainWindow, Ui_MainWindow):
             else:
                 self.upload_books(paths, list(map(sanitize_file_name, names)), infos, on_card=on_card)
         finally:
-            progress.setValue(len(paths))
+            progress.setValue(progress.maximum())
+            progress.hide()
             progress.close()
 
     def upload_books(self, files, names, metadata, on_card=False, memory=None):
@@ -1391,7 +1424,12 @@ in which you want to store your books files. Any existing books will be automati
             self.memory_view.write_settings()
     
     def quit(self, checked, restart=False):
-        if self.shutdown():
+        if not self.confirm_quit():
+            return
+        try:
+            self.shutdown()
+        except:
+            pass
         self.restart_after_quit = restart
         QApplication.instance().quit()
             
@@ -1424,22 +1462,26 @@ in which you want to store your books files. Any existing books will be automati
         QDesktopServices.openUrl(QUrl.fromLocalFile(pt.name))
             
     
-    def shutdown(self):
+    def confirm_quit(self):
+        if self.job_manager.has_jobs():
             msg = _('There are active jobs. Are you sure you want to quit?')
             if self.job_manager.has_device_jobs():
                 msg = '<p>'+__appname__ + _(''' is communicating with the device!<br>
                       'Quitting may cause corruption on the device.<br>
                       'Are you sure you want to quit?''')+'</p>'
-        if self.job_manager.has_jobs():
+            
             d = QMessageBox(QMessageBox.Warning, _('WARNING: Active jobs'), msg,
                             QMessageBox.Yes|QMessageBox.No, self)
             d.setIconPixmap(QPixmap(':/images/dialog_warning.svg'))
             d.setDefaultButton(QMessageBox.No)
             if d.exec_() != QMessageBox.Yes:
                 return False
+        return True
 
-        self.job_manager.terminate_all_jobs()
+    
+    def shutdown(self):
         self.write_settings()
+        self.job_manager.terminate_all_jobs()
         self.device_manager.keep_going = False
         self.cover_cache.stop()
         self.hide()
@@ -1465,7 +1507,11 @@ in which you want to store your books files. Any existing books will be automati
             self.hide()
             e.ignore()
         else:
-            if self.shutdown():
+            if self.confirm_quit():
+                try:
+                    self.shutdown()
+                except:
+                    pass
                 e.accept()
             else:
                 e.ignore()

src/calibre/gui2/tools.py

@@ -13,7 +13,7 @@ from calibre.utils.config import prefs
 from calibre.gui2.dialogs.lrf_single import LRFSingleDialog, LRFBulkDialog
 from calibre.gui2.dialogs.epub import Config as EPUBConvert
 import calibre.gui2.dialogs.comicconf as ComicConf
-from calibre.gui2 import warning_dialog, dynamic
+from calibre.gui2 import warning_dialog
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.ebooks.lrf import preferred_source_formats as LRF_PREFERRED_SOURCE_FORMATS
 from calibre.ebooks.metadata.opf import OPFCreator
@@ -22,7 +22,9 @@ from calibre.ebooks.epub.from_any import SOURCE_FORMATS as EPUB_PREFERRED_SOURCE
 def convert_single_epub(parent, db, comics, others):
     changed = False
     jobs = []
-    for row in others:
+    others_ids = [db.id(row) for row in others]
+    comics_ids = [db.id(row) for row in comics]
+    for row, row_id in zip(others, others_ids):
         temp_files = []
         d = EPUBConvert(parent, db, row)
         if d.source_format is not None:
@@ -44,10 +46,10 @@ def convert_single_epub(parent, db, comics, others):
                     opts.cover = d.cover_file.name
                 temp_files.extend([d.opf_file, pt, of])
                 jobs.append(('any2epub', args, _('Convert book: ')+d.mi.title, 
-                             'EPUB', db.id(row), temp_files))
+                             'EPUB', row_id, temp_files))
                 changed = True
                 
-    for row in comics:
+    for row, row_id in zip(comics, comics_ids):
         mi = db.get_metadata(row)
         title = author = _('Unknown')
         if mi.title:
@@ -76,7 +78,7 @@ def convert_single_epub(parent, db, comics, others):
         args = [pt.name, opts]
         changed = True
         jobs.append(('comic2epub', args, _('Convert comic: ')+opts.title, 
-                     'EPUB', db.id(row), [pt, of]))
+                     'EPUB', row_id, [pt, of]))
         
     return jobs, changed
     
@@ -85,7 +87,9 @@ def convert_single_epub(parent, db, comics, others):
 def convert_single_lrf(parent, db, comics, others):
     changed = False
     jobs = []
-    for row in others:
+    others_ids = [db.id(row) for row in others]
+    comics_ids = [db.id(row) for row in comics]
+    for row, row_id in zip(others, others_ids):
         temp_files = []
         d = LRFSingleDialog(parent, db, row)
         if d.selected_format:
@@ -104,10 +108,10 @@ def convert_single_lrf(parent, db, comics, others):
                     temp_files.append(d.cover_file)
                 temp_files.extend([pt, of])
                 jobs.append(('any2lrf', [cmdline], _('Convert book: ')+d.title(), 
-                             'LRF', db.id(row), temp_files))
+                             'LRF', row_id, temp_files))
                 changed = True
                 
-    for row in comics:
+    for row, row_id in zip(comics, comics_ids):
         mi = db.get_metadata(row)
         title = author = _('Unknown')
         if mi.title:
@@ -138,7 +142,7 @@ def convert_single_lrf(parent, db, comics, others):
         args = [pt.name, opts]
         changed = True
         jobs.append(('comic2lrf', args, _('Convert comic: ')+opts.title, 
-                     'LRF', db.id(row), [pt, of]))
+                     'LRF', row_id, [pt, of]))
         
     return jobs, changed
 
@@ -162,6 +166,7 @@ def convert_bulk_epub(parent, db, comics, others):
     parent.status_bar.showMessage(_('Starting Bulk conversion of %d books')%total, 2000)
     
     for i, row in enumerate(others+comics):
+        row_id = db.id(row)
         if row in others:
             data = None
             for fmt in EPUB_PREFERRED_SOURCE_FORMATS:
@@ -198,7 +203,7 @@ def convert_bulk_epub(parent, db, comics, others):
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
             temp_files = [cf] if cf is not None else []
             temp_files.extend([opf_file, pt, of])
-            jobs.append(('any2epub', args, desc, 'EPUB', db.id(row), temp_files))
+            jobs.append(('any2epub', args, desc, 'EPUB', row_id, temp_files))
         else:
             options = comic_opts.copy()
             mi = db.get_metadata(row)
@@ -224,7 +229,7 @@ def convert_bulk_epub(parent, db, comics, others):
             options.verbose = 1
             args = [pt.name, options]
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
-            jobs.append(('comic2epub', args, desc, 'EPUB', db.id(row), [pt, of]))        
+            jobs.append(('comic2epub', args, desc, 'EPUB', row_id, [pt, of]))        
         
     if bad_rows:
         res = []
@@ -255,6 +260,7 @@ def convert_bulk_lrf(parent, db, comics, others):
     parent.status_bar.showMessage(_('Starting Bulk conversion of %d books')%total, 2000)
     
     for i, row in enumerate(others+comics):
+        row_id = db.id(row)
         if row in others:
             cmdline = list(d.cmdline)
             mi = db.get_metadata(row)
@@ -294,7 +300,7 @@ def convert_bulk_lrf(parent, db, comics, others):
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
             temp_files = [cf] if cf is not None else []
             temp_files.extend([pt, of])
-            jobs.append(('any2lrf', [cmdline], desc, 'LRF', db.id(row), temp_files))
+            jobs.append(('any2lrf', [cmdline], desc, 'LRF', row_id, temp_files))
         else:
             options = comic_opts.copy()
             mi = db.get_metadata(row)
@@ -320,7 +326,7 @@ def convert_bulk_lrf(parent, db, comics, others):
             options.verbose = 1
             args = [pt.name, options]
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
-            jobs.append(('comic2lrf', args, desc, 'LRF', db.id(row), [pt, of]))        
+            jobs.append(('comic2lrf', args, desc, 'LRF', row_id, [pt, of]))        
         
     if bad_rows:
         res = []

src/calibre/library/cli.py

@@ -85,7 +85,9 @@ STANZA_TEMPLATE='''\
           <div xmlns="http://www.w3.org/1999/xhtml">
               <py:for each="f in ('authors', 'publisher', 'rating', 'tags', 'series', 'isbn')">
               <py:if test="record[f]">
-              ${f.capitalize()}:${unicode(', '.join(record[f]) if f=='tags' else record[f])}<br/>
+              ${f.capitalize()}:${unicode(', '.join(record[f]) if f=='tags' else record[f])}
+              <py:if test="f =='series'"># ${str(record['series_index'])}</py:if>
+              <br/>
               </py:if>
               </py:for>
               <py:if test="record['comments']">
@@ -231,7 +233,7 @@ NULL = DevNull()
 
 def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
     orig = sys.stdout
-    sys.stdout = NULL
+    #sys.stdout = NULL
     try:
         files, dirs = [], []
         for path in paths:

src/calibre/library/database2.py

@@ -878,6 +878,14 @@ class LibraryDatabase2(LibraryDatabase):
         if notify:
             self.notify('metadata', [id])
             
+    def set_timestamp(self, id, dt, notify=True):
+        if dt:
+            self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id))
+            self.data.set(id, FIELD_MAP['timestamp'], dt, row_is_id=True)
+            self.conn.commit()
+            if notify:
+                self.notify('metadata', [id])
+    
     def set_publisher(self, id, publisher, notify=True):
         self.conn.execute('DELETE FROM books_publishers_link WHERE book=?',(id,))
         self.conn.execute('DELETE FROM publishers WHERE (SELECT COUNT(id) FROM books_publishers_link WHERE publisher=publishers.id) < 1')

src/calibre/manual/customize.rst

@@ -46,7 +46,7 @@ Create a file name :file:`my_plugin.py` (the file name must end with plugin.py)
       ext  = os.path.splitext(path_to_ebook)[-1][1:].lower()
       mi = get_metadata(file, ext)
       mi.publisher = 'Hello World'
-      set_metadata(file, ext, mi)
+      set_metadata(file, mi, ext)
       return path_to_ebook
 
 That's all. To add this code to |app| as a plugin, simply create a zip file with::

src/calibre/parallel.py

@@ -615,7 +615,7 @@ class Job(object):
                 self.log = unicode(self.log, 'utf-8', 'replace')
             ans.extend(self.log.split('\n'))
         
-        ans = [x.decode(preferred_encoding, 'replace') if isinstance(x, 'str') else x for x in ans]
+        ans = [x.decode(preferred_encoding, 'replace') if isinstance(x, str) else x for x in ans]
         
         return u'<br>'.join(ans)
 

src/calibre/utils/config.py

@@ -17,7 +17,9 @@ from calibre.constants import terminal_controller, iswindows, isosx, \
 from calibre.utils.lock import LockError, ExclusiveFile 
 from collections import defaultdict
 
-if iswindows:
+if os.environ.has_key('CALIBRE_CONFIG_DIRECTORY'):
+    config_dir = os.path.abspath(os.environ['CALIBRE_CONFIG_DIRECTORY'])
+elif iswindows:
     config_dir = plugins['winutil'][0].special_folder_path(plugins['winutil'][0].CSIDL_APPDATA)
     if not os.access(config_dir, os.W_OK|os.X_OK):
         config_dir = os.path.expanduser('~')

src/calibre/utils/fontconfig.py

@@ -22,7 +22,7 @@ match to a given font specification. The main functions in this module are:
 .. autofunction:: match
 '''
 
-import sys, os, locale, codecs
+import sys, os, locale, codecs, subprocess, re
 from ctypes import cdll, c_void_p, Structure, c_int, POINTER, c_ubyte, c_char, util, \
                    pointer, byref, create_string_buffer, Union, c_char_p, c_double
 
@@ -34,6 +34,7 @@ except:
 
 iswindows = 'win32' in sys.platform or 'win64' in sys.platform
 isosx     = 'darwin' in sys.platform
+isbsd     = 'bsd' in sys.platform
 DISABLED  = False 
 #if isosx:
 #    libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))
@@ -57,6 +58,13 @@ def load_library():
         return cdll.LoadLibrary(lib)
     elif iswindows:
         return cdll.LoadLibrary('libfontconfig-1')
+    elif isbsd:
+        raw = subprocess.Popen('pkg-config --libs-only-L fontconfig'.split(), 
+                         stdout=subprocess.PIPE).stdout.read().strip()
+        match = re.search(r'-L([^\s,]+)', raw)
+        if not match:
+            return cdll.LoadLibrary('libfontconfig.so')
+        return cdll.LoadLibrary(match.group(1)+'/libfontconfig.so')
     else:
         try:
             return cdll.LoadLibrary(util.find_library('fontconfig'))

src/calibre/web/feeds/recipes/__init__.py

@@ -20,7 +20,8 @@ recipe_modules = ['recipe_' + r for r in (
            'science_news', 'the_nation', 'lrb', 'harpers_full', 'liberation',
            'linux_magazine', 'telegraph_uk', 'utne', 'sciencedaily', 'forbes',
            'time_magazine', 'endgadget', 'fudzilla', 'nspm_int', 'nspm', 'pescanik',
-           'spiegel_int', 'themarketticker', 'tomshardware',
+           'spiegel_int', 'themarketticker', 'tomshardware', 'xkcd', 'ftd', 'zdnet',
+           'joelonsoftware',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_ftd.py

@@ -0,0 +1,49 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
+'''
+Fetch FTD.
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class FTheiseDe(BasicNewsRecipe):
+    
+    title = 'FTD'
+    description = 'Financial Times Deutschland'
+    __author__ = 'Oliver Niesner'
+    use_embedded_content   = False
+    timefmt = ' [%d %b %Y]'
+    max_articles_per_feed = 40
+    no_stylesheets = True
+    
+    remove_tags = [dict(id='navi_top'),
+		   dict(id='topbanner'),
+		   dict(id='seitenkopf'),
+		   dict(id='footer'),
+		   dict(id='rating_open'),
+		   dict(id='ADS_Top'),
+		   dict(id='ADS_Middle1'),
+		   #dict(id='IDMS_ajax_chart_price_information_table'),
+		   dict(id='ivwimg'),
+		   dict(name='span', attrs={'class':'rsaquo'}),
+		   dict(name='p', attrs={'class':'zwischenhead'}),
+		   dict(name='div', attrs={'class':'chartBox'}),
+		   dict(name='span', attrs={'class':'vote_455857'}),
+		   dict(name='div', attrs={'class':'relatedhalb'}),
+		   dict(name='div', attrs={'class':'bpoll'}),
+		   dict(name='div', attrs={'class':'pollokknopf'}),
+		   dict(name='div', attrs={'class':'videohint'}),
+		   dict(name='div', attrs={'class':'videoshadow'}),
+		   dict(name='div', attrs={'class':'boxresp videorahmen'}),
+		   dict(name='div', attrs={'class':'boxresp'}),
+		   dict(name='div', attrs={'class':'abspielen'}),
+		   dict(name='div', attrs={'class':'wertungoben'}),
+		   dict(name='div', attrs={'class':'artikelfuss'}),
+		   dict(name='div', attrs={'class':'artikelsplitfaq'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'artikelfuss'})]
+    
+    feeds =  [ ('FTD', 'http://www.ftd.de/static/ticker/ftd-topnews.rdf') ] 
+    
+

src/calibre/web/feeds/recipes/recipe_joelonsoftware.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+'''
+joelonsoftware.com
+'''
+
+class Joelonsoftware(BasicNewsRecipe):
+    
+    title       = 'Joel on Software'
+    __author__  = 'Darko Miletic'
+    description = 'Painless Software Management'
+    no_stylesheets = True
+    use_embedded_content  = True
+    
+    cover_url = 'http://www.joelonsoftware.com/RssJoelOnSoftware.jpg'
+    
+    html2lrf_options = [  '--comment'       , description
+                        , '--category'      , 'blog,software,news'
+                        , '--author'        , 'Joel Spolsky'
+                       ]
+    
+    feeds = [(u'Articles', u'http://www.joelonsoftware.com/rss.xml')]

src/calibre/web/feeds/recipes/recipe_time_magazine.py

@@ -6,22 +6,28 @@ __copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
 time.com
 '''
 
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Time(BasicNewsRecipe):
     title                 = u'Time'
-    __author__            = 'Darko Miletic'
+    __author__            = 'Kovid Goyal'
     description           = 'Weekly magazine'    
     oldest_article        = 7
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
     
-    #cover_url = 'http://img.timeinc.net/time/rd/trunk/www/web/feds/i/logo_time_home.gif'
-    
     keep_only_tags = [dict(name='div', attrs={'class':'tout1'})]
-    remove_tags    = [dict(name='ul', attrs={'class':['button', 'find']})]
+    remove_tags_after = [dict(id='connectStory')]
+    remove_tags    = [
+                      dict(name='ul', attrs={'class':['button', 'find']}),
+                      dict(name='div', attrs={'class':['nav', 'header', 'sectheader', 
+                                                       'searchWrap', 'subNav', 
+                                                       'artTools', 'connect',
+                                                       'similarrecs']}),
+                      dict(name='div', id=['articleSideBar', 'connectStory']),
+                      dict(name='dl', id=['links']),                                 
+                      ]
 
     feeds          = [
                        (u'Top Stories', u'http://feedproxy.google.com/time/topstories')
@@ -34,17 +40,20 @@ class Time(BasicNewsRecipe):
                        ,(u'Travel', u'http://feedproxy.google.com/time/travel')
                      ]
     
+    def get_article_url(self, article):
+        return article.get('guid',  article['link'])
+    
     def get_cover_url(self):
         soup = self.index_to_soup('http://www.time.com/time/')
         img = soup.find('img', alt='Current Time.com Cover', width='107')
         if img is not None:
             return img.get('src', None)
         
-    
     def print_version(self, url):
-        raw = self.browser.open(url).read()
+        try:
-        soup = BeautifulSoup(raw.decode('utf8', 'replace'))
+            soup = self.index_to_soup(url)
             print_link = soup.find('a', {'id':'prt'})
-        if print_link is None:
-            return ''
             return 'http://www.time.com' + print_link['href']
+        except:
+            self.log_exception('Failed to find print version for '+url)
+        return ''

src/calibre/web/feeds/recipes/recipe_xkcd.py

@@ -0,0 +1,36 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
+'''
+Fetch xkcd.
+'''
+
+import time
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class XkcdCom(BasicNewsRecipe):
+    title = 'xkcd'
+    description = 'A webcomic of romance and math humor.'
+    __author__ = 'Martin Pitt'
+    use_embedded_content   = False
+    oldest_article = 60
+    keep_only_tags = [dict(id='middleContent')]
+    remove_tags = [dict(name='ul'), dict(name='h3'), dict(name='br')]
+    no_stylesheets = True
+    
+    def parse_index(self):
+        INDEX = 'http://xkcd.com/archive/'
+
+        soup = self.index_to_soup(INDEX) 
+        articles = []
+        for item in soup.findAll('a', title=True):
+            articles.append({
+                'date': item['title'],
+                'timestamp': time.mktime(time.strptime(item['title'], '%Y-%m-%d'))+1,
+                'url': 'http://xkcd.com' + item['href'],
+                'title': self.tag_to_string(item).encode('UTF-8'),
+                'description': '',
+                'content': '',
+            })
+
+        return [('xkcd', articles)]

src/calibre/web/feeds/recipes/recipe_zdnet.py

@@ -0,0 +1,46 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
+'''
+Fetch zdnet.
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+import re
+
+
+class cdnet(BasicNewsRecipe):
+    
+    title = 'zdnet'
+    description = 'zdnet security'
+    __author__ = 'Oliver Niesner'
+    use_embedded_content   = False
+    timefmt = ' [%d %b %Y]'
+    max_articles_per_feed = 40
+    no_stylesheets = True
+    encoding = 'iso-8859-1'
+
+    #preprocess_regexps = \
+#	[(re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in
+#		[
+#		(r'<84>', lambda match: ''),
+#		(r'<93>', lambda match: ''),
+#		]
+#	]
+    
+    remove_tags = [dict(id='eyebrows'),
+		   dict(id='header'),
+		   dict(id='search'),
+		   dict(id='nav'),
+		   dict(id=''),
+		   dict(name='div', attrs={'class':'banner'}),
+		   dict(name='p', attrs={'class':'tags'}),
+		   dict(name='div', attrs={'class':'special1'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'bloggerDesc clear'})]
+    
+    feeds =  [ ('zdnet', 'http://feeds.feedburner.com/zdnet/security') ] 
+    
+
+
+
+

src/odf/namespaces.py

@@ -20,9 +20,10 @@
 TOOLSVERSION = u"ODFPY/0.8.1dev"
 
 ANIMNS         = u"urn:oasis:names:tc:opendocument:xmlns:animation:1.0"
+DBNS           = u"urn:oasis:names:tc:opendocument:xmlns:database:1.0"
 CHARTNS        = u"urn:oasis:names:tc:opendocument:xmlns:chart:1.0"
 CONFIGNS       = u"urn:oasis:names:tc:opendocument:xmlns:config:1.0"
-DBNS           = u"http://openoffice.org/2004/database"
+#DBNS           = u"http://openoffice.org/2004/database"
 DCNS           = u"http://purl.org/dc/elements/1.1/"
 DOMNS          = u"http://www.w3.org/2001/xml-events"
 DR3DNS         = u"urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0"
@@ -39,6 +40,7 @@ OOONS          = u"http://openoffice.org/2004/office"
 OOOWNS         = u"http://openoffice.org/2004/writer"
 OOOCNS         = u"http://openoffice.org/2004/calc"
 PRESENTATIONNS = u"urn:oasis:names:tc:opendocument:xmlns:presentation:1.0"
+RDFANS         = u"http://docs.oasis-open.org/opendocument/meta/rdfa#"
 SCRIPTNS       = u"urn:oasis:names:tc:opendocument:xmlns:script:1.0"
 SMILNS         = u"urn:oasis:names:tc:opendocument:xmlns:smil-compatible:1.0"
 STYLENS        = u"urn:oasis:names:tc:opendocument:xmlns:style:1.0"
@@ -47,6 +49,7 @@ TABLENS        = u"urn:oasis:names:tc:opendocument:xmlns:table:1.0"
 TEXTNS         = u"urn:oasis:names:tc:opendocument:xmlns:text:1.0"
 XFORMSNS       = u"http://www.w3.org/2002/xforms"
 XLINKNS        = u"http://www.w3.org/1999/xlink"
+XMLNS          = "http://www.w3.org/XML/1998/namespace"
 
 
 nsdict = {
@@ -70,6 +73,7 @@ nsdict = {
    OOOWNS: u'ooow',
    OOOCNS: u'ooc',
    PRESENTATIONNS: u'presentation',
+   RDFANS: u'rdfa',
    SCRIPTNS: u'script',
    SMILNS: u'smil',
    STYLENS: u'style',
@@ -78,4 +82,5 @@ nsdict = {
    TEXTNS: u'text',
    XFORMSNS: u'xforms',
    XLINKNS: u'xlink',
+   XMLNS: u'xml',
 }

src/odf/odf2xhtml.py

@@ -22,7 +22,7 @@
 #pdb.set_trace()
 import zipfile
 import xml.sax
-from xml.sax import handler
+from xml.sax import handler, expatreader
 from xml.sax.xmlreader import InputSource
 from xml.sax.saxutils import escape, quoteattr
 
@@ -206,10 +206,10 @@ class StyleToCSS:
         if hpos == "center":
             sdict['margin-left'] = "auto"
             sdict['margin-right'] = "auto"
-        else:
+#       else:
-            # force it to be *something* then delete it
+#           # force it to be *something* then delete it
-            sdict['margin-left'] = sdict['margin-right'] = ''
+#           sdict['margin-left'] = sdict['margin-right'] = ''
-            del sdict['margin-left'], sdict['margin-right']
+#           del sdict['margin-left'], sdict['margin-right']
 
         if hpos in ("right","outside"):
             if wrap in ( "left", "parallel","dynamic"):
@@ -336,8 +336,9 @@ special_styles = {
 class ODF2XHTML(handler.ContentHandler):
     """ The ODF2XHTML parses an ODF file and produces XHTML"""
 
-    def __init__(self):
+    def __init__(self, generate_css=True, embedable=False):
         # Tags
+        self.generate_css = generate_css
         self.elements = {
         (DCNS, 'title'): (self.s_processcont, self.e_dc_title),
         (DCNS, 'language'): (self.s_processcont, self.e_dc_contentlanguage),
@@ -349,6 +350,7 @@ class ODF2XHTML(handler.ContentHandler):
         (DRAWNS, 'fill-image'): (self.s_draw_fill_image, None),
         (DRAWNS, "layer-set"):(self.s_ignorexml, None),
         (DRAWNS, 'page'): (self.s_draw_page, self.e_draw_page),
+        (DRAWNS, 'text-box'): (self.s_draw_textbox, self.e_draw_textbox),
         (METANS, 'creation-date'):(self.s_processcont, self.e_dc_metatag),
         (METANS, 'generator'):(self.s_processcont, self.e_dc_metatag),
         (METANS, 'initial-creator'): (self.s_processcont, self.e_dc_metatag),
@@ -421,6 +423,12 @@ class ODF2XHTML(handler.ContentHandler):
         (TEXTNS, "table-of-content-source"):(self.s_text_x_source, self.e_text_x_source),
         (TEXTNS, "user-index-source"):(self.s_text_x_source, self.e_text_x_source),
         }
+        if embedable:
+            self.elements[(OFFICENS, u"text")] = (None,None)
+            self.elements[(OFFICENS, u"spreadsheet")] = (None,None)
+            self.elements[(OFFICENS, u"presentation")] = (None,None)
+            self.elements[(OFFICENS, u"document-content")] = (None,None)
+
 
     def writeout(self, s):
         if s != '':
@@ -548,14 +556,18 @@ class ODF2XHTML(handler.ContentHandler):
         """ A <draw:frame> is made into a <div> in HTML which is then styled
         """
         anchor_type = attrs.get((TEXTNS,'anchor-type'),'char')
+        htmltag = 'div'
         name = "G-" + attrs.get( (DRAWNS,'style-name'), "")
         if name == 'G-':
             name = "PR-" + attrs.get( (PRESENTATIONNS,'style-name'), "")
         name = name.replace(".","_")
         if anchor_type == "paragraph":
-            style = ""
+            style = 'position:relative;'
         elif anchor_type == 'char':
             style = "position:relative;"
+        elif anchor_type == 'as-char':
+            htmltag = 'div'
+            style = ''
         else:
             style = "position: absolute;"
         if attrs.has_key( (SVGNS,"width") ):
@@ -566,7 +578,10 @@ class ODF2XHTML(handler.ContentHandler):
             style = style + "left:" +  attrs[(SVGNS,"x")] + ";"
         if attrs.has_key( (SVGNS,"y") ):
             style = style + "top:" +  attrs[(SVGNS,"y")] + ";"
-        self.opentag('div', {'class': name, 'style': style})
+        if self.generate_css:
+            self.opentag(htmltag, {'class': name, 'style': style})
+        else:
+            self.opentag(htmltag)
 
     def e_draw_frame(self, tag, attrs):
         """ End the <draw:frame>
@@ -593,6 +608,7 @@ class ODF2XHTML(handler.ContentHandler):
         imghref = attrs[(XLINKNS,"href")]
         imghref = self.rewritelink(imghref)
         htmlattrs = {'alt':"", 'src':imghref }
+        if self.generate_css:
             if anchor_type != "char":
                 htmlattrs['style'] = "display: block;"
         self.emptytag('img', htmlattrs)
@@ -607,7 +623,10 @@ class ODF2XHTML(handler.ContentHandler):
         stylename = stylename.replace(".","_")
         masterpage = attrs.get( (DRAWNS,'master-page-name'),"")
         masterpage = masterpage.replace(".","_")
+        if self.generate_css:
             self.opentag('fieldset', {'class':"DP-%s MP-%s" % (stylename, masterpage) })
+        else:
+            self.opentag('fieldset')
         self.opentag('legend')
         self.writeout(escape(name))
         self.closetag('legend')
@@ -615,12 +634,25 @@ class ODF2XHTML(handler.ContentHandler):
     def e_draw_page(self, tag, attrs):
         self.closetag('fieldset')
 
+    def s_draw_textbox(self, tag, attrs):
+        style = ''
+        if attrs.has_key( (FONS,"min-height") ):
+            style = style + "min-height:" +  attrs[(FONS,"min-height")] + ";"
+        self.opentag('div')
+#       self.opentag('div', {'style': style})
+
+    def e_draw_textbox(self, tag, attrs):
+        """ End the <draw:text-box>
+        """
+        self.closetag('div')
+
     def html_body(self, tag, attrs):
         self.writedata()
+        if self.generate_css:
             self.opentag('style', {'type':"text/css"}, True)
             self.writeout('/*<![CDATA[*/\n')
             self.writeout('\nimg { width: 100%; height: 100%; }\n')
-        self.writeout('* { padding: 0; margin: 0; }\n')
+            self.writeout('* { padding: 0; margin: 0;  background-color:white; }\n')
             self.writeout('body { margin: 0 1em; }\n')
             self.writeout('ol, ul { padding-left: 2em; }\n')
             self.generate_stylesheet()
@@ -660,7 +692,10 @@ class ODF2XHTML(handler.ContentHandler):
     def generate_footnotes(self):
         if self.currentnote == 0:
             return
+        if self.generate_css:
             self.opentag('ol', {'style':'border-top: 1px solid black'}, True)
+        else:
+            self.opentag('ol')
         for key in range(1,self.currentnote+1):
             note = self.notedict[key]
 #       for key,note in self.notedict.items():
@@ -731,6 +766,8 @@ class ODF2XHTML(handler.ContentHandler):
         """ Copy all attributes to a struct.
             We will later convert them to CSS2
         """
+        if self.currentstyle is None:
+            return
         for key,attr in attrs.items():
             self.styledict[self.currentstyle][key] = attr
 
@@ -874,7 +911,7 @@ class ODF2XHTML(handler.ContentHandler):
         """ Start a table
         """
         c = attrs.get( (TABLENS,'style-name'), None)
-        if c:
+        if c and self.generate_css:
             c = c.replace(".","_")
             self.opentag('table',{ 'class': "T-%s" % c })
         else:
@@ -958,7 +995,7 @@ class ODF2XHTML(handler.ContentHandler):
         for x in range(level + 1,10):
             self.headinglevels[x] = 0
         special = special_styles.get("P-"+name)
-        if special:
+        if special or not self.generate_css:
             self.opentag('h%s' % level)
         else:
             self.opentag('h%s' % level, {'class':"P-%s" % name })
@@ -997,7 +1034,10 @@ class ODF2XHTML(handler.ContentHandler):
             # textbox itself may be nested within another list.
             level = self.tagstack.count_tags(tag) + 1
             name = self.tagstack.rfindattr( (TEXTNS,'style-name') )
+        if self.generate_css:
             self.opentag('%s' % self.listtypes.get(name), {'class':"%s_%d" % (name, level) })
+        else:
+            self.opentag('%s' % self.listtypes.get(name))
         self.purgedata()
 
     def e_text_list(self, tag, attrs):
@@ -1113,6 +1153,7 @@ class ODF2XHTML(handler.ContentHandler):
             specialtag = special_styles.get("P-"+c)
             if specialtag is None:
                 specialtag = 'p'
+                if self.generate_css:
                     htmlattrs['class'] = "P-%s" % c
         self.opentag(specialtag, htmlattrs)
         self.purgedata()
@@ -1149,7 +1190,7 @@ class ODF2XHTML(handler.ContentHandler):
         if c:
             c = c.replace(".","_")
             special = special_styles.get("S-"+c)
-            if special is None:
+            if special is None and self.generate_css:
                 htmlattrs['class'] = "S-%s" % c
         self.opentag('span', htmlattrs)
         self.purgedata()
@@ -1219,7 +1260,10 @@ class ODF2XHTML(handler.ContentHandler):
         # Extract the interesting files
         z = zipfile.ZipFile(self._odffile)
 
-        parser = xml.sax.make_parser()
+        # For some reason Trac has trouble when xml.sax.make_parser() is used.
+        # Could it be because PyXML is installed, and therefore a different parser
+        # might be chosen? By calling expatreader directly we avoid this issue
+        parser = expatreader.create_parser()
         parser.setFeature(handler.feature_namespaces, 1)
         parser.setContentHandler(self)
         parser.setErrorHandler(handler.ErrorHandler())

src/odf/opendocument.py

@@ -287,7 +287,7 @@ class OpenDocument:
             else:
                 ext = mimetypes.guess_extension(mediatype)
             manifestfn = "Pictures/%0.0f%s" % ((time.time()*10000000000), ext)
-            self.Pictures[manifestfn] = (IS_FILENAME, fileobj, mediatype)
+            self.Pictures[manifestfn] = (IS_FILENAME, filename, mediatype)
         else:
             manifestfn = filename
             self.Pictures[manifestfn] = (IS_IMAGE, content, mediatype)