sync to trunk

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.124'
+__version__   = '0.4.125'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/ebooks/epub/from_html.py

@@ -35,7 +35,7 @@ Conversion of HTML/OPF files follows several stages:
 import os, sys, cStringIO, logging, re, functools, shutil
 
 from lxml.etree import XPath
-from lxml import html
+from lxml import html, etree
 from PyQt4.Qt import QApplication, QPixmap
 
 from calibre.ebooks.html import Processor, merge_metadata, get_filelist,\
@@ -61,7 +61,7 @@ def remove_bad_link(element, attribute, link, pos):
             element.set(attribute, '')
             del element.attrib[attribute]
 
-def check(opf_path, pretty_print):
+def check_links(opf_path, pretty_print):
     '''
     Find and remove all invalid links in the HTML files 
     '''
@@ -284,6 +284,16 @@ def find_oeb_cover(htmlfile):
     if match:
         return match.group(1)
 
+def condense_ncx(ncx_path):
+    tree = etree.parse(ncx_path)
+    for tag in tree.getroot().iter(tag=etree.Element):
+        if tag.text:
+            tag.text = tag.text.strip()
+        if tag.tail:
+            tag.tail = tag.tail.strip()
+    compressed = etree.tostring(tree.getroot(), encoding='utf-8')
+    open(ncx_path, 'wb').write(compressed)
+
 def convert(htmlfile, opts, notification=None, create_epub=True, 
             oeb_cover=False, extract_to=None):
     htmlfile = os.path.abspath(htmlfile)
@@ -366,7 +376,8 @@ def convert(htmlfile, opts, notification=None, create_epub=True,
             if opts.show_ncx:
                 print toc
         split(opf_path, opts, stylesheet_map)
-        check(opf_path, opts.pretty_print)
+        check_links(opf_path, opts.pretty_print)
+        
         opf = OPF(opf_path, tdir)
         opf.remove_guide()
         oeb_cover_file = None
@@ -387,6 +398,13 @@ def convert(htmlfile, opts, notification=None, create_epub=True,
             if not raw.startswith('<?xml '):
                 raw = '<?xml version="1.0"  encoding="UTF-8"?>\n'+raw
             f.write(raw)
+        ncx_path = os.path.join(os.path.dirname(opf_path), 'toc.ncx')
+        if os.path.exists(ncx_path) and os.stat(ncx_path).st_size > opts.profile.flow_size:
+            logger.info('Condensing NCX from %d bytes...'%os.stat(ncx_path).st_size)
+            condense_ncx(ncx_path)
+            if os.stat(ncx_path).st_size > opts.profile.flow_size:
+                logger.warn('NCX still larger than allowed size at %d bytes. Menu based Table of Contents may not work on device.'%os.stat(ncx_path).st_size)
+            
         if create_epub:
             epub = initialize_container(opts.output)
             epub.add_dir(tdir)

src/calibre/ebooks/html.py

@@ -458,6 +458,8 @@ class Parser(PreProcessor, LoggingInterface):
     def parse_html(self):
         ''' Create lxml ElementTree from HTML '''
         self.log_info('\tParsing '+os.sep.join(self.htmlfile.path.split(os.sep)[-3:]))
+        if self.htmlfile.is_binary:
+            raise ValueError('Not a valid HTML file: '+self.htmlfile.path)
         src = open(self.htmlfile.path, 'rb').read().decode(self.htmlfile.encoding, 'replace').strip()
         src = src.replace('\x00', '')
         src = self.preprocess(src)

src/calibre/ebooks/metadata/meta.py

@@ -83,7 +83,7 @@ def get_metadata(stream, stream_type='lrf', use_libprs_metadata=False):
     return base
 
 def set_metadata(stream, mi, stream_type='lrf'):
-    if stream_type: 
+    if stream_type:
         stream_type = stream_type.lower()
     set_file_type_metadata(stream, mi, stream_type)
     

src/calibre/ebooks/mobi/reader.py

@@ -257,6 +257,8 @@ class MobiReader(object):
                 pass
             try:
                 styles.append('text-indent: %s' % tag['width'])
+                if tag['width'].startswith('-'):
+                    styles.append('margin-left: %s'%(tag['width'][1:]))
                 del tag['width']
             except KeyError:
                 pass

src/calibre/gui2/__init__.py

@@ -54,8 +54,12 @@ def _config():
     c.add_opt('autolaunch_server', default=False, help=_('Automatically launch content server on application startup'))
     c.add_opt('oldest_news', default=60, help=_('Oldest news kept in database'))
     c.add_opt('systray_icon', default=True, help=_('Show system tray icon'))
-    c.add_opt('upload_news_to_device', default=True, help=_('Upload downloaded news to device'))
+    c.add_opt('upload_news_to_device', default=True, 
-    c.add_opt('delete_news_from_library_on_upload', default=False, help=_('Delete books from library after uploading to device'))
+              help=_('Upload downloaded news to device'))
+    c.add_opt('delete_news_from_library_on_upload', default=False, 
+              help=_('Delete books from library after uploading to device'))
+    c.add_opt('separate_cover_flow', default=False, 
+              help=_('Show the cover flow in a separate window instead of in the main calibre window'))
     return ConfigProxy(c)
     
 config = _config()

src/calibre/gui2/cover_flow.py

@@ -69,11 +69,11 @@ if pictureflow is not None:
             
     class CoverFlow(pictureflow.PictureFlow):
         
-        def __init__(self, height=300, parent=None):
+        def __init__(self, height=300, parent=None, text_height=25):
             pictureflow.PictureFlow.__init__(self, parent, 
                                 config['cover_flow_queue_length']+1)
             self.setSlideSize(QSize(int(2/3. * height), height))
-            self.setMinimumSize(QSize(int(2.35*0.67*height), (5/3.)*height+25))
+            self.setMinimumSize(QSize(int(2.35*0.67*height), (5/3.)*height+text_height))
             self.setFocusPolicy(Qt.WheelFocus)
             self.setSizePolicy(QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum))
             

src/calibre/gui2/dialogs/config.py

@@ -244,6 +244,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
         self.connect(self.remove_plugin, SIGNAL('clicked()'), lambda : self.modify_plugin(op='remove'))
         self.connect(self.button_plugin_browse, SIGNAL('clicked()'), self.find_plugin)
         self.connect(self.button_plugin_add, SIGNAL('clicked()'), self.add_plugin)
+        self.separate_cover_flow.setChecked(config['separate_cover_flow'])
     
     def add_plugin(self):
         path = unicode(self.plugin_path.text())
@@ -392,6 +393,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
         config['column_map'] = cols
         config['toolbar_icon_size'] = self.ICON_SIZES[self.toolbar_button_size.currentIndex()]
         config['show_text_in_toolbar'] = bool(self.show_toolbar_text.isChecked())
+        config['separate_cover_flow'] = bool(self.separate_cover_flow.isChecked())
         pattern = self.filename_pattern.commit()
         prefs['filename_pattern'] = pattern
         p = {0:'normal', 1:'high', 2:'low'}[self.priority.currentIndex()]

src/calibre/gui2/dialogs/config.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>800</width>
-    <height>563</height>
+    <height>570</height>
    </rect>
   </property>
   <property name="windowTitle" >
@@ -356,7 +356,7 @@
            </item>
           </layout>
          </item>
-         <item row="5" column="0" >
+         <item row="7" column="0" >
           <widget class="QGroupBox" name="groupBox_2" >
            <property name="title" >
             <string>Toolbar</string>
@@ -404,7 +404,7 @@
            </layout>
           </widget>
          </item>
-         <item row="6" column="0" >
+         <item row="8" column="0" >
           <widget class="QGroupBox" name="groupBox" >
            <property name="title" >
             <string>Select visible &amp;columns in library view</string>
@@ -492,20 +492,27 @@
            </property>
           </widget>
          </item>
-         <item row="3" column="0" >
+         <item row="4" column="0" >
           <widget class="QCheckBox" name="sync_news" >
            <property name="text" >
             <string>Automatically send downloaded &amp;news to ebook reader</string>
            </property>
           </widget>
          </item>
-         <item row="4" column="0" >
+         <item row="5" column="0" >
           <widget class="QCheckBox" name="delete_news" >
            <property name="text" >
             <string>&amp;Delete news from library when it is sent to reader</string>
            </property>
           </widget>
          </item>
+         <item row="3" column="0" >
+          <widget class="QCheckBox" name="separate_cover_flow" >
+           <property name="text" >
+            <string>Show cover &amp;browser in a separate window (needs restart)</string>
+           </property>
+          </widget>
+         </item>
         </layout>
        </widget>
        <widget class="QWidget" name="page_2" >

src/calibre/gui2/library.py

@@ -8,9 +8,9 @@ from math import cos, sin, pi
 from PyQt4.QtGui import QTableView, QAbstractItemView, QColor, \
                         QItemDelegate, QPainterPath, QLinearGradient, QBrush, \
                         QPen, QStyle, QPainter, QLineEdit, \
-                        QPalette, QImage, QApplication, QMenu
+                        QPalette, QImage, QApplication, QMenu, QStyledItemDelegate
 from PyQt4.QtCore import QAbstractTableModel, QVariant, Qt, QString, \
-                         SIGNAL, QObject, QSize, QModelIndex
+                         SIGNAL, QObject, QSize, QModelIndex, QDate
 
 from calibre import strftime
 from calibre.ptempfile import PersistentTemporaryFile
@@ -82,6 +82,17 @@ class LibraryDelegate(QItemDelegate):
         sb.setMaximum(5)
         return sb
 
+class DateDelegate(QStyledItemDelegate):
+    
+    def displayText(self, val, locale):
+        d = val.toDate()
+        return d.toString('dd MMM yyyy')
+        if d.isNull():
+            return ''
+        d = datetime(d.year(), d.month(), d.day())
+        return strftime(BooksView.TIME_FMT, d.timetuple())
+        
+
 class BooksModel(QAbstractTableModel):
     coding = zip(
     [1000,900,500,400,100,90,50,40,10,9,5,4,1],
@@ -114,7 +125,8 @@ class BooksModel(QAbstractTableModel):
         QAbstractTableModel.__init__(self, parent)
         self.db = None
         self.column_map = config['column_map']
-        self.editable_cols = ['title', 'authors', 'rating', 'publisher', 'tags', 'series']
+        self.editable_cols = ['title', 'authors', 'rating', 'publisher', 
+                              'tags', 'series', 'timestamp']
         self.default_image = QImage(':/images/book.svg')
         self.sorted_on = ('timestamp', Qt.AscendingOrder)
         self.last_search = '' # The last search performed on this model
@@ -136,7 +148,12 @@ class BooksModel(QAbstractTableModel):
                 idx = self.column_map.index('rating')
             except ValueError:
                 idx = -1
-            self.emit(SIGNAL('columns_sorted(int)'), idx)
+            try:
+                tidx = self.column_map.index('timestamp')
+            except ValueError:
+                tidx = -1
+            
+            self.emit(SIGNAL('columns_sorted(int,int)'), idx, tidx)
         
     
     def set_database(self, db):
@@ -443,7 +460,7 @@ class BooksModel(QAbstractTableModel):
             dt = self.db.data[r][tmdx]
             if dt:
                 dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight)
-                return strftime(BooksView.TIME_FMT, dt.timetuple())
+                return QDate(dt.year, dt.month, dt.day)
         
         def rating(r):
             r = self.db.data[r][ridx]
@@ -508,35 +525,40 @@ class BooksModel(QAbstractTableModel):
         return flags
 
     def setData(self, index, value, role):
-        done = False
         if role == Qt.EditRole:
             row, col = index.row(), index.column()
             column = self.column_map[col]
             if column not in self.editable_cols:
                 return False
-            val = unicode(value.toString().toUtf8(), 'utf-8').strip() if column != 'rating' else \
+            val = int(value.toInt()[0]) if column == 'rating' else \
-                  int(value.toInt()[0])
+                  value.toDate() if column == 'timestamp' else \
+                  unicode(value.toString())
+            id = self.db.id(row)  
             if column == 'rating':
                 val = 0 if val < 0 else 5 if val > 5 else val
                 val *= 2
-            if column == 'series':
+            elif column == 'series':
                 pat = re.compile(r'\[(\d+)\]')
                 match = pat.search(val)
-                id = self.db.id(row)
                 if match is not None:
                     self.db.set_series_index(id, int(match.group(1)))
                     val = pat.sub('', val)
                 val = val.strip()
                 if val:
                     self.db.set_series(id, val)
+            elif column == 'timestamp':
+                if val.isNull() or not val.isValid():
+                    return False
+                dt = datetime(val.year(), val.month(), val.day()) + timedelta(seconds=time.timezone) - timedelta(hours=time.daylight)
+                self.db.set_timestamp(id, dt)
             else:
                 self.db.set(row, column, val)
             self.emit(SIGNAL("dataChanged(QModelIndex, QModelIndex)"), \
                                 index, index)
             if column == self.sorted_on[0]:
                 self.resort()
-            done = True
+            
-        return done
+        return True
 
 class BooksView(TableView):
     TIME_FMT = '%d %b %Y'
@@ -555,25 +577,29 @@ class BooksView(TableView):
     def __init__(self, parent, modelcls=BooksModel):
         TableView.__init__(self, parent)
         self.rating_delegate = LibraryDelegate(self)
+        self.timestamp_delegate = DateDelegate(self)
         self.display_parent = parent
         self._model = modelcls(self)
         self.setModel(self._model)
         self.setSelectionBehavior(QAbstractItemView.SelectRows)
         self.setSortingEnabled(True)
         try:
-            self.columns_sorted(self._model.column_map.index('rating'))
+            self.columns_sorted(self._model.column_map.index('rating'),
+                                self._model.column_map.index('timestamp'))
         except ValueError:
             pass
         QObject.connect(self.selectionModel(), SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'),
                         self._model.current_changed)
-        self.connect(self._model, SIGNAL('columns_sorted(int)'), self.columns_sorted, Qt.QueuedConnection)
+        self.connect(self._model, SIGNAL('columns_sorted(int, int)'), self.columns_sorted, Qt.QueuedConnection)
         
-    def columns_sorted(self, col):
+    def columns_sorted(self, rating_col, timestamp_col):
         for i in range(self.model().columnCount(None)):
             if self.itemDelegateForColumn(i) == self.rating_delegate:
                 self.setItemDelegateForColumn(i, self.itemDelegate())
-        if col > -1:
+        if rating_col > -1:
-            self.setItemDelegateForColumn(col, self.rating_delegate)
+            self.setItemDelegateForColumn(rating_col, self.rating_delegate)
+        if timestamp_col > -1:
+            self.setItemDelegateForColumn(timestamp_col, self.timestamp_delegate)
             
     def set_context_menu(self, edit_metadata, send_to_device, convert, view, 
                          save, open_folder, book_details, similar_menu=None):

src/calibre/gui2/main.py

@@ -7,7 +7,7 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
                      QModelIndex, QPixmap, QColor, QPainter, QMenu, QIcon, \
                      QToolButton, QDialog, QDesktopServices, QFileDialog, \
                      QSystemTrayIcon, QApplication, QKeySequence, QAction, \
-                     QProgressDialog, QMessageBox
+                     QProgressDialog, QMessageBox, QStackedLayout
 from PyQt4.QtSvg import QSvgRenderer
 
 from calibre import __version__, __appname__, islinux, sanitize_file_name, \
@@ -22,7 +22,8 @@ from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \
                            pixmap_to_data, choose_dir, ORG_NAME, \
                            set_sidebar_directories, Dispatcher, \
                            SingleApplication, Application, available_height, \
-                           max_available_height, config, info_dialog
+                           max_available_height, config, info_dialog, \
+                           available_width
 from calibre.gui2.cover_flow import CoverFlow, DatabaseImages, pictureflowerror
 from calibre.library.database import LibraryDatabase
 from calibre.gui2.dialogs.scheduler import Scheduler
@@ -342,9 +343,16 @@ class Main(MainWindow, Ui_MainWindow):
         ########################### Cover Flow ################################
         self.cover_flow = None
         if CoverFlow is not None:
-            self.cover_flow = CoverFlow(height=220 if available_height() > 950 else 170 if available_height() > 850 else 140)
+            text_height = 40 if config['separate_cover_flow'] else 25
+            ah = available_height()
+            cfh = ah-100
+            cfh = 3./5 * cfh - text_height
+            if not config['separate_cover_flow']:
+                cfh = 220 if ah > 950 else 170 if ah > 850 else 140
+            self.cover_flow = CoverFlow(height=cfh, text_height=text_height)
             self.cover_flow.setVisible(False)
-            self.library.layout().addWidget(self.cover_flow)
+            if not config['separate_cover_flow']:
+                self.library.layout().addWidget(self.cover_flow)
             self.connect(self.cover_flow, SIGNAL('currentChanged(int)'), self.sync_cf_to_listview)
             self.connect(self.cover_flow, SIGNAL('itemActivated(int)'), self.show_book_info)
             self.connect(self.status_bar.cover_flow_button, SIGNAL('toggled(bool)'), self.toggle_cover_flow)
@@ -410,17 +418,40 @@ class Main(MainWindow, Ui_MainWindow):
                 
     
     def toggle_cover_flow(self, show):
-        if show:
+        if config['separate_cover_flow']:
-            self.library_view.setCurrentIndex(self.library_view.currentIndex())
+            if show:
-            self.cover_flow.setVisible(True)
+                d = QDialog(self)
-            self.cover_flow.setFocus(Qt.OtherFocusReason)
+                ah, aw = available_height(), available_width()
-            #self.status_bar.book_info.book_data.setMaximumHeight(100)
+                d.resize(int(aw/2.), ah-60)
-            #self.status_bar.setMaximumHeight(120)
+                d._layout = QStackedLayout()
-            self.library_view.scrollTo(self.library_view.currentIndex())
+                d.setLayout(d._layout)
+                d.setWindowTitle(_('Browse by covers'))
+                d.layout().addWidget(self.cover_flow)
+                self.cover_flow.setVisible(True)
+                self.cover_flow.setFocus(Qt.OtherFocusReason)
+                self.library_view.scrollTo(self.library_view.currentIndex())
+                d.show()
+                self.connect(d, SIGNAL('finished(int)'), 
+                             lambda x: self.status_bar.cover_flow_button.setChecked(False))
+                self.cf_dialog = d
+            else:
+                cfd = getattr(self, 'cf_dialog', None)
+                if cfd is not None:
+                    self.cover_flow.setVisible(False)
+                    cfd.hide()
+                    self.cf_dialog = None
         else:
-            self.cover_flow.setVisible(False)
+            if show:
-            #self.status_bar.book_info.book_data.setMaximumHeight(1000)
+                self.library_view.setCurrentIndex(self.library_view.currentIndex())
-        self.setMaximumHeight(available_height())
+                self.cover_flow.setVisible(True)
+                self.cover_flow.setFocus(Qt.OtherFocusReason)
+                #self.status_bar.book_info.book_data.setMaximumHeight(100)
+                #self.status_bar.setMaximumHeight(120)
+                self.library_view.scrollTo(self.library_view.currentIndex())
+            else:
+                self.cover_flow.setVisible(False)
+                #self.status_bar.book_info.book_data.setMaximumHeight(1000)
+            self.setMaximumHeight(available_height())
 
     def toggle_tags_view(self, show):
         if show:
@@ -583,7 +614,8 @@ class Main(MainWindow, Ui_MainWindow):
         try:
             duplicates = self.library_view.model().db.recursive_import(root, single, callback=callback)
         finally:
-            progress.close()
+            progress.hide()
+            progress.close()            
         if duplicates:
             files = _('<p>Books with the same title as the following already exist in the database. Add them anyway?<ul>')
             for mi, formats in duplicates:
@@ -702,7 +734,8 @@ class Main(MainWindow, Ui_MainWindow):
             else:
                 self.upload_books(paths, list(map(sanitize_file_name, names)), infos, on_card=on_card)
         finally:
-            progress.setValue(len(paths))
+            progress.setValue(progress.maximum())
+            progress.hide()
             progress.close()
 
     def upload_books(self, files, names, metadata, on_card=False, memory=None):

src/calibre/library/database2.py

@@ -877,6 +877,14 @@ class LibraryDatabase2(LibraryDatabase):
         self.conn.commit()
         if notify:
             self.notify('metadata', [id])
+            
+    def set_timestamp(self, id, dt, notify=True):
+        if dt:
+            self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id))
+            self.data.set(id, FIELD_MAP['timestamp'], dt, row_is_id=True)
+            self.conn.commit()
+            if notify:
+                self.notify('metadata', [id])
     
     def set_publisher(self, id, publisher, notify=True):
         self.conn.execute('DELETE FROM books_publishers_link WHERE book=?',(id,))

src/calibre/manual/customize.rst

@@ -46,7 +46,7 @@ Create a file name :file:`my_plugin.py` (the file name must end with plugin.py)
       ext  = os.path.splitext(path_to_ebook)[-1][1:].lower()
       mi = get_metadata(file, ext)
       mi.publisher = 'Hello World'
-      set_metadata(file, ext, mi)
+      set_metadata(file, mi, ext)
       return path_to_ebook
 
 That's all. To add this code to |app| as a plugin, simply create a zip file with::

src/calibre/parallel.py

@@ -615,7 +615,7 @@ class Job(object):
                 self.log = unicode(self.log, 'utf-8', 'replace')
             ans.extend(self.log.split('\n'))
         
-        ans = [x.decode(preferred_encoding, 'replace') if isinstance(x, 'str') else x for x in ans]
+        ans = [x.decode(preferred_encoding, 'replace') if isinstance(x, str) else x for x in ans]
         
         return u'<br>'.join(ans)
 

src/calibre/web/feeds/recipes/__init__.py

@@ -20,7 +20,7 @@ recipe_modules = ['recipe_' + r for r in (
            'science_news', 'the_nation', 'lrb', 'harpers_full', 'liberation',
            'linux_magazine', 'telegraph_uk', 'utne', 'sciencedaily', 'forbes',
            'time_magazine', 'endgadget', 'fudzilla', 'nspm_int', 'nspm', 'pescanik',
-           'spiegel_int', 'themarketticker', 'tomshardware',
+           'spiegel_int', 'themarketticker', 'tomshardware', 'xkcd',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_time_magazine.py

@@ -6,22 +6,28 @@ __copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
 time.com
 '''
 
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Time(BasicNewsRecipe):
     title                 = u'Time'
-    __author__            = 'Darko Miletic'
+    __author__            = 'Kovid Goyal'
     description           = 'Weekly magazine'    
     oldest_article        = 7
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
     
-    #cover_url = 'http://img.timeinc.net/time/rd/trunk/www/web/feds/i/logo_time_home.gif'
-    
     keep_only_tags = [dict(name='div', attrs={'class':'tout1'})]
-    remove_tags    = [dict(name='ul', attrs={'class':['button', 'find']})]
+    remove_tags_after = [dict(id='connectStory')]
+    remove_tags    = [
+                      dict(name='ul', attrs={'class':['button', 'find']}),
+                      dict(name='div', attrs={'class':['nav', 'header', 'sectheader', 
+                                                       'searchWrap', 'subNav', 
+                                                       'artTools', 'connect',
+                                                       'similarrecs']}),
+                      dict(name='div', id=['articleSideBar', 'connectStory']),
+                      dict(name='dl', id=['links']),                                 
+                      ]
 
     feeds          = [
                        (u'Top Stories', u'http://feedproxy.google.com/time/topstories')
@@ -34,17 +40,20 @@ class Time(BasicNewsRecipe):
                        ,(u'Travel', u'http://feedproxy.google.com/time/travel')
                      ]
     
+    def get_article_url(self, article):
+        return article.get('guid',  article['link'])
+    
     def get_cover_url(self):
         soup = self.index_to_soup('http://www.time.com/time/')
         img = soup.find('img', alt='Current Time.com Cover', width='107')
         if img is not None:
             return img.get('src', None)
         
-    
     def print_version(self, url):
-        raw = self.browser.open(url).read()
+        try:
-        soup = BeautifulSoup(raw.decode('utf8', 'replace'))
+            soup = self.index_to_soup(url)
-        print_link = soup.find('a', {'id':'prt'})
+            print_link = soup.find('a', {'id':'prt'})
-        if print_link is None:
+            return 'http://www.time.com' + print_link['href']
-            return ''
+        except:
-        return 'http://www.time.com' + print_link['href']
+            self.log_exception('Failed to find print version for '+url)
+        return ''

src/calibre/web/feeds/recipes/recipe_xkcd.py

@@ -0,0 +1,36 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
+'''
+Fetch xkcd.
+'''
+
+import time
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class XkcdCom(BasicNewsRecipe):
+    title = 'xkcd'
+    description = 'A webcomic of romance and math humor.'
+    __author__ = 'Martin Pitt'
+    use_embedded_content   = False
+    oldest_article = 60
+    keep_only_tags = [dict(id='middleContent')]
+    remove_tags = [dict(name='ul'), dict(name='h3'), dict(name='br')]
+    no_stylesheets = True
+    
+    def parse_index(self):
+        INDEX = 'http://xkcd.com/archive/'
+
+        soup = self.index_to_soup(INDEX) 
+        articles = []
+        for item in soup.findAll('a', title=True):
+            articles.append({
+                'date': item['title'],
+                'timestamp': time.mktime(time.strptime(item['title'], '%Y-%m-%d'))+1,
+                'url': 'http://xkcd.com' + item['href'],
+                'title': self.tag_to_string(item).encode('UTF-8'),
+                'description': '',
+                'content': '',
+            })
+
+        return [('xkcd', articles)]

src/odf/namespaces.py

@@ -20,9 +20,10 @@
 TOOLSVERSION = u"ODFPY/0.8.1dev"
 
 ANIMNS         = u"urn:oasis:names:tc:opendocument:xmlns:animation:1.0"
+DBNS           = u"urn:oasis:names:tc:opendocument:xmlns:database:1.0"
 CHARTNS        = u"urn:oasis:names:tc:opendocument:xmlns:chart:1.0"
 CONFIGNS       = u"urn:oasis:names:tc:opendocument:xmlns:config:1.0"
-DBNS           = u"http://openoffice.org/2004/database"
+#DBNS           = u"http://openoffice.org/2004/database"
 DCNS           = u"http://purl.org/dc/elements/1.1/"
 DOMNS          = u"http://www.w3.org/2001/xml-events"
 DR3DNS         = u"urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0"
@@ -39,6 +40,7 @@ OOONS          = u"http://openoffice.org/2004/office"
 OOOWNS         = u"http://openoffice.org/2004/writer"
 OOOCNS         = u"http://openoffice.org/2004/calc"
 PRESENTATIONNS = u"urn:oasis:names:tc:opendocument:xmlns:presentation:1.0"
+RDFANS         = u"http://docs.oasis-open.org/opendocument/meta/rdfa#"
 SCRIPTNS       = u"urn:oasis:names:tc:opendocument:xmlns:script:1.0"
 SMILNS         = u"urn:oasis:names:tc:opendocument:xmlns:smil-compatible:1.0"
 STYLENS        = u"urn:oasis:names:tc:opendocument:xmlns:style:1.0"
@@ -47,6 +49,7 @@ TABLENS        = u"urn:oasis:names:tc:opendocument:xmlns:table:1.0"
 TEXTNS         = u"urn:oasis:names:tc:opendocument:xmlns:text:1.0"
 XFORMSNS       = u"http://www.w3.org/2002/xforms"
 XLINKNS        = u"http://www.w3.org/1999/xlink"
+XMLNS          = "http://www.w3.org/XML/1998/namespace"
 
 
 nsdict = {
@@ -70,6 +73,7 @@ nsdict = {
    OOOWNS: u'ooow',
    OOOCNS: u'ooc',
    PRESENTATIONNS: u'presentation',
+   RDFANS: u'rdfa',
    SCRIPTNS: u'script',
    SMILNS: u'smil',
    STYLENS: u'style',
@@ -78,4 +82,5 @@ nsdict = {
    TEXTNS: u'text',
    XFORMSNS: u'xforms',
    XLINKNS: u'xlink',
+   XMLNS: u'xml',
 }

src/odf/odf2xhtml.py

@@ -22,7 +22,7 @@
 #pdb.set_trace()
 import zipfile
 import xml.sax
-from xml.sax import handler
+from xml.sax import handler, expatreader
 from xml.sax.xmlreader import InputSource
 from xml.sax.saxutils import escape, quoteattr
 
@@ -206,10 +206,10 @@ class StyleToCSS:
         if hpos == "center":
             sdict['margin-left'] = "auto"
             sdict['margin-right'] = "auto"
-        else:
+#       else:
-            # force it to be *something* then delete it
+#           # force it to be *something* then delete it
-            sdict['margin-left'] = sdict['margin-right'] = ''
+#           sdict['margin-left'] = sdict['margin-right'] = ''
-            del sdict['margin-left'], sdict['margin-right']
+#           del sdict['margin-left'], sdict['margin-right']
 
         if hpos in ("right","outside"):
             if wrap in ( "left", "parallel","dynamic"):
@@ -336,8 +336,9 @@ special_styles = {
 class ODF2XHTML(handler.ContentHandler):
     """ The ODF2XHTML parses an ODF file and produces XHTML"""
 
-    def __init__(self):
+    def __init__(self, generate_css=True, embedable=False):
         # Tags
+        self.generate_css = generate_css
         self.elements = {
         (DCNS, 'title'): (self.s_processcont, self.e_dc_title),
         (DCNS, 'language'): (self.s_processcont, self.e_dc_contentlanguage),
@@ -349,6 +350,7 @@ class ODF2XHTML(handler.ContentHandler):
         (DRAWNS, 'fill-image'): (self.s_draw_fill_image, None),
         (DRAWNS, "layer-set"):(self.s_ignorexml, None),
         (DRAWNS, 'page'): (self.s_draw_page, self.e_draw_page),
+        (DRAWNS, 'text-box'): (self.s_draw_textbox, self.e_draw_textbox),
         (METANS, 'creation-date'):(self.s_processcont, self.e_dc_metatag),
         (METANS, 'generator'):(self.s_processcont, self.e_dc_metatag),
         (METANS, 'initial-creator'): (self.s_processcont, self.e_dc_metatag),
@@ -421,6 +423,12 @@ class ODF2XHTML(handler.ContentHandler):
         (TEXTNS, "table-of-content-source"):(self.s_text_x_source, self.e_text_x_source),
         (TEXTNS, "user-index-source"):(self.s_text_x_source, self.e_text_x_source),
         }
+        if embedable:
+            self.elements[(OFFICENS, u"text")] = (None,None)
+            self.elements[(OFFICENS, u"spreadsheet")] = (None,None)
+            self.elements[(OFFICENS, u"presentation")] = (None,None)
+            self.elements[(OFFICENS, u"document-content")] = (None,None)
+
 
     def writeout(self, s):
         if s != '':
@@ -548,14 +556,18 @@ class ODF2XHTML(handler.ContentHandler):
         """ A <draw:frame> is made into a <div> in HTML which is then styled
         """
         anchor_type = attrs.get((TEXTNS,'anchor-type'),'char')
+        htmltag = 'div'
         name = "G-" + attrs.get( (DRAWNS,'style-name'), "")
         if name == 'G-':
             name = "PR-" + attrs.get( (PRESENTATIONNS,'style-name'), "")
         name = name.replace(".","_")
         if anchor_type == "paragraph":
-            style = ""
+            style = 'position:relative;'
         elif anchor_type == 'char':
-            style = "position: relative;"
+            style = "position:relative;"
+        elif anchor_type == 'as-char':
+            htmltag = 'div'
+            style = ''
         else:
             style = "position: absolute;"
         if attrs.has_key( (SVGNS,"width") ):
@@ -566,7 +578,10 @@ class ODF2XHTML(handler.ContentHandler):
             style = style + "left:" +  attrs[(SVGNS,"x")] + ";"
         if attrs.has_key( (SVGNS,"y") ):
             style = style + "top:" +  attrs[(SVGNS,"y")] + ";"
-        self.opentag('div', {'class': name, 'style': style})
+        if self.generate_css:
+            self.opentag(htmltag, {'class': name, 'style': style})
+        else:
+            self.opentag(htmltag)
 
     def e_draw_frame(self, tag, attrs):
         """ End the <draw:frame>
@@ -593,8 +608,9 @@ class ODF2XHTML(handler.ContentHandler):
         imghref = attrs[(XLINKNS,"href")]
         imghref = self.rewritelink(imghref)
         htmlattrs = {'alt':"", 'src':imghref }
-        if anchor_type != "char":
+        if self.generate_css:
-            htmlattrs['style'] = "display: block;"
+            if anchor_type != "char":
+                htmlattrs['style'] = "display: block;"
         self.emptytag('img', htmlattrs)
 
     def s_draw_page(self, tag, attrs):
@@ -607,7 +623,10 @@ class ODF2XHTML(handler.ContentHandler):
         stylename = stylename.replace(".","_")
         masterpage = attrs.get( (DRAWNS,'master-page-name'),"")
         masterpage = masterpage.replace(".","_")
-        self.opentag('fieldset', {'class':"DP-%s MP-%s" % (stylename, masterpage) })
+        if self.generate_css:
+            self.opentag('fieldset', {'class':"DP-%s MP-%s" % (stylename, masterpage) })
+        else:
+            self.opentag('fieldset')
         self.opentag('legend')
         self.writeout(escape(name))
         self.closetag('legend')
@@ -615,17 +634,30 @@ class ODF2XHTML(handler.ContentHandler):
     def e_draw_page(self, tag, attrs):
         self.closetag('fieldset')
 
+    def s_draw_textbox(self, tag, attrs):
+        style = ''
+        if attrs.has_key( (FONS,"min-height") ):
+            style = style + "min-height:" +  attrs[(FONS,"min-height")] + ";"
+        self.opentag('div')
+#       self.opentag('div', {'style': style})
+
+    def e_draw_textbox(self, tag, attrs):
+        """ End the <draw:text-box>
+        """
+        self.closetag('div')
+
     def html_body(self, tag, attrs):
         self.writedata()
-        self.opentag('style', {'type':"text/css"}, True)
+        if self.generate_css:
-        self.writeout('/*<![CDATA[*/\n')
+            self.opentag('style', {'type':"text/css"}, True)
-        self.writeout('\nimg { width: 100%; height: 100%; }\n')
+            self.writeout('/*<![CDATA[*/\n')
-        self.writeout('* { padding: 0; margin: 0; }\n')
+            self.writeout('\nimg { width: 100%; height: 100%; }\n')
-        self.writeout('body { margin: 0 1em; }\n')
+            self.writeout('* { padding: 0; margin: 0;  background-color:white; }\n')
-        self.writeout('ol, ul { padding-left: 2em; }\n')
+            self.writeout('body { margin: 0 1em; }\n')
-        self.generate_stylesheet()
+            self.writeout('ol, ul { padding-left: 2em; }\n')
-        self.writeout('/*]]>*/\n')
+            self.generate_stylesheet()
-        self.closetag('style')
+            self.writeout('/*]]>*/\n')
+            self.closetag('style')
         self.purgedata()
         self.closetag('head')
         self.opentag('body', block=True)
@@ -660,7 +692,10 @@ class ODF2XHTML(handler.ContentHandler):
     def generate_footnotes(self):
         if self.currentnote == 0:
             return
-        self.opentag('ol', {'style':'border-top: 1px solid black'}, True)
+        if self.generate_css:
+            self.opentag('ol', {'style':'border-top: 1px solid black'}, True)
+        else:
+            self.opentag('ol')
         for key in range(1,self.currentnote+1):
             note = self.notedict[key]
 #       for key,note in self.notedict.items():
@@ -731,6 +766,8 @@ class ODF2XHTML(handler.ContentHandler):
         """ Copy all attributes to a struct.
             We will later convert them to CSS2
         """
+        if self.currentstyle is None:
+            return
         for key,attr in attrs.items():
             self.styledict[self.currentstyle][key] = attr
 
@@ -874,7 +911,7 @@ class ODF2XHTML(handler.ContentHandler):
         """ Start a table
         """
         c = attrs.get( (TABLENS,'style-name'), None)
-        if c:
+        if c and self.generate_css:
             c = c.replace(".","_")
             self.opentag('table',{ 'class': "T-%s" % c })
         else:
@@ -958,7 +995,7 @@ class ODF2XHTML(handler.ContentHandler):
         for x in range(level + 1,10):
             self.headinglevels[x] = 0
         special = special_styles.get("P-"+name)
-        if special:
+        if special or not self.generate_css:
             self.opentag('h%s' % level)
         else:
             self.opentag('h%s' % level, {'class':"P-%s" % name })
@@ -997,7 +1034,10 @@ class ODF2XHTML(handler.ContentHandler):
             # textbox itself may be nested within another list.
             level = self.tagstack.count_tags(tag) + 1
             name = self.tagstack.rfindattr( (TEXTNS,'style-name') )
-        self.opentag('%s' % self.listtypes.get(name), {'class':"%s_%d" % (name, level) })
+        if self.generate_css:
+            self.opentag('%s' % self.listtypes.get(name), {'class':"%s_%d" % (name, level) })
+        else:
+            self.opentag('%s' % self.listtypes.get(name))
         self.purgedata()
 
     def e_text_list(self, tag, attrs):
@@ -1113,7 +1153,8 @@ class ODF2XHTML(handler.ContentHandler):
             specialtag = special_styles.get("P-"+c)
             if specialtag is None:
                 specialtag = 'p'
-                htmlattrs['class'] = "P-%s" % c
+                if self.generate_css:
+                    htmlattrs['class'] = "P-%s" % c
         self.opentag(specialtag, htmlattrs)
         self.purgedata()
 
@@ -1149,7 +1190,7 @@ class ODF2XHTML(handler.ContentHandler):
         if c:
             c = c.replace(".","_")
             special = special_styles.get("S-"+c)
-            if special is None:
+            if special is None and self.generate_css:
                 htmlattrs['class'] = "S-%s" % c
         self.opentag('span', htmlattrs)
         self.purgedata()
@@ -1219,7 +1260,10 @@ class ODF2XHTML(handler.ContentHandler):
         # Extract the interesting files
         z = zipfile.ZipFile(self._odffile)
 
-        parser = xml.sax.make_parser()
+        # For some reason Trac has trouble when xml.sax.make_parser() is used.
+        # Could it be because PyXML is installed, and therefore a different parser
+        # might be chosen? By calling expatreader directly we avoid this issue
+        parser = expatreader.create_parser()
         parser.setFeature(handler.feature_namespaces, 1)
         parser.setContentHandler(self)
         parser.setErrorHandler(handler.ErrorHandler())

src/odf/opendocument.py

@@ -287,7 +287,7 @@ class OpenDocument:
             else:
                 ext = mimetypes.guess_extension(mediatype)
             manifestfn = "Pictures/%0.0f%s" % ((time.time()*10000000000), ext)
-            self.Pictures[manifestfn] = (IS_FILENAME, fileobj, mediatype)
+            self.Pictures[manifestfn] = (IS_FILENAME, filename, mediatype)
         else:
             manifestfn = filename
             self.Pictures[manifestfn] = (IS_IMAGE, content, mediatype)