Merge upstream changes

.pydevproject

@@ -2,7 +2,7 @@
 <?eclipse-pydev version="1.0"?>
 
 <pydev_project>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.5</pydev_property>
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
 <path>/calibre/src</path>
 </pydev_pathproperty>

installer/linux/freeze.py

@@ -38,6 +38,10 @@ def freeze():
                        '/usr/lib/libxml2.so.2',
                        '/usr/lib/libxslt.so.1',
                        '/usr/lib/libxslt.so.1',
+                       '/usr/lib/libgthread-2.0.so.0',
+                       '/usr/lib/libglib-2.0.so.0',
+                       '/usr/lib/gcc/i686-pc-linux-gnu/4.3.3/libstdc++.so.6',
+                       '/usr/lib/libpng12.so.0',
                        '/usr/lib/libexslt.so.0',
                        '/usr/lib/libMagickWand.so',
                        '/usr/lib/libMagickCore.so',
@@ -81,7 +85,8 @@ def freeze():
                 'PyQt4.QtScript.so', 'PyQt4.QtSql.so', 'PyQt4.QtTest.so', 'qt',
                 'glib', 'gobject']
     
-    packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg']
+    packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg', 
+                'dateutil']
     
     includes += ['calibre.web.feeds.recipes.'+r for r in recipe_modules]
     

installer/osx/freeze.py

@@ -342,6 +342,7 @@ def main():
                                        'calibre.ebooks.lrf.any.*', 'calibre.ebooks.lrf.feeds.*',
                                        'keyword', 'codeop', 'pydoc', 'readline',
                                        'BeautifulSoup', 'calibre.ebooks.lrf.fonts.prs500.*',
+                                       'dateutil',
                                        ],
                          'packages' : ['PIL', 'Authorization', 'lxml'],
                          'excludes' : ['IPython'],

installer/windows/freeze.py

@@ -179,7 +179,8 @@ def main(args=sys.argv):
                                              'calibre.ebooks.lrf.fonts.prs500.*',
                                              'PyQt4.QtWebKit', 'PyQt4.QtNetwork',
                                              ],
-                                  'packages'  : ['PIL', 'lxml', 'cherrypy'],
+                                  'packages'  : ['PIL', 'lxml', 'cherrypy',
+                                                 'dateutil'],
                                   'excludes'  : ["Tkconstants", "Tkinter", "tcl",
                                                  "_imagingtk", "ImageTk", "FixTk"
                                                 ],

src/calibre/__init__.py

@@ -69,7 +69,7 @@ def sanitize_file_name(name, substitute='_', as_unicode=False):
     one = re.sub(r'^\.+$', '_', one)
     if as_unicode:
         one = one.decode(filesystem_encoding)
-    return one
+    return one.replace('..', '_')
 
 
 class CommandLineError(Exception):
@@ -382,8 +382,10 @@ def walk(dir):
         for f in record[-1]:
             yield os.path.join(record[0], f)
 
-def strftime(fmt, t=time.localtime()):
+def strftime(fmt, t=None):
     ''' A version of strtime that returns unicode strings. '''
+    if t is None:
+        t = time.localtime()
     if iswindows:
         if isinstance(fmt, unicode):
             fmt = fmt.encode('mbcs')

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.135'
+__version__   = '0.4.137'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/customize/builtins.py

@@ -132,7 +132,7 @@ class HTMLMetadataReader(MetadataReaderPlugin):
 class MOBIMetadataReader(MetadataReaderPlugin):
     
     name        = 'Read MOBI metadata'
-    file_types  = set(['mobi', 'prc'])
+    file_types  = set(['mobi', 'prc', '.azw'])
     description = _('Read metadata from %s files')%'MOBI'
     
     def get_metadata(self, stream, ftype):

src/calibre/devices/cybookg3/driver.py

@@ -17,8 +17,8 @@ class CYBOOKG3(USBMS):
     # Be sure these have an entry in calibre.devices.mime
     FORMATS     = ['mobi', 'prc', 'html', 'pdf', 'rtf', 'txt']
     
-    VENDOR_ID   = 0x0bda
-    PRODUCT_ID  = 0x0703
+    VENDOR_ID   = [0x0bda, 0x3034]
+    PRODUCT_ID  = [0x0703, 0x1795]
     BCD         = [0x110, 0x132]
     
     VENDOR_NAME = 'BOOKEEN'

src/calibre/devices/kindle/driver.py

@@ -12,8 +12,8 @@ class KINDLE(USBMS):
     # Ordered list of supported formats
     FORMATS     = ['azw', 'mobi', 'prc', 'txt']
     
-    VENDOR_ID   = 0x1949
-    PRODUCT_ID  = 0x0001
+    VENDOR_ID   = [0x1949]
+    PRODUCT_ID  = [0x0001]
     BCD         = [0x399]
     
     VENDOR_NAME = 'KINDLE'

src/calibre/devices/prs505/books.py

@@ -186,7 +186,10 @@ class BookList(_BookList):
         node = self.document.createElement(self.prefix + "text")
         mime = MIME_MAP[name.rpartition('.')[-1].lower()]
         cid = self.max_id()+1
-        sourceid = str(self[0].sourceid) if len(self) else "1"
+        try:
+            sourceid = str(self[0].sourceid) if len(self) else '1'
+        except:
+            sourceid = '1'
         attrs = {
                  "title"  : info["title"], 
                  'titleSorter' : sortable_title(info['title']),

src/calibre/devices/prs505/driver.py

@@ -32,7 +32,7 @@ class PRS505(Device):
     BCD          = [0x229]  #: Needed to disambiguate 505 and 700 on linux
     PRODUCT_NAME = 'PRS-505'
     VENDOR_NAME  = 'SONY'
-    FORMATS      = ['lrf', 'epub', 'lrx', 'rtf', 'pdf', 'txt']
+    FORMATS      = ['epub', 'lrf', 'lrx', 'rtf', 'pdf', 'txt']
     
     MEDIA_XML    = 'database/cache/media.xml'
     CACHE_XML    = 'Sony Reader/database/cache.xml'
@@ -248,6 +248,7 @@ class PRS505(Device):
                 time.sleep(3)
                 self.open_osx()
         if self._card_prefix is not None:
+            try:
                 cachep = os.path.join(self._card_prefix, self.CACHE_XML)
                 if not os.path.exists(cachep):
                     os.makedirs(os.path.dirname(cachep), mode=0777)
@@ -257,6 +258,10 @@ class PRS505(Device):
 </cache>
 '''.encode('utf8'))
                     f.close()
+            except:
+                self._card_prefix = None
+                import traceback
+                traceback.print_exc()
             
     def set_progress_reporter(self, pr):
         self.report_progress = pr

src/calibre/devices/usbms/device.py

@@ -74,18 +74,21 @@ class Device(_Device):
     def get_fdi(cls):
         fdi = ''
 
+        for vid in cls.VENDOR_ID:
+            for pid in cls.PRODUCT_ID:
                 fdi_base_values = dict(
                                        app=__appname__,
                                        deviceclass=cls.__name__,
-                               vendor_id=hex(cls.VENDOR_ID),
-                               product_id=hex(cls.PRODUCT_ID),
+                                       vendor_id=hex(vid),
+                                       product_id=hex(pid),
                                        main_memory=cls.MAIN_MEMORY_VOLUME_LABEL,
                                        storage_card=cls.STORAGE_CARD_VOLUME_LABEL,
                                   )
+
                 if cls.BCD is None:
                     fdi_base_values['BCD_start'] = ''
                     fdi_base_values['BCD_end'] = ''
-            fdi = cls.FDI_TEMPLATE % fdi_base_values
+                    fdi += cls.FDI_TEMPLATE % fdi_base_values
                 else:
                     for bcd in cls.BCD:
                         fdi_bcd_values = fdi_base_values

src/calibre/ebooks/epub/from_any.py

@@ -124,6 +124,7 @@ MAP = {
        'lit'  : lit2opf,
        'mobi' : mobi2opf,
        'prc'  : mobi2opf,
+       'azw'  : mobi2opf,
        'fb2'  : fb22opf,
        'rtf'  : rtf2opf,
        'txt'  : txt2opf,
@@ -131,7 +132,8 @@ MAP = {
        'epub' : epub2opf,
        'odt'  : odt2epub,
        }
-SOURCE_FORMATS = ['lit', 'mobi', 'prc', 'fb2', 'odt', 'rtf', 'txt', 'pdf', 'rar', 'zip', 'oebzip', 'htm', 'html', 'epub']
+SOURCE_FORMATS = ['lit', 'mobi', 'prc', 'azw', 'fb2', 'odt', 'rtf', 
+                  'txt', 'pdf', 'rar', 'zip', 'oebzip', 'htm', 'html', 'epub']
 
 def unarchive(path, tdir):
     extract(path, tdir)

src/calibre/ebooks/epub/from_html.py

@@ -74,7 +74,9 @@ def check_links(opf_path, pretty_print):
         html_files = []
         for item in opf.itermanifest():
             if 'html' in item.get('media-type', '').lower():
-                f = item.get('href').split('/')[-1].decode('utf-8')
+                f = item.get('href').split('/')[-1]
+                if isinstance(f, str):
+                    f = f.decode('utf-8')
                 html_files.append(os.path.abspath(content(f)))
         
         for path in html_files:

src/calibre/ebooks/html.py

@@ -330,7 +330,10 @@ class PreProcessor(object):
                    sanitize_head),
                   # Convert all entities, since lxml doesn't handle them well
                   (re.compile(r'&(\S+?);'), convert_entities),
-                  
+                  # Remove the <![if/endif tags inserted by everybody's darling, MS Word
+                  (re.compile(r'(?i)<{0,1}!\[(end){0,1}if[^>]*>'), lambda match: ''),
+                  # Strip all comments since Adobe DE is petrified of them
+                  (re.compile(r'<!--[^>]*>'), lambda match : ''),
                   ]
                      
     # Fix pdftohtml markup
@@ -490,6 +493,16 @@ class Parser(PreProcessor, LoggingInterface):
             self.root.insert(0, head)
 
         self.head = head 
+        try:
+            self.body = self.root.body
+        except:
+            import traceback
+            err = traceback.format_exc()
+            self.root = fromstring(u'<html><head/><body><p>This page was too '
+                                   'severely malformed for calibre to handle. '
+                                   'It has been replaced by this error message.'
+                                   '</p><pre>%s</pre></body></html>'%err)
+            self.head = self.root.xpath('./head')[0]
             self.body = self.root.body
         for a in self.root.xpath('//a[@name]'):
             a.set('id', a.get('name'))

src/calibre/ebooks/lrf/__init__.py

@@ -29,6 +29,7 @@ preferred_source_formats = [
                             'XHTM',
                             'XHTML',
                             'PRC',
+                            'AZW',
                             'RTF',
                             'PDF',
                             'TXT',

src/calibre/ebooks/lrf/any/convert_from.py

@@ -154,7 +154,7 @@ def process_file(path, options, logger=None):
             convertor = txt2lrf
         elif 'epub' == ext:
             convertor = epub2lrf
-        elif ext in ['mobi', 'prc']:
+        elif ext in ['mobi', 'prc', 'azw']:
             convertor = mobi2lrf
         elif ext == 'fb2':
             convertor = fb22lrf

src/calibre/ebooks/metadata/__init__.py

@@ -192,7 +192,8 @@ class MetaInformation(object):
         for attr in ('author_sort', 'title_sort', 'comments', 'category',
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'tags', 'cover_data', 'application_id', 'guide',
-                     'manifest', 'spine', 'toc', 'cover', 'language', 'book_producer'):
+                     'manifest', 'spine', 'toc', 'cover', 'language', 
+                     'book_producer', 'timestamp'):
             if hasattr(mi, attr):
                 setattr(ans, attr, getattr(mi, attr))
         
@@ -217,7 +218,7 @@ class MetaInformation(object):
         for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher',
                   'series', 'series_index', 'rating', 'isbn', 'language',
                   'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
-                  'book_producer',
+                  'book_producer', 'timestamp'
                   ):
             setattr(self, x, getattr(mi, x, None))
     
@@ -235,7 +236,8 @@ class MetaInformation(object):
         for attr in ('author_sort', 'title_sort', 'comments', 'category',
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'application_id', 'manifest', 'spine', 'toc', 
-                     'cover', 'language', 'guide', 'book_producer'):
+                     'cover', 'language', 'guide', 'book_producer', 
+                     'timestamp'):
             if hasattr(mi, attr):
                 val = getattr(mi, attr)
                 if val is not None:
@@ -276,6 +278,8 @@ class MetaInformation(object):
             ans += u'Series   : '+unicode(self.series) + ' #%s\n'%self.format_series_index()  
         if self.language:
             ans += u'Language : '     + unicode(self.language) + u'\n'
+        if self.timestamp is not None:
+            ans += u'Timestamp : ' + self.timestamp.isoformat(' ')
         return ans.strip()
     
     def to_html(self):
@@ -289,12 +293,12 @@ class MetaInformation(object):
         if self.series:
             ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
         ans += [(_('Language'), unicode(self.language))]
+        if self.timestamp is not None:
+            ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
         for i, x in enumerate(ans):
             ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
         return u'<table>%s</table>'%u'\n'.join(ans)
         
-        
-    
     def __str__(self):
         return self.__unicode__().encode('utf-8')
     

src/calibre/ebooks/metadata/meta.py

@@ -31,8 +31,14 @@ def metadata_from_formats(formats):
     mi = MetaInformation(None, None)
     formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)],
                                      METADATA_PRIORITIES[path_to_ext(y)]))
-    for path in formats:
-        ext = path_to_ext(path)
+    extensions = list(map(path_to_ext, formats))
+    if 'opf' in extensions:
+        opf = formats[extensions.index('opf')]
+        mi2 = opf_metadata(opf)
+        if mi2 is not None and mi2.title:
+            return mi2
+    
+    for path, ext in zip(formats, extensions):
         stream = open(path, 'rb')
         try:
             mi.smart_update(get_metadata(stream, stream_type=ext, use_libprs_metadata=True))

src/calibre/ebooks/metadata/ncx.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0"  encoding="UTF-8"?>
 <?python
 from uuid import uuid4
+import re
 ?>
 <ncx version="2005-1" 
      xml:lang="en" 
@@ -19,7 +20,7 @@ from uuid import uuid4
     <py:def function="navpoint(np, level)">
         ${'%*s'%(4*level,'')}<navPoint id="${str(uuid4())}" playOrder="${str(np.play_order)}">
             ${'%*s'%(4*level,'')}<navLabel>
-                ${'%*s'%(4*level,'')}<text>${np.text}</text>
+                ${'%*s'%(4*level,'')}<text>${re.sub(r'\s+', ' ', np.text)}</text>
             ${'%*s'%(4*level,'')}</navLabel>
             ${'%*s'%(4*level,'')}<content src="${unicode(np.href)+(('#' + unicode(np.fragment)) if np.fragment else '')}" />
             <py:for each="np2 in np">${navpoint(np2, level+1)}</py:for>

src/calibre/ebooks/metadata/opf.xml

@@ -10,7 +10,7 @@
         <dc:creator opf:role="aut" py:for="i, author in enumerate(mi.authors)" py:attrs="{'opf:file-as':mi.author_sort} if mi.author_sort and i == 0 else {}">${author}</dc:creator>
         <dc:contributor opf:role="bkp" py:with="attrs={'opf:file-as':__appname__}" py:attrs="attrs">${'%s (%s)'%(__appname__, __version__)} [http://${__appname__}.kovidgoyal.net]</dc:contributor> 
         <dc:identifier opf:scheme="${__appname__}" id="${__appname__}_id">${mi.application_id}</dc:identifier>
-    
+        <dc:date py:if="getattr(mi, 'timestamp', None) is not None">${mi.timestamp.isoformat()}</dc:date>    
         <dc:language>${mi.language if mi.language else 'UND'}</dc:language>
         <dc:type py:if="mi.category">${mi.category}</dc:type>
         <dc:description py:if="mi.comments">${mi.comments}</dc:description>

src/calibre/ebooks/metadata/opf2.py

@@ -12,6 +12,7 @@ from urllib import unquote
 from urlparse import urlparse
 
 from lxml import etree
+from dateutil import parser
 
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre import relpath
@@ -436,6 +437,7 @@ class OPF(object):
     series          = MetadataField('series', is_dc=False)
     series_index    = MetadataField('series_index', is_dc=False, formatter=int, none_is=1)
     rating          = MetadataField('rating', is_dc=False, formatter=int)
+    timestamp       = MetadataField('date', formatter=parser.parse)
 
 
     def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True):

src/calibre/ebooks/mobi/reader.py

@@ -186,6 +186,8 @@ class MobiReader(object):
         self.processed_html = self.processed_html.decode(self.book_header.codec, 'ignore')
         for pat in ENCODING_PATS:
             self.processed_html = pat.sub('', self.processed_html)
+        self.processed_html = re.sub(r'&(\S+?);', entity_to_unicode,
+                                     self.processed_html)
         self.extract_images(processed_records, output_dir)
         self.replace_page_breaks()
         self.cleanup_html()
@@ -287,7 +289,8 @@ class MobiReader(object):
                 align = attrib.pop('align').strip()
                 if align:
                     styles.append('text-align: %s' % align)
-            if mobi_version == 1 and tag.tag == 'hr':
+            if tag.tag == 'hr':
+                if mobi_version == 1:
                     tag.tag = 'div'
                     styles.append('page-break-before: always')
                     styles.append('display: block')
@@ -311,6 +314,9 @@ class MobiReader(object):
                     recindex = attrib.pop(attr, None) or recindex
                 if recindex is not None:
                     attrib['src'] = 'images/%s.jpg' % recindex
+            elif tag.tag == 'pre':
+                if not tag.text:
+                    tag.tag = 'div'
             if styles:
                 attrib['style'] = '; '.join(styles)
             if 'filepos-id' in attrib:
@@ -454,6 +460,7 @@ class MobiReader(object):
             pos = end
         self.processed_html += self.mobi_html[pos:]
         
+    
     def extract_images(self, processed_records, output_dir):
         if self.verbose:
             print 'Extracting images...'

src/calibre/gui2/__init__.py

@@ -1,7 +1,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 """ The GUI """
-import sys, os, re, StringIO, traceback
+import sys, os, re, StringIO, traceback, time
 from PyQt4.QtCore import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, QSize, \
                          QByteArray, QLocale, QUrl, QTranslator, QCoreApplication, \
                          QModelIndex
@@ -14,6 +14,9 @@ from calibre import __author__, islinux, iswindows, isosx
 from calibre.startup import get_lang
 from calibre.utils.config import Config, ConfigProxy, dynamic
 import calibre.resources as resources
+from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
+from calibre.ebooks.metadata import MetaInformation
+
 
 NONE = QVariant() #: Null value to return from the data function of item models
 
@@ -149,6 +152,40 @@ class Dispatcher(QObject):
     def dispatch(self, args, kwargs):
         self.func(*args, **kwargs)
 
+class GetMetadata(QObject):
+    '''
+    Convenience class to ensure that metadata readers are used only in the
+    GUI thread. Must be instantiated in the GUI thread.
+    '''
+    
+    def __init__(self):
+        QObject.__init__(self)
+        self.connect(self, SIGNAL('edispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                     self._get_metadata, Qt.QueuedConnection)
+        self.connect(self, SIGNAL('idispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                     self._from_formats, Qt.QueuedConnection)
+        
+    def __call__(self, id, *args, **kwargs):
+        self.emit(SIGNAL('edispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                  id, args, kwargs)
+    
+    def from_formats(self, id, *args, **kwargs):
+        self.emit(SIGNAL('idispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                  id, args, kwargs)
+    
+    def _from_formats(self, id, args, kwargs):
+        try:
+            mi = metadata_from_formats(*args, **kwargs)
+        except:
+            mi = MetaInformation('', [_('Unknown')])
+        self.emit(SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'), id, mi)
+    
+    def _get_metadata(self, id, args, kwargs):
+        try:
+            mi = get_metadata(*args, **kwargs)
+        except:
+            mi = MetaInformation('', [_('Unknown')])
+        self.emit(SIGNAL('metadata(PyQt_PyObject, PyQt_PyObject)'), id, mi)
 
 class TableView(QTableView):
     def __init__(self, parent):

src/calibre/gui2/add.py

@@ -0,0 +1,224 @@
+'''
+UI for adding books to the database
+'''
+import os
+
+from PyQt4.Qt import QThread, SIGNAL, QMutex, QWaitCondition, Qt
+
+from calibre.gui2.dialogs.progress import ProgressDialog
+from calibre.constants import preferred_encoding
+from calibre.gui2.widgets import WarningDialog
+
+class Add(QThread):
+    
+    def __init__(self):
+        QThread.__init__(self)
+        self._lock = QMutex()
+        self._waiting = QWaitCondition()
+    
+    def is_canceled(self):
+        if self.pd.canceled:
+            self.canceled = True
+        return self.canceled
+    
+    def wait_for_condition(self):
+        self._lock.lock()
+        self._waiting.wait(self._lock)
+        self._lock.unlock()
+        
+    def wake_up(self):
+        self._waiting.wakeAll()
+ 
+class AddFiles(Add):
+    
+    def __init__(self, paths, default_thumbnail, get_metadata, db=None):
+        Add.__init__(self)
+        self.paths = paths
+        self.get_metadata = get_metadata
+        self.default_thumbnail = default_thumbnail
+        self.db = db
+        self.formats, self.metadata, self.names, self.infos = [], [], [], []
+        self.duplicates = []
+        self.number_of_books_added = 0
+        self.connect(self.get_metadata, 
+                     SIGNAL('metadata(PyQt_PyObject, PyQt_PyObject)'),
+                     self.metadata_delivered)
+    
+    def metadata_delivered(self, id, mi):
+        if self.is_canceled():
+            self.reading.wakeAll()
+            return
+        if not mi.title:
+            mi.title = os.path.splitext(self.names[id])[0]
+        mi.title = mi.title if isinstance(mi.title, unicode) else \
+                   mi.title.decode(preferred_encoding, 'replace')
+        self.metadata.append(mi)
+        self.infos.append({'title':mi.title, 
+                           'authors':', '.join(mi.authors),
+                           'cover':self.default_thumbnail, 'tags':[]})
+        if self.db is not None:
+            duplicates, num = self.db.add_books(self.paths[id:id+1], 
+                                self.formats[id:id+1], [mi], 
+                                add_duplicates=False)
+            self.number_of_books_added += num
+            if duplicates:
+                if not self.duplicates:
+                    self.duplicates = [[], [], [], []]
+                for i in range(4):
+                    self.duplicates[i] += duplicates[i]
+        self.emit(SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), 
+                      mi.title, id)
+        self.wake_up()
+    
+    def create_progress_dialog(self, title, msg, parent):
+        self._parent = parent
+        self.pd = ProgressDialog(title, msg, -1, len(self.paths)-1, parent)
+        self.connect(self, SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'),
+                     self.update_progress_dialog)
+        self.pd.setModal(True)
+        self.pd.show()
+        self.connect(self, SIGNAL('finished()'), self.pd.hide)
+        return self.pd
+        
+       
+    def update_progress_dialog(self, title, count):
+        self.pd.set_value(count)
+        if self.db is not None:
+            self.pd.set_msg(_('Added %s to library')%title)
+        else:
+            self.pd.set_msg(_('Read metadata from ')+title)
+        
+        
+    def run(self):
+        self.canceled = False
+        for c, book in enumerate(self.paths):
+            if self.pd.canceled:
+                self.canceled = True
+                break
+            format = os.path.splitext(book)[1]
+            format = format[1:] if format else None
+            stream = open(book, 'rb')
+            self.formats.append(format)
+            self.names.append(os.path.basename(book))
+            self.get_metadata(c, stream, stream_type=format, 
+                                  use_libprs_metadata=True)
+            self.wait_for_condition()
+        
+                    
+    def process_duplicates(self):
+        if self.duplicates:
+            files = _('<p>Books with the same title as the following already '
+                      'exist in the database. Add them anyway?<ul>')
+            for mi in self.duplicates[2]:
+                files += '<li>'+mi.title+'</li>\n'
+            d = WarningDialog (_('Duplicates found!'), 
+                              _('Duplicates found!'), 
+                              files+'</ul></p>', parent=self._parent)
+            if d.exec_() == d.Accepted:
+                num = self.db.add_books(*self.duplicates, 
+                                  **dict(add_duplicates=True))[1]
+                self.number_of_books_added += num
+                
+            
+class AddRecursive(Add):
+    
+    def __init__(self, path, db, get_metadata, single_book_per_directory, parent):
+        self.path = path
+        self.db = db
+        self.get_metadata = get_metadata
+        self.single_book_per_directory = single_book_per_directory
+        self.duplicates, self.books, self.metadata = [], [], []
+        self.number_of_books_added = 0
+        self.canceled = False
+        Add.__init__(self)
+        self.connect(self.get_metadata, 
+                     SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'),
+                     self.metadata_delivered, Qt.QueuedConnection)
+        self.connect(self, SIGNAL('searching_done()'), self.searching_done,
+                     Qt.QueuedConnection)    
+        self._parent = parent
+        self.pd = ProgressDialog(_('Adding books recursively...'),
+                                 _('Searching for books in all sub-directories...'), 
+                                 0, 0, parent)
+        self.connect(self, SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'),
+                     self.update_progress_dialog)
+        self.connect(self, SIGNAL('update(PyQt_PyObject)'), self.pd.set_msg,
+                     Qt.QueuedConnection)
+        self.connect(self, SIGNAL('pupdate(PyQt_PyObject)'), self.pd.set_value,
+                     Qt.QueuedConnection)
+        self.pd.setModal(True)
+        self.pd.show()
+        self.connect(self, SIGNAL('finished()'), self.pd.hide)
+        
+    def update_progress_dialog(self, title, count):
+        self.pd.set_value(count)
+        if title:
+            self.pd.set_msg(_('Read metadata from ')+title)
+    
+    def metadata_delivered(self, id, mi):
+        if self.is_canceled():
+            self.reading.wakeAll()
+            return
+        self.emit(SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'),
+                          mi.title, id)
+        self.metadata.append((mi if mi.title else None, self.books[id]))
+        if len(self.metadata) >= len(self.books):
+            self.metadata = [x for x in self.metadata if x[0] is not None]
+            self.pd.set_min(-1)
+            self.pd.set_max(len(self.metadata)-1)
+            self.pd.set_value(-1)
+            self.pd.set_msg(_('Adding books to database...'))
+        self.wake_up()
+    
+    def searching_done(self):
+        self.pd.set_min(-1)
+        self.pd.set_max(len(self.books)-1)
+        self.pd.set_value(-1)
+        self.pd.set_msg(_('Reading metadata...'))
+    
+   
+    def run(self):
+        root = os.path.abspath(self.path)
+        for dirpath in os.walk(root):
+            if self.is_canceled():
+                return
+            self.emit(SIGNAL('update(PyQt_PyObject)'), 
+                      _('Searching in')+' '+dirpath[0])
+            self.books += list(self.db.find_books_in_directory(dirpath[0], 
+                                            self.single_book_per_directory))
+        self.books = [formats for formats in self.books if formats]
+        # Reset progress bar
+        self.emit(SIGNAL('searching_done()'))
+        
+        for c, formats in enumerate(self.books):
+            self.get_metadata.from_formats(c, formats)
+            self.wait_for_condition()
+            
+        # Add books to database
+        for c, x in enumerate(self.metadata):
+            mi, formats = x
+            if self.is_canceled():
+                break
+            if self.db.has_book(mi):
+                self.duplicates.append((mi, formats))
+            else:
+                self.db.import_book(mi, formats, notify=False)
+                self.number_of_books_added += 1
+            self.emit(SIGNAL('pupdate(PyQt_PyObject)'), c)
+ 
+        
+    def process_duplicates(self):
+        if self.duplicates:
+            files = _('<p>Books with the same title as the following already '
+                      'exist in the database. Add them anyway?<ul>')
+            for mi in self.duplicates:
+                files += '<li>'+mi[0].title+'</li>\n'
+            d = WarningDialog (_('Duplicates found!'), 
+                              _('Duplicates found!'), 
+                              files+'</ul></p>', parent=self._parent)
+            if d.exec_() == d.Accepted:
+                for mi, formats in self.duplicates:
+                    self.db.import_book(mi, formats, notify=False)
+                    self.number_of_books_added += 1
+        
+        

src/calibre/gui2/dialogs/confirm_delete.py

@@ -5,7 +5,7 @@ __docformat__ = 'restructuredtext en'
 
 from calibre.gui2 import dynamic
 from calibre.gui2.dialogs.confirm_delete_ui import Ui_Dialog
-from PyQt4.Qt import QDialog, SIGNAL
+from PyQt4.Qt import QDialog, SIGNAL, Qt
 
 def _config_name(name):
     return name + '_again'
@@ -19,6 +19,7 @@ class Dialog(QDialog, Ui_Dialog):
         self.msg.setText(msg)
         self.name = name
         self.connect(self.again, SIGNAL('stateChanged(int)'), self.toggle)
+        self.buttonBox.setFocus(Qt.OtherFocusReason)
         
 
     def toggle(self, x):

src/calibre/gui2/dialogs/metadata_single.py

@@ -113,6 +113,10 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
     def set_cover(self):
         row = self.formats.currentRow()
         fmt = self.formats.item(row)
+        if fmt is None:
+            error_dialog(self, _('No format selected'),
+                    _('No format selected')).exec_()
+            return
         ext = fmt.ext.lower()
         if fmt.path is None:
             stream = self.db.format(self.row, ext, as_file=True)
@@ -265,7 +269,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
     def cover_dropped(self):
         self.cover_changed = True
     
-    def initialize_series_and_publisher(self):
+    def initialize_series(self):
         all_series = self.db.all_series()
         all_series.sort(cmp=lambda x, y : cmp(x[1], y[1]))
         series_id = self.db.series_id(self.row)
@@ -289,6 +293,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                 l.invalidate()
                 l.activate()
         
+    def initialize_series_and_publisher(self):
+        self.initialize_series()
         all_publishers = self.db.all_publishers()
         all_publishers.sort(cmp=lambda x, y : cmp(x[1], y[1]))
         publisher_id = self.db.publisher_id(self.row)

src/calibre/gui2/dialogs/progress.py

@@ -20,6 +20,7 @@ class ProgressDialog(QDialog, Ui_Dialog):
         self.setWindowModality(Qt.ApplicationModal)
         self.set_min(min)
         self.set_max(max)
+        self.bar.setValue(min)
         self.canceled = False
         
         self.connect(self.button_box, SIGNAL('rejected()'), self._canceled)

src/calibre/gui2/library.py

@@ -1,3 +1,4 @@
+from calibre.ebooks.metadata import authors_to_string
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import os, textwrap, traceback, time, re
@@ -371,35 +372,24 @@ class BooksModel(QAbstractTableModel):
         if not rows_are_ids:
             rows = [self.db.id(row.row()) for row in rows]
         for id in rows:
-            au = self.db.authors(id, index_is_id=True)
-            tags = self.db.tags(id, index_is_id=True)
-            if not au:
-                au = _('Unknown')
-            au = au.split(',')
-            if len(au) > 1:
-                t = ', '.join(au[:-1])
-                t += ' & ' + au[-1]
-                au = t
-            else:
-                au = ' & '.join(au)
-            if not tags:
-                tags = []
-            else:
-                tags = tags.split(',')
-            series = self.db.series(id, index_is_id=True)
-            if series is not None:
-                tags.append(series)
-            mi = {
-                  'title'   : self.db.title(id, index_is_id=True),
+            mi = self.db.get_metadata(id, index_is_id=True)
+            au = authors_to_string(mi.authors if mi.authors else [_('Unknown')])
+            tags = mi.tags if mi.tags else []
+            if mi.series is not None:
+                tags.append(mi.series)
+            info = {
+                  'title'   : mi.title,
                   'authors' : au,
                   'cover'   : self.db.cover(id, index_is_id=True),
                   'tags'    : tags,
-                  'comments': self.db.comments(id, index_is_id=True),
+                  'comments': mi.comments,
+                  }
+            if mi.series is not None:
+                info['tag order'] = {
+                    mi.series:self.db.books_in_series_of(id, index_is_id=True)
                 }
-            if series is not None:
-                mi['tag order'] = {series:self.db.books_in_series_of(id, index_is_id=True)}
 
-            metadata.append(mi)
+            metadata.append(info)
         return metadata
 
     def get_preferred_formats_from_ids(self, ids, all_formats, mode='r+b'):
@@ -424,8 +414,11 @@ class BooksModel(QAbstractTableModel):
                      
             
     
-    def get_preferred_formats(self, rows, formats, paths=False, set_metadata=False):
+    def get_preferred_formats(self, rows, formats, paths=False, 
+                              set_metadata=False, specific_format=None):
         ans = []
+        if specific_format is not None:
+            formats = [specific_format.lower()]
         for row in (row.row() for row in rows):
             format = None
             fmts = self.db.formats(row)

src/calibre/gui2/main.py

@@ -13,7 +13,6 @@ from PyQt4.QtSvg import QSvgRenderer
 from calibre import __version__, __appname__, islinux, sanitize_file_name, \
                     iswindows, isosx, preferred_encoding
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.ebooks.metadata.meta import get_metadata
 from calibre.devices.errors import FreeSpaceError
 from calibre.devices.interface import Device
 from calibre.utils.config import prefs, dynamic
@@ -23,7 +22,7 @@ from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \
                            set_sidebar_directories, Dispatcher, \
                            SingleApplication, Application, available_height, \
                            max_available_height, config, info_dialog, \
-                           available_width
+                           available_width, GetMetadata
 from calibre.gui2.cover_flow import CoverFlow, DatabaseImages, pictureflowerror
 from calibre.gui2.dialogs.scheduler import Scheduler
 from calibre.gui2.update import CheckForUpdates
@@ -49,7 +48,6 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.library.database2 import LibraryDatabase2, CoverCache
 from calibre.parallel import JobKilled
 from calibre.utils.filenames import ascii_filename
-from calibre.gui2.widgets import WarningDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 
 class Main(MainWindow, Ui_MainWindow):
@@ -78,6 +76,7 @@ class Main(MainWindow, Ui_MainWindow):
         self.setupUi(self)
         self.setWindowTitle(__appname__)
         self.verbose = opts.verbose
+        self.get_metadata = GetMetadata()
         self.read_settings()
         self.job_manager = JobManager()
         self.jobs_dialog = JobsDialog(self, self.job_manager)
@@ -157,6 +156,9 @@ class Main(MainWindow, Ui_MainWindow):
         sm.addAction(QIcon(':/images/sd.svg'), _('Send to storage card'))
         sm.addAction(QIcon(':/images/reader.svg'), _('Send to main memory')+' '+_('and delete from library'))
         sm.addAction(QIcon(':/images/sd.svg'), _('Send to storage card')+' '+_('and delete from library'))
+        sm.addAction(self.action_send_specific_format_to_device)
+        self.connect(self.action_send_specific_format_to_device, 
+                     SIGNAL('triggered()'), self.send_specific_format_to_device)
         sm.addSeparator()
         sm.addAction(_('Send to storage card by default'))
         sm.actions()[-1].setCheckable(True)
@@ -331,11 +333,16 @@ class Main(MainWindow, Ui_MainWindow):
             self.cover_flow.setVisible(False)
             if not config['separate_cover_flow']:
                 self.library.layout().addWidget(self.cover_flow)
-            self.connect(self.cover_flow, SIGNAL('currentChanged(int)'), self.sync_cf_to_listview)
-            self.connect(self.cover_flow, SIGNAL('itemActivated(int)'), self.show_book_info)
-            self.connect(self.status_bar.cover_flow_button, SIGNAL('toggled(bool)'), self.toggle_cover_flow)
-            self.connect(self.cover_flow, SIGNAL('stop()'), self.status_bar.cover_flow_button.toggle)
-            QObject.connect(self.library_view.selectionModel(), SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'),
+            self.connect(self.cover_flow, SIGNAL('currentChanged(int)'), 
+                         self.sync_cf_to_listview)
+            self.connect(self.cover_flow, SIGNAL('itemActivated(int)'), 
+                         self.show_book_info)
+            self.connect(self.status_bar.cover_flow_button, 
+                         SIGNAL('toggled(bool)'), self.toggle_cover_flow)
+            self.connect(self.cover_flow, SIGNAL('stop()'), 
+                         self.status_bar.cover_flow_button.toggle)
+            QObject.connect(self.library_view.selectionModel(), 
+                            SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'),
                             self.sync_cf_to_listview)
             self.db_images = DatabaseImages(self.library_view.model())
             self.cover_flow.setImages(self.db_images)
@@ -391,7 +398,7 @@ class Main(MainWindow, Ui_MainWindow):
     def change_output_format(self, x):
         of = unicode(x).strip()
         if of != prefs['output_format']:
-            if of not in ('LRF',):
+            if of not in ('LRF', 'EPUB'):
                 warning_dialog(self, 'Warning', 
                                '<p>%s support is still in beta. If you find bugs, please report them by opening a <a href="http://calibre.kovidgoyal.net">ticket</a>.'%of).exec_()
             prefs.set('output_format', of)
@@ -483,7 +490,8 @@ class Main(MainWindow, Ui_MainWindow):
         if not hasattr(index, 'row') and self.library_view.currentIndex().row() != index:
             index = self.library_view.model().index(index, 0)
             self.library_view.setCurrentIndex(index)
-        if hasattr(index, 'row') and self.cover_flow.isVisible() and self.cover_flow.currentSlide() != index.row():
+        if hasattr(index, 'row') and self.cover_flow.isVisible() and \
+                self.cover_flow.currentSlide() != index.row():
             self.cover_flow.setCurrentSlide(index.row())
 
     def another_instance_wants_to_talk(self, msg):
@@ -608,35 +616,25 @@ class Main(MainWindow, Ui_MainWindow):
     ################################# Add books ################################
 
     def add_recursive(self, single):
-        root = choose_dir(self, 'recursive book import root dir dialog', 'Select root folder')
+        root = choose_dir(self, 'recursive book import root dir dialog', 
+                          'Select root folder')
         if not root:
             return
-        progress = ProgressDialog(_('Adding books recursively...'),
-                                   min=0, max=0, parent=self)
-        progress.show()
-        def callback(msg):
-            if msg != '.':
-                progress.set_msg((_('Added ')+msg) if msg else _('Searching...'))
-            QApplication.processEvents()
-            QApplication.sendPostedEvents()
-            QApplication.flush()
-            return progress.canceled
-        try:
-            duplicates = self.library_view.model().db.recursive_import(root, single, callback=callback)
-        finally:
-            progress.hide()            
-        if duplicates:
-            files = _('<p>Books with the same title as the following already exist in the database. Add them anyway?<ul>')
-            for mi, formats in duplicates:
-                files += '<li>'+mi.title+'</li>\n'
-            d = WarningDialog(_('Duplicates found!'), _('Duplicates found!'), 
-                              files+'</ul></p>', self)
-            if d.exec_() == QDialog.Accepted:
-                for mi, formats in duplicates:
-                    self.library_view.model().db.import_book(mi, formats )
+        from calibre.gui2.add import AddRecursive
+        self._add_recursive_thread = AddRecursive(root, 
+                                self.library_view.model().db, self.get_metadata,
+                                single, self)
+        self.connect(self._add_recursive_thread, SIGNAL('finished()'),
+                     self._recursive_files_added)
+        self._add_recursive_thread.start()
     
-        self.library_view.model().resort()
+    def _recursive_files_added(self):
+        self._add_recursive_thread.process_duplicates()
+        if self._add_recursive_thread.number_of_books_added > 0:
+            self.library_view.model().resort(reset=False)
             self.library_view.model().research()
+            self.library_view.model().count_changed()
+        self._add_recursive_thread = None
         
     def add_recursive_single(self, checked):
         '''
@@ -686,65 +684,41 @@ class Main(MainWindow, Ui_MainWindow):
             return
         to_device = self.stack.currentIndex() != 0
         self._add_books(books, to_device)
-        if to_device:
-            self.status_bar.showMessage(_('Uploading books to device.'), 2000)
+        
 
     def _add_books(self, paths, to_device, on_card=None):
         if on_card is None:
             on_card = self.stack.currentIndex() == 2
         if not paths:
             return
-        # Get format and metadata information
-        formats, metadata, names, infos = [], [], [], []
-        progress = ProgressDialog(_('Adding books...'), _('Reading metadata...'),
-                                  min=0, max=len(paths), parent=self)
-        progress.show()
-        try:
-            for c, book in enumerate(paths):
-                progress.set_value(c+1)
-                if progress.canceled:
-                    return
-                format = os.path.splitext(book)[1]
-                format = format[1:] if format else None
-                stream = open(book, 'rb')
-                try:
-                    mi = get_metadata(stream, stream_type=format, use_libprs_metadata=True)
-                except:
-                    mi = MetaInformation(None, None)
-                if not mi.title:
-                    mi.title = os.path.splitext(os.path.basename(book))[0]
-                if not mi.authors:
-                    mi.authors = [_('Unknown')]
-                formats.append(format)
-                metadata.append(mi)
-                names.append(os.path.basename(book))
-                infos.append({'title':mi.title, 'authors':', '.join(mi.authors),
-                              'cover':self.default_thumbnail, 'tags':[]})
-                title = mi.title if isinstance(mi.title, unicode) else mi.title.decode(preferred_encoding, 'replace')
-                progress.set_msg(_('Read metadata from ')+title)
-                QApplication.processEvents()
+        from calibre.gui2.add import AddFiles
+        self._add_files_thread = AddFiles(paths, self.default_thumbnail, 
+                                          self.get_metadata,
+                                          None if to_device else \
+                                          self.library_view.model().db
+                                          )
+        self._add_files_thread.send_to_device = to_device
+        self._add_files_thread.on_card = on_card
+        self._add_files_thread.create_progress_dialog(_('Adding books...'),
+                                                _('Reading metadata...'), self)
+        self.connect(self._add_files_thread, SIGNAL('finished()'),
+                     self._files_added)
+        self._add_files_thread.start()
     
-            if not to_device:
-                progress.set_msg(_('Adding books to database...'))
-                QApplication.processEvents()
-                model = self.library_view.model()
-                
-                paths = list(paths)
-                duplicates, number_added = model.add_books(paths, formats, metadata)
-                if duplicates:
-                    files = _('<p>Books with the same title as the following already exist in the database. Add them anyway?<ul>')
-                    for mi in duplicates[2]:
-                        files += '<li>'+mi.title+'</li>\n'
-                    d = WarningDialog(_('Duplicates found!'), _('Duplicates found!'), files+'</ul></p>', parent=self)
-                    if d.exec_() == QDialog.Accepted:
-                        num = model.add_books(*duplicates, **dict(add_duplicates=True))[1]
-                        number_added += num
-                model.books_added(number_added)
+    def _files_added(self):
+        t = self._add_files_thread
+        self._add_files_thread = None
+        if not t.canceled:
+            if t.send_to_device:
+                self.upload_books(t.paths, 
+                                  list(map(sanitize_file_name, t.names)), 
+                                  t.infos, on_card=t.on_card)
+                self.status_bar.showMessage(_('Uploading books to device.'), 2000)
             else:
-                self.upload_books(paths, list(map(sanitize_file_name, names)), infos, on_card=on_card)
-        finally:
-            QApplication.processEvents()
-            progress.hide()
+                t.process_duplicates()
+        if t.number_of_books_added > 0:
+            self.library_view.model().books_added(t.number_of_books_added)
+            self.db_images.reset()
         
     def upload_books(self, files, names, metadata, on_card=False, memory=None):
         '''
@@ -801,7 +775,10 @@ class Main(MainWindow, Ui_MainWindow):
         if not rows or len(rows) == 0:
             return
         if self.stack.currentIndex() == 0:
-            if not confirm('<p>'+_('The selected books will be <b>permanently deleted</b> and the files removed from your computer. Are you sure?')+'</p>', 'library_delete_books', self):
+            if not confirm('<p>'+_('The selected books will be '
+                                   '<b>permanently deleted</b> and the files '
+                                   'removed from your computer. Are you sure?')
+                                +'</p>', 'library_delete_books', self):
                 return
             view.model().delete_books(rows)
         else:
@@ -919,8 +896,16 @@ class Main(MainWindow, Ui_MainWindow):
                 self.upload_books(files, names, metadata, on_card=on_card, memory=[[f.name for f in files], remove])
                 self.status_bar.showMessage(_('Sending news to device.'), 5000)
                 
+    def send_specific_format_to_device(self):
+        d = ChooseFormatDialog(self, _('Choose format to send to device'), 
+                               self.device_manager.device_class.FORMATS)
+        d.exec_()
+        fmt = d.format().lower()
+        on_card = config['send_to_storage_card_by_default']
+        self.sync_to_device(on_card, False, specific_format=fmt)
         
-    def sync_to_device(self, on_card, delete_from_library):
+    
+    def sync_to_device(self, on_card, delete_from_library, specific_format=None):
         rows = self.library_view.selectionModel().selectedRows()
         if not self.device_manager or not rows or len(rows) == 0:
             return
@@ -933,7 +918,8 @@ class Main(MainWindow, Ui_MainWindow):
         metadata = iter(metadata)
         _files   = self.library_view.model().get_preferred_formats(rows,
                                     self.device_manager.device_class.FORMATS, 
-                                    paths=True, set_metadata=True)
+                                    paths=True, set_metadata=True,
+                                    specific_format=specific_format)
         files = [getattr(f, 'name', None) for f in _files]
         bad, good, gf, names, remove_ids = [], [], [], [], []
         for f in files:
@@ -1410,8 +1396,15 @@ class Main(MainWindow, Ui_MainWindow):
     def initialize_database(self):
         self.library_path = prefs['library_path']
         if self.library_path is None: # Need to migrate to new database layout
+            base = os.path.expanduser('~')
+            if iswindows:
+                from calibre import plugins
+                from PyQt4.Qt import QDir
+                base = plugins['winutil'][0].special_folder_path(plugins['winutil'][0].CSIDL_PERSONAL)
+                if not base or not os.path.exists(base):
+                    base = unicode(QDir.homePath()).replace('/', os.sep)
             dir = unicode(QFileDialog.getExistingDirectory(self, 
-                            _('Choose a location for your ebook library.'), os.getcwd()))
+                            _('Choose a location for your ebook library.'), base))
             if not dir:
                 dir = os.path.expanduser('~/Library')
             self.library_path = os.path.abspath(dir)
@@ -1597,6 +1590,11 @@ def main(args=sys.argv):
             print 'Restarting with:', e, sys.argv
             os.execvp(e, sys.argv)
         else:
+            if iswindows:
+                try:
+                    main.system_tray_icon.hide()
+                except:
+                    pass
             return ret
     return 0
 

src/calibre/gui2/main.ui

@@ -656,6 +656,15 @@
     <string>Books with the same tags</string>
    </property>
   </action>
+  <action name="action_send_specific_format_to_device" >
+   <property name="icon" >
+    <iconset resource="images.qrc" >
+     <normaloff>:/images/book.svg</normaloff>:/images/book.svg</iconset>
+   </property>
+   <property name="text" >
+    <string>Send specific format to device</string>
+   </property>
+  </action>
  </widget>
  <customwidgets>
   <customwidget>

src/calibre/library/database.py

@@ -4,13 +4,10 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Backend that implements storage of ebooks in an sqlite database.
 '''
 import sqlite3 as sqlite
-import datetime, re, os, cPickle, sre_constants
+import datetime, re, cPickle, sre_constants
 from zlib import compress, decompress
 
-from calibre import sanitize_file_name
-from calibre.ebooks.metadata.meta import set_metadata, metadata_from_formats
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.web.feeds.recipes import migrate_automatic_profile_to_automatic_recipe
 
 class Concatenate(object):
@@ -1391,116 +1388,11 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
                          
 
 
-    def import_book(self, mi, formats):
-        series_index = 1 if mi.series_index is None else mi.series_index
-        if not mi.authors:
-            mi.authors = [_('Unknown')]
-        aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors)
-        obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)',
-                          (mi.title, None, series_index, aus))
-        id = obj.lastrowid
-        self.conn.commit()
-        self.set_metadata(id, mi)
-        for path in formats:
-            ext = os.path.splitext(path)[1][1:].lower()
-            stream = open(path, 'rb')
-            stream.seek(0, 2)
-            usize = stream.tell()
-            stream.seek(0)
-            data = sqlite.Binary(compress(stream.read()))
-            try:
-                self.conn.execute('INSERT INTO data(book, format, uncompressed_size, data) VALUES (?,?,?,?)',
-                              (id, ext, usize, data))
-            except sqlite.IntegrityError:
-                self.conn.execute('UPDATE data SET uncompressed_size=?, data=? WHERE book=? AND format=?',
-                                  (usize, data, id, ext))
-        self.conn.commit()
-
-    def import_book_directory_multiple(self, dirpath, callback=None):
-        dirpath = os.path.abspath(dirpath)
-        duplicates = []
-        books = {}
-        for path in os.listdir(dirpath):
-            if callable(callback):
-                callback('.')
-            path = os.path.abspath(os.path.join(dirpath, path))
-            if os.path.isdir(path) or not os.access(path, os.R_OK):
-                continue
-            ext = os.path.splitext(path)[1]
-            if not ext:
-                continue
-            ext = ext[1:].lower()
-            if ext not in BOOK_EXTENSIONS:
-                continue
-
-            key = os.path.splitext(path)[0]
-            if not books.has_key(key):
-                books[key] = []
-
-            books[key].append(path)
-
-        for formats in books.values():
-            mi = metadata_from_formats(formats)
-            if mi.title is None:
-                continue
-            if self.has_book(mi):
-                duplicates.append((mi, formats))
-                continue
-            self.import_book(mi, formats)
-            if callable(callback):
-                if callback(mi.title):
-                    break
-        return duplicates
-
-
-    def import_book_directory(self, dirpath, callback=None):
-        dirpath = os.path.abspath(dirpath)
-        formats = []
-        for path in os.listdir(dirpath):
-            if callable(callback):
-                callback('.')
-            path = os.path.abspath(os.path.join(dirpath, path))
-            if os.path.isdir(path) or not os.access(path, os.R_OK):
-                continue
-            ext = os.path.splitext(path)[1]
-            if not ext:
-                continue
-            ext = ext[1:].lower()
-            if ext not in BOOK_EXTENSIONS:
-                continue
-            formats.append(path)
-
-        if not formats:
-            return
-        
-        mi = metadata_from_formats(formats)
-        if mi.title is None:
-            return
-        if self.has_book(mi):
-            return [(mi, formats)]
-        self.import_book(mi, formats)
-        if callable(callback):
-            callback(mi.title)
-            
     
 
     def has_id(self, id):
         return self.conn.get('SELECT id FROM books where id=?', (id,), all=False) is not None
 
-    def recursive_import(self, root, single_book_per_directory=True, callback=None):
-        root = os.path.abspath(root)
-        duplicates  = []
-        for dirpath in os.walk(root):
-            res = self.import_book_directory(dirpath[0], callback=callback) if \
-                single_book_per_directory else \
-                  self.import_book_directory_multiple(dirpath[0], callback=callback)
-            if res is not None:
-                duplicates.extend(res)
-            if callable(callback):
-                if callback(''):
-                    break
-            
-        return duplicates
     
     
 

src/calibre/library/database2.py

@@ -12,20 +12,24 @@ from itertools import repeat
 from datetime import datetime
 
 from PyQt4.QtCore import QCoreApplication, QThread, QReadWriteLock
-from PyQt4.QtGui import QApplication, QPixmap, QImage
+from PyQt4.QtGui import QApplication, QImage
 __app = None
 
 from calibre.library import title_sort
 from calibre.library.database import LibraryDatabase
 from calibre.library.sqlite import connect, IntegrityError
 from calibre.utils.search_query_parser import SearchQueryParser
-from calibre.ebooks.metadata import string_to_authors, authors_to_string, MetaInformation
-from calibre.ebooks.metadata.meta import get_metadata, set_metadata
+from calibre.ebooks.metadata import string_to_authors, authors_to_string, \
+                                    MetaInformation, authors_to_sort_string
+from calibre.ebooks.metadata.meta import get_metadata, set_metadata, \
+    metadata_from_formats
 from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.customize.ui import run_plugins_on_import
+
 from calibre import sanitize_file_name
+from calibre.ebooks import BOOK_EXTENSIONS
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 
@@ -377,8 +381,10 @@ class LibraryDatabase2(LibraryDatabase):
             return row[loc]
         
         for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn', 
-                     'publisher', 'rating', 'series', 'series_index', 'tags', 'title'):
-            setattr(self, prop, functools.partial(get_property, loc=FIELD_MAP['comments' if prop == 'comment' else prop]))
+                     'publisher', 'rating', 'series', 'series_index', 'tags', 
+                     'title', 'timestamp'):
+            setattr(self, prop, functools.partial(get_property, 
+                    loc=FIELD_MAP['comments' if prop == 'comment' else prop]))
         
     def initialize_database(self):
         from calibre.resources import metadata_sqlite
@@ -587,6 +593,7 @@ class LibraryDatabase2(LibraryDatabase):
         mi.author_sort = self.author_sort(idx, index_is_id=index_is_id)
         mi.comments    = self.comments(idx, index_is_id=index_is_id)
         mi.publisher   = self.publisher(idx, index_is_id=index_is_id)
+        mi.timestamp   = self.timestamp(idx, index_is_id=index_is_id)
         tags = self.tags(idx, index_is_id=index_is_id)
         if tags:
             mi.tags = [i.strip() for i in tags.split(',')]
@@ -627,7 +634,7 @@ class LibraryDatabase2(LibraryDatabase):
             if not QCoreApplication.instance():
                 global __app
                 __app = QApplication([])
-            p = QPixmap()
+            p = QImage()
             if callable(getattr(data, 'read', None)):
                 data = data.read()
             p.loadFromData(data)
@@ -881,6 +888,8 @@ class LibraryDatabase2(LibraryDatabase):
             self.set_isbn(id, mi.isbn, notify=False)
         if mi.series_index and mi.series_index > 0:
             self.set_series_index(id, mi.series_index, notify=False)
+        if getattr(mi, 'timestamp', None) is not None:
+            self.set_timestamp(id, mi.timestamp, notify=False)
         self.set_path(id, True)
         self.notify('metadata', [id])
         
@@ -1142,7 +1151,7 @@ class LibraryDatabase2(LibraryDatabase):
     def add_books(self, paths, formats, metadata, uris=[], add_duplicates=True):
         '''
         Add a book to the database. The result cache is not updated.
-        @param paths: List of paths to book files or file-like objects
+        :param:`paths` List of paths to book files or file-like objects
         '''
         formats, metadata, uris = iter(formats), iter(metadata), iter(uris)
         duplicates = []
@@ -1180,30 +1189,39 @@ class LibraryDatabase2(LibraryDatabase):
         self.conn.commit()
         self.data.refresh_ids(self.conn, ids) # Needed to update format list and size
         if duplicates:
-            paths    = tuple(duplicate[0] for duplicate in duplicates)
-            formats  = tuple(duplicate[1] for duplicate in duplicates)
-            metadata = tuple(duplicate[2] for duplicate in duplicates)
-            uris     = tuple(duplicate[3] for duplicate in duplicates)
+            paths    = list(duplicate[0] for duplicate in duplicates)
+            formats  = list(duplicate[1] for duplicate in duplicates)
+            metadata = list(duplicate[2] for duplicate in duplicates)
+            uris     = list(duplicate[3] for duplicate in duplicates)
             return (paths, formats, metadata, uris), len(ids)
         return None, len(ids)
      
-    def import_book(self, mi, formats):
+    def import_book(self, mi, formats, notify=True):
         series_index = 1 if mi.series_index is None else mi.series_index
+        if not mi.title:
+            mi.title = _('Unknown')
         if not mi.authors:
-            mi.authors = ['Unknown']
-        aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors)
+            mi.authors = [_('Unknown')]
+        aus = mi.author_sort if mi.author_sort else authors_to_sort_string(mi.authors)
+        if isinstance(aus, str):
+            aus = aus.decode(preferred_encoding, 'replace')
+        title = mi.title if isinstance(mi.title, unicode) else \
+                mi.title.decode(preferred_encoding, 'replace')
         obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)', 
-                          (mi.title, None, series_index, aus))
+                          (title, None, series_index, aus))
         id = obj.lastrowid
         self.data.books_added([id], self.conn)
         self.set_path(id, True)
         self.set_metadata(id, mi)
         for path in formats:
             ext = os.path.splitext(path)[1][1:].lower()
+            if ext == 'opf':
+                continue
             stream = open(path, 'rb')
             self.add_format(id, ext, stream, index_is_id=True)
         self.conn.commit()
         self.data.refresh_ids(self.conn, [id]) # Needed to update format list and size
+        if notify:
             self.notify('add', [id])
         
     def move_library_to(self, newloc, progress=None):
@@ -1388,6 +1406,11 @@ books_series_link      feeds
                 f = open(os.path.join(base, sanitize_file_name(name)+'.opf'), 'wb')
                 if not mi.authors:
                     mi.authors = [_('Unknown')]
+                cdata = self.cover(int(id), index_is_id=True)
+                if cdata is not None:
+                    cname = sanitize_file_name(name)+'.jpg'
+                    open(os.path.join(base, cname), 'wb').write(cdata)
+                    mi.cover = cname
                 opf = OPFCreator(base, mi)
                 opf.render(f)
                 f.close()
@@ -1452,5 +1475,87 @@ books_series_link      feeds
                     break
         return failures
     
+    def find_books_in_directory(self, dirpath, single_book_per_directory):
+        dirpath = os.path.abspath(dirpath)
+        if single_book_per_directory:
+            formats = []
+            for path in os.listdir(dirpath):
+                path = os.path.abspath(os.path.join(dirpath, path))
+                if os.path.isdir(path) or not os.access(path, os.R_OK):
+                    continue
+                ext = os.path.splitext(path)[1]
+                if not ext:
+                    continue
+                ext = ext[1:].lower()
+                if ext not in BOOK_EXTENSIONS and ext != 'opf':
+                    continue
+                formats.append(path)
+            yield formats
+        else:
+            books = {}
+            for path in os.listdir(dirpath):
+                path = os.path.abspath(os.path.join(dirpath, path))
+                if os.path.isdir(path) or not os.access(path, os.R_OK):
+                    continue
+                ext = os.path.splitext(path)[1]
+                if not ext:
+                    continue
+                ext = ext[1:].lower()
+                if ext not in BOOK_EXTENSIONS:
+                    continue
+    
+                key = os.path.splitext(path)[0]
+                if not books.has_key(key):
+                    books[key] = []
+                books[key].append(path)
+            
+            for formats in books.values():
+                yield formats
+
+    def import_book_directory_multiple(self, dirpath, callback=None):
+        duplicates = []
+        for formats in self.find_books_in_directory(dirpath, False):
+            mi = metadata_from_formats(formats)
+            if mi.title is None:
+                continue
+            if self.has_book(mi):
+                duplicates.append((mi, formats))
+                continue
+            self.import_book(mi, formats)
+            if callable(callback):
+                if callback(mi.title):
+                    break
+        return duplicates
+
+    def import_book_directory(self, dirpath, callback=None):
+        dirpath = os.path.abspath(dirpath)
+        formats = self.find_books_in_directory(dirpath, True)
+        if not formats:
+            return
+        
+        mi = metadata_from_formats(formats)
+        if mi.title is None:
+            return
+        if self.has_book(mi):
+            return [(mi, formats)]
+        self.import_book(mi, formats)
+        if callable(callback):
+            callback(mi.title)
+            
+    def recursive_import(self, root, single_book_per_directory=True, callback=None):
+        root = os.path.abspath(root)
+        duplicates  = []
+        for dirpath in os.walk(root):
+            res = self.import_book_directory(dirpath[0], callback=callback) if \
+                single_book_per_directory else \
+                  self.import_book_directory_multiple(dirpath[0], callback=callback)
+            if res is not None:
+                duplicates.extend(res)
+            if callable(callback):
+                if callback(''):
+                    break
+            
+        return duplicates
+
 
         

src/calibre/library/sqlite.py

@@ -12,11 +12,50 @@ from sqlite3 import IntegrityError
 from threading import Thread
 from Queue import Queue
 from threading import RLock
+from datetime import tzinfo, datetime, timedelta
 
 from calibre.library import title_sort
 
 global_lock = RLock()
 
+def convert_timestamp(val):
+    datepart, timepart = val.split(' ')
+    tz, mult = None, 1
+    x = timepart.split('+')
+    if len(x) > 1:
+        timepart, tz = x
+    else:
+        x = timepart.split('-')
+        if len(x) > 1:
+            timepart, tz = x
+            mult = -1
+
+    year, month, day = map(int, datepart.split("-"))
+    timepart_full = timepart.split(".")
+    hours, minutes, seconds = map(int, timepart_full[0].split(":"))
+    if len(timepart_full) == 2:
+        microseconds = int(timepart_full[1])
+    else:
+        microseconds = 0
+    if tz is not None:
+        h, m = map(int, tz.split(':'))
+        delta = timedelta(minutes=mult*(60*h + m))
+        tz = type('CustomTZ', (tzinfo,), {'utcoffset':lambda self, dt:delta, 
+                                          'dst':lambda self,dt:timedelta(0)})()
+
+    val = datetime(year, month, day, hours, minutes, seconds, microseconds, 
+                   tzinfo=tz)
+    if tz is not None:
+        val = datetime(*(val.utctimetuple()[:6]))
+    return val
+
+def adapt_datetime(dt):
+    dt = datetime(*(dt.utctimetuple()[:6]))
+    return dt.isoformat(' ')
+
+sqlite.register_adapter(datetime, adapt_datetime)
+sqlite.register_converter('timestamp', convert_timestamp)
+
 class Concatenate(object):
     '''String concatenation aggregator for sqlite'''
     def __init__(self, sep=','):

src/calibre/linux.py

@@ -501,6 +501,7 @@ VIEWER = '''\
 Version=%s
 Type=Application
 Name=LRF Viewer
+GenericName=Viewer for LRF files
 Comment=Viewer for LRF files (SONY ebook format files)
 TryExec=lrfviewer
 Exec=lrfviewer %%F
@@ -513,8 +514,9 @@ EVIEWER = '''\
 [Desktop Entry]
 Version=%s
 Type=Application
-Name=Ebook Viewer
-Comment=Viewer for Ebooks
+Name=E-book Viewer
+GenericName=Viewer for E-books
+Comment=Viewer for E-books
 TryExec=ebook-viewer
 Exec=ebook-viewer %%F
 Icon=calibre-viewer
@@ -527,7 +529,8 @@ GUI = '''\
 [Desktop Entry]
 Version=%s
 Type=Application
-Name=calibre - Ebook library management
+Name=calibre
+GenericName=E-book library management
 Comment=E-book library management
 TryExec=calibre
 Exec=calibre

src/calibre/manual/faq.rst

@@ -28,7 +28,7 @@ What formats does |app| support conversion to/from?
 |                   |        |                  |                       |                       |
 |                   |  LIT   |       ✔          |          ✔            |          ✔            |
 |                   |        |                  |                       |                       |
-|                   |  PRC   |       ✔          |          ✔            |          ✔            |
+|                   |  PRC** |       ✔          |          ✔            |          ✔            |
 |                   |        |                  |                       |                       |
 |                   |  EPUB  |       ✔          |          ✔            |          ✔            |
 |                   |        |                  |                       |                       |
@@ -49,6 +49,7 @@ What formats does |app| support conversion to/from?
 |                   |  LRS   |                  |          ✔            |                       |
 +-------------------+--------+------------------+-----------------------+-----------------------+
 
+** PRC is a generic format, |app| supports PRC files with TextRead and MOBIBook headers
 
 
 What are the best source formats to convert?
@@ -146,7 +147,7 @@ When you first run |app|, it will ask you for a folder in which to store your bo
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The whole point if |app|'s library management features is that they provide an interface for locating books that is *much* more efficient than any possible directory scheme you could come up with for your collection. Indeed, once you become comfortable using |app|'s interface to find, sort and browse your collection, you wont ever feel the need to hunt through the files on your disk to find a book again. By managing books in its own directory struture of Author -> Title -> Book files, |app| is able to achieve a high level of reliability and standardization.  
+The whole point of |app|'s library management features is that they provide an interface for locating books that is *much* more efficient than any possible directory scheme you could come up with for your collection. Indeed, once you become comfortable using |app|'s interface to find, sort and browse your collection, you wont ever feel the need to hunt through the files on your disk to find a book again. By managing books in its own directory struture of Author -> Title -> Book files, |app| is able to achieve a high level of reliability and standardization.  
 
 Why doesn't |app| have a column for foo?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/manual/templates/layout.html

@@ -0,0 +1,12 @@
+{% extends "!layout.html" %}
+{% block sidebarlogo %}
+{{ super() }}
+         <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
+               <input type="hidden" name="cmd" value="_s-xclick" />
+               <input type="hidden" name="hosted_button_id" value="3028915" />
+               <input type="image" src="https://www.paypal.com/en_US/i/btn/btn_donate_LG.gif" border="0" name="submit" alt="Donate to support calibre development" style="border:0pt" />
+               <img alt="" border="0" src="https://www.paypal.com/en_US/i/scr/pixel.gif" width="1" height="1"/>
+           </form>
+           <hr/>
+{% endblock %}
+

src/calibre/trac/donations/server.py

@@ -197,6 +197,7 @@ class Server(object):
     def calculate_month_trend(self, days=31):
         stats = self.get_slice(date.today()-timedelta(days=days-1), date.today())
         fig = plt.figure(2, (12, 4), 96)#, facecolor, edgecolor, frameon, FigureClass)
+        fig.clear()
         ax = fig.add_subplot(111)
         x = list(range(days-1, -1, -1))
         y = stats.daily_totals
@@ -235,6 +236,7 @@ Donors per day: %(dpd).2f
         y = [m.total for m in _months]
         ml   = mdates.MonthLocator() # every month
         fig = plt.figure(1, (8, 4), 96)#, facecolor, edgecolor, frameon, FigureClass)
+        fig.clear()
         ax = fig.add_subplot(111)
         average = sum(y)/len(y)
         ax.bar(x, y, align='center', width=20, color='g')

src/calibre/trac/plugins/download.py

@@ -1,26 +1,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import re
-from pkg_resources import resource_filename
-
-from trac.core import Component, implements
-from trac.web.chrome import INavigationContributor, ITemplateProvider, add_stylesheet
-from trac.web.main import IRequestHandler
-from trac.util import Markup
-
-
 __appname__ = 'calibre'
-DOWNLOAD_DIR = '/var/www/calibre.kovidgoyal.net/htdocs/downloads'
-MOBILEREAD = 'https://dev.mobileread.com/dist/kovid/calibre/'
-
-class OS(dict):
-    """Dictionary with a default value for unknown keys."""
-    def __init__(self, dict):
-        self.update(dict)
-        if not dict.has_key('img'):
-            self['img'] = self['name']
-
-class Distribution(object):
+import re, textwrap
 
 DEPENDENCIES = [
             #(Generic, version, gentoo, ubuntu, fedora)
@@ -30,60 +11,69 @@ class Distribution(object):
             ('libusb', '0.1.12', None, None, None),
             ('Qt', '4.4.0', 'qt', 'libqt4-core libqt4-gui', 'qt4'),
             ('PyQt', '4.4.2', 'PyQt4', 'python-qt4', 'PyQt4'),
-        ('mechanize for python', '0.1.11', 'dev-python/mechanize', 'python-mechanize', 'python-mechanize'),
+            ('python-mechanize', '0.1.11', 'dev-python/mechanize', 'python-mechanize', 'python-mechanize'),
             ('ImageMagick', '6.3.5', 'imagemagick', 'imagemagick', 'ImageMagick'),
             ('xdg-utils', '1.0.2', 'xdg-utils', 'xdg-utils', 'xdg-utils'),
             ('dbus-python', '0.82.2', 'dbus-python', 'python-dbus', 'dbus-python'),
             ('lxml', '2.0.5', 'lxml', 'python-lxml', 'python-lxml'),
+            ('python-dateutil', '1.4.1', 'python-dateutil', 'python-dateutil', 'python-dateutil'),
             ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'),
             ('help2man', '1.36.4', 'help2man', 'help2man', 'help2man'),
             ]
         
-    DISTRO_MAP = {'gentoo':2, 'ubuntu':3, 'fedora':4, 'debian':3}
 
-    INSTALLERS = ('emerge -avn', 'apt-get install', 'yum install')
-    AS_ROOT    = (True, False, True)
+class CoolDistro:
     
-    TITLEMAP = {'gentoo':'Gentoo', 'ubuntu':'Ubuntu Intrepid Ibex',
-                'fedora':'Fedora 10', 'debian':'Debian sid', 'generic': 'Install from source'}
+    def __init__(self, name, title, prefix=''):
+        self.title = title
+        url = prefix + '/chrome/dl/images/%s_logo.png'
+        self.img = url%name
 
-    MANUAL_MAP = {
-                  'fedora' : '''<li>You have to upgrade Qt to at least 4.4.0 and PyQt to at least 4.4.2</li>''',
-                  }
+def get_linux_data(version='1.0.0'):
+    data = {'version':version, 'app':__appname__}
+    data['title'] = 'Download calibre for linux'
+    data['supported'] = []
+    for name, title in [
+                        ('ubuntu', 'Ubuntu Jaunty Jackalope'),
+                        ('debian', 'Debian Sid'),
+                        ('exherbo', 'Exherbo'),
+                        ]:
+        data['supported'].append(CoolDistro(name, title, 
+                                        prefix='http://calibre.kovidgoyal.net'))
+        data['dependencies'] = DEPENDENCIES
+    return data
 
-    def __init__(self, os):
-        self.os = os
-        self.img = os
-        self.title = self.TITLEMAP[os]
-        self.app = __appname__
-        self.is_generic = os == 'generic'
-        offset = 0
-        if not self.is_generic:
-            index = self.DISTRO_MAP[self.os]
-            if os == 'debian':
-                self.as_root = True  
-            else: self.as_root = self.AS_ROOT[index-2]
-            prefix = '' 
-            if not self.as_root: prefix =  'sudo '
-            cmd = prefix + self.INSTALLERS[index-2]
-            pre = ' \\\n '.ljust(len(cmd)+4)
-            for dep in self.DEPENDENCIES:
-                if len(cmd) > 70+offset:
-                    offset += 70
-                    cmd += pre
-                cmd += ' ' 
-                if dep[index]: cmd += dep[index]
-            self.command = cmd.strip()
-            easy_install = 'easy_install'
-            if os == 'debian':
-                easy_install = 'easy_install-2.5'
-            self.command += '\n'+prefix+easy_install+' -U calibre \n'+prefix+'calibre_postinstall'
-            try:
-                self.manual = Markup(self.MANUAL_MAP[os])
-            except KeyError:
-                self.manual = None
+if __name__ == '__main__':
+    import os
+    from calibre.utils.genshi.template import MarkupTemplate
+    import cherrypy
+    class Test:
+        def index(self):
+            raw = open(os.path.dirname(os.path.abspath(__file__))+'/templates/linux.html').read()
+            return MarkupTemplate(raw).generate(**get_linux_data()).render('xhtml')
+        index.exposed = True
+    t = Test()
+    t.index()    
+    cherrypy.quickstart(t)
 else:
-            self.img = 'linux'
+    from pkg_resources import resource_filename
+    
+    from trac.core import Component, implements
+    from trac.web.chrome import INavigationContributor, ITemplateProvider, add_stylesheet
+    from trac.web.main import IRequestHandler
+    from trac.util import Markup
+    
+    
+    
+    DOWNLOAD_DIR = '/var/www/calibre.kovidgoyal.net/htdocs/downloads'
+    MOBILEREAD = 'https://dev.mobileread.com/dist/kovid/calibre/'
+    
+    class OS(dict):
+        """Dictionary with a default value for unknown keys."""
+        def __init__(self, dict):
+            self.update(dict)
+            if not dict.has_key('img'):
+                self['img'] = self['name']
     
         
     class Download(Component):
@@ -124,16 +114,6 @@ class Download(Component):
                         return self.osx(req)
                     elif os == 'linux':
                         return self.linux(req)
-                elif 'binary' in os:
-                    return self.linux_binary(req)
-                else:
-                    return self.linux_distro(req, os)       
-    
-    def linux_distro(self, req, os):
-        version = self.version_from_filename()
-        distro = Distribution(os)
-        data = dict(distro=distro,title=distro.title, version=version)
-        return 'distro.html', data, None
         
         def top_level(self, req):
             operating_systems = [
@@ -188,10 +168,6 @@ select Install.</li>
     '''%dict(appname=__appname__)))
             return 'binary.html', data, None
         
-    def linux_binary(self, req):
-        version = self.version_from_filename()
-        return 'pyinstaller.html', {'app':__appname__, 'version':version}, None
-    
         def osx(self, req):
             version = self.version_from_filename()
             file = 'calibre-%s.dmg'%(version,) 
@@ -211,20 +187,11 @@ select Install.</li>
             return 'binary.html', data, None
         
         def linux(self, req):
-        operating_systems = [
-            OS({'name' : 'binary', 'title': 'Binary Installer'}),
-            OS({'name' : 'gentoo', 'title': 'Gentoo'}),
-            OS({'name' : 'ubuntu', 'title': 'Ubuntu'}),
-            OS({'name' : 'fedora', 'title': 'Fedora'}),
-            OS({'name' : 'debian', 'title': 'Debian'}),
-            OS({'name' : 'generic','title': 'Install from source', 'img':'linux'}),
-                             ]
-        data = dict(title='Choose linux distribution', width=100,
-                    operating_systems=operating_systems, font_size='x-large', top_level=False)
-        return 'download.html', data, None
+            data = get_linux_data(version=self.version_from_filename())
+            return 'linux.html', data, None
                                 
     
-LINUX_INSTALLER = r'''
+    LINUX_INSTALLER = textwrap.dedent(r'''
     import sys, os, shutil, tarfile, subprocess, tempfile, urllib2, re, stat
     
     MOBILEREAD='https://dev.mobileread.com/dist/kovid/calibre/'
@@ -415,4 +382,5 @@ def main():
         pi = os.path.join(destdir, 'calibre_postinstall')
         subprocess.call(pi, shell=True)
         return 0
-'''
+    ''')
+    

src/calibre/trac/plugins/templates/distro.html

@@ -1,68 +0,0 @@
-<!DOCTYPE html
-    PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml"
-      xmlns:py="http://genshi.edgewall.org/"
-      xmlns:xi="http://www.w3.org/2001/XInclude">
-  <xi:include href="layout.html" />
-  <head>
-    <title>$title</title>
-  </head>
-
-  <body>
-    <div id="ctxtnav" class="nav"></div>
-
-    <div id="content" class="download">
-      <h1><img src="${href.chrome('/dl/images/%s_logo.png'%(distro.img,))}" valign="middle" width="60" height="80"/> $title</h1>
-      See the <a href="/wiki/Changelog">Changelog</a> for the changes in the latest version. <span py:if="not distro.is_generic"><b>Note:</b> As of 0.4.80 this install
-      will not work on 64-bit CPUs. Try the precompiled binary available <a href="/download_binary">here</a> instead.</span>
-      <div py:if="not distro.is_generic">
-        First verify that you have a sufficiently new installation of python
-        <pre class="wiki">python --version</pre> should return at least 2.5.1<br />
-        <p py:if="distro.os == 'gentoo'">
-            Make sure your python is compiled with the sqlite USE flag. 
-        </p>
-        <br />
-        Run the following commands in a terminal <span py:if="distro.as_root">as root</span>
-        <pre class="wiki">${distro.command}</pre>
-        <div py:if="distro.manual">
-            <h2>Manual steps</h2>
-            <ul>
-                ${distro.manual}
-            </ul>
-        </div>
-      </div>
-      <div py:if="distro.is_generic">
-        <ol>
-            <li>Make sure that your system has <code>python &gt;= 2.5</code></li>
-            <li>Install the various dependencies listed below: Make sure that any python packages are installed into python2.5 (e.g. setuptools, python-imaging, PyQt4, etc). You will also have to install the development versions of the packages (these packages usually have  -dev added to their names).</li>
-            <li>Run the following commands in a terminal:
-            <pre class="wiki">
-wget -O- http://calibre.kovidgoyal.net/downloads/calibre-${version}.tar.gz | tar xvz 
-cd calibre*
-python setup.py build &amp;&amp; sudo python setup.py install
-            </pre></li>
-        </ol>
-        <h2>Dependencies</h2>
-        <table border="1" cellpadding="10">
-            <tr><th style="font-weight:bold">Name</th><th style="font-weight:bold">Minimum version</th></tr>
-            <tr py:for="dep in distro.DEPENDENCIES">
-                <td>${dep[0]}</td><td>${dep[1]}</td>
-            </tr>
-            <tr>
-                <td>HAL</td><td>0.5.10</td>
-            </tr>
-        </table>
-      </div>
-      While you wait for the installation to complete, please consider donating to support the development of ${distro.app}.
-        <div>
-            <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
-                <input type="hidden" name="cmd" value="_s-xclick" />
-                <input type="hidden" name="hosted_button_id" value="3029289" />
-                <input type="image" src="https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="Donate to support calibre development" />
-                <img alt="" border="0" src="https://www.paypal.com/en_US/i/scr/pixel.gif" width="1" height="1" />
-          </form>
-        </div>
-    </div>
-  </body>
-</html>

src/calibre/trac/plugins/templates/linux.html

@@ -0,0 +1,155 @@
+<!DOCTYPE html
+    PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml"
+      xmlns:py="http://genshi.edgewall.org/"
+      xmlns:xi="http://www.w3.org/2001/XInclude">
+  <xi:include href="layout.html" />
+  <head>
+    <title>${title}</title>
+    <style type="text/css">
+        table tr th.distro_type { 
+            font-family: monospace;
+            font-weight: bold;
+            font-size: larger;
+            vertical-align:middle;
+            text-align: center;
+            border-bottom: solid 1pt black;
+         }
+        
+        .right {
+            margin-left:2em;
+            border-left: solid 1pt black;
+            }
+        
+        table#install_info {
+            border-collapse: collapse;
+            border-width: 1pt;
+            border-style: solid;
+            table-layout: fixed;
+            width: 95%;
+            background: #F7F7F0 none repeat scroll 0 0;
+        }
+        table#dependencies {
+            border-collapse: collapse;
+            border-width: 0pt;
+            font-family:monospace;
+        }
+        .tdata {vertical-align: top;}
+    </style>
+  </head>
+
+  <body>
+    <div id="ctxtnav" class="nav"></div>
+    <div id="content" class="download">
+        <h1>${title}</h1>
+        <p>
+            The latest release of ${app} is ${version}. See the 
+            <a href="/wiki/Changelog">Changelog</a> for a list of new features.
+        </p>
+        <p>
+            ${app} is available in the software repositories of the following 
+            linux distributions:
+            <table id="install_info">
+                <col width="150" /><col width="*" />
+                <tr>
+                    <th class="left distro_type">Supported<br/>distributions</th>
+                    <th class="right distro_type">Unsupported distributions</th>
+                </tr>
+                <tr class="tdata">
+                    <td class="left" style="overflow-y:scroll">
+                        <div py:for="distro in supported" 
+                        style="text-align:center;margin-top:2ex;">
+                            <div>
+                                <img width="64px" height="64px" 
+                                    src="${distro.img}" alt="${distro.title}" />
+                            </div>
+                            ${distro.title}
+                        </div>
+                    </td>
+                    <td class="right">
+                        <div style="margin-left:2em">
+                            <h3>Binary install</h3>
+                            <p>
+                                ${app} has a binary installer that has been 
+                                tested on a number of distributions on both 
+                                32-bit and 64-bit x86 machines. To install, 
+                                copy paste the following command into a terminal 
+                                and press Enter: 
+                            </p>
+                            <pre class="wiki">
+sudo python -c "import urllib2; exec urllib2.urlopen('http://calibre.kovidgoyal.net/download_linux_binary_installer').read(); main()"
+                            </pre>
+                            <h4>Note</h4>
+                            <ul>
+                                <li>
+                                    When running the command line utilities,
+                                    they will segfault after completion. This can
+                                    be ignored.
+                                </li>
+                                <li>
+                                    You must have help2man and xdg-utils installed 
+                                    on your system before running the installer.
+                                </li>
+                                <li>
+                                    On a 64bit machine, you must have 32-bit versions
+                                    of common libraries like X11, freetype, fontconfig,
+                                    expat and their varios dependencies installed.
+                                </li>
+                            </ul>
+                            <h3>Source install</h3>
+                            <p>
+                                <ol>
+                                    <li>
+                                        Make sure your system has python &ge; ${dependencies[0][1]}
+                                    </li>
+                                    <li>
+                                        Install the various dependencies listed below
+                                    </li>
+                                    <li>
+                                        Run the following commands in a terminal:
+                                    </li>
+                                </ol>
+                                <pre class="wiki">
+wget -O- http://calibre.kovidgoyal.net/downloads/${app}-${version}.tar.gz | tar xvz 
+cd calibre*
+python setup.py build &amp;&amp; sudo python setup.py install
+                                </pre>
+                                Note that if your distribution does not have a
+                                correctly compiled libunrar.so, ${app} will not 
+                                support rar files.
+                            </p>
+                        </div>
+                    </td>
+                </tr>
+            </table>
+        </p>
+        <p>
+            While you wait for the download to complete, please consider 
+            donating to support the development of ${app}.
+        </p>
+      <div>
+        <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
+            <input type="hidden" name="cmd" value="_s-xclick" />
+            <input type="image" src="https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!" />
+            <img alt="" border="0" src="https://www.paypal.com/en_US/i/scr/pixel.gif" width="1" height="1" />
+            <input type="hidden" name="encrypted" value="-----BEGIN PKCS7-----MIIHbwYJKoZIhvcNAQcEoIIHYDCCB1wCAQExggEwMIIBLAIBADCBlDCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20CAQAwDQYJKoZIhvcNAQEBBQAEgYBn7jneGiSLVO8rcDrBtOUXL+HftY+CiC47hTntwICio6qqpLKezIryyG8tKcjY58Rcocur/kDwljEutIafVG7XRA7BJL9eZdHAZsZdX04f4dApzkWwR9w6GQhj0kwmO2ZNE878UcgGZBve4qQKWM8bf2pMY7vJwCNoo6ozpIi3VTELMAkGBSsOAwIaBQAwgewGCSqGSIb3DQEHATAUBggqhkiG9w0DBwQIBTALt7s1gJmAgcjEAwUMRYeIdIOE/yi0Y5vrVKBFxOUCbqTx/lu3Rk4EHsODZXLHT+BDA5WSWYO3AXfv2Lmlv1kJ7jWrjUVirYoQ5M4qdIhY9DtvPioIMMRoTJmYM9JKH8n2TWcjJ1XIzIuDP4zn8/Ya9hap3RHOrj2RBj89g7iSuFRsjoA0PYZgtWAKwR7g3LLpjRachn041JO55BEd3YWUgorNQeo3WEHgowLFfTWgFFePkm8OoWA1klWkYp4S07IhX5NaRc8OegkdshpkiIHGAKCCA4cwggODMIIC7KADAgECAgEAMA0GCSqGSIb3DQEBBQUAMIGOMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxFDASBgNVBAoTC1BheVBhbCBJbmMuMRMwEQYDVQQLFApsaXZlX2NlcnRzMREwDwYDVQQDFAhsaXZlX2FwaTEcMBoGCSqGSIb3DQEJARYNcmVAcGF5cGFsLmNvbTAeFw0wNDAyMTMxMDEzMTVaFw0zNTAyMTMxMDEzMTVaMIGOMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxFDASBgNVBAoTC1BheVBhbCBJbmMuMRMwEQYDVQQLFApsaXZlX2NlcnRzMREwDwYDVQQDFAhsaXZlX2FwaTEcMBoGCSqGSIb3DQEJARYNcmVAcGF5cGFsLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAwUdO3fxEzEtcnI7ZKZL412XvZPugoni7i7D7prCe0AtaHTc97CYgm7NsAtJyxNLixmhLV8pyIEaiHXWAh8fPKW+R017+EmXrr9EaquPmsVvTywAAE1PMNOKqo2kl4Gxiz9zZqIajOm1fZGWcGS0f5JQ2kBqNbvbg2/Za+GJ/qwUCAwEAAaOB7jCB6zAdBgNVHQ4EFgQUlp98u8ZvF71ZP1LXChvsENZklGswgbsGA1UdIwSBszCBsIAUlp98u8ZvF71ZP1LXChvsENZklGuhgZSkgZEwgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEUMBIGA1UEChMLUGF5UGFsIEluYy4xEzARBgNVBAsUCmxpdmVfY2VydHMxETAPBgNVBAMUCGxpdmVfYXBpMRwwGgYJKoZIhvcNAQkBFg1yZUBwYXlwYWwuY29tggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAgV86VpqAWuXvX6Oro4qJ1tYVIT5DgWpE692Ag422H7yRIr/9j/iKG4Thia/Oflx4TdL+IFJBAyPK9v6zZNZtBgPBynXb048hsP16l2vi0k5Q2JKiPDsEfBhGI+HnxLXEaUWAcVfCsQFvd2A1sxRr67ip5y2wwBelUecP3AjJ+YcxggGaMIIBlgIBATCBlDCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20CAQAwCQYFKw4DAhoFAKBdMBgGCSqGSIb3DQEJAzELBgkqhkiG9w0BBwEwHAYJKoZIhvcNAQkFMQ8XDTA4MDQzMDE1MzkyMlowIwYJKoZIhvcNAQkEMRYEFJSI9/zWx7TUlKPY7kLjnvzB1h6sMA0GCSqGSIb3DQEBAQUABIGAikZNCmQdkWPdfmYnGqOb1f65ViaK0zjHf50azvsigWQLlhHqJ3PgB+jEJH3JU9Pm9M4wgiK23Bg2oIGuIsAfQkYO9mw/HjtDtOQHqXyZZbrM32YGtNWUD4ynakLYnaz7OnPl40aTPD4iDApgsGcj1oMdmw7KA2E9J0l2J9iJXF4=-----END PKCS7-----" />
+        </form>
+      </div>
+      <hr/>
+      <h3>Dependencies</h3>
+      ${app} has the following dependencies (the listed version is the minimum version)
+      <br/><br/>
+      <table id="dependencies">
+          <tr>
+              <th class="distro_type" style="margin-right:2em">Package</th>
+              <th class="distro_type">Version</th>
+          </tr>
+              
+          <tr class="dependency" py:for="dep in dependencies">
+              <td style="margin-right:2em">${dep[0]}</td><td>${dep[1]}</td>
+          </tr>
+      </table>    
+    </div>
+  </body>
+</html>

src/calibre/trac/plugins/templates/pyinstaller.html

@@ -1,47 +0,0 @@
-<!DOCTYPE html
-    PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml"
-      xmlns:py="http://genshi.edgewall.org/"
-      xmlns:xi="http://www.w3.org/2001/XInclude">
-  <xi:include href="layout.html" />
-  <head>
-    <title>Download $app for Linux</title>
-  </head>
-
-  <body>
-    <div id="ctxtnav" class="nav"></div>
-
-    <div id="content" class="binary">
-      <h1>Download $app for Linux</h1>
-      <p>This binary package is compatible with most recent linux distributions running on Intel 32 bit CPUs. It needs testing on 64 bit CPUs. There have been reports of its working on some 64bit machines. </p>
-      <p>
-      <img width="50" height="50" style="border:1px red solid" src="${href.chrome('/dl/images/binary_logo.png')}" /> 
-      (Version: $version <a href="/wiki/Changelog">Changelog</a>)
-      </p>
-      <p>To install, copy paste the following command into a terminal and press Enter:
-      </p>
-      <pre class="wiki">sudo python -c "import urllib2; exec urllib2.urlopen('http://calibre.kovidgoyal.net/download_linux_binary_installer').read(); main()"</pre>
-      <p>
-      While you wait for the download to complete, please consider donating to support the development
-      of ${app}.</p>
-      <div>
-        <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
-            <input type="hidden" name="cmd" value="_s-xclick" />
-            <input type="image" src="https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!" />
-            <img alt="" border="0" src="https://www.paypal.com/en_US/i/scr/pixel.gif" width="1" height="1" />
-            <input type="hidden" name="encrypted" value="-----BEGIN PKCS7-----MIIHbwYJKoZIhvcNAQcEoIIHYDCCB1wCAQExggEwMIIBLAIBADCBlDCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20CAQAwDQYJKoZIhvcNAQEBBQAEgYBn7jneGiSLVO8rcDrBtOUXL+HftY+CiC47hTntwICio6qqpLKezIryyG8tKcjY58Rcocur/kDwljEutIafVG7XRA7BJL9eZdHAZsZdX04f4dApzkWwR9w6GQhj0kwmO2ZNE878UcgGZBve4qQKWM8bf2pMY7vJwCNoo6ozpIi3VTELMAkGBSsOAwIaBQAwgewGCSqGSIb3DQEHATAUBggqhkiG9w0DBwQIBTALt7s1gJmAgcjEAwUMRYeIdIOE/yi0Y5vrVKBFxOUCbqTx/lu3Rk4EHsODZXLHT+BDA5WSWYO3AXfv2Lmlv1kJ7jWrjUVirYoQ5M4qdIhY9DtvPioIMMRoTJmYM9JKH8n2TWcjJ1XIzIuDP4zn8/Ya9hap3RHOrj2RBj89g7iSuFRsjoA0PYZgtWAKwR7g3LLpjRachn041JO55BEd3YWUgorNQeo3WEHgowLFfTWgFFePkm8OoWA1klWkYp4S07IhX5NaRc8OegkdshpkiIHGAKCCA4cwggODMIIC7KADAgECAgEAMA0GCSqGSIb3DQEBBQUAMIGOMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxFDASBgNVBAoTC1BheVBhbCBJbmMuMRMwEQYDVQQLFApsaXZlX2NlcnRzMREwDwYDVQQDFAhsaXZlX2FwaTEcMBoGCSqGSIb3DQEJARYNcmVAcGF5cGFsLmNvbTAeFw0wNDAyMTMxMDEzMTVaFw0zNTAyMTMxMDEzMTVaMIGOMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxFDASBgNVBAoTC1BheVBhbCBJbmMuMRMwEQYDVQQLFApsaXZlX2NlcnRzMREwDwYDVQQDFAhsaXZlX2FwaTEcMBoGCSqGSIb3DQEJARYNcmVAcGF5cGFsLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAwUdO3fxEzEtcnI7ZKZL412XvZPugoni7i7D7prCe0AtaHTc97CYgm7NsAtJyxNLixmhLV8pyIEaiHXWAh8fPKW+R017+EmXrr9EaquPmsVvTywAAE1PMNOKqo2kl4Gxiz9zZqIajOm1fZGWcGS0f5JQ2kBqNbvbg2/Za+GJ/qwUCAwEAAaOB7jCB6zAdBgNVHQ4EFgQUlp98u8ZvF71ZP1LXChvsENZklGswgbsGA1UdIwSBszCBsIAUlp98u8ZvF71ZP1LXChvsENZklGuhgZSkgZEwgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEUMBIGA1UEChMLUGF5UGFsIEluYy4xEzARBgNVBAsUCmxpdmVfY2VydHMxETAPBgNVBAMUCGxpdmVfYXBpMRwwGgYJKoZIhvcNAQkBFg1yZUBwYXlwYWwuY29tggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAgV86VpqAWuXvX6Oro4qJ1tYVIT5DgWpE692Ag422H7yRIr/9j/iKG4Thia/Oflx4TdL+IFJBAyPK9v6zZNZtBgPBynXb048hsP16l2vi0k5Q2JKiPDsEfBhGI+HnxLXEaUWAcVfCsQFvd2A1sxRr67ip5y2wwBelUecP3AjJ+YcxggGaMIIBlgIBATCBlDCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20CAQAwCQYFKw4DAhoFAKBdMBgGCSqGSIb3DQEJAzELBgkqhkiG9w0BBwEwHAYJKoZIhvcNAQkFMQ8XDTA4MDQzMDE1MzkyMlowIwYJKoZIhvcNAQkEMRYEFJSI9/zWx7TUlKPY7kLjnvzB1h6sMA0GCSqGSIb3DQEBAQUABIGAikZNCmQdkWPdfmYnGqOb1f65ViaK0zjHf50azvsigWQLlhHqJ3PgB+jEJH3JU9Pm9M4wgiK23Bg2oIGuIsAfQkYO9mw/HjtDtOQHqXyZZbrM32YGtNWUD4ynakLYnaz7OnPl40aTPD4iDApgsGcj1oMdmw7KA2E9J0l2J9iJXF4=-----END PKCS7-----" />
-        </form>
-        
-      </div>        
-      <h2>Note</h2>
-      <div class="note">
-      <ul>
-      <li>This installer is very new and has only been tested on a couple of systems, so if you encounter
-      problems, please report them.</li>
-      <li>You shoud have help2man and xdg-utils installed on your system.</li>
-      </ul>
-      </div>
-    </div>
-  </body>
-</html>

src/calibre/utils/config.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 '''
 Manage application-wide preferences.
 '''
-import os, re, cPickle, textwrap
+import os, re, cPickle, textwrap, traceback
 from copy import deepcopy
 from functools import partial
 from optparse import OptionParser as _OptionParser
@@ -314,7 +314,12 @@ class OptionSet(object):
         if not isinstance(src, unicode):
             src = src.decode('utf-8')
         if src is not None:
+            try:
                 exec src in options
+            except:
+                print 'Failed to parse options string:'
+                print repr(src)
+                traceback.print_exc()
         opts = OptionValues()
         for pref in self.preferences:
             val = options.get(pref.name, pref.default)
@@ -539,7 +544,7 @@ def _prefs():
               help=_('Path to directory in which your library of books is stored'))
     c.add_opt('language', default=None,
               help=_('The language in which to display the user interface'))
-    c.add_opt('output_format', default='LRF', 
+    c.add_opt('output_format', default='EPUB', 
               help=_('The default output format for ebook conversions.'))
     c.add_opt('read_file_metadata', default=True,
               help=_('Read metadata from files'))

src/calibre/web/feeds/news.py

@@ -190,7 +190,7 @@ class BasicNewsRecipe(object, LoggingInterface):
     #: For the format for specifying a tag see :attr:`BasicNewsRecipe.remove_tags`.
     #: For example::
     #:
-    #:     remove_tags_before = [dict(id='content')]
+    #:     remove_tags_before = dict(id='content')
     #:
     #: will remove all
     #: tags before the first element with `id="content"`.

src/calibre/web/feeds/recipes/__init__.py

@@ -28,6 +28,8 @@ recipe_modules = ['recipe_' + r for r in (
            'la_tercera', 'el_mercurio_chile', 'la_cuarta', 'lanacion_chile', 'la_segunda',
            'jb_online', 'estadao', 'o_globo', 'vijesti', 'elmundo', 'the_oz',
            'honoluluadvertiser', 'starbulletin', 'exiled', 'indy_star', 'dna',
+           'pobjeda', 'chicago_breaking_news', 'glasgow_herald', 'linuxdevices',
+           'hindu', 'cincinnati_enquirer',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_ars_technica.py

@@ -1,50 +1,60 @@
 #!/usr/bin/env  python
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
-__docformat__ = 'restructuredtext en'
 
+__license__   = 'GPL v3'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 arstechnica.com
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class ArsTechnica(BasicNewsRecipe):
-    title          = 'Ars Technica'
-    description    = 'The art of technology'
-    oldest_article = 7
+class ArsTechnica2(BasicNewsRecipe):
+    title                 = u'Ars Technica'
     language              = _('English')
-    no_stylesheets = True
-    __author__     = 'Michael Warner'
+    __author__            = 'Darko Miletic'
+    description           = 'The art of technology'    
+    publisher             = 'Ars Technica'
+    category              = 'news, IT, technology'    
+    language              = _('English')
+    oldest_article        = 2
     max_articles_per_feed = 100
-    extra_css = """
-body {
-   font: normal 19px/180% Times, serif;
-}
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    remove_javascript     = True
+    use_embedded_content  = False
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+
+    keep_only_tags = [dict(name='div', attrs={'id':['news-item-info','news-item']})]
 
-h1, h2, h3, h4 {
-   font: bold 28px/100% Verdana, Arial, Helvetica, sans-serif;
-   margin-top: 19px
-}
-"""
     remove_tags = [
-                   dict(id="Masthead"),
-                   dict(id="Banner"),
-                   dict(id="Nav"),
-                   dict(name='div', attrs={'class':'ContentHeader'}), 
-                   dict(name='img'), 
-                   dict(name='div', attrs={'class':'Inset RelatedStories'}), 
-                   dict(name='div', attrs={'class':'Tags'}), 
-                   dict(name='div', attrs={'class':'PostOptions flat'}),
-                   dict(name='div', attrs={'class':'ContentFooter'}),
-                   dict(id="Sidebar"),
-                   dict(id="LatestPosts"),
-                   dict(id="Footer")]
-    feeds       = [(u'News and Features', u'http://feeds.arstechnica.com/arstechnica/BAaf'),
-                   (u'Nobel Intent (Science)', u'http://arstechnica.com/journals/science.rssx'),
-                   (u'Infinite Loop (Apple)', u'http://arstechnica.com/journals/apple.rssx'),
-                   (u'M-Dollar (Microsoft)', u'http://arstechnica.com/journals/microsoft.rssx'),
-                   (u'Open Ended (Linux)', u'http://arstechnica.com/journals/linux.rssx'),
-                   (u'Opposable Thumbs (Games)', u'http://arstechnica.com/journals/thumbs.rssx'),
-                   (u'Kit (Hardware)', u'http://arstechnica.com/journals/hardware.rssx'),
-                   (u'Journals', u'http://arstechnica.com/journals.rssx')]
+                     dict(name=['object','link','embed'])
+                    ,dict(name='div', attrs={'class':'related-stories'})
+                  ]
+
+    
+    feeds = [
+              (u'Infinite Loop (Apple content)'        , u'http://feeds.arstechnica.com/arstechnica/apple/'      )
+             ,(u'Opposable Thumbs (Gaming content)'    , u'http://feeds.arstechnica.com/arstechnica/gaming/'     )
+             ,(u'Gear and Gadgets'                     , u'http://feeds.arstechnica.com/arstechnica/gadgets/'    )
+             ,(u'Chipster (Hardware content)'          , u'http://feeds.arstechnica.com/arstechnica/hardware/'   )
+             ,(u'Uptime (IT content)'                  , u'http://feeds.arstechnica.com/arstechnica/business/'   )
+             ,(u'Open Ended (Open Source content)'     , u'http://feeds.arstechnica.com/arstechnica/open-source/')
+             ,(u'One Microsoft Way'                    , u'http://feeds.arstechnica.com/arstechnica/microsoft/'  )
+             ,(u'Nobel Intent (Science content)'       , u'http://feeds.arstechnica.com/arstechnica/science/'    )
+             ,(u'Law & Disorder (Tech policy content)' , u'http://feeds.arstechnica.com/arstechnica/tech-policy/')
+            ]
+
+    def preprocess_html(self, soup):
+        ftag = soup.find('div', attrs={'class':'news-item-byline'})
+        if ftag:
+           ftag.insert(4,'<br /><br />')
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup

src/calibre/web/feeds/recipes/recipe_b92.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 b92.net
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class B92(BasicNewsRecipe):
@@ -22,19 +21,22 @@ class B92(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     cover_url = 'http://static.b92.net/images/fp/logo.gif'
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
-
-    keep_only_tags = [ dict(name='div', attrs={'class':'sama_vest'}) ]
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment'  , description
                         , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
+
+    keep_only_tags = [ dict(name='div', attrs={'class':'sama_vest'}) ]
         
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+
     feeds          = [
                         (u'Vesti', u'http://www.b92.net/info/rss/vesti.xml')
                        ,(u'Biz'  , u'http://www.b92.net/info/rss/biz.xml'  )
@@ -54,9 +56,10 @@ class B92(BasicNewsRecipe):
         return nurl
 
     def preprocess_html(self, soup):
-        soup.html['xml:lang'] = 'sr-Latn'
-        soup.html['lang']     = 'sr-Latn'
-        mtag = '<meta http-equiv="Content-Language" content="sr-Latn"/>'
+        lng = 'sr-Latn-RS'
+        soup.html['xml:lang'] = lng
+        soup.html['lang']     = lng
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)    
         for item in soup.findAll(style=True):
             del item['style']
@@ -64,4 +67,3 @@ class B92(BasicNewsRecipe):
             del item['align']
             item.insert(0,'<br /><br />')
         return soup
-    language = _('Serbian')

src/calibre/web/feeds/recipes/recipe_blic.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 blic.rs
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Blic(BasicNewsRecipe):
@@ -21,15 +20,17 @@ class Blic(BasicNewsRecipe):
     remove_javascript     = True
     no_stylesheets        = True
     use_embedded_content  = False
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment'  , description
                         , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
     
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
 
@@ -44,10 +45,9 @@ class Blic(BasicNewsRecipe):
         return u'http://www.blic.rs/_print.php?' + rest_url
 
     def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Language" content="sr-Latn"/>'
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)    
         for item in soup.findAll(style=True):
             del item['style']
         return soup
         
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_chicago_breaking_news.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+chicagobreakingnews.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class ChicagoBreakingNews(BasicNewsRecipe):
+    title                 = 'Chicago Breaking News'
+    __author__            = 'Darko Miletic'
+    description           = 'Breaking News from Chicago'    
+    oldest_article        = 1
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = True
+    publisher             = 'Chicago Breaking News'
+    category              = 'news, politics, USA, Chicago'
+    encoding              = 'utf8'
+    language              = _('English')
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+
+    feeds = [(u'Breaking news', u'http://feeds2.feedburner.com/ChicagoBreakingNews/')]
+
+    def preprocess_html(self, soup):
+        links = soup.findAll('a')
+        for item in soup.findAll('a'):
+            if item['href'].find('http://feedads.googleadservices.com') > -1:
+               item.extract()
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll(color=True):
+            del item['color']
+        for item in soup.findAll(size=True):
+            del item['size']
+        return soup

src/calibre/web/feeds/recipes/recipe_cincinnati_enquirer.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+__license__   = 'GPL v3'
+__copyright__ = '2009 Kovid Goyal <kovid at kovidgoyal.net>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1234144423(BasicNewsRecipe):
+    title          = u'Cincinnati Enquirer'
+    oldest_article = 7
+    language       = _('English')
+    __author__     = 'Joseph Kitzmiller'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    remove_javascript     = True
+    encoding = 'cp1252'
+    extra_css = ' p {font-size: medium; font-weight: normal;} '
+    
+    keep_only_tags = [dict(name='div', attrs={'class':'padding'})]
+    
+    remove_tags = [
+                     dict(name=['object','link','table','embed'])
+                    ,dict(name='div',attrs={'id':'pluckcomments'})
+                    ,dict(name='div',attrs={'class':'articleflex-container'})
+                  ]
+   
+    feeds          = [(u'Cincinnati Enquirer', u'http://rss.cincinnati.com/apps/pbcs.dll/section?category=rssenq01&mime=xml')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll(face=True):
+            del item['face']
+        return soup

src/calibre/web/feeds/recipes/recipe_danas.py

@@ -1,12 +1,11 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 danas.rs
 '''
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Danas(BasicNewsRecipe):
@@ -20,15 +19,17 @@ class Danas(BasicNewsRecipe):
     no_stylesheets        = False
     remove_javascript     = True
     use_embedded_content  = False
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment'  , description
                         , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
  
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
@@ -43,9 +44,8 @@ class Danas(BasicNewsRecipe):
     feeds          = [ (u'Vesti', u'http://www.danas.rs/rss/rss.asp')]
 
     def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Language" content="sr-Latn"/>'
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)    
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_elargentino.py

@@ -5,7 +5,6 @@ __copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 elargentino.com
 '''
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class ElArgentino(BasicNewsRecipe):
@@ -21,6 +20,7 @@ class ElArgentino(BasicNewsRecipe):
     use_embedded_content  = False
     encoding              = 'utf8'
     cover_url             = 'http://www.elargentino.com/TemplateWeb/MediosFooter/tapa_elargentino.png'
+    language              = _('Spanish')
 
     html2lrf_options = [
                           '--comment', description
@@ -59,5 +59,3 @@ class ElArgentino(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']        
         return soup
-    
-    language = _('Spanish')

src/calibre/web/feeds/recipes/recipe_glasgow_herald.py

@@ -0,0 +1,34 @@
+import re
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class GlasgowHerald(BasicNewsRecipe):
+    title          = u'Glasgow Herald'
+    oldest_article = 1
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    language       = _('English')
+    __author__     = 'McCande' 
+
+    preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in
+    [
+                (r'<center><h3>', lambda match : '<h3>'),
+                (r'Click here to comment on this story...', lambda match : ''),
+                (r'<h3>Related links</h3>.*?</head>', lambda match : '</head>'),
+        ]
+        ]
+
+
+
+
+    feeds          = [
+                        (u'News', u'http://www.theherald.co.uk/news/news/rss.xml'),
+                        (u'Politics', u'http://www.theherald.co.uk/politics/news/rss.xml'),
+                        (u'Features', u'http://www.theherald.co.uk/features/features/rss.xml'),
+                        (u'Business', u'http://www.theherald.co.uk/business/news/rss.xml')]
+
+    def print_version(self, url):
+        (beginning,end)=url.split(".var.")
+        num=end[0:7]
+        main="http://www.theherald.co.uk/misc/print.php?artid="+num
+        return main

src/calibre/web/feeds/recipes/recipe_granma.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 granma.cubaweb.cu
 '''
@@ -21,6 +21,7 @@ class Granma(BasicNewsRecipe):
     use_embedded_content  = False
     encoding              = 'cp1252'
     cover_url             = 'http://www.granma.cubaweb.cu/imagenes/granweb229d.jpg'
+    language              = _('Spanish')
     remove_javascript     = True
     
     html2lrf_options = [
@@ -30,10 +31,12 @@ class Granma(BasicNewsRecipe):
                         , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
     keep_only_tags = [dict(name='table', attrs={'height':'466'})]
     
+    remove_tags = [dict(name=['embed','link','object'])]
+
     feeds = [(u'Noticias', u'http://www.granma.cubaweb.cu/noticias.xml' )]
 
     
@@ -49,4 +52,3 @@ class Granma(BasicNewsRecipe):
             del item['style']
         return soup
     
-    language = _('Spanish')

src/calibre/web/feeds/recipes/recipe_hindu.py

@@ -0,0 +1,47 @@
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TheHindu(BasicNewsRecipe):
+    title                 = u'The Hindu'
+    language              = _('English')
+    oldest_article        = 7
+    __author__            = _('Kovid Goyal')
+    max_articles_per_feed = 100
+    
+    remove_tags_before = {'name':'font', 'class':'storyhead'}
+    preprocess_regexps = [
+                (re.compile(r'<!-- story ends -->.*', re.DOTALL), 
+                 lambda match: '</body></html>'),                                                    
+                          ]
+    
+    feeds          = [
+      (u'Main - Font Page', u'http://www.hindu.com/rss/01hdline.xml'), 
+      (u'Main - National', u'http://www.hindu.com/rss/02hdline.xml'), 
+      (u'Main - International', u'http://www.hindu.com/rss/03hdline.xml'), 
+      (u'Main - Opinion', u'http://www.hindu.com/rss/05hdline.xml'), 
+      (u'Main - Business', u'http://www.hindu.com/rss/06hdline.xml'), 
+      (u'Main - Sport', u'http://www.hindu.com/rss/07hdline.xml'), 
+      (u'Main - Weather / Religion / Crossword / Cartoon', 
+       u'http://www.hindu.com/rss/10hdline.xml'), 
+      (u'Main - Engagements', u'http://www.hindu.com/rss/26hdline.xml'), 
+      (u'Supplement - Literary Review', 
+       u'http://www.hindu.com/rss/lrhdline.xml'), 
+      (u'Supplement - Sunday Magazine', 
+       u'http://www.hindu.com/rss/maghdline.xml'), 
+      (u'Supplement - Open Page', u'http://www.hindu.com/rss/ophdline.xml'), 
+      (u'Supplement - Business Review', 
+       u'http://www.hindu.com/rss/bizhdline.xml'), 
+      (u'Supplement - Book Review', 
+       u'http://www.hindu.com/rss/brhdline.xml'), 
+      (u'Supplement - Science & Technology', 
+       u'http://www.hindu.com/rss/setahdline.xml')
+      ]
+    
+    def postprocess_html(self, soup, first_fetch):
+        for t in soup.findAll(['table', 'tr', 'td']):
+            t.name = 'div'
+        return soup

src/calibre/web/feeds/recipes/recipe_infobae.py

@@ -14,10 +14,11 @@ class Infobae(BasicNewsRecipe):
     description           = 'Informacion Libre las 24 horas'
     publisher             = 'Infobae.com'
     category              = 'news, politics, Argentina'     
-    oldest_article        = 2
+    oldest_article        = 1
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
+    language              = _('Spanish')
     encoding              = 'iso-8859-1'
     cover_url             = 'http://www.infobae.com/imgs/header/header.gif'
     remove_javascript     = True
@@ -26,9 +27,15 @@ class Infobae(BasicNewsRecipe):
                           '--comment'  , description
                         , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
+
+    remove_tags = [
+                     dict(name=['embed','link','object'])
+                    ,dict(name='a', attrs={'onclick':'javascript:window.print()'})
+                  ]
     
     feeds = [ 
               (u'Noticias'  , u'http://www.infobae.com/adjuntos/html/RSS/hoy.xml'       )
@@ -48,5 +55,3 @@ class Infobae(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-
-    language = _('Spanish')

src/calibre/web/feeds/recipes/recipe_jutarnji.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 jutarnji.hr
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Jutarnji(BasicNewsRecipe):
@@ -16,32 +15,32 @@ class Jutarnji(BasicNewsRecipe):
     description           = u'Hrvatski portal'
     publisher             = 'Jutarnji.hr'
     category              = 'news, politics, Croatia'    
-    oldest_article        = 2
+    oldest_article        = 1
     max_articles_per_feed = 100
-    simultaneous_downloads = 1
+    simultaneous_downloads = 2
     delay                 = 1
     language              = _('Croatian')
     no_stylesheets        = True
     use_embedded_content  = False
     remove_javascript     = True
     encoding              = 'cp1250'
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment'  , description
                         , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     
     remove_tags = [ 
-                    dict(name='embed')
+                    dict(name=['embed','hr','link','object'])
                    ,dict(name='a', attrs={'class':'a11'})
-                   ,dict(name='hr')
                   ]
     
     feeds = [
@@ -60,9 +59,7 @@ class Jutarnji(BasicNewsRecipe):
         return 'http://www.jutarnji.hr/ispis_clanka.jl?artid=' + rrest
 
     def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
-        soup.head.insert(0,mtag)
-        mtag = '<meta http-equiv="Content-Language" content="hr"/>'
+        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n<meta http-equiv="Content-Language" content="hr-HR"/>'
         soup.head.insert(0,mtag)
         for item in soup.findAll(style=True):
             del item['style']        

src/calibre/web/feeds/recipes/recipe_juventudrebelde.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 juventudrebelde.cu
 '''
-from calibre import strftime
 
+from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Juventudrebelde(BasicNewsRecipe):
@@ -20,6 +20,7 @@ class Juventudrebelde(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'cp1252'
+    language              = _('Spanish')
     cover_url             = strftime('http://www.juventudrebelde.cu/UserFiles/File/impreso/iportada-%Y-%m-%d.jpg')
     remove_javascript     = True
     
@@ -30,7 +31,7 @@ class Juventudrebelde(BasicNewsRecipe):
                         , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
     keep_only_tags = [dict(name='div', attrs={'id':'noticia'})]
 
@@ -51,4 +52,3 @@ class Juventudrebelde(BasicNewsRecipe):
             del item['style']
         return soup
              
-    language = _('Spanish')

src/calibre/web/feeds/recipes/recipe_linuxdevices.py

@@ -0,0 +1,80 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
+'''
+Fetch Linuxdevices.
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class Sueddeutsche(BasicNewsRecipe):
+    
+    title = u'Linuxdevices'
+    description = 'News about Linux driven Hardware'
+    __author__ = 'Oliver Niesner'
+    use_embedded_content   = False
+    timefmt = ' [%a, %d %b %Y]'
+    language = _('English')
+    max_articles_per_feed = 50
+    no_stylesheets = True
+    encoding = 'latin1'
+
+    remove_tags_after = [dict(id='nointelliTXT')]
+    filter_regexps = [r'ad\.doubleclick\.net']
+
+
+    remove_tags = [dict(name='div', attrs={'class':'bannerSuperBanner'}),
+                   dict(name='div', attrs={'class':'bannerSky'}),
+                   dict(name='div', attrs={'class':'footerLinks'}),
+                   dict(name='div', attrs={'class':'seitenanfang'}),
+                   dict(name='td', attrs={'class':'mar5'}),
+                   dict(name='td', attrs={'class':'mar5'}),
+                   dict(name='table', attrs={'class':'pageAktiv'}),
+                   dict(name='table', attrs={'class':'xartable'}),
+                   dict(name='table', attrs={'class':'wpnavi'}),
+                   dict(name='table', attrs={'class':'bgcontent absatz'}),
+                   dict(name='table', attrs={'class':'footer'}),
+                   dict(name='table', attrs={'class':'artikelBox'}),
+                   dict(name='table', attrs={'class':'kommentare'}),
+                   dict(name='table', attrs={'class':'pageBoxBot'}),
+                   #dict(name='table', attrs={'with':'100%'}),
+                   dict(name='td', attrs={'nowrap':'nowrap'}),
+                   dict(name='td', attrs={'valign':'middle'}),
+                   dict(name='td', attrs={'align':'left'}),
+                   dict(name='td', attrs={'align':'center'}),
+                   dict(name='td', attrs={'height':'5'}),
+                   dict(name='div', attrs={'class':'artikelBox navigatorBox'}),
+                   dict(name='div', attrs={'class':'similar-article-box'}),
+                   dict(name='div', attrs={'class':'videoBigHack'}),
+                   dict(name='td', attrs={'class':'artikelDruckenRight'}),
+                   dict(name='td', attrs={'class':'width="200"'}),
+                   dict(name='a', attrs={'href':'/news'}),
+                   dict(name='a', attrs={'href':'/'}),
+                   dict(name='a', attrs={'href':'/articles'}),
+                   dict(name='a', attrs={'href':'/cgi-bin/survey/survey.cgi'}),
+                   dict(name='a', attrs={'href':'/cgi-bin/board/UltraBoard.pl'}),
+                   dict(name='iframe'),
+                   dict(name='form'),
+                   #dict(name='tr', attrs={'td':'Click here to learn'}),
+                   dict(name='span', attrs={'class':'hidePrint'}),
+                   dict(id='headerLBox'),
+                   dict(id='nointelliTXT'),
+                   dict(id='rechteSpalte'),
+                   dict(id='newsticker-list-small'),
+                   dict(id='ntop5'),
+                   dict(id='ntop5send'),
+                   dict(id='ntop5commented'),
+                   dict(id='nnav-bgheader'),
+                   dict(id='nnav-headerteaser'),
+                   dict(id='nnav-head'),
+                   dict(id='nnav-top'),
+                   dict(id='nnav-logodiv'),
+                   dict(id='nnav-logo'),
+                   dict(id='nnav-oly'),
+                   dict(id='readcomment')]
+    
+
+
+    feeds =  [ (u'Linuxdevices', u'http://www.linuxdevices.com/backend/headlines.rss') ] 
+

src/calibre/web/feeds/recipes/recipe_nin.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 nin.co.yu
 '''
 
 import re, urllib
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Nin(BasicNewsRecipe):    
@@ -27,15 +26,17 @@ class Nin(BasicNewsRecipe):
     LOGIN                  = PREFIX + '/?logout=true'
     remove_javascript      = True
     use_embedded_content   = False
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment'  , description
                         , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
                           
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     
@@ -69,5 +70,3 @@ class Nin(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_novosti.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 novosti.rs
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Novosti(BasicNewsRecipe):
@@ -22,15 +21,17 @@ class Novosti(BasicNewsRecipe):
     use_embedded_content  = False
     encoding              = 'utf8'
     remove_javascript     = True
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment'  , description
                         , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options  = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
 
@@ -40,10 +41,8 @@ class Novosti(BasicNewsRecipe):
     feeds              = [(u'Vesti', u'http://www.novosti.rs/php/vesti/rss.php')]
 
     def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Language" content="sr-Latn"/>'
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)    
         for item in soup.findAll(style=True):
             del item['style']        
         return soup
-
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_nspm.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 nspm.rs
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Nspm(BasicNewsRecipe):
@@ -16,14 +15,15 @@ class Nspm(BasicNewsRecipe):
     description           = 'Casopis za politicku teoriju i drustvena istrazivanja'    
     publisher             = 'NSPM'
     category              = 'news, politics, Serbia'    
-    oldest_article        = 7
+    oldest_article        = 2
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
     INDEX                 = 'http://www.nspm.rs/?alphabet=l'
     encoding              = 'utf8'
     remove_javascript     = True
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment'  , description
@@ -32,10 +32,13 @@ class Nspm(BasicNewsRecipe):
                         , '--ignore-tables'
                         ]
     
-    html2epub_options  = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
-    remove_tags        = [dict(name='a')]
+    remove_tags        = [
+                            dict(name=['a','img','link','object','embed'])
+                           ,dict(name='td', attrs={'class':'buttonheading'})
+                         ]
     
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
@@ -48,13 +51,12 @@ class Nspm(BasicNewsRecipe):
         return url.replace('.html','/stampa.html')
 
     def preprocess_html(self, soup):
-        soup.html['xml:lang'] = 'sr-Latn-RS'
-        soup.html['lang']     = 'sr-Latn-RS'
+        lng = 'sr-Latn-RS'
+        soup.html['xml:lang'] = lng
+        soup.html['lang']     = lng
         ftag = soup.find('meta',attrs={'http-equiv':'Content-Language'})
         if ftag:
-           ftag['content'] = 'sr-Latn-RS'
+           ftag['content'] = lng
         for item in soup.findAll(style=True):
             del item['style']     
         return soup
-
-    language = _('Serbian')

src/calibre/web/feeds/recipes/recipe_pescanik.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 pescanik.net
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Pescanik(BasicNewsRecipe):
@@ -16,30 +15,32 @@ class Pescanik(BasicNewsRecipe):
     description           = 'Pescanik'
     publisher             = 'Pescanik'
     category              = 'news, politics, Serbia'    
-    oldest_article        = 7
+    oldest_article        = 5
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
     remove_javascript     = True
     encoding              = 'utf8'
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    cover_url             = "http://pescanik.net/templates/ja_teline/images/logo.png"
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment'  , description
                         , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
     
-    cover_url = "http://pescanik.net/templates/ja_teline/images/logo.png"
     
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     
     remove_tags = [
                      dict(name='td'  , attrs={'class':'buttonheading'})
                     ,dict(name='span', attrs={'class':'article_seperator'})
-                    ,dict(name=['object','link'])
+                    ,dict(name=['object','link','img','h4','ul'])
                   ]
 
     feeds       = [(u'Pescanik Online', u'http://pescanik.net/index.php?option=com_rd_rss&id=12')]
@@ -54,5 +55,3 @@ class Pescanik(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_pobjeda.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+
+'''
+pobjeda.co.me
+'''
+
+import re
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Pobjeda(BasicNewsRecipe):
+    title                 = 'Pobjeda Online'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Montenegro'
+    publisher             = 'Pobjeda a.d.'
+    category              = 'news, politics, Montenegro'    
+    language              = _('Serbian')
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    remove_javascript     = True
+    encoding              = 'utf8'
+    remove_javascript     = True
+    use_embedded_content  = False
+    INDEX                 = u'http://www.pobjeda.co.me'
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: serif1, serif}'
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+     
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+
+    keep_only_tags = [dict(name='div', attrs={'class':'vijest'})]
+
+    remove_tags = [dict(name=['object','link'])]
+
+    feeds = [
+               (u'Politika'          , u'http://www.pobjeda.co.me/rubrika.php?rubrika=1'  )
+              ,(u'Ekonomija'         , u'http://www.pobjeda.co.me/rubrika.php?rubrika=2'  )
+              ,(u'Drustvo'           , u'http://www.pobjeda.co.me/rubrika.php?rubrika=3'  )
+              ,(u'Crna Hronika'      , u'http://www.pobjeda.co.me/rubrika.php?rubrika=4'  )
+              ,(u'Kultura'           , u'http://www.pobjeda.co.me/rubrika.php?rubrika=5'  )
+              ,(u'Hronika Podgorice' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=7'  )
+              ,(u'Feljton'           , u'http://www.pobjeda.co.me/rubrika.php?rubrika=8'  )
+              ,(u'Crna Gora'         , u'http://www.pobjeda.co.me/rubrika.php?rubrika=9'  )
+              ,(u'Svijet'            , u'http://www.pobjeda.co.me/rubrika.php?rubrika=202')
+              ,(u'Ekonomija i Biznis', u'http://www.pobjeda.co.me/dodatak.php?rubrika=11' )
+              ,(u'Djeciji Svijet'    , u'http://www.pobjeda.co.me/dodatak.php?rubrika=12' )
+              ,(u'Kultura i Drustvo' , u'http://www.pobjeda.co.me/dodatak.php?rubrika=13' )
+              ,(u'Agora'             , u'http://www.pobjeda.co.me/dodatak.php?rubrika=133')
+              ,(u'Ekologija'         , u'http://www.pobjeda.co.me/dodatak.php?rubrika=252')
+            ]
+
+    def preprocess_html(self, soup):
+        soup.html['xml:lang'] = 'sr-Latn-ME'
+        soup.html['lang']     = 'sr-Latn-ME'
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-ME"/>'
+        soup.head.insert(0,mtag)
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup
+
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('img',attrs={'alt':'Naslovna strana'})
+        if cover_item:
+           cover_url = self.INDEX + cover_item.parent['href']
+        return cover_url
+    
+    def parse_index(self):
+        totalfeeds = []
+        lfeeds = self.get_feeds()
+        for feedobj in lfeeds:
+            feedtitle, feedurl = feedobj
+            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
+            articles = []
+            soup = self.index_to_soup(feedurl)
+            for item in soup.findAll('div', attrs={'class':'vijest'}):
+                description = self.tag_to_string(item.h2)
+                atag = item.h1.find('a')
+                if atag:
+                    url         = self.INDEX + '/' + atag['href']
+                    title       = self.tag_to_string(atag)
+                    date        = strftime(self.timefmt)
+                    articles.append({
+                                      'title'      :title
+                                     ,'date'       :date
+                                     ,'url'        :url
+                                     ,'description':description
+                                    })
+            totalfeeds.append((feedtitle, articles))
+        return totalfeeds
+        

src/calibre/web/feeds/recipes/recipe_politika.py

@@ -6,7 +6,6 @@ __copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
 politika.rs
 '''
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Politika(BasicNewsRecipe):
@@ -21,7 +20,8 @@ class Politika(BasicNewsRecipe):
     use_embedded_content  = False
     remove_javascript     = True
     encoding              = 'utf8'
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment', description
@@ -60,6 +60,6 @@ class Politika(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']
         ftag = soup.find('div',attrs={'class':'content_center_border'})
-        if ftag:
-           ftag['align'] = 'left'
+        if ftag.has_key('align'):
+           del ftag['align']
         return soup

src/calibre/web/feeds/recipes/recipe_vijesti.py

@@ -4,11 +4,10 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
 
 '''
-vijesti.cg.yu
+vijesti.me
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Vijesti(BasicNewsRecipe):
@@ -22,10 +21,11 @@ class Vijesti(BasicNewsRecipe):
     no_stylesheets        = True
     remove_javascript     = True
     encoding              = 'cp1250'
-    cover_url             = 'http://www.vijesti.cg.yu/img/logo.gif'
+    cover_url             = 'http://www.vijesti.me/img/logo.gif'
     remove_javascript     = True
     use_embedded_content  = False
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment', description
@@ -39,12 +39,9 @@ class Vijesti(BasicNewsRecipe):
 
     keep_only_tags = [dict(name='div', attrs={'id':'mainnews'})]
 
-    remove_tags = [
-                     dict(name='div', attrs={'align':'right'})
-                    ,dict(name=['object','link'])
-                  ]
+    remove_tags = [dict(name=['object','link','embed'])]
 
-    feeds = [(u'Sve vijesti', u'http://www.vijesti.cg.yu/rss.php' )]
+    feeds = [(u'Sve vijesti', u'http://www.vijesti.me/rss.php' )]
 
     def preprocess_html(self, soup):
         soup.html['xml:lang'] = 'sr-Latn-ME'
@@ -56,5 +53,3 @@ class Vijesti(BasicNewsRecipe):
                del item['align']
                item.insert(0,'<br /><br />')
         return soup
-
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_vreme.py

@@ -1,14 +1,13 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 vreme.com
 '''
 
 import re
 from calibre import strftime
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Vreme(BasicNewsRecipe):    
@@ -24,15 +23,17 @@ class Vreme(BasicNewsRecipe):
     LOGIN = 'http://www.vreme.com/account/index.php'
     remove_javascript     = True
     use_embedded_content  = False
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
                           '--comment'  , description
                         , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
     
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
 
@@ -87,7 +88,12 @@ class Vreme(BasicNewsRecipe):
         del soup.body['text'   ]
         del soup.body['bgcolor']
         del soup.body['onload' ]
-        mtag = '<meta http-equiv="Content-Language" content="sr-Latn"/>'
+        for item in soup.findAll('table'):
+            if item.has_key('width'):
+               del item['width']
+            if item.has_key('height'):
+               del item['height']                    
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)
         tbl = soup.body.table
         tbbb = soup.find('td')
@@ -104,5 +110,3 @@ class Vreme(BasicNewsRecipe):
         if cover_item:
            cover_url = self.INDEX + cover_item['src']
         return cover_url
-
-    language              = _('Serbian')

src/calibre/web/fetch/simple.py

@@ -8,7 +8,7 @@ Fetch a webpage and its links recursively. The webpages are saved to disk in
 UTF-8 encoding with any charset declarations removed.
 '''
 import sys, socket, os, urlparse, logging, re, time, copy, urllib2, threading, traceback
-from urllib import url2pathname
+from urllib import url2pathname, quote
 from threading import RLock
 from httplib import responses
 from PIL import Image
@@ -179,6 +179,8 @@ class RecursiveFetcher(object, LoggingInterface):
         delta = time.time() - self.last_fetch_at 
         if  delta < self.delay:
             time.sleep(delta)
+        if re.search(r'\s+', url) is not None:
+            url = quote(url)
         with self.browser_lock:
             try:
                 with closing(self.browser.open(url)) as f:

src/cherrypy/_cpchecker.py

@@ -58,7 +58,7 @@ class Checker(object):
                             "specific sections. You must explicitly pass "
                             "application config via "
                             "cherrypy.tree.mount(..., config=app_config)")
-                warnings.warn(msg)
+                    warnings.warn(msg[:5])
                 return
     
     def check_static_paths(self):