Sync to pluginize

installer/windows/freeze.py

@@ -14,7 +14,20 @@ IMAGEMAGICK_DIR  = 'C:\\ImageMagick'
 FONTCONFIG_DIR   = 'C:\\fontconfig'
 VC90             = r'C:\VC90.CRT'
 
-import sys, os, py2exe, shutil, zipfile, glob, re
+# ModuleFinder can't handle runtime changes to __path__, but win32com uses them
+import sys
+import py2exe.mf as modulefinder
+import win32com
+for p in win32com.__path__[1:]:
+    modulefinder.AddPackagePath("win32com", p)
+for extra in ["win32com.shell"]: #,"win32com.mapi"
+    __import__(extra)
+    m = sys.modules[extra]
+    for p in m.__path__[1:]:
+        modulefinder.AddPackagePath(extra, p)
+
+
+import os, py2exe, shutil, zipfile, glob, re
 from distutils.core import setup
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 sys.path.insert(0, BASE_DIR)

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.5.5'
+__version__   = '0.5.6'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/customize/builtins.py

@@ -18,7 +18,7 @@ every time you add an HTML file to the library.\
     file_types = set(['html', 'htm', 'xhtml', 'xhtm'])
     supported_platforms = ['windows', 'osx', 'linux']
     on_import = True
-    
+
     def run(self, htmlfile):
         of = self.temporary_file('_plugin_html2zip.zip')
         from calibre.ebooks.html import gui_main as html2oeb
@@ -26,172 +26,173 @@ every time you add an HTML file to the library.\
         return of.name
 
 class OPFMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read OPF metadata'
     file_types  = set(['opf'])
     description = _('Read metadata from %s files')%'OPF'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.opf2 import OPF
         from calibre.ebooks.metadata import MetaInformation
         return MetaInformation(OPF(stream, os.getcwd()))
 
 class RTFMetadataReader(MetadataReaderPlugin):
-    
+
-    name        = 'Read RTF metadata' 
+    name        = 'Read RTF metadata'
     file_types  = set(['rtf'])
     description = _('Read metadata from %s files')%'RTF'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.rtf import get_metadata
         return get_metadata(stream)
 
 class FB2MetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read FB2 metadata'
     file_types  = set(['fb2'])
     description = _('Read metadata from %s files')%'FB2'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.fb2 import get_metadata
         return get_metadata(stream)
 
 
 class LRFMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read LRF metadata'
     file_types  = set(['lrf'])
     description = _('Read metadata from %s files')%'LRF'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.lrf.meta import get_metadata
         return get_metadata(stream)
 
 class PDFMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read PDF metadata'
     file_types  = set(['pdf'])
     description = _('Read metadata from %s files')%'PDF'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.pdf import get_metadata
         return get_metadata(stream)
 
 class LITMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read LIT metadata'
     file_types  = set(['lit'])
     description = _('Read metadata from %s files')%'LIT'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.lit import get_metadata
         return get_metadata(stream)
 
 class IMPMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read IMP metadata'
     file_types  = set(['imp'])
     description = _('Read metadata from %s files')%'IMP'
     author      = 'Ashish Kulkarni'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.imp import get_metadata
         return get_metadata(stream)
 
 class RBMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read RB metadata'
     file_types  = set(['rb'])
     description = _('Read metadata from %s files')%'RB'
     author      = 'Ashish Kulkarni'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.rb import get_metadata
         return get_metadata(stream)
 
 class EPUBMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read EPUB metadata'
     file_types  = set(['epub'])
     description = _('Read metadata from %s files')%'EPUB'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.epub import get_metadata
         return get_metadata(stream)
 
 class HTMLMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read HTML metadata'
     file_types  = set(['html'])
     description = _('Read metadata from %s files')%'HTML'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.html import get_metadata
         return get_metadata(stream)
 
 class MOBIMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read MOBI metadata'
     file_types  = set(['mobi', 'prc', 'azw'])
     description = _('Read metadata from %s files')%'MOBI'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.mobi.reader import get_metadata
         return get_metadata(stream)
 
 
 class TOPAZMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read Topaz metadata'
     file_types  = set(['tpz', 'azw1'])
     description = _('Read metadata from %s files')%'MOBI'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.topaz import get_metadata
         return get_metadata(stream)
 
 class ODTMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read ODT metadata'
     file_types  = set(['odt'])
     description = _('Read metadata from %s files')%'ODT'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.odt import get_metadata
         return get_metadata(stream)
-        
+
 class TXTMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read TXT metadata'
     file_types  = set(['txt'])
     description = _('Read metadata from %s files') % 'TXT'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.txt import get_metadata
         return get_metadata(stream)
 
 class LRXMetadataReader(MetadataReaderPlugin):
-    
+
     name        = 'Read LRX metadata'
     file_types  = set(['lrx'])
     description = _('Read metadata from %s files')%'LRX'
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.lrx import get_metadata
         return get_metadata(stream)
 
 class ComicMetadataReader(MetadataReaderPlugin):
-    
+
     name = 'Read comic metadata'
     file_types = set(['cbr', 'cbz'])
     description = _('Extract cover from comic files')
-    
+
     def get_metadata(self, stream, ftype):
         if ftype == 'cbr':
             from calibre.libunrar import extract_member as extract_first
+            extract_first
         else:
             from calibre.libunzip import extract_member as extract_first
-        from calibre.ebooks.metadata import MetaInformation 
+        from calibre.ebooks.metadata import MetaInformation
         ret = extract_first(stream)
         mi = MetaInformation(None, None)
         if ret is not None:
@@ -199,65 +200,65 @@ class ComicMetadataReader(MetadataReaderPlugin):
             ext = os.path.splitext(path)[1][1:]
             mi.cover_data = (ext.lower(), data)
         return mi
-    
+
 class ZipMetadataReader(MetadataReaderPlugin):
-    
+
     name = 'Read ZIP metadata'
     file_types = set(['zip', 'oebzip'])
     description = _('Read metadata from ebooks in ZIP archives')
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.zip import get_metadata
         return get_metadata(stream)
 
 class RARMetadataReader(MetadataReaderPlugin):
-    
+
     name = 'Read RAR metadata'
     file_types = set(['rar'])
     description = _('Read metadata from ebooks in RAR archives')
-    
+
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.rar import get_metadata
         return get_metadata(stream)
 
 
 class EPUBMetadataWriter(MetadataWriterPlugin):
-    
+
     name = 'Set EPUB metadata'
     file_types = set(['epub'])
     description = _('Set metadata in %s files')%'EPUB'
-    
+
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.metadata.epub import set_metadata
         set_metadata(stream, mi)
-        
+
 class LRFMetadataWriter(MetadataWriterPlugin):
-    
+
     name = 'Set LRF metadata'
     file_types = set(['lrf'])
     description = _('Set metadata in %s files')%'LRF'
-    
+
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.lrf.meta import set_metadata
         set_metadata(stream, mi)
 
 class RTFMetadataWriter(MetadataWriterPlugin):
-    
+
     name = 'Set RTF metadata'
     file_types = set(['rtf'])
     description = _('Set metadata in %s files')%'RTF'
-    
+
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.metadata.rtf import set_metadata
         set_metadata(stream, mi)
 
 class MOBIMetadataWriter(MetadataWriterPlugin):
-    
+
     name        = 'Set MOBI metadata'
     file_types  = set(['mobi', 'prc', 'azw'])
     description = _('Set metadata in %s files')%'MOBI'
     author      = 'Marshall T. Vandegrift'
-    
+
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.metadata.mobi import set_metadata
         set_metadata(stream, mi)
@@ -267,14 +268,16 @@ from calibre.ebooks.epub.input import EPUBInput
 from calibre.ebooks.mobi.input import MOBIInput
 from calibre.ebooks.pdf.input import PDFInput
 from calibre.ebooks.txt.input import TXTInput
+from calibre.ebooks.html.input import HTMLInput
 from calibre.ebooks.oeb.output import OEBOutput
 from calibre.ebooks.txt.output import TXTOutput
 from calibre.ebooks.pdf.output import PDFOutput
 from calibre.customize.profiles import input_profiles, output_profiles
 
-plugins = [HTML2ZIP, EPUBInput, MOBIInput, PDFInput, TXTInput, OEBOutput, TXTOutput, PDFOutput]
+plugins = [HTML2ZIP, EPUBInput, MOBIInput, PDFInput, HTMLInput,
+        TXTInput, OEBOutput, TXTOutput, PDFOutput]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
                                         x.__name__.endswith('MetadataReader')]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
                                         x.__name__.endswith('MetadataWriter')]
-plugins += input_profiles + output_profiles
+plugins += input_profiles + output_profiles

src/calibre/customize/conversion.py

@@ -163,9 +163,9 @@ class InputFormatPlugin(Plugin):
             for x in os.listdir('.'):
                 shutil.rmtree(x) if os.path.isdir(x) else os.remove(x)
 
-
             ret = self.convert(stream, options, file_ext,
                                log, accelerators)
+
         if options.debug_input is not None:
             options.debug_input = os.path.abspath(options.debug_input)
             if not os.path.exists(options.debug_input):

src/calibre/devices/__init__.py

@@ -13,12 +13,14 @@ def devices():
     from calibre.devices.kindle.driver import KINDLE
     from calibre.devices.kindle.driver import KINDLE2
     from calibre.devices.blackberry.driver import BLACKBERRY
-    return (PRS500, PRS505, PRS700, CYBOOKG3, KINDLE, KINDLE2, BLACKBERRY)
+    from calibre.devices.eb600.driver import EB600
+    return (PRS500, PRS505, PRS700, CYBOOKG3, KINDLE, KINDLE2,
+            BLACKBERRY, EB600)
 
 import time
 
 DAY_MAP   = dict(Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6)
-MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12) 
+MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12)
 INVERSE_DAY_MAP = dict(zip(DAY_MAP.values(), DAY_MAP.keys()))
 INVERSE_MONTH_MAP = dict(zip(MONTH_MAP.values(), MONTH_MAP.keys()))
 

src/calibre/devices/cybookg3/driver.py

@@ -11,37 +11,36 @@ from calibre.ebooks.metadata import authors_to_string
 from calibre.devices.errors import FreeSpaceError
 from calibre.devices.usbms.driver import USBMS
 import calibre.devices.cybookg3.t2b as t2b
-from calibre.devices.errors import FreeSpaceError
 
 class CYBOOKG3(USBMS):
     # Ordered list of supported formats
     # Be sure these have an entry in calibre.devices.mime
     FORMATS     = ['mobi', 'prc', 'html', 'pdf', 'rtf', 'txt']
-    
+
     VENDOR_ID   = [0x0bda, 0x3034]
     PRODUCT_ID  = [0x0703, 0x1795]
     BCD         = [0x110, 0x132]
-    
+
     VENDOR_NAME = 'BOOKEEN'
     WINDOWS_MAIN_MEM = 'CYBOOK_GEN3__-FD'
     WINDOWS_CARD_MEM = 'CYBOOK_GEN3__-SD'
-    
+
     OSX_MAIN_MEM = 'Bookeen Cybook Gen3 -FD Media'
     OSX_CARD_MEM = 'Bookeen Cybook Gen3 -SD Media'
-    
+
     MAIN_MEMORY_VOLUME_LABEL  = 'Cybook Gen 3 Main Memory'
     STORAGE_CARD_VOLUME_LABEL = 'Cybook Gen 3 Storage Card'
-    
+
     EBOOK_DIR_MAIN = "eBooks"
     EBOOK_DIR_CARD = "eBooks"
     THUMBNAIL_HEIGHT = 144
     SUPPORTS_SUB_DIRS = True
-    
+
-    def upload_books(self, files, names, on_card=False, end_session=True, 
+    def upload_books(self, files, names, on_card=False, end_session=True,
                      metadata=None):
         if on_card and not self._card_prefix:
             raise ValueError(_('The reader has no storage card connected.'))
-            
+
         if not on_card:
             path = os.path.join(self._main_prefix, self.EBOOK_DIR_MAIN)
         else:
@@ -66,7 +65,7 @@ class CYBOOKG3(USBMS):
         paths = []
         names = iter(names)
         metadata = iter(metadata)
-        
+
         for infile in files:
             newpath = path
             mdata = metadata.next()
@@ -83,20 +82,20 @@ class CYBOOKG3(USBMS):
                             newpath += tag
                             newpath = os.path.normpath(newpath)
                             break
-                            
+
                 if newpath == path:
                     newpath = os.path.join(newpath, authors_to_string(mdata.get('authors', '')))
                     newpath = os.path.join(newpath, mdata.get('title', ''))
 
             if not os.path.exists(newpath):
                 os.makedirs(newpath)
-            
+
             filepath = os.path.join(newpath, names.next())
             paths.append(filepath)
-            
+
             if hasattr(infile, 'read'):
                 infile.seek(0)
-                
+
                 dest = open(filepath, 'wb')
                 shutil.copyfileobj(infile, dest, 10*1024*1024)
 
@@ -104,35 +103,35 @@ class CYBOOKG3(USBMS):
                 dest.close()
             else:
                 shutil.copy2(infile, filepath)
-                
+
-            coverdata = None            
+            coverdata = None
             if 'cover' in mdata.keys():
                 if mdata['cover'] != None:
                     coverdata = mdata['cover'][2]
-            
+
             t2bfile = open('%s_6090.t2b' % (os.path.splitext(filepath)[0]), 'wb')
             t2b.write_t2b(t2bfile, coverdata)
             t2bfile.close()
-    
+
         return zip(paths, cycle([on_card]))
 
     def delete_books(self, paths, end_session=True):
         for path in paths:
             if os.path.exists(path):
                 os.unlink(path)
-                
+
                 filepath, ext = os.path.splitext(path)
-                
+
-                # Delete the ebook auxiliary files
+                # Delete the ebook auxiliary file
                 if os.path.exists(filepath + '.mbp'):
                     os.unlink(filepath + '.mbp')
                 if os.path.exists(filepath + '.dat'):
                     os.unlink(filepath + '.dat')
-                
+
                 # Delete the thumbnails file auto generated for the ebook
                 if os.path.exists(filepath + '_6090.t2b'):
                     os.unlink(filepath + '_6090.t2b')
-                    
+
                 try:
                     os.removedirs(os.path.dirname(path))
                 except:

src/calibre/devices/eb600/__init__.py

@@ -0,0 +1,2 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'

src/calibre/devices/eb600/driver.py

@@ -0,0 +1,41 @@
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+'''
+Device driver for the Netronix EB600
+'''
+
+from calibre.devices.usbms.driver import USBMS
+
+class EB600(USBMS):
+    # Ordered list of supported formats
+    FORMATS     = ['epub', 'prc', 'chm', 'djvu', 'html', 'rtf', 'txt', 'pdf']
+    DRM_FORMATS = ['prc', 'mobi', 'html', 'pdf', 'txt']
+
+    VENDOR_ID   = [0x1f85]
+    PRODUCT_ID  = [0x1688]
+    BCD         = [0x110]
+
+    VENDOR_NAME      = 'NETRONIX'
+    WINDOWS_MAIN_MEM = 'EBOOK'
+    WINDOWS_CARD_MEM = 'EBOOK'
+
+    OSX_MAIN_MEM = 'EB600 Internal Storage Media'
+    OSX_CARD_MEM = 'EB600 Card Storage Media'
+
+    MAIN_MEMORY_VOLUME_LABEL  = 'EB600 Main Memory'
+    STORAGE_CARD_VOLUME_LABEL = 'EB600 Storage Card'
+
+    EBOOK_DIR_MAIN = ''
+    EBOOK_DIR_CARD = ''
+    SUPPORTS_SUB_DIRS = True
+
+    def windows_sort_drives(self, drives):
+        main = drives['main']
+        card = drives['card']
+        if card and main and card < main:
+            drives['main'] = card
+            drives['card'] = main
+
+        return drives
+
+

src/calibre/devices/usbms/device.py

@@ -174,6 +174,14 @@ class Device(_Device):
 
         return prefix
 
+    def windows_sort_drives(self, drives):
+        '''
+        Called to disambiguate main memory and storage card for devices that
+        do not distinguish between them on the basis of `WINDOWS_CARD_NAME`.
+        For e.g.: The EB600
+        '''
+        return drives
+
     def open_windows(self):
         time.sleep(6)
         drives = {}
@@ -188,11 +196,14 @@ class Device(_Device):
             if 'main' in drives.keys() and 'card' in drives.keys():
                 break
 
+        drives = self.windows_sort_drives(drives)
         self._main_prefix = drives.get('main')
         self._card_prefix = drives.get('card')
 
         if not self._main_prefix:
-            raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.') % self.__class__.__name__)
+            raise DeviceError(
+                _('Unable to detect the %s disk drive. Try rebooting.') %
+                self.__class__.__name__)
 
     def get_osx_mountpoints(self, raw=None):
         if raw is None:

src/calibre/ebooks/epub/from_html.py

@@ -36,7 +36,7 @@ import os, sys, cStringIO, logging, re, functools, shutil
 
 from lxml.etree import XPath
 from lxml import html, etree
-from PyQt4.Qt import QApplication, QPixmap
+from PyQt4.Qt import QApplication, QPixmap, Qt
 
 from calibre.ebooks.html_old import Processor, merge_metadata, get_filelist,\
     opf_traverse, create_metadata, rebase_toc, Link, parser
@@ -50,7 +50,7 @@ from calibre.ebooks.epub.pages import add_page_map
 from calibre.ebooks.epub.fonts import Rationalizer
 from calibre.constants import preferred_encoding
 from calibre.customize.ui import run_plugins_on_postprocess
-from calibre import walk, CurrentDir, to_unicode
+from calibre import walk, CurrentDir, to_unicode, fit_image
 
 content = functools.partial(os.path.join, u'content')
 
@@ -112,6 +112,31 @@ def find_html_index(files):
                 return f, os.path.splitext(f)[1].lower()[1:]
     return html_files[-1], os.path.splitext(html_files[-1])[1].lower()[1:]
 
+def rescale_images(imgdir, screen_size, log):
+    pwidth, pheight = screen_size
+    if QApplication.instance() is None:
+        QApplication([])
+    for f in os.listdir(imgdir):
+        path = os.path.join(imgdir, f)
+        if os.path.splitext(f)[1] in ('.css', '.js'):
+            continue
+
+        p = QPixmap()
+        p.load(path)
+        if p.isNull():
+            continue
+        width, height = p.width(), p.height()
+        scaled, new_width, new_height = fit_image(width, height, pwidth,
+                pheight)
+        if scaled:
+            log.info('Rescaling image: '+f)
+            p.scaled(new_width, new_height, Qt.IgnoreAspectRatio,
+                    Qt.SmoothTransformation).save(path, 'JPEG')
+
+
+
+
+
 class HTMLProcessor(Processor, Rationalizer):
 
     def __init__(self, htmlfile, opts, tdir, resource_map, htmlfiles, stylesheets):
@@ -482,6 +507,10 @@ def convert(htmlfile, opts, notification=None, create_epub=True,
             if os.stat(ncx_path).st_size > opts.profile.flow_size:
                 logger.warn('NCX still larger than allowed size at %d bytes. Menu based Table of Contents may not work on device.'%os.stat(ncx_path).st_size)
 
+        if opts.profile.screen_size is not None:
+            rescale_images(os.path.join(tdir, 'content', 'resources'),
+                    opts.profile.screen_size, logger)
+
         if create_epub:
             epub = initialize_container(opts.output)
             epub.add_dir(tdir)

src/calibre/ebooks/html/__init__.py

@@ -17,7 +17,7 @@ def tostring(root, strip_comments=False, pretty_print=False):
     root.set('xmlns', 'http://www.w3.org/1999/xhtml')
     root.set('{http://www.w3.org/1999/xhtml}xlink', 'http://www.w3.org/1999/xlink')
     for x in root.iter():
-        if x.tag.rpartition('}')[-1].lower() == 'svg':
+        if hasattr(x.tag, 'rpartition') and x.tag.rpartition('}')[-1].lower() == 'svg':
             x.set('xmlns', 'http://www.w3.org/2000/svg')
 
     ans = _tostring(root, encoding='utf-8', pretty_print=pretty_print)

src/calibre/ebooks/html/input.py

@@ -11,14 +11,12 @@ __docformat__ = 'restructuredtext en'
 Input plugin for HTML or OPF ebooks.
 '''
 
-import os, re, sys, cStringIO
+import os, re, sys
 from urlparse import urlparse, urlunparse
 from urllib import unquote
 
 from calibre.customize.conversion import InputFormatPlugin
-from calibre.ebooks.metadata.meta import get_metadata
+from calibre.ebooks.metadata.opf2 import OPFCreator
-from calibre.ebooks.metadata.opf2 import OPF, OPFCreator
-from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.customize.conversion import OptionRecommendation
 from calibre import unicode_path
@@ -213,72 +211,21 @@ def traverse(path_to_html_file, max_levels=sys.maxint, verbose=0, encoding=None)
         sys.setrecursionlimit(orec)
 
 
-def opf_traverse(opf_reader, verbose=0, encoding=None):
-    '''
-    Return a list of :class:`HTMLFile` objects in the order specified by the
-    `<spine>` element of the OPF.
-
-    :param opf_reader: An :class:`calibre.ebooks.metadata.opf2.OPF` instance.
-    :param encoding:   Specify character encoding of HTML files. If `None` it is
-                       auto-detected.
-    '''
-    if not opf_reader.spine:
-        raise ValueError('OPF does not have a spine')
-    flat = []
-    for path in opf_reader.spine.items():
-        path = os.path.abspath(path)
-        if path not in flat:
-            flat.append(os.path.abspath(path))
-    for item in opf_reader.manifest:
-        if 'html' in item.mime_type:
-            path = os.path.abspath(item.path)
-            if path not in flat:
-                flat.append(path)
-    for i, path in enumerate(flat):
-        if not os.path.exists(path):
-            path = path.replace('&', '%26')
-            if os.path.exists(path):
-                flat[i] = path
-                for item in opf_reader.itermanifest():
-                    item.set('href', item.get('href').replace('&', '%26'))
-    ans = []
-    for path in flat:
-        if os.path.exists(path):
-            ans.append(HTMLFile(path, 0, encoding, verbose))
-        else:
-            print 'WARNING: OPF spine item %s does not exist'%path
-    ans = [f for f in ans if not f.is_binary]
-    return ans
-
-def search_for_opf(dir):
-    for f in os.listdir(dir):
-        if f.lower().endswith('.opf'):
-            return OPF(open(os.path.join(dir, f), 'rb'), dir)
-
 def get_filelist(htmlfile, dir, opts, log):
     '''
     Build list of files referenced by html file or try to detect and use an
     OPF file instead.
     '''
-    print 'Building file list...'
+    log.info('Building file list...')
-    opf = search_for_opf(dir)
+    filelist = traverse(htmlfile, max_levels=int(opts.max_levels),
-    filelist = None
+                        verbose=opts.verbose,
-    if opf is not None:
+                        encoding=opts.input_encoding)\
-        try:
+                [0 if opts.breadth_first else 1]
-            filelist = opf_traverse(opf, verbose=opts.verbose,
-                    encoding=opts.input_encoding)
-        except:
-            pass
-    if not filelist:
-        filelist = traverse(htmlfile, max_levels=int(opts.max_levels),
-                            verbose=opts.verbose,
-                            encoding=opts.input_encoding)\
-                    [0 if opts.breadth_first else 1]
     if opts.verbose:
         log.debug('\tFound files...')
         for f in filelist:
             log.debug('\t\t', f)
-    return opf, filelist
+    return filelist
 
 
 class HTMLInput(InputFormatPlugin):
@@ -309,34 +256,32 @@ class HTMLInput(InputFormatPlugin):
 
     def convert(self, stream, opts, file_ext, log,
                 accelerators):
+        from calibre.ebooks.metadata.meta import get_metadata
+
         basedir = os.getcwd()
+
         if hasattr(stream, 'name'):
             basedir = os.path.dirname(stream.name)
         if file_ext == 'opf':
-            opf = OPF(stream, basedir)
+            opfpath = stream.name
-            filelist = opf_traverse(opf, verbose=opts.verbose,
-                    encoding=opts.input_encoding)
-            mi = MetaInformation(opf)
         else:
-            opf, filelist = get_filelist(stream.name, basedir, opts, log)
+            filelist = get_filelist(stream.name, basedir, opts, log)
-            mi = MetaInformation(opf)
+            mi = get_metadata(stream, 'html')
-            mi.smart_update(get_metadata(stream, 'html'))
+            mi = OPFCreator(os.getcwdu(), mi)
+            mi.guide = None
+            entries = [(f.path, 'application/xhtml+xml') for f in filelist]
+            mi.create_manifest(entries)
+            mi.create_spine([f.path for f in filelist])
 
-        mi = OPFCreator(os.getcwdu(), mi)
+            mi.render(open('metadata.opf', 'wb'))
-        mi.guide = None
+            opfpath = os.path.abspath('metadata.opf')
-        entries = [(f.path, 'application/xhtml+xml') for f in filelist]
-        mi.create_manifest(entries)
-        mi.create_spine([f.path for f in filelist])
-
-        tocbuf = cStringIO.StringIO()
-        mi.render(open('metadata.opf', 'wb'), tocbuf, 'toc.ncx')
-        toc = tocbuf.getvalue()
-        if toc:
-            open('toc.ncx', 'wb').write(toc)
 
         from calibre.ebooks.conversion.plumber import create_oebbook
-        return create_oebbook(log, os.path.abspath('metadata.opf'))
+        oeb = create_oebbook(log, opfpath)
-
+
-
+        from calibre.ebooks.oeb.transforms.package import Package
+        Package(os.getcwdu())(oeb, opts)
+
+        return oeb
 
 

src/calibre/ebooks/metadata/__init__.py

@@ -11,9 +11,7 @@ from urllib import unquote, quote
 from urlparse import urlparse
 
 
-from calibre.constants import __version__ as VERSION
 from calibre import relpath
-from calibre.utils.config import OptionParser
 
 def string_to_authors(raw):
     raw = raw.replace('&&', u'\uffff')
@@ -189,11 +187,11 @@ class MetaInformation(object):
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'tags', 'cover_data', 'application_id', 'guide',
                      'manifest', 'spine', 'toc', 'cover', 'language',
-                     'book_producer', 'timestamp'):
+                     'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc'):
             if hasattr(mi, attr):
                 setattr(ans, attr, getattr(mi, attr))
 
-    def __init__(self, title, authors=[_('Unknown')]):
+    def __init__(self, title, authors=(_('Unknown'),)):
         '''
         @param title: title or ``_('Unknown')`` or a MetaInformation object
         @param authors: List of strings or []
@@ -204,9 +202,9 @@ class MetaInformation(object):
             title = mi.title
             authors = mi.authors
         self.title = title
-        self.author = authors # Needed for backward compatibility
+        self.author = list(authors) if authors else []# Needed for backward compatibility
         #: List of strings or []
-        self.authors = authors
+        self.authors = list(authors) if authors else []
         self.tags = getattr(mi, 'tags', [])
         #: mi.cover_data = (ext, data)
         self.cover_data   = getattr(mi, 'cover_data', (None, None))
@@ -214,7 +212,7 @@ class MetaInformation(object):
         for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher',
                   'series', 'series_index', 'rating', 'isbn', 'language',
                   'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
-                  'book_producer', 'timestamp'
+                  'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc'
                   ):
             setattr(self, x, getattr(mi, x, None))
 
@@ -229,15 +227,15 @@ class MetaInformation(object):
         if mi.authors and mi.authors[0] != _('Unknown'):
             self.authors = mi.authors
 
-            
         for attr in ('author_sort', 'title_sort', 'category',
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'application_id', 'manifest', 'spine', 'toc',
                      'cover', 'language', 'guide', 'book_producer',
-                     'timestamp'):
+                     'timestamp', 'lccn', 'lcc', 'ddc'):
-            val = getattr(mi, attr, None)
+            if hasattr(mi, attr):
-            if val is not None:
+                val = getattr(mi, attr)
-                setattr(self, attr, val)
+                if val is not None:
+                    setattr(self, attr, val)
 
         if mi.tags:
             self.tags += mi.tags
@@ -245,7 +243,7 @@ class MetaInformation(object):
 
         if getattr(mi, 'cover_data', None) and mi.cover_data[0] is not None:
             self.cover_data = mi.cover_data
-            
+
         my_comments = getattr(self, 'comments', '')
         other_comments = getattr(mi, 'comments', '')
         if not my_comments:
@@ -254,7 +252,7 @@ class MetaInformation(object):
             other_comments = ''
         if len(other_comments.strip()) > len(my_comments.strip()):
             self.comments = other_comments
-            
+
     def format_series_index(self):
         try:
             x = float(self.series_index)
@@ -293,6 +291,13 @@ class MetaInformation(object):
             fmt('Rating', self.rating)
         if self.timestamp is not None:
             fmt('Timestamp', self.timestamp.isoformat(' '))
+        if self.lccn:
+            fmt('LCCN', unicode(self.lccn))
+        if self.lcc:
+            fmt('LCC', unicode(self.lcc))
+        if self.ddc:
+            fmt('DDC', unicode(self.ddc))
+
         return u'\n'.join(ans)
 
     def to_html(self):
@@ -302,6 +307,12 @@ class MetaInformation(object):
         ans += [(_('Producer'), unicode(self.book_producer))]
         ans += [(_('Comments'), unicode(self.comments))]
         ans += [('ISBN', unicode(self.isbn))]
+        if self.lccn:
+            ans += [('LCCN', unicode(self.lccn))]
+        if self.lcc:
+            ans += [('LCC', unicode(self.lcc))]
+        if self.ddc:
+            ans += [('DDC', unicode(self.ddc))]
         ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
         if self.series:
             ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]

src/calibre/ebooks/mobi/reader.py

@@ -59,8 +59,9 @@ class EXTHHeader(object):
             elif id == 502:
                 # last update time
                 pass
-            elif id == 503 and (not title or title == _('Unknown')):
+            elif id == 503: # Long title
-                title = content
+                if not title or title == _('Unknown'):
+                    title = content
             #else:
             #    print 'unknown record', id, repr(content)
         if title:
@@ -87,6 +88,8 @@ class EXTHHeader(object):
                         content, '%Y-%m-%d',).date()
             except:
                 pass
+        elif id == 108:
+            pass # Producer
         #else:
         #    print 'unhandled metadata record', id, repr(content)
 
@@ -522,7 +525,8 @@ class MobiReader(object):
         else:
             raise MobiError('Unknown compression algorithm: %s'%repr(self.book_header.compression_type))
         if self.book_header.ancient and '<html' not in self.mobi_html[:300].lower():
-            self.mobi_html = self.mobi_html.replace('\r ', '\n\n').replace('\0', '')
+            self.mobi_html = self.mobi_html.replace('\r ', '\n\n ')
+        self.mobi_html = self.mobi_html.replace('\0', '')
         return processed_records
 
 

src/calibre/ebooks/oeb/reader.py

@@ -573,7 +573,7 @@ class OEBReader(object):
         item = self._find_ncx(opf)
         self._toc_from_opf(opf, item)
         self._pages_from_opf(opf, item)
-        self._ensure_cover_image()
+        #self._ensure_cover_image()
 
 
 def main(argv=sys.argv):

src/calibre/ebooks/oeb/transforms/package.py

@@ -6,13 +6,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os
+import os, re
 from urllib import unquote as urlunquote
 from functools import partial
 
 from lxml import etree
 import cssutils
 
+from calibre import sanitize_file_name
 from calibre.constants import islinux
 from calibre.ebooks.oeb.base import OEB_DOCS, urlnormalize, urldefrag, \
                                     rewrite_links
@@ -36,15 +37,21 @@ class Package(object):
         self.new_base_path = os.path.abspath(base)
 
     def rewrite_links_in(self, item):
-        base = os.path.join(self.new_base_path, *item.href.split('/'))
+        old_href = item.old_href.split('#')[0]
+        new_href = item.href.split('#')[0]
+        base = os.path.join(self.old_base_path, *old_href.split('/'))
         base = os.path.dirname(base)
+        self.log.debug('\tRewriting links in', base+'/'+
+                item.href.rpartition('/')[-1])
+        new_base = os.path.join(self.new_base_path, *new_href.split('/'))
+        new_base = os.path.dirname(new_base)
 
         if etree.iselement(item.data):
-            self.rewrite_links_in_xml(item.data, base)
+            self.rewrite_links_in_xml(item.data, base, new_base)
         elif hasattr(item.data, 'cssText'):
-            self.rewrite_links_in_css(item.data, base)
+            self.rewrite_links_in_css(item.data, base, new_base)
 
-    def link_replacer(self, link_, base=''):
+    def link_replacer(self, link_, base='', new_base=''):
         link = urlnormalize(link_)
         link, frag = urldefrag(link)
         link = urlunquote(link).replace('/', os.sep)
@@ -55,20 +62,33 @@ class Package(object):
             link = link.lower()
         if link not in self.map:
             return link_
-        nlink = os.path.relpath(self.map[link], base)
+        nlink = os.path.relpath(self.map[link], new_base)
         if frag:
-            nlink = '#'.join(nlink, frag)
+            nlink = '#'.join((nlink, frag))
         return nlink.replace(os.sep, '/')
 
-    def rewrite_links_in_css(self, sheet, base):
+    def rewrite_links_in_css(self, sheet, base, new_base):
-        repl = partial(self.link_replacer, base=base)
+        repl = partial(self.link_replacer, base=base, new_base=new_base)
         cssutils.replaceUrls(sheet, repl)
 
-    def rewrite_links_in_xml(self, root, base):
+    def rewrite_links_in_xml(self, root, base, new_base):
-        repl = partial(self.link_replacer, base=base)
+        repl = partial(self.link_replacer, base=base, new_base=new_base)
         rewrite_links(root, repl)
 
-    def move_manifest_item(self, item):
+    def uniqify_name(self, new_href, hrefs):
+        c = 0
+        while new_href in hrefs:
+            c += 1
+            parts = new_href.split('/')
+            name, ext = os.path.splitext(parts[-1])
+            name = re.sub(r'_\d+$', '', name)
+            name += '_%d'%c
+            parts[-1] = name + ext
+            new_href = '/'.join(parts)
+        return new_href
+
+
+    def move_manifest_item(self, item, hrefs):
         item.data # Make sure the data has been loaded and cached
         old_abspath = os.path.join(self.old_base_path,
                 *(urldefrag(item.href)[0].split('/')))
@@ -79,11 +99,17 @@ class Package(object):
             new_href = 'content/'
         elif item.href.lower().endswith('.ncx'):
             new_href = ''
-        new_href += bname
+        new_href += sanitize_file_name(bname)
+
+        if new_href in hrefs:
+            new_href = self.uniqify_name(new_href, hrefs)
+        hrefs.add(new_href)
 
         new_abspath = os.path.join(self.new_base_path, *new_href.split('/'))
         new_abspath = os.path.abspath(new_abspath)
+        item.old_href = self.oeb.manifest.hrefs.pop(item.href).href
         item.href   = new_href
+        self.oeb.manifest.hrefs[item.href] = item
         if not islinux:
             old_abspath, new_abspath = old_abspath.lower(), new_abspath.lower()
         if old_abspath != new_abspath:
@@ -91,25 +117,33 @@ class Package(object):
 
     def rewrite_links_in_toc(self, toc):
         if toc.href:
-            toc.href = self.link_replacer(toc.href, base=self.new_base_path)
+            toc.href = self.link_replacer(toc.href, base=self.old_base_path,
+                    new_base=self.new_base_path)
 
         for x in toc:
             self.rewrite_links_in_toc(x)
 
     def __call__(self, oeb, context):
         self.map = {}
-        self.log = self.oeb.log
+        self.log = oeb.log
+        self.oeb = oeb
         self.old_base_path = os.path.abspath(oeb.container.rootdir)
 
+        hrefs = set([])
         for item in self.oeb.manifest:
-            self.move_manifest_item(item)
+            self.move_manifest_item(item, hrefs)
 
+        self.log.debug('Rewriting links in OEB documents...')
         for item in self.oeb.manifest:
             self.rewrite_links_in(item)
 
         if getattr(oeb.toc, 'nodes', False):
+            self.log.debug('Rewriting links in TOC...')
             self.rewrite_links_in_toc(oeb.toc)
 
         if hasattr(oeb, 'guide'):
+            self.log.debug('Rewriting links in guide...')
             for ref in oeb.guide.values():
-                ref.href = self.link_replacer(ref.href, base=self.new_base_path)
+                ref.href = self.link_replacer(ref.href,
+                        base=self.old_base_path,
+                        new_base=self.new_base_path)

src/calibre/ebooks/oeb/writer.py

@@ -48,7 +48,8 @@ class OEBWriter(object):
                    pretty_print=pretty_print)
 
     def __call__(self, oeb, path):
-        """Read the book in the :class:`OEBBook` object :param:`oeb` to a file
+        """
+        Read the book in the :class:`OEBBook` object :param:`oeb` to a file
         at :param:`path`.
         """
         version = int(self.version[0])

src/calibre/gui2/__init__.py

@@ -466,5 +466,3 @@ class Application(QApplication):
                 self.translator.loadFromData(data)
                 self.installTranslator(self.translator)
 
-
-

src/calibre/gui2/dialogs/config.py

@@ -199,7 +199,7 @@ class EmailAccounts(QAbstractTableModel):
             return (account, self.accounts[account])
         if role == Qt.ToolTipRole:
             return self.tooltips[col]
-        if role == Qt.DisplayRole:
+        if role in [Qt.DisplayRole, Qt.EditRole]:
             if col == 0:
                 return QVariant(account)
             if col ==  1:
@@ -397,6 +397,9 @@ class ConfigDialog(QDialog, Ui_Dialog):
         self.separate_cover_flow.setChecked(config['separate_cover_flow'])
         self.setup_email_page()
         self.category_view.setCurrentIndex(self.category_view.model().index(0))
+        self.delete_news.setEnabled(bool(self.sync_news.isChecked()))
+        self.connect(self.sync_news, SIGNAL('toggled(bool)'),
+                self.delete_news.setEnabled)
 
     def setup_email_page(self):
         opts = smtp_prefs().parse()

src/calibre/gui2/dialogs/config.ui

@@ -371,7 +371,7 @@
          <item>
           <widget class="QCheckBox" name="delete_news">
            <property name="text">
-            <string>&amp;Delete news from library when it is sent to reader</string>
+            <string>&amp;Delete news from library when it is automatically sent to reader</string>
            </property>
           </widget>
          </item>

src/calibre/gui2/dialogs/metadata_single.ui

@@ -324,7 +324,7 @@
                     <string>Book </string>
                    </property>
                    <property name="minimum">
-                    <number>1</number>
+                    <number>0</number>
                    </property>
                    <property name="maximum">
                     <number>10000</number>

src/calibre/gui2/dialogs/scheduler.py

@@ -25,7 +25,7 @@ from calibre.gui2.dialogs.user_profiles import UserProfiles
 config = DynamicConfig('scheduler')
 
 class Recipe(object):
-    
+
     def __init__(self, id=None, recipe_class=None, builtin=True):
         self.id                 = id
         self.title              = getattr(recipe_class, 'title', None)
@@ -39,14 +39,14 @@ class Recipe(object):
         if self.author == _('Unknown') and not builtin:
             self.author = _('You')
         self.needs_subscription = getattr(recipe_class, 'needs_subscription', False)
-        
+
     def pickle(self):
         return self.__dict__.copy()
-    
+
     def unpickle(self, dict):
         self.__dict__.update(dict)
         return self
-        
+
     def __cmp__(self, other):
         if self.id == getattr(other, 'id', None):
             return 0
@@ -59,38 +59,39 @@ class Recipe(object):
         if not self.builtin and getattr(other, 'builtin', True):
             return -1
         return english_sort(self.title, getattr(other, 'title', ''))
-    
+
     def __hash__(self):
         return hash(self.id)
-    
+
     def __eq__(self, other):
         return self.id == getattr(other, 'id', None)
-    
+
     def __repr__(self):
         schedule = self.schedule
         if schedule and schedule > 1e5:
             schedule = decode_schedule(schedule)
         return u'%s|%s|%s|%s'%(self.id, self.title, self.last_downloaded.ctime(), schedule)
-    
+
 builtin_recipes = [Recipe(m, r, True) for r, m in zip(recipes, recipe_modules)]
 
 def save_recipes(recipes):
     config['scheduled_recipes'] = [r.pickle() for r in recipes]
-    
+
 def load_recipes():
     config.refresh()
     recipes = []
     for r in config.get('scheduled_recipes', []):
         r = Recipe().unpickle(r)
-        if r.builtin and not str(r.id).startswith('recipe_'):
+        if r.builtin and \
+            (not str(r.id).startswith('recipe_') or not str(r.id) in recipe_modules):
             continue
         recipes.append(r)
     return recipes
 
 class RecipeModel(QAbstractItemModel, SearchQueryParser):
-    
+
     LOCATIONS = ['all']
-    
+
     def __init__(self, db, *args):
         QAbstractItemModel.__init__(self, *args)
         SearchQueryParser.__init__(self)
@@ -104,18 +105,18 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
         self.bold_font = QFont()
         self.bold_font.setBold(True)
         self.bold_font = QVariant(self.bold_font)
-        
+
-    
+
     def refresh(self):
         sr = load_recipes()
         for recipe in self.recipes:
             if recipe in sr:
                 recipe.schedule = sr[sr.index(recipe)].schedule
                 recipe.last_downloaded = sr[sr.index(recipe)].last_downloaded
-        
+
         self.recipes.sort()
         self.num_of_recipes = len(self.recipes)
-        
+
         self.category_map = {}
         for r in self.recipes:
             category = getattr(r, 'language', _('Unknown'))
@@ -126,12 +127,12 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
             if category not in self.category_map.keys():
                 self.category_map[category] = []
             self.category_map[category].append(r)
-            
+
         self.categories = sorted(self.category_map.keys(), cmp=self.sort_categories)
         self._map = dict(self.category_map)
-        
+
     def sort_categories(self, x, y):
-        
+
         def decorate(x):
             if x == _('Scheduled'):
                 x = '0' + x
@@ -140,13 +141,13 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
             else:
                 x = '2' + x
             return x
-        
+
         return cmp(decorate(x), decorate(y))
-                
+
-    
+
     def universal_set(self):
         return set(self.recipes)
-    
+
     def get_matches(self, location, query):
         query = query.strip().lower()
         if not query:
@@ -154,9 +155,9 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
         results = set([])
         for recipe in self.recipes:
             if query in recipe.title.lower() or query in recipe.description.lower():
-                results.add(recipe) 
+                results.add(recipe)
         return results
-    
+
     def search(self, query):
         try:
             results = self.parse(unicode(query))
@@ -170,24 +171,24 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
                     if recipe in results:
                         self._map[category].append(recipe)
         self.reset()
-    
+
     def resort(self):
         self.recipes.sort()
         self.reset()
-    
+
     def index(self, row, column, parent):
         return self.createIndex(row, column, parent.row() if parent.isValid() else -1)
-    
+
     def parent(self, index):
         if index.internalId() == -1:
             return QModelIndex()
         return self.createIndex(index.internalId(), 0, -1)
-    
+
     def columnCount(self, parent):
         if not parent.isValid() or not parent.parent().isValid():
             return 1
         return 0
-    
+
     def rowCount(self, parent):
         if not parent.isValid():
             return len(self.categories)
@@ -195,7 +196,7 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
             category = self.categories[parent.row()]
             return len(self._map[category])
         return 0
-        
+
     def data(self, index, role):
         if index.parent().isValid():
             category = self.categories[index.parent().row()]
@@ -206,7 +207,7 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
                 return recipe
             elif role == Qt.DecorationRole:
                 icon = self.default_icon
-                icon_path = (':/images/news/%s.png'%recipe.id).replace('recipe_', '') 
+                icon_path = (':/images/news/%s.png'%recipe.id).replace('recipe_', '')
                 if not recipe.builtin:
                     icon = self.custom_icon
                 elif QFile().exists(icon_path):
@@ -222,18 +223,18 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
             elif role == Qt.ForegroundRole and category == _('Scheduled'):
                 return QVariant(QColor(0, 255, 0))
         return NONE
-    
+
     def update_recipe_schedule(self, recipe):
         for srecipe in self.recipes:
             if srecipe == recipe:
                 srecipe.schedule = recipe.schedule
-            
+
 
 class Search(QLineEdit):
-    
+
     HELP_TEXT = _('Search')
     INTERVAL = 500 #: Time to wait before emitting search signal
-    
+
     def __init__(self, *args):
         QLineEdit.__init__(self, *args)
         self.default_palette = QApplication.palette(self)
@@ -244,20 +245,20 @@ class Search(QLineEdit):
         self.clear_to_help_mode()
         self.timer = None
         self.connect(self, SIGNAL('textEdited(QString)'), self.text_edited_slot)
-            
+
     def focusInEvent(self, ev):
         self.setPalette(QApplication.palette(self))
         if self.in_help_mode():
             self.setText('')
         return QLineEdit.focusInEvent(self, ev)
-    
+
     def in_help_mode(self):
         return unicode(self.text()) == self.HELP_TEXT
-    
+
     def clear_to_help_mode(self):
         self.setPalette(self.gray)
         self.setText(self.HELP_TEXT)
-        
+
     def text_edited_slot(self, text):
         text = unicode(text)
         self.timer = self.startTimer(self.INTERVAL)
@@ -281,7 +282,7 @@ def decode_schedule(num):
     return day-1, hour-1, minute-1
 
 class SchedulerDialog(QDialog, Ui_Dialog):
-    
+
     def __init__(self, db, *args):
         QDialog.__init__(self, *args)
         self.setupUi(self)
@@ -308,25 +309,25 @@ class SchedulerDialog(QDialog, Ui_Dialog):
         self.search.setFocus(Qt.OtherFocusReason)
         self.old_news.setValue(gconf['oldest_news'])
         self.rnumber.setText(_('%d recipes')%self._model.num_of_recipes)
-        for day in (_('day'), _('Monday'), _('Tuesday'), _('Wednesday'), 
+        for day in (_('day'), _('Monday'), _('Tuesday'), _('Wednesday'),
                     _('Thursday'), _('Friday'), _('Saturday'), _('Sunday')):
             self.day.addItem(day)
-    
+
     def currentChanged(self, current, previous):
         if current.parent().isValid():
             self.show_recipe(current)
-        
+
     def download_now(self):
         recipe = self._model.data(self.recipes.currentIndex(), Qt.UserRole)
         self.emit(SIGNAL('download_now(PyQt_PyObject)'), recipe)
-        
+
     def set_account_info(self, *args):
         username, password = map(unicode, (self.username.text(), self.password.text()))
         username, password = username.strip(), password.strip()
         recipe = self._model.data(self.recipes.currentIndex(), Qt.UserRole)
         key = 'recipe_account_info_%s'%recipe.id
         config[key] = (username, password) if username and password else None
-        
+
     def do_schedule(self, *args):
         if not getattr(self, 'allow_scheduling', False):
             return
@@ -342,7 +343,7 @@ class SchedulerDialog(QDialog, Ui_Dialog):
                 recipe.last_downloaded = datetime.fromordinal(1)
                 recipes.append(recipe)
             if recipe.needs_subscription and not config['recipe_account_info_%s'%recipe.id]:
-                error_dialog(self, _('Must set account information'), 
+                error_dialog(self, _('Must set account information'),
                              _('This recipe requires a username and password')).exec_()
                 self.schedule.setCheckState(Qt.Unchecked)
                 return
@@ -364,7 +365,7 @@ class SchedulerDialog(QDialog, Ui_Dialog):
         save_recipes(recipes)
         self._model.update_recipe_schedule(recipe)
         self.emit(SIGNAL('new_schedule(PyQt_PyObject)'), recipes)
-                
+
     def show_recipe(self, index):
         recipe = self._model.data(index, Qt.UserRole)
         self.current_recipe = recipe
@@ -395,7 +396,7 @@ class SchedulerDialog(QDialog, Ui_Dialog):
             self.interval_button.setChecked(False)
             self.interval.setEnabled(False)
         self.schedule.setChecked(recipe.schedule is not None)
-        self.allow_scheduling = True    
+        self.allow_scheduling = True
         self.detail_box.setVisible(True)
         self.account.setVisible(recipe.needs_subscription)
         self.interval.setEnabled(self.schedule.checkState() == Qt.Checked)
@@ -417,11 +418,11 @@ class SchedulerDialog(QDialog, Ui_Dialog):
             self.last_downloaded.setText(_('Last downloaded')+': '+tm)
         else:
             self.last_downloaded.setText(_('Last downloaded: never'))
-            
+
 class Scheduler(QObject):
-    
+
     INTERVAL = 1 # minutes
-    
+
     def __init__(self, main):
         self.main = main
         self.verbose = main.verbose
@@ -439,7 +440,7 @@ class Scheduler(QObject):
         self.oldest = gconf['oldest_news']
         self.oldest_timer.start(int(60 * 60000))
         self.oldest_check()
-        
+
         self.news_menu = QMenu()
         self.news_icon = QIcon(':/images/news.svg')
         self.scheduler_action = QAction(QIcon(':/images/scheduler.svg'), _('Schedule news download'), self)
@@ -448,27 +449,27 @@ class Scheduler(QObject):
         self.cac = QAction(QIcon(':/images/user_profile.svg'), _('Add a custom news source'), self)
         self.connect(self.cac, SIGNAL('triggered(bool)'), self.customize_feeds)
         self.news_menu.addAction(self.cac)
-        
+
     def oldest_check(self):
         if self.oldest > 0:
             delta = timedelta(days=self.oldest)
             ids = self.main.library_view.model().db.tags_older_than(_('News'), delta)
             if ids:
                 self.main.library_view.model().delete_books_by_id(ids)
-    
+
     def customize_feeds(self, *args):
         main = self.main
         d = UserProfiles(main, main.library_view.model().db.get_feeds())
         d.exec_()
         feeds = tuple(d.profiles())
         main.library_view.model().db.set_feeds(feeds)
-            
+
-    
+
     def debug(self, *args):
         if self.verbose:
             sys.stdout.write(' '.join(map(unicode, args))+'\n')
             sys.stdout.flush()
-    
+
     def check(self):
         if not self.lock.tryLock():
             return
@@ -494,15 +495,15 @@ class Scheduler(QObject):
                     matches = day_matches and (hour*60+minute) < tnow
                     if matches and recipe.last_downloaded.toordinal() < date.today().toordinal():
                         needs_downloading.add(recipe)
-                
+
             self.debug('Needs downloading:', needs_downloading)
-        
+
             needs_downloading = [r for r in needs_downloading if r not in self.queue]
             for recipe in needs_downloading:
                 self.do_download(recipe)
         finally:
             self.lock.unlock()
-            
+
     def do_download(self, recipe):
         try:
             id = int(recipe.id)
@@ -538,7 +539,7 @@ class Scheduler(QObject):
         finally:
             self.lock.unlock()
         self.debug('Downloaded:', recipe)
-            
+
     def download(self, recipe):
         self.lock.lock()
         try:
@@ -548,10 +549,10 @@ class Scheduler(QObject):
                 self.do_download(recipe)
         finally:
             self.lock.unlock()
-    
+
     def refresh_schedule(self, recipes):
         self.recipes = recipes
-    
+
     def show_dialog(self, *args):
         self.lock.lock()
         try:

src/calibre/gui2/lrf_renderer/config.ui

@@ -1,72 +1,73 @@
-<ui version="4.0" >
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
  <class>ViewerConfig</class>
- <widget class="QDialog" name="ViewerConfig" >
+ <widget class="QDialog" name="ViewerConfig">
-  <property name="geometry" >
+  <property name="geometry">
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>281</width>
+    <width>373</width>
-    <height>214</height>
+    <height>264</height>
    </rect>
   </property>
-  <property name="windowTitle" >
+  <property name="windowTitle">
    <string>Configure Viewer</string>
   </property>
-  <property name="windowIcon" >
+  <property name="windowIcon">
-   <iconset resource="../images.qrc" >
+   <iconset resource="../images.qrc">
     <normaloff>:/images/config.svg</normaloff>:/images/config.svg</iconset>
   </property>
-  <layout class="QGridLayout" >
+  <layout class="QGridLayout" name="gridLayout">
-   <item row="0" column="0" colspan="2" >
+   <item row="0" column="0">
-    <widget class="QCheckBox" name="white_background" >
+    <widget class="QCheckBox" name="white_background">
-     <property name="text" >
+     <property name="text">
       <string>Use white background</string>
      </property>
     </widget>
    </item>
-   <item row="1" column="0" >
+   <item row="1" column="0">
-    <widget class="QCheckBox" name="hyphenate" >
+    <widget class="QCheckBox" name="hyphenate">
-     <property name="text" >
+     <property name="text">
       <string>Hyphenate</string>
      </property>
-     <property name="checked" >
+     <property name="checked">
       <bool>true</bool>
      </property>
     </widget>
    </item>
-   <item row="3" column="1" >
+   <item row="2" column="0">
-    <widget class="QDialogButtonBox" name="buttonBox" >
+    <widget class="QLabel" name="label">
-     <property name="orientation" >
+     <property name="frameShape">
-      <enum>Qt::Horizontal</enum>
-     </property>
-     <property name="standardButtons" >
-      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
-     </property>
-    </widget>
-   </item>
-   <item row="2" column="0" colspan="2" >
-    <widget class="QLabel" name="label" >
-     <property name="frameShape" >
       <enum>QFrame::Box</enum>
      </property>
-     <property name="text" >
+     <property name="text">
       <string>&lt;b&gt;Changes will only take effect after a restart.&lt;/b&gt;</string>
      </property>
-     <property name="textFormat" >
+     <property name="textFormat">
       <enum>Qt::RichText</enum>
      </property>
-     <property name="alignment" >
+     <property name="alignment">
       <set>Qt::AlignCenter</set>
      </property>
-     <property name="wordWrap" >
+     <property name="wordWrap">
       <bool>true</bool>
      </property>
     </widget>
    </item>
+   <item row="3" column="0">
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <resources>
-  <include location="../images.qrc" />
+  <include location="../images.qrc"/>
  </resources>
  <connections>
   <connection>
@@ -75,11 +76,11 @@
    <receiver>ViewerConfig</receiver>
    <slot>accept()</slot>
    <hints>
-    <hint type="sourcelabel" >
+    <hint type="sourcelabel">
      <x>248</x>
      <y>254</y>
     </hint>
-    <hint type="destinationlabel" >
+    <hint type="destinationlabel">
      <x>157</x>
      <y>274</y>
     </hint>
@@ -91,11 +92,11 @@
    <receiver>ViewerConfig</receiver>
    <slot>reject()</slot>
    <hints>
-    <hint type="sourcelabel" >
+    <hint type="sourcelabel">
      <x>316</x>
      <y>260</y>
     </hint>
-    <hint type="destinationlabel" >
+    <hint type="destinationlabel">
      <x>286</x>
      <y>274</y>
     </hint>

src/calibre/gui2/main.py

@@ -1110,27 +1110,30 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             return
         self._view_file(job.result)
 
-    def _view_file(self, name):
+    def _launch_viewer(self, name=None, viewer='ebook-viewer', internal=True):
         self.setCursor(Qt.BusyCursor)
         try:
-            ext = os.path.splitext(name)[1].upper().replace('.', '')
+            if internal:
-            if ext in config['internally_viewed_formats']:
+                args = [viewer]
-                if ext == 'LRF':
+                if isosx and 'ebook' in viewer:
-                    args = ['lrfviewer', name]
+                    args.append('--raise-window')
-                    self.job_manager.server.run_free_job('lrfviewer',
+                if name is not None:
-                                                        kwdargs=dict(args=args))
+                    args.append(name)
-                else:
+                self.job_manager.server.run_free_job(viewer,
-                    args = ['ebook-viewer', name]
+                        kwdargs=dict(args=args))
-                    if isosx:
-                        args.append('--raise-window')
-                    self.job_manager.server.run_free_job('ebook-viewer',
-                                                        kwdargs=dict(args=args))
             else:
-                QDesktopServices.openUrl(QUrl('file:'+name))#launch(name)
+                QDesktopServices.openUrl(QUrl.fromLocalFile(name))#launch(name)
+
             time.sleep(5) # User feedback
         finally:
             self.unsetCursor()
 
+    def _view_file(self, name):
+        ext = os.path.splitext(name)[1].upper().replace('.', '')
+        viewer = 'lrfviewer' if ext == 'LRF' else 'ebook-viewer'
+        internal = ext in config['internally_viewed_formats']
+        self._launch_viewer(name, viewer, internal)
+
     def view_specific_format(self, triggered):
         rows = self.library_view.selectionModel().selectedRows()
         if not rows or len(rows) == 0:
@@ -1165,8 +1168,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         rows = self.current_view().selectionModel().selectedRows()
         if self.current_view() is self.library_view:
             if not rows or len(rows) == 0:
-                d = error_dialog(self, _('Cannot view'), _('No book selected'))
+                self._launch_viewer()
-                d.exec_()
                 return
 
             row = rows[0].row()

src/calibre/library/cli.py

@@ -15,6 +15,7 @@ from calibre import terminal_controller, preferred_encoding
 from calibre.utils.config import OptionParser, prefs
 try:
     from calibre.utils.single_qt_application import send_message
+    send_message
 except:
     send_message = None
 from calibre.ebooks.metadata.meta import get_metadata
@@ -55,7 +56,7 @@ XML_TEMPLATE = '''\
         </py:for>
         </formats>
     </record>
-</py:for> 
+</py:for>
 </calibredb>
 '''
 
@@ -114,7 +115,7 @@ def get_db(dbpath, options):
     dbpath = os.path.abspath(dbpath)
     return LibraryDatabase2(dbpath)
 
-def do_list(db, fields, sort_by, ascending, search_text, line_width, separator, 
+def do_list(db, fields, sort_by, ascending, search_text, line_width, separator,
             prefix, output_format, subtitle='Books in the calibre database'):
     if sort_by:
         db.sort(sort_by, ascending)
@@ -134,13 +135,13 @@ def do_list(db, fields, sort_by, ascending, search_text, line_width, separator,
         for i in data:
             for j, field in enumerate(fields):
                 widths[j] = max(widths[j], len(unicode(i[str(field)])))
-        
+
         screen_width = terminal_controller.COLS if line_width < 0 else line_width
         if not screen_width:
             screen_width = 80
         field_width = screen_width//len(fields)
         base_widths = map(lambda x: min(x+1, field_width), widths)
-    
+
         while sum(base_widths) < screen_width:
             adjusted = False
             for i in range(len(widths)):
@@ -150,14 +151,14 @@ def do_list(db, fields, sort_by, ascending, search_text, line_width, separator,
                     break
             if not adjusted:
                 break
-    
+
         widths = list(base_widths)
         titles = map(lambda x, y: '%-*s'%(x, y), widths, fields)
         print terminal_controller.GREEN + ''.join(titles)+terminal_controller.NORMAL
-    
+
         wrappers = map(lambda x: TextWrapper(x-1), widths)
         o = cStringIO.StringIO()
-    
+
         for record in data:
             text = [wrappers[i].wrap(unicode(record[field]).encode('utf-8')) for i, field in enumerate(fields)]
             lines = max(map(len, text))
@@ -178,9 +179,9 @@ def do_list(db, fields, sort_by, ascending, search_text, line_width, separator,
             if isinstance(x['fmt_epub'], unicode):
                 x['fmt_epub'] = x['fmt_epub'].encode('utf-8')
         template = MarkupTemplate(STANZA_TEMPLATE)
-        return template.generate(id="urn:calibre:main", data=data, subtitle=subtitle, 
+        return template.generate(id="urn:calibre:main", data=data, subtitle=subtitle,
                 sep=os.sep, quote=quote, updated=db.last_modified()).render('xml')
-            
+
 
 
 def command_list(args, dbpath):
@@ -199,7 +200,7 @@ List the books available in the calibre database.
                       help=_('Sort results in ascending order'))
     parser.add_option('-s', '--search', default=None,
                       help=_('Filter the results by the search query. For the format of the search query, please see the search related documentation in the User Manual. Default is to do no filtering.'))
-    parser.add_option('-w', '--line-width', default=-1, type=int, 
+    parser.add_option('-w', '--line-width', default=-1, type=int,
                       help=_('The maximum width of a single line in the output. Defaults to detecting screen size.'))
     parser.add_option('--separator', default=' ', help=_('The string used to separate fields. Default is a space.'))
     parser.add_option('--prefix', default=None, help=_('The prefix for all file paths. Default is the absolute path to the library folder.'))
@@ -264,14 +265,14 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
 
             formats.append(format)
             metadata.append(mi)
-        
+
         file_duplicates = []
         if files:
-            file_duplicates = db.add_books(files, formats, metadata, 
+            file_duplicates = db.add_books(files, formats, metadata,
                                            add_duplicates=add_duplicates)
             if file_duplicates:
                 file_duplicates = file_duplicates[0]
-                
+
 
         dir_dups = []
         for dir in dirs:

src/calibre/library/database2.py

@@ -31,6 +31,21 @@ from calibre.customize.ui import run_plugins_on_import
 from calibre import sanitize_file_name
 from calibre.ebooks import BOOK_EXTENSIONS
 
+if iswindows:
+    import calibre.utils.winshell as winshell
+
+def delete_file(path):
+    try:
+        winshell.delete_file(path, silent=True, no_confirm=True)
+    except:
+        os.remove(path)
+
+def delete_tree(path):
+    try:
+        winshell.delete_file(path, silent=True, no_confirm=True)
+    except:
+        shutil.rmtree(path)
+
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 
 FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'publisher':3, 'rating':4, 'timestamp':5,
@@ -499,7 +514,7 @@ class LibraryDatabase2(LibraryDatabase):
 
     def rmtree(self, path):
         if not self.normpath(self.library_path).startswith(self.normpath(path)):
-            shutil.rmtree(path)
+            delete_tree(path)
 
     def normpath(self, path):
         path = os.path.abspath(os.path.realpath(path))
@@ -745,7 +760,10 @@ class LibraryDatabase2(LibraryDatabase):
         path = os.path.join(self.library_path, self.path(id, index_is_id=True))
         self.data.remove(id)
         if os.path.exists(path):
-            self.rmtree(path)
+            if iswindows:
+                winshell.delete_file(path, no_confirm=True, silent=True)
+            else:
+                self.rmtree(path)
             parent = os.path.dirname(path)
             if len(os.listdir(parent)) == 0:
                 self.rmtree(parent)
@@ -764,7 +782,7 @@ class LibraryDatabase2(LibraryDatabase):
             ext = ('.' + format.lower()) if format else ''
             path = os.path.join(path, name+ext)
             try:
-                os.remove(path)
+                delete_file(path)
             except:
                 traceback.print_exc()
             self.conn.execute('DELETE FROM data WHERE book=? AND format=?', (id, format.upper()))

src/calibre/manual/faq.rst

@@ -105,7 +105,7 @@ Device Integration
 
 What devices does |app| support?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-At the moment |app| has full support for the SONY PRS 500/505/700, Cybook Gen 3, Amazon Kindle 1 and 2 as well as the iPhone. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk.
+At the moment |app| has full support for the SONY PRS 500/505/700, Cybook Gen 3, Amazon Kindle 1/2, Netronix EB600 and the iPhone. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk.
 
 I used |app| to transfer some books to my reader, and now the SONY software hangs every time I connect the reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/startup.py

@@ -24,7 +24,7 @@ from calibre.translations.msgfmt import make
 _run_once = False
 if not _run_once:
     _run_once = True
-    
+
     ################################################################################
     # Setup translations
 
@@ -32,7 +32,8 @@ if not _run_once:
         lang = prefs['language']
         if lang is not None:
             return lang
-        lang = locale.getdefaultlocale()[0]
+        lang = locale.getdefaultlocale(['LANGUAGE', 'LC_ALL', 'LC_CTYPE',
+                                       'LC_MESSAGES', 'LANG'])[0]
         if lang is None and os.environ.has_key('LANG'): # Needed for OS X
             try:
                 lang = os.environ['LANG']

src/calibre/trac/plugins/download.py

@@ -38,6 +38,7 @@ def get_linux_data(version='1.0.0'):
                         ('exherbo', 'Exherbo'),
                         ('foresight', 'Foresight 2.1'),
                         ('ubuntu', 'Ubuntu Jaunty Jackalope'),
+                        ('linux_mint', 'Linux Mint Gloria'),
                         ]:
         data['supported'].append(CoolDistro(name, title,
                                         prefix='http://calibre.kovidgoyal.net'))

src/calibre/utils/winshell.py

@@ -0,0 +1,400 @@
+"""winshell - convenience functions to access Windows shell functionality
+
+Certain aspects of the Windows user interface are grouped by
+ Microsoft as Shell functions. These include the Desktop, shortcut
+ icons, special folders (such as My Documents) and a few other things.
+
+These are mostly available via the shell module of the win32all
+ extensions, but whenever I need to use them, I've forgotten the
+ various constants and so on.
+
+Several of the shell items have two variants: personal and common,
+ or User and All Users. These refer to systems with profiles in use:
+ anything from NT upwards, and 9x with Profiles turned on. Where
+ relevant, the Personal/User version refers to that owned by the
+ logged-on user and visible only to that user; the Common/All Users
+ version refers to that maintained by an Administrator and visible
+ to all users of the system.
+
+(c) Tim Golden <winshell@timgolden.me.uk> 25th November 2003
+Licensed under the (GPL-compatible) MIT License:
+http://www.opensource.org/licenses/mit-license.php
+
+9th Nov 2005  0.2  . License changed to MIT
+                   . Added functionality using SHFileOperation
+25th Nov 2003 0.1  . Initial release by Tim Golden
+"""
+
+__VERSION__ = "0.2"
+
+import os
+from win32com import storagecon
+from win32com.shell import shell, shellcon
+import pythoncom
+
+class x_winshell (Exception):
+  pass
+
+#
+# Although this can be done in one call, Win9x didn't
+#  support it, so I added this workaround.
+#
+def get_path (folder_id):
+  return shell.SHGetPathFromIDList (shell.SHGetSpecialFolderLocation (0, folder_id))
+
+def desktop (common=0):
+  "What folder is equivalent to the current desktop?"
+  return get_path ((shellcon.CSIDL_DESKTOP, shellcon.CSIDL_COMMON_DESKTOPDIRECTORY)[common])
+
+def common_desktop ():
+#
+# Only here because already used in code
+#
+  return desktop (common=1)
+
+def application_data (common=0):
+  "What folder holds application configuration files?"
+  return get_path ((shellcon.CSIDL_APPDATA, shellcon.CSIDL_COMMON_APPDATA)[common])
+
+def favourites (common=0):
+  "What folder holds the Explorer favourites shortcuts?"
+  return get_path ((shellcon.CSIDL_FAVORITES, shellcon.CSIDL_COMMON_FAVORITES)[common])
+bookmarks = favourites
+
+def start_menu (common=0):
+  "What folder holds the Start Menu shortcuts?"
+  return get_path ((shellcon.CSIDL_STARTMENU, shellcon.CSIDL_COMMON_STARTMENU)[common])
+
+def programs (common=0):
+  "What folder holds the Programs shortcuts (from the Start Menu)?"
+  return get_path ((shellcon.CSIDL_PROGRAMS, shellcon.CSIDL_COMMON_PROGRAMS)[common])
+
+def startup (common=0):
+  "What folder holds the Startup shortcuts (from the Start Menu)?"
+  return get_path ((shellcon.CSIDL_STARTUP, shellcon.CSIDL_COMMON_STARTUP)[common])
+
+def personal_folder ():
+  "What folder holds the My Documents files?"
+  return get_path (shellcon.CSIDL_PERSONAL)
+my_documents = personal_folder
+
+def recent ():
+  "What folder holds the Documents shortcuts (from the Start Menu)?"
+  return get_path (shellcon.CSIDL_RECENT)
+
+def sendto ():
+  "What folder holds the SendTo shortcuts (from the Context Menu)?"
+  return get_path (shellcon.CSIDL_SENDTO)
+
+#
+# Internally abstracted function to handle one
+#  of several shell-based file manipulation
+#  routines. Not all the possible parameters
+#  are covered which might be passed to the
+#  underlying SHFileOperation API call, but
+#  only those which seemed useful to me at
+#  the time.
+#
+def _file_operation (
+  operation,
+  source_path,
+  target_path=None,
+  allow_undo=True,
+  no_confirm=False,
+  rename_on_collision=True,
+  silent=False,
+  hWnd=None
+):
+  #
+  # At present the Python wrapper around SHFileOperation doesn't
+  # allow lists of files. Hopefully it will at some point, so
+  # take account of it here.
+  # If you pass this shell function a "/"-separated path with
+  # a wildcard, eg c:/temp/*.tmp, it gets confused. It's ok
+  # with a backslash, so convert here.
+  #
+  source_path = source_path or ""
+  if isinstance (source_path, basestring):
+    source_path = os.path.abspath (source_path)
+  else:
+    source_path = [os.path.abspath (i) for i in source_path]
+
+  target_path = target_path or ""
+  if isinstance (target_path, basestring):
+    target_path = os.path.abspath (target_path)
+  else:
+    target_path = [os.path.abspath (i) for i in target_path]
+
+  flags = 0
+  if allow_undo: flags |= shellcon.FOF_ALLOWUNDO
+  if no_confirm: flags |= shellcon.FOF_NOCONFIRMATION
+  if rename_on_collision: flags |= shellcon.FOF_RENAMEONCOLLISION
+  if silent: flags |= shellcon.FOF_SILENT
+
+  result, n_aborted = shell.SHFileOperation (
+    (hWnd or 0, operation, source_path, target_path, flags, None, None)
+  )
+  if result <> 0:
+    raise x_winshell, result
+  elif n_aborted:
+    raise x_winshell, "%d operations were aborted by the user" % n_aborted
+
+def copy_file (
+  source_path,
+  target_path,
+  allow_undo=True,
+  no_confirm=False,
+  rename_on_collision=True,
+  silent=False,
+  hWnd=None
+):
+  """Perform a shell-based file copy. Copying in
+   this way allows the possibility of undo, auto-renaming,
+   and showing the "flying file" animation during the copy.
+
+  The default options allow for undo, don't automatically
+   clobber on a name clash, automatically rename on collision
+   and display the animation.
+  """
+  _file_operation (
+    shellcon.FO_COPY,
+    source_path,
+    target_path,
+    allow_undo,
+    no_confirm,
+    rename_on_collision,
+    silent,
+    hWnd
+  )
+
+def move_file (
+  source_path,
+  target_path,
+  allow_undo=True,
+  no_confirm=False,
+  rename_on_collision=True,
+  silent=False,
+  hWnd=None
+):
+  """Perform a shell-based file move. Moving in
+   this way allows the possibility of undo, auto-renaming,
+   and showing the "flying file" animation during the copy.
+
+  The default options allow for undo, don't automatically
+   clobber on a name clash, automatically rename on collision
+   and display the animation.
+  """
+  _file_operation (
+    shellcon.FO_MOVE,
+    source_path,
+    target_path,
+    allow_undo,
+    no_confirm,
+    rename_on_collision,
+    silent,
+    hWnd
+  )
+
+def rename_file (
+  source_path,
+  target_path,
+  allow_undo=True,
+  no_confirm=False,
+  rename_on_collision=True,
+  silent=False,
+  hWnd=None
+):
+  """Perform a shell-based file rename. Renaming in
+   this way allows the possibility of undo, auto-renaming,
+   and showing the "flying file" animation during the copy.
+
+  The default options allow for undo, don't automatically
+   clobber on a name clash, automatically rename on collision
+   and display the animation.
+  """
+  _file_operation (
+    shellcon.FO_RENAME,
+    source_path,
+    target_path,
+    allow_undo,
+    no_confirm,
+    rename_on_collision,
+    silent,
+    hWnd
+  )
+
+def delete_file (
+  source_path,
+  allow_undo=True,
+  no_confirm=False,
+  rename_on_collision=True,
+  silent=False,
+  hWnd=None
+):
+  """Perform a shell-based file delete. Deleting in
+   this way uses the system recycle bin, allows the
+   possibility of undo, and showing the "flying file"
+   animation during the delete.
+
+  The default options allow for undo, don't automatically
+   clobber on a name clash, automatically rename on collision
+   and display the animation.
+  """
+  _file_operation (
+    shellcon.FO_DELETE,
+    source_path,
+    None,
+    allow_undo,
+    no_confirm,
+    rename_on_collision,
+    silent,
+    hWnd
+  )
+
+def CreateShortcut (Path, Target, Arguments = "", StartIn = "", Icon = ("",0), Description = ""):
+  """Create a Windows shortcut:
+
+  Path - As what file should the shortcut be created?
+  Target - What command should the desktop use?
+  Arguments - What arguments should be supplied to the command?
+  StartIn - What folder should the command start in?
+  Icon - (filename, index) What icon should be used for the shortcut?
+  Description - What description should the shortcut be given?
+
+  eg
+  CreateShortcut (
+    Path=os.path.join (desktop (), "PythonI.lnk"),
+    Target=r"c:\python\python.exe",
+    Icon=(r"c:\python\python.exe", 0),
+    Description="Python Interpreter"
+  )
+  """
+  sh = pythoncom.CoCreateInstance (
+    shell.CLSID_ShellLink,
+    None,
+    pythoncom.CLSCTX_INPROC_SERVER,
+    shell.IID_IShellLink
+  )
+
+  sh.SetPath (Target)
+  sh.SetDescription (Description)
+  sh.SetArguments (Arguments)
+  sh.SetWorkingDirectory (StartIn)
+  sh.SetIconLocation (Icon[0], Icon[1])
+
+  persist = sh.QueryInterface (pythoncom.IID_IPersistFile)
+  persist.Save (Path, 1)
+
+#
+# Constants for structured storage
+#
+# These come from ObjIdl.h
+FMTID_USER_DEFINED_PROPERTIES = "{F29F85E0-4FF9-1068-AB91-08002B27B3D9}"
+FMTID_CUSTOM_DEFINED_PROPERTIES = "{D5CDD505-2E9C-101B-9397-08002B2CF9AE}"
+
+PIDSI_TITLE               = 0x00000002
+PIDSI_SUBJECT             = 0x00000003
+PIDSI_AUTHOR              = 0x00000004
+PIDSI_CREATE_DTM          = 0x0000000c
+PIDSI_KEYWORDS            = 0x00000005
+PIDSI_COMMENTS            = 0x00000006
+PIDSI_TEMPLATE            = 0x00000007
+PIDSI_LASTAUTHOR          = 0x00000008
+PIDSI_REVNUMBER           = 0x00000009
+PIDSI_EDITTIME            = 0x0000000a
+PIDSI_LASTPRINTED         = 0x0000000b
+PIDSI_LASTSAVE_DTM        = 0x0000000d
+PIDSI_PAGECOUNT           = 0x0000000e
+PIDSI_WORDCOUNT           = 0x0000000f
+PIDSI_CHARCOUNT           = 0x00000010
+PIDSI_THUMBNAIL           = 0x00000011
+PIDSI_APPNAME             = 0x00000012
+PROPERTIES = (
+  PIDSI_TITLE,
+  PIDSI_SUBJECT,
+  PIDSI_AUTHOR,
+  PIDSI_CREATE_DTM,
+  PIDSI_KEYWORDS,
+  PIDSI_COMMENTS,
+  PIDSI_TEMPLATE,
+  PIDSI_LASTAUTHOR,
+  PIDSI_EDITTIME,
+  PIDSI_LASTPRINTED,
+  PIDSI_LASTSAVE_DTM,
+  PIDSI_PAGECOUNT,
+  PIDSI_WORDCOUNT,
+  PIDSI_CHARCOUNT,
+  PIDSI_APPNAME
+)
+
+#
+# This was taken from someone else's example,
+#  but I can't find where. If you know, please
+#  tell me so I can give due credit.
+#
+def structured_storage (filename):
+  """Pick out info from MS documents with embedded
+   structured storage (typically MS Word docs etc.)
+
+  Returns a dictionary of information found
+  """
+
+  if not pythoncom.StgIsStorageFile (filename):
+    return {}
+
+  flags = storagecon.STGM_READ | storagecon.STGM_SHARE_EXCLUSIVE
+  storage = pythoncom.StgOpenStorage (filename, None, flags)
+  try:
+    properties_storage = storage.QueryInterface (pythoncom.IID_IPropertySetStorage)
+  except pythoncom.com_error:
+    return {}
+
+  property_sheet = properties_storage.Open (FMTID_USER_DEFINED_PROPERTIES)
+  try:
+    data = property_sheet.ReadMultiple (PROPERTIES)
+  finally:
+    property_sheet = None
+
+  title, subject, author, created_on, keywords, comments, template_used, \
+   updated_by, edited_on, printed_on, saved_on, \
+   n_pages, n_words, n_characters, \
+   application = data
+
+  result = {}
+  if title: result['title'] = title
+  if subject: result['subject'] = subject
+  if author: result['author'] = author
+  if created_on: result['created_on'] = created_on
+  if keywords: result['keywords'] = keywords
+  if comments: result['comments'] = comments
+  if template_used: result['template_used'] = template_used
+  if updated_by: result['updated_by'] = updated_by
+  if edited_on: result['edited_on'] = edited_on
+  if printed_on: result['printed_on'] = printed_on
+  if saved_on: result['saved_on'] = saved_on
+  if n_pages: result['n_pages'] = n_pages
+  if n_words: result['n_words'] = n_words
+  if n_characters: result['n_characters'] = n_characters
+  if application: result['application'] = application
+  return result
+
+if __name__ == '__main__':
+  try:
+    print 'Desktop =>', desktop ()
+    print 'Common Desktop =>', desktop (1)
+    print 'Application Data =>', application_data ()
+    print 'Common Application Data =>', application_data (1)
+    print 'Bookmarks =>', bookmarks ()
+    print 'Common Bookmarks =>', bookmarks (1)
+    print 'Start Menu =>', start_menu ()
+    print 'Common Start Menu =>', start_menu (1)
+    print 'Programs =>', programs ()
+    print 'Common Programs =>', programs (1)
+    print 'Startup =>', startup ()
+    print 'Common Startup =>', startup (1)
+    print 'My Documents =>', my_documents ()
+    print 'Recent =>', recent ()
+    print 'SendTo =>', sendto ()
+  finally:
+    raw_input ("Press enter...")
+

src/calibre/web/feeds/main.py

@@ -1,11 +1,11 @@
-#!/usr/bin/env  python 
+#!/usr/bin/env  python
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
 CLI for downloading feeds.
 '''
 
-import sys, os, logging
+import sys, os
 from calibre.web.feeds.recipes import get_builtin_recipe, compile_recipe, titles
 from calibre.web.fetch.simple import option_parser as _option_parser
 from calibre.web.feeds.news import BasicNewsRecipe
@@ -14,13 +14,13 @@ from calibre.utils.config import Config, StringConfig
 def config(defaults=None):
     desc = _('Options to control the fetching of periodical content from the web.')
     c = Config('feeds2disk', desc) if defaults is None else StringConfig(defaults, desc)
-    
+
     web2disk = c.add_group('web2disk', _('Customize the download engine'))
-    web2disk('timeout', ['-t', '--timeout'], default=10.0, 
+    web2disk('timeout', ['-t', '--timeout'], default=10.0,
               help=_('Timeout in seconds to wait for a response from the server. Default: %default s'),)
-    web2disk('delay', ['--delay'], default=0, 
+    web2disk('delay', ['--delay'], default=0,
               help=_('Minimum interval in seconds between consecutive fetches. Default is %default s'))
-    web2disk('encoding', ['--encoding'], default=None, 
+    web2disk('encoding', ['--encoding'], default=None,
               help=_('The character encoding for the websites you are trying to download. The default is to try and guess the encoding.'))
     web2disk('match_regexps', ['--match-regexp'], default=[], action='append',
               help=_('Only links that match this regular expression will be followed. This option can be specified multiple times, in which case as long as a link matches any one regexp, it will be followed. By default all links are followed.'))
@@ -28,42 +28,42 @@ def config(defaults=None):
               help=_('Any link that matches this regular expression will be ignored. This option can be specified multiple times, in which case as long as any regexp matches a link, it will be ignored.By default, no links are ignored. If both --filter-regexp and --match-regexp are specified, then --filter-regexp is applied first.'))
     web2disk('no_stylesheets', ['--dont-download-stylesheets'], action='store_true', default=False,
               help=_('Do not download CSS stylesheets.'))
-    
+
     c.add_opt('feeds', ['--feeds'], default=None,
-                 help=_('''Specify a list of feeds to download. For example: 
+                 help=_('''Specify a list of feeds to download. For example:
 "['http://feeds.newsweek.com/newsweek/TopNews', 'http://feeds.newsweek.com/headlines/politics']"
 If you specify this option, any argument to %prog is ignored and a default recipe is used to download the feeds.'''))
     c.add_opt('verbose', ['-v', '--verbose'], default=0, action='count',
                  help=_('''Be more verbose while processing.'''))
     c.add_opt('title', ['--title'], default=None,
                  help=_('The title for this recipe. Used as the title for any ebooks created from the downloaded feeds.'))
-    c.add_opt('username', ['-u', '--username'], default=None, 
+    c.add_opt('username', ['-u', '--username'], default=None,
                  help=_('Username for sites that require a login to access content.'))
-    c.add_opt('password', ['-p', '--password'], default=None, 
+    c.add_opt('password', ['-p', '--password'], default=None,
                  help=_('Password for sites that require a login to access content.'))
-    c.add_opt('lrf', ['--lrf'], default=False, action='store_true', 
+    c.add_opt('lrf', ['--lrf'], default=False, action='store_true',
                  help='Optimize fetching for subsequent conversion to LRF.')
-    c.add_opt('epub', ['--epub'], default=False, action='store_true', 
+    c.add_opt('epub', ['--epub'], default=False, action='store_true',
                  help='Optimize fetching for subsequent conversion to EPUB.')
-    c.add_opt('mobi', ['--mobi'], default=False, action='store_true', 
+    c.add_opt('mobi', ['--mobi'], default=False, action='store_true',
                  help='Optimize fetching for subsequent conversion to MOBI.')
     c.add_opt('recursions', ['--recursions'], default=0,
                  help=_('Number of levels of links to follow on webpages that are linked to from feeds. Defaul %default'))
-    c.add_opt('output_dir', ['--output-dir'], default='.', 
+    c.add_opt('output_dir', ['--output-dir'], default='.',
                  help=_('The directory in which to store the downloaded feeds. Defaults to the current directory.'))
     c.add_opt('no_progress_bar', ['--no-progress-bar'], default=False, action='store_true',
                  help=_("Don't show the progress bar"))
     c.add_opt('debug', ['--debug'], action='store_true', default=False,
                  help=_('Very verbose output, useful for debugging.'))
-    c.add_opt('test', ['--test'], action='store_true', default=False, 
+    c.add_opt('test', ['--test'], action='store_true', default=False,
                  help=_('Useful for recipe development. Forces max_articles_per_feed to 2 and downloads at most 2 feeds.'))
-    
+
     return c
-    
+
 USAGE=_('''\
 %%prog [options] ARG
 
-%%prog parses an online source of articles, like an RSS or ATOM feed and 
+%%prog parses an online source of articles, like an RSS or ATOM feed and
 fetches the article contents organized in a nice hierarchy.
 
 ARG can be one of:
@@ -85,9 +85,9 @@ def option_parser(usage=USAGE):
     p.remove_option('--verbose')
     p.remove_option('--max-files')
     p.subsume('WEB2DISK OPTIONS', _('Options to control web2disk (used to fetch websites linked from feeds)'))
-    
+
     p.add_option('--feeds', default=None,
-                 help=_('''Specify a list of feeds to download. For example: 
+                 help=_('''Specify a list of feeds to download. For example:
 "['http://feeds.newsweek.com/newsweek/TopNews', 'http://feeds.newsweek.com/headlines/politics']"
 If you specify this option, any argument to %prog is ignored and a default recipe is used to download the feeds.'''))
     p.add_option('--verbose', default=False, action='store_true',
@@ -99,70 +99,62 @@ If you specify this option, any argument to %prog is ignored and a default recip
     p.add_option('--lrf', default=False, action='store_true', help='Optimize fetching for subsequent conversion to LRF.')
     p.add_option('--recursions', default=0, type='int',
                  help=_('Number of levels of links to follow on webpages that are linked to from feeds. Defaul %default'))
-    p.add_option('--output-dir', default=os.getcwd(), 
+    p.add_option('--output-dir', default=os.getcwd(),
                  help=_('The directory in which to store the downloaded feeds. Defaults to the current directory.'))
     p.add_option('--no-progress-bar', dest='no_progress_bar', default=False, action='store_true',
                  help=_('Dont show the progress bar'))
     p.add_option('--debug', action='store_true', default=False,
                  help=_('Very verbose output, useful for debugging.'))
-    p.add_option('--test', action='store_true', default=False, 
+    p.add_option('--test', action='store_true', default=False,
                  help=_('Useful for recipe development. Forces max_articles_per_feed to 2 and downloads at most 2 feeds.'))
-    
+
     return p
-    
+
 class RecipeError(Exception):
     pass
 
-def run_recipe(opts, recipe_arg, parser, notification=None, handler=None):
+def run_recipe(opts, recipe_arg, parser, notification=None):
     if notification is None:
         from calibre.utils.terminfo import TerminalController, ProgressBar
         term = TerminalController(sys.stdout)
         pb = ProgressBar(term, _('Fetching feeds...'), no_progress_bar=opts.no_progress_bar)
         notification = pb.update
-    
+
     recipe = None
     if opts.feeds is not None:
         recipe = BasicNewsRecipe
     else:
         try:
             if os.access(recipe_arg, os.R_OK):
-                recipe = compile_recipe(open(recipe_arg).read())                
+                recipe = compile_recipe(open(recipe_arg).read())
             else:
                 raise Exception('not file')
         except:
             recipe = get_builtin_recipe(recipe_arg)
             if recipe is None:
                 recipe = compile_recipe(recipe_arg)
-    
+
     if recipe is None:
         raise RecipeError(recipe_arg+ ' is an invalid recipe')
-        
+
-    
-    if handler is None:
-        from calibre import ColoredFormatter
-        handler = logging.StreamHandler(sys.stdout)
-        handler.setLevel(logging.DEBUG if opts.debug else logging.INFO if opts.verbose else logging.WARN)
-        handler.setFormatter(ColoredFormatter('%(levelname)s: %(message)s\n')) # The trailing newline is need because of the progress bar
-        logging.getLogger('feeds2disk').addHandler(handler)
-    
     recipe = recipe(opts, parser, notification)
-    
+
     if not os.path.exists(recipe.output_dir):
         os.makedirs(recipe.output_dir)
     recipe.download(for_lrf=True)
-    
+
     return recipe
 
-def main(args=sys.argv, notification=None, handler=None):
+def main(args=sys.argv, notification=None):
     p = option_parser()
     opts, args = p.parse_args(args=args[1:])
-    
+
     if len(args) != 1 and opts.feeds is None:
         p.print_help()
         return 1
     recipe_arg = args[0] if len(args) > 0 else None
-    run_recipe(opts, recipe_arg, p, notification=notification, handler=handler)    
+    run_recipe(opts, recipe_arg, p, notification=notification)
-            
+
     return 0
 
 if __name__ == '__main__':

src/calibre/web/feeds/news.py

@@ -7,7 +7,7 @@ Defines various abstract base classes that can be subclassed to create powerful
 __docformat__ = "restructuredtext en"
 
 
-import logging, os, cStringIO, time, traceback, re, urlparse, sys
+import os, time, traceback, re, urlparse, sys
 from collections import defaultdict
 from functools import partial
 from contextlib import nested, closing
@@ -27,6 +27,7 @@ from calibre.web.feeds import feed_from_xml, templates, feeds_from_index, Feed
 from calibre.web.fetch.simple import option_parser as web2disk_option_parser
 from calibre.web.fetch.simple import RecursiveFetcher
 from calibre.utils.threadpool import WorkRequest, ThreadPool, NoResultsPending
+from calibre.utils.logging import Log
 from calibre.ptempfile import PersistentTemporaryFile, \
                               PersistentTemporaryDirectory
 
@@ -423,7 +424,7 @@ class BasicNewsRecipe(object):
         '''
         raise NotImplementedError
 
-    def get_obfuscated_article(self, url, logger):
+    def get_obfuscated_article(self, url):
         '''
         If you set :member:`articles_are_obfuscated` this method is called with
         every article URL. It should return the path to a file on the filesystem
@@ -443,6 +444,7 @@ class BasicNewsRecipe(object):
         :param parser:  Command line option parser. Used to intelligently merge options.
         :param progress_reporter: A Callable that takes two arguments: progress (a number between 0 and 1) and a string message. The message should be optional.
         '''
+        self.log = Log()
         if not isinstance(self.title, unicode):
             self.title = unicode(self.title, 'utf-8', 'replace')
 
@@ -455,7 +457,6 @@ class BasicNewsRecipe(object):
 
 
         if self.debug:
-            logging.getLogger('feeds2disk').setLevel(logging.DEBUG)
             self.verbose = True
         self.report_progress = progress_reporter
 
@@ -560,20 +561,20 @@ class BasicNewsRecipe(object):
             res = self.build_index()
             self.report_progress(1, _('Download finished'))
             if self.failed_downloads:
-                self.log_warning(_('Failed to download the following articles:'))
+                self.log.warning(_('Failed to download the following articles:'))
                 for feed, article, debug in self.failed_downloads:
-                    self.log_warning(article.title+_(' from ')+feed.title)
+                    self.log.warning(article.title+_(' from ')+feed.title)
-                    self.log_debug(article.url)
+                    self.log.debug(article.url)
-                    self.log_debug(debug)
+                    self.log.debug(debug)
             if self.partial_failures:
-                self.log_warning(_('Failed to download parts of the following articles:'))
+                self.log.warning(_('Failed to download parts of the following articles:'))
                 for feed, atitle, aurl, debug in self.partial_failures:
-                    self.log_warning(atitle + _(' from ') + feed)
+                    self.log.warning(atitle + _(' from ') + feed)
-                    self.log_debug(aurl)
+                    self.log.debug(aurl)
-                    self.log_warning(_('\tFailed links:'))
+                    self.log.warning(_('\tFailed links:'))
                     for l, tb in debug:
-                        self.log_warning(l)
+                        self.log.warning(l)
-                        self.log_debug(tb)
+                        self.log.debug(tb)
             return res
         finally:
             self.cleanup()
@@ -636,20 +637,11 @@ class BasicNewsRecipe(object):
                               extra_css=self.extra_css).render(doctype='xhtml')
 
 
-    def create_logger(self, feed_number, article_number):
+    def _fetch_article(self, url, dir, f, a, num_of_feeds):
-        logger = logging.getLogger('feeds2disk.article_%d_%d'%(feed_number, article_number))
-        out = cStringIO.StringIO()
-        handler = logging.StreamHandler(out)
-        handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
-        handler.setLevel(logging.INFO if self.verbose else logging.WARNING)
-        if self.debug:
-            handler.setLevel(logging.DEBUG)
-        logger.addHandler(handler)
-        return logger, out
-
-    def _fetch_article(self, url, dir, logger, f, a, num_of_feeds):
         self.web2disk_options.browser = self.get_browser() if self.multithreaded_fetch else self.browser
-        fetcher = RecursiveFetcher(self.web2disk_options, logger, self.image_map, self.css_map, (url, f, a, num_of_feeds))
+        fetcher = RecursiveFetcher(self.web2disk_options, self.log,
+                self.image_map, self.css_map,
+                (url, f, a, num_of_feeds))
         fetcher.base_dir = dir
         fetcher.current_dir = dir
         fetcher.show_progress = False
@@ -661,21 +653,21 @@ class BasicNewsRecipe(object):
             raise Exception(_('Could not fetch article. Run with --debug to see the reason'))
         return res, path, failures
 
-    def fetch_article(self, url, dir, logger, f, a, num_of_feeds):
+    def fetch_article(self, url, dir, f, a, num_of_feeds):
-        return self._fetch_article(url, dir, logger, f, a, num_of_feeds)
+        return self._fetch_article(url, dir, f, a, num_of_feeds)
 
-    def fetch_obfuscated_article(self, url, dir, logger, f, a, num_of_feeds):
+    def fetch_obfuscated_article(self, url, dir, f, a, num_of_feeds):
-        path = os.path.abspath(self.get_obfuscated_article(url, logger))
+        path = os.path.abspath(self.get_obfuscated_article(url))
         url = ('file:'+path) if iswindows else ('file://'+path)
-        return self._fetch_article(url, dir, logger, f, a, num_of_feeds)
+        return self._fetch_article(url, dir, f, a, num_of_feeds)
 
-    def fetch_embedded_article(self, article, dir, logger, f, a, num_of_feeds):
+    def fetch_embedded_article(self, article, dir, f, a, num_of_feeds):
         templ = templates.EmbeddedContent()
         raw = templ.generate(article).render('html')
         with PersistentTemporaryFile('_feeds2disk.html') as pt:
             pt.write(raw)
             url = ('file:'+pt.name) if iswindows else ('file://'+pt.name)
-        return self._fetch_article(url, dir, logger, f, a, num_of_feeds)
+        return self._fetch_article(url, dir,  f, a, num_of_feeds)
 
 
     def build_index(self):
@@ -716,7 +708,6 @@ class BasicNewsRecipe(object):
                 art_dir = os.path.join(feed_dir, 'article_%d'%a)
                 if not os.path.isdir(art_dir):
                     os.makedirs(art_dir)
-                logger, stream = self.create_logger(f, a)
                 try:
                     url = self.print_version(article.url)
                 except NotImplementedError:
@@ -726,10 +717,9 @@ class BasicNewsRecipe(object):
                 func, arg = (self.fetch_embedded_article, article) if self.use_embedded_content else \
                             ((self.fetch_obfuscated_article if self.articles_are_obfuscated \
                               else self.fetch_article), url)
-                req = WorkRequest(func, (arg, art_dir, logger, f, a, len(feed)),
+                req = WorkRequest(func, (arg, art_dir, f, a, len(feed)),
                                       {}, (f, a), self.article_downloaded,
                                       self.error_in_article_download)
-                req.stream = stream
                 req.feed = feed
                 req.article = article
                 req.feed_dir = feed_dir
@@ -768,8 +758,8 @@ class BasicNewsRecipe(object):
             cu = self.get_cover_url()
         except Exception, err:
             cu = None
-            self.log_error(_('Could not download cover: %s')%str(err))
+            self.log.error(_('Could not download cover: %s')%str(err))
-            self.log_debug(traceback.format_exc())
+            self.log.debug(traceback.format_exc())
         if cu is not None:
             ext = cu.rpartition('.')[-1]
             if '?' in ext:
@@ -841,8 +831,8 @@ class BasicNewsRecipe(object):
             f.write(html.encode('utf-8'))
         renderer = render_html(hf)
         if renderer.tb is not None:
-            self.logger.warning('Failed to render default cover')
+            self.log.warning('Failed to render default cover')
-            self.logger.debug(renderer.tb)
+            self.log.debug(renderer.tb)
         else:
             cover_file.write(renderer.data)
             cover_file.flush()
@@ -863,7 +853,7 @@ class BasicNewsRecipe(object):
         manifest.append(os.path.join(dir, 'index.ncx'))
         cpath = getattr(self, 'cover_path', None)
         if cpath is None:
-            pf = PersistentTemporaryFile('_recipe_cover.jpg')
+            pf = open(os.path.join(dir, 'cover.jpg'), 'wb')
             self.default_cover(pf)
             cpath =  pf.name
         if cpath is not None and os.access(cpath, os.R_OK):
@@ -944,7 +934,7 @@ class BasicNewsRecipe(object):
         a = request.requestID[1]
 
         article = request.article
-        self.log_debug(_('\nDownloaded article %s from %s\n%s')%(article.title, article.url, request.stream.getvalue().decode('utf-8', 'ignore')))
+        self.log.debug(_('\nDownloaded article %s from %s')%(article.title, article.url))
         article.orig_url = article.url
         article.url = 'article_%d/index.html'%a
         article.downloaded = True
@@ -956,11 +946,11 @@ class BasicNewsRecipe(object):
 
     def error_in_article_download(self, request, traceback):
         self.jobs_done += 1
-        self.log_error(_('Failed to download article: %s from %s\n')%(request.article.title, request.article.url))
+        self.log.error(_('Failed to download article: %s from %s\n')%(request.article.title, request.article.url))
         debug = request.stream.getvalue().decode('utf-8', 'ignore')
-        self.log_debug(debug)
+        self.log.debug(debug)
-        self.log_debug(traceback)
+        self.log.debug(traceback)
-        self.log_debug('\n')
+        self.log.debug('\n')
         self.report_progress(float(self.jobs_done)/len(self.jobs), _('Article download failed: %s')%request.article.title)
         self.failed_downloads.append((request.feed, request.article, debug))
 
@@ -990,7 +980,7 @@ class BasicNewsRecipe(object):
                 feed.populate_from_preparsed_feed(msg, [])
                 feed.description = unicode(err)
                 parsed_feeds.append(feed)
-                self.log_exception(msg)
+                self.log.exception(msg)
 
 
         return parsed_feeds
@@ -1033,6 +1023,28 @@ class BasicNewsRecipe(object):
         nmassage.extend(entity_replace)
         return BeautifulSoup(raw, markupMassage=nmassage)
 
+    @classmethod
+    def adeify_images(cls, soup):
+         '''
+         If your recipe when converted to EPUB has problems with images when
+         viewed in Adobe Digital Editions, call this method from within
+         :method:`postprocess_html`.
+         '''
+         for item in soup.findAll('img'):
+             for attrib in ['height','width','border','align','style']:
+                 if item.has_key(attrib):
+                    del item[attrib]
+             oldParent = item.parent
+             myIndex = oldParent.contents.index(item)
+             item.extract()
+             divtag = Tag(soup,'div')
+             brtag  = Tag(soup,'br')
+             oldParent.insert(myIndex,divtag)
+             divtag.append(item)
+             divtag.append(brtag)
+         return soup
+
+
 class CustomIndexRecipe(BasicNewsRecipe):
 
     def custom_index(self):
@@ -1057,7 +1069,7 @@ class CustomIndexRecipe(BasicNewsRecipe):
         index = os.path.abspath(self.custom_index())
         url = 'file:'+index if iswindows else 'file://'+index
         self.web2disk_options.browser = self.browser
-        fetcher = RecursiveFetcher(self.web2disk_options, self.logger)
+        fetcher = RecursiveFetcher(self.web2disk_options, self.log)
         fetcher.base_dir = self.output_dir
         fetcher.current_dir = self.output_dir
         fetcher.show_progress = False
@@ -1069,7 +1081,7 @@ class AutomaticNewsRecipe(BasicNewsRecipe):
 
     keep_only_tags = [dict(name=['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'])]
 
-    def fetch_embedded_article(self, article, dir, logger, f, a, num_of_feeds):
+    def fetch_embedded_article(self, article, dir, f, a, num_of_feeds):
         if self.use_embedded_content:
             self.web2disk_options.keep_only_tags = []
-        return BasicNewsRecipe.fetch_embedded_article(self, article, dir, logger, f, a, num_of_feeds)
+        return BasicNewsRecipe.fetch_embedded_article(self, article, dir, f, a, num_of_feeds)

src/calibre/web/feeds/recipes/__init__.py

@@ -8,7 +8,7 @@ recipe_modules = ['recipe_' + r for r in (
            'newsweek', 'atlantic', 'economist', 'portfolio', 'the_register',
            'usatoday', 'outlook_india', 'bbc', 'greader', 'wsj',
            'wired', 'globe_and_mail', 'smh', 'espn', 'business_week', 'miami_herald',
-           'ars_technica', 'upi', 'new_yorker', 'irish_times', 'iht', 'lanacion',
+           'ars_technica', 'upi', 'new_yorker', 'irish_times', 'lanacion',
            'discover_magazine', 'scientific_american', 'new_york_review_of_books',
            'daily_telegraph', 'guardian', 'el_pais', 'new_scientist', 'b92',
            'politika', 'moscow_times', 'latimes', 'japan_times', 'san_fran_chronicle',
@@ -37,7 +37,8 @@ recipe_modules = ['recipe_' + r for r in (
            'new_york_review_of_books_no_sub', 'politico', 'adventuregamers',
            'mondedurable', 'instapaper', 'dnevnik_cro', 'vecernji_list',
            'nacional_cro', '24sata', 'dnevni_avaz', 'glas_srpske', '24sata_rs',
-           'krstarica', 'krstarica_en', 'tanjug', 'laprensa_ni',
+           'krstarica', 'krstarica_en', 'tanjug', 'laprensa_ni', 'azstarnet',
+           'corriere_della_sera_it', 'corriere_della_sera_en', 'msdnmag_en',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_24sata_rs.py

@@ -9,6 +9,7 @@ __copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
 
 import re
 from calibre.web.feeds.recipes import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
 
 class Ser24Sata(BasicNewsRecipe):
     title                 = '24 Sata - Sr'
@@ -39,14 +40,30 @@ class Ser24Sata(BasicNewsRecipe):
 
     feeds = [(u'Vesti Dana', u'http://www.24sata.rs/rss.php')]
 
+    def cleanup_image_tags(self,soup):
+        for item in soup.findAll('img'):
+            for attrib in ['height','width','border','align']:
+                if item.has_key(attrib):
+                   del item[attrib]
+            oldParent = item.parent
+            myIndex = oldParent.contents.index(item)
+            item.extract()
+            divtag = Tag(soup,'div')
+            brtag  = Tag(soup,'br')
+            oldParent.insert(myIndex,divtag)
+            divtag.append(item)
+            divtag.append(brtag)
+        return soup
+
     def preprocess_html(self, soup):
         soup.html['xml:lang'] = 'sr-Latn-RS'
         soup.html['lang']     = 'sr-Latn-RS'
         mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
         soup.head.insert(0,mtag)
-        return soup
+        return self.cleanup_image_tags(soup)
 
     def print_version(self, url):
         article, sep, rest = url.partition('#')
-        return article.replace('/show.php','/_print.php')
+        article_base, sep2, article_id = article.partition('id=')
+        return 'http://www.24sata.co.rs/_print.php?id=' + article_id
 

src/calibre/web/feeds/recipes/recipe_azstarnet.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.azstarnet.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Azstarnet(BasicNewsRecipe):
+    title                 = 'Arizona  Daily Star'
+    __author__            = 'Darko Miletic'
+    description           = 'news from Arizona'
+    publisher             = 'azstarnet.com'
+    category              = 'news, politics, Arizona, USA'
+    delay                 = 1    
+    oldest_article        = 1
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    needs_subscription    = True
+    remove_javascript     = True
+
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open('http://azstarnet.com/registration/retro.php')
+            br.select_form(nr=1)
+            br['email'] = self.username
+            br['pass' ] = self.password
+            br.submit()
+        return br
+    
+    
+    keep_only_tags = [dict(name='div', attrs={'id':'storycontent'})]
+
+    remove_tags = [
+                     dict(name=['object','link','iframe','base','img'])
+                    ,dict(name='div',attrs={'class':'bannerinstory'})
+                  ]
+
+    
+    feeds = [(u'Tucson Region', u'http://rss.azstarnet.com/index.php?site=metro')]
+
+    def preprocess_html(self, soup):
+        soup.html['dir' ] = 'ltr'
+        soup.html['lang'] = 'en-US'
+        mtag = '\n<meta http-equiv="Content-Language" content="en-US"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n'
+        soup.head.insert(0,mtag)    
+        for item in soup.findAll(style=True):
+            del item['style']    
+        return soup
+    

src/calibre/web/feeds/recipes/recipe_blic.py

@@ -8,11 +8,12 @@ blic.rs
 
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
 
 class Blic(BasicNewsRecipe):
-    title                 = u'Blic'
+    title                 = 'Blic'
-    __author__            = u'Darko Miletic'
+    __author__            = 'Darko Miletic'
-    description           = u'Blic.co.yu online verzija najtiraznije novine u Srbiji donosi najnovije vesti iz Srbije i sveta, komentare, politicke analize, poslovne i ekonomske vesti, vesti iz regiona, intervjue, informacije iz kulture, reportaze, pokriva sve sportske dogadjaje, detaljan tv program, nagradne igre, zabavu, fenomenalni Blic strip, dnevni horoskop, arhivu svih dogadjaja'    
+    description           = 'Blic.co.yu online verzija najtiraznije novine u Srbiji donosi najnovije vesti iz Srbije i sveta, komentare, politicke analize, poslovne i ekonomske vesti, vesti iz regiona, intervjue, informacije iz kulture, reportaze, pokriva sve sportske dogadjaje, detaljan tv program, nagradne igre, zabavu, fenomenalni Blic strip, dnevni horoskop, arhivu svih dogadjaja'    
     publisher             = 'RINGIER d.o.o.'
     category              = 'news, politics, Serbia'
     oldest_article        = 2
@@ -21,7 +22,7 @@ class Blic(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     language              = _('Serbian')
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif} '
     
     html2lrf_options = [
                           '--comment'  , description
@@ -30,7 +31,7 @@ class Blic(BasicNewsRecipe):
                         , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} "' 
     
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
 
@@ -44,10 +45,26 @@ class Blic(BasicNewsRecipe):
         start_url, question, rest_url = url.partition('?')
         return u'http://www.blic.rs/_print.php?' + rest_url
 
+    def cleanup_image_tags(self,soup):
+        for item in soup.findAll('img'):
+            for attrib in ['height','width','border','align']:
+                if item.has_key(attrib):
+                   del item[attrib]
+            oldParent = item.parent
+            myIndex = oldParent.contents.index(item)
+            item.extract()
+            divtag = Tag(soup,'div')
+            brtag  = Tag(soup,'br')
+            oldParent.insert(myIndex,divtag)
+            divtag.append(item)
+            divtag.append(brtag)
+        return soup
+      
+    
     def preprocess_html(self, soup):
         mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)    
         for item in soup.findAll(style=True):
             del item['style']
-        return soup
+        return self.cleanup_image_tags(soup)
         

src/calibre/web/feeds/recipes/recipe_corriere_della_sera_en.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.corriere.it/english
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Corriere_en(BasicNewsRecipe):
+    title                 = 'Corriere della Sera in English'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Milan and Italy'
+    oldest_article        = 15
+    publisher             = 'Corriere della Sera'
+    category              = 'news, politics, Italy'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'cp1252'
+    remove_javascript     = True
+    language              = _('English')
+
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        , '--ignore-tables'
+                        ]
+
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True'
+
+    keep_only_tags = [dict(name='div', attrs={'class':['news-dettaglio article','article']})]
+
+    remove_tags = [
+                    dict(name=['base','object','link','embed','img'])
+                   ,dict(name='div', attrs={'class':'news-goback'})
+                   ,dict(name='ul', attrs={'class':'toolbar'})
+                  ]
+
+    remove_tags_after = dict(name='p', attrs={'class':'footnotes'})
+
+    feeds = [(u'Italian Life', u'http://www.corriere.it/rss/english.xml')]
+

src/calibre/web/feeds/recipes/recipe_corriere_della_sera_it.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.corriere.it
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+class Corriere_it(BasicNewsRecipe):
+    title                 = 'Corriere della Sera'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Milan and Italy'    
+    oldest_article        = 7
+    publisher             = 'Corriere della Sera'
+    category              = 'news, politics, Italy'        
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'cp1252'
+    remove_javascript     = True
+    language              = _('Italian')
+
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        , '--ignore-tables'
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
+
+    keep_only_tags = [dict(name='div', attrs={'class':['news-dettaglio article','article']})]
+
+    remove_tags = [
+                    dict(name=['base','object','link','embed','img'])
+                   ,dict(name='div', attrs={'class':'news-goback'})
+                   ,dict(name='ul', attrs={'class':'toolbar'})
+                  ]
+
+    remove_tags_after = dict(name='p', attrs={'class':'footnotes'})
+    
+    feeds = [ 
+              (u'Ultimora'  , u'http://www.corriere.it/rss/ultimora.xml'  )
+             ,(u'Cronache'  , u'http://www.corriere.it/rss/cronache.xml'  )
+             ,(u'Economia'  , u'http://www.corriere.it/rss/economia.xml'  )
+             ,(u'Editoriali', u'http://www.corriere.it/rss/editoriali.xml')
+             ,(u'Esteri'    , u'http://www.corriere.it/rss/esteri.xml'    )
+             ,(u'Politica'  , u'http://www.corriere.it/rss/politica.xml'  )
+             ,(u'Salute'    , u'http://www.corriere.it/rss/salute.xml'    )
+             ,(u'Scienze'   , u'http://www.corriere.it/rss/scienze.xml'   )
+             ,(u'Spettacolo', u'http://www.corriere.it/rss/spettacoli.xml')
+             ,(u'Sport'     , u'http://www.corriere.it/rss/sport.xml'     )
+            ]
+

src/calibre/web/feeds/recipes/recipe_iht.py

@@ -12,7 +12,7 @@ from calibre.ptempfile import PersistentTemporaryFile
 class InternationalHeraldTribune(BasicNewsRecipe):
     title          = u'The International Herald Tribune'
     __author__     = 'Derry FitzGerald'
-    language = _('English')
+    language       = _('English')
     oldest_article = 1
     max_articles_per_feed = 10
     no_stylesheets = True
@@ -20,13 +20,13 @@ class InternationalHeraldTribune(BasicNewsRecipe):
     remove_tags    = [dict(name='div', attrs={'class':'footer'}),
                       dict(name=['form'])]
     preprocess_regexps = [
-            (re.compile(r'<!-- webtrends.*', re.DOTALL), 
+            (re.compile(r'<!-- webtrends.*', re.DOTALL),
              lambda m:'</body></html>')
                           ]
-    extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }' 
+    extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }'
 
     feeds          = [
-                      (u'Frontpage', u'http://www.iht.com/rss/frontpage.xml'), 
+                      (u'Frontpage', u'http://www.iht.com/rss/frontpage.xml'),
                       (u'Business', u'http://www.iht.com/rss/business.xml'),
                       (u'Americas', u'http://www.iht.com/rss/america.xml'),
                       (u'Europe', u'http://www.iht.com/rss/europe.xml'),
@@ -45,7 +45,7 @@ class InternationalHeraldTribune(BasicNewsRecipe):
                     ]
     temp_files = []
     articles_are_obfuscated = True
-    
+
     def get_obfuscated_article(self, url, logger):
         br = self.get_browser()
         br.open(url)
@@ -55,4 +55,4 @@ class InternationalHeraldTribune(BasicNewsRecipe):
         self.temp_files.append(PersistentTemporaryFile('_iht.html'))
         self.temp_files[-1].write(html)
         self.temp_files[-1].close()
-        return self.temp_files[-1].name
+        return self.temp_files[-1].name

src/calibre/web/feeds/recipes/recipe_msdnmag_en.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+msdn.microsoft.com/en-us/magazine
+'''
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class MSDNMagazine_en(BasicNewsRecipe):
+    title                 = 'MSDN Magazine'
+    __author__            = 'Darko Miletic'
+    description           = 'The Microsoft Journal for Developers'
+    publisher             = 'Microsoft Press'
+    category              = 'news, IT, Microsoft, programming, windows'        
+    oldest_article        = 31
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    remove_javascript     = True
+    current_issue         = 'http://msdn.microsoft.com/en-us/magazine/default.aspx'
+    language              = _('English')
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+                            
+    feeds = [(u'Articles', u'http://msdn.microsoft.com/en-us/magazine/rss/default.aspx?z=z&iss=1')]
+
+    keep_only_tags = [dict(name='div', attrs={'class':'topic'})]
+
+    remove_tags = [
+                     dict(name=['object','link','base','table'])
+                    ,dict(name='div', attrs={'class':'MTPS_CollapsibleRegion'})
+                  ]
+    
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup(self.current_issue)
+        link_item = soup.find('span',attrs={'class':'ContentsImageSpacer'})
+        if link_item:
+           imgt = link_item.find('img')
+           if imgt:
+              cover_url = imgt['src']
+        return cover_url
+
+    
+    def preprocess_html(self, soup):
+        for item in soup.findAll('div',attrs={'class':['FeatureSmallHead','ColumnTypeSubTitle']}):
+            item.name="h2"
+        for item in soup.findAll('div',attrs={'class':['FeatureHeadline','ColumnTypeTitle']}):
+            item.name="h1"
+        for item in soup.findAll('div',attrs={'class':'ArticleTypeTitle'}):
+            item.name="h3"
+        return soup
+    

src/calibre/web/feeds/recipes/recipe_new_yorker.py

@@ -9,11 +9,12 @@ newyorker.com
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class NewYorker(BasicNewsRecipe):
+
     title                 = u'The New Yorker'
     __author__            = 'Darko Miletic'
-    description           = 'The best of US journalism'    
+    description           = 'The best of US journalism'
     oldest_article        = 7
-    language = _('English')
+    language              = _('English')
     max_articles_per_feed = 100
     no_stylesheets        = False
     use_embedded_content  = False
@@ -24,7 +25,7 @@ class NewYorker(BasicNewsRecipe):
     .calibre_recipe_title {font-size:normal}
     .calibre_feed_description {font-size:xx-small}
     '''
-                    
+
 
     keep_only_tags = [
                         dict(name='div'  , attrs={'id':'printbody'   })
@@ -41,3 +42,12 @@ class NewYorker(BasicNewsRecipe):
 
     def print_version(self, url):
         return url + '?printable=true'
+
+    def postprocess_html(self, soup, x):
+        body = soup.find('body')
+        if body:
+            html = soup.find('html')
+            if html:
+                body.extract()
+                html.insert(-1, body)
+        return soup

src/calibre/web/feeds/recipes/recipe_nspm.py

@@ -8,9 +8,10 @@ nspm.rs
 
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
 
 class Nspm(BasicNewsRecipe):
-    title                 = u'Nova srpska politicka misao'
+    title                 = 'Nova srpska politicka misao'
     __author__            = 'Darko Miletic'
     description           = 'Casopis za politicku teoriju i drustvena istrazivanja'    
     publisher             = 'NSPM'
@@ -36,7 +37,7 @@ class Nspm(BasicNewsRecipe):
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     remove_tags        = [
-                            dict(name=['a','img','link','object','embed'])
+                            dict(name=['link','object','embed'])
                            ,dict(name='td', attrs={'class':'buttonheading'})
                          ]
     
@@ -50,6 +51,21 @@ class Nspm(BasicNewsRecipe):
     def print_version(self, url):
         return url.replace('.html','/stampa.html')
 
+    def cleanup_image_tags(self,soup):
+        for item in soup.findAll('img'):
+            for attrib in ['height','width','border','align']:
+                if item.has_key(attrib):
+                   del item[attrib]
+            oldParent = item.parent
+            myIndex = oldParent.contents.index(item)
+            item.extract()
+            divtag = Tag(soup,'div')
+            brtag  = Tag(soup,'br')
+            oldParent.insert(myIndex,divtag)
+            divtag.append(item)
+            divtag.append(brtag)
+        return soup
+
     def preprocess_html(self, soup):
         lng = 'sr-Latn-RS'
         soup.html['xml:lang'] = lng
@@ -59,4 +75,4 @@ class Nspm(BasicNewsRecipe):
            ftag['content'] = lng
         for item in soup.findAll(style=True):
             del item['style']     
-        return soup
+        return self.cleanup_image_tags(soup)

src/calibre/web/feeds/recipes/recipe_tomshardware.py

@@ -1,38 +1,47 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
-tomshardware.com
+tomshardware.com/us
 '''
 
+import urllib
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
-class Tomshardware(BasicNewsRecipe):
+class Tomshardware(BasicNewsRecipe):    
+    title               = "Tom's Hardware US"
+    __author__          = 'Darko Miletic'
+    description         = 'Hardware reviews and News'
+    publisher           = "Tom's Hardware"
+    category            = 'news, IT, hardware, USA'        
+    no_stylesheets      = True
+    needs_subscription  = True
+    language            = _('English')    
+    INDEX               = 'http://www.tomshardware.com'
+    LOGIN               = INDEX + '/membres/'
+    remove_javascript   = True
+    use_embedded_content= False
     
-    title       = "Tom's Hardware US"
+    html2lrf_options = [
-    __author__  = 'Darko Miletic'
+                          '--comment', description
-    description = 'Hardware reviews and News'
+                        , '--category', category
-    no_stylesheets = True
+                        , '--publisher', publisher
-    needs_subscription = True
+                        ]
-    language = _('English')
-    INDEX = 'http://www.tomshardware.com'
-    LOGIN = 'http://www.tomshardware.com/membres/?r=%2Fus%2F#loginForm'
-    cover_url = 'http://img.bestofmedia.com/img/tomshardware/design/tomshardware.jpg'
     
-    html2lrf_options = [  '--comment'       , description
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
-                        , '--category'      , 'hardware,news'
-                        , '--base-font-size', '10'
-                       ]
 
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
+        br.open(self.INDEX+'/us/')
         if self.username is not None and self.password is not None:
-            br.open(self.LOGIN)
+            data = urllib.urlencode({ 'action':'login_action'
-            br.select_form(name='connexion')
+                                     ,'r':self.INDEX+'/us/'
-            br['login'] = self.username
+                                     ,'login':self.username
-            br['mdp'  ] = self.password
+                                     ,'mdp':self.password
-            br.submit()
+                                   })
+            br.open(self.LOGIN,data)
         return br
 
     remove_tags = [
@@ -41,18 +50,30 @@ class Tomshardware(BasicNewsRecipe):
                   ]
     
     feeds = [
-              (u'Latest Articles', u'http://www.tomshardware.com/feeds/atom/tom-s-hardware-us,18-2.xml')
+              (u'Latest Articles', u'http://www.tomshardware.com/feeds/atom/tom-s-hardware-us,18-2.xml'          )
              ,(u'Latest News'    , u'http://www.tomshardware.com/feeds/atom/tom-s-hardware-us,18-1.xml')
             ]
-                  
+
     def print_version(self, url):
         main, sep, rest = url.rpartition('.html')
         rmain, rsep, article_id = main.rpartition(',')
         tmain, tsep, trest = rmain.rpartition('/reviews/')
+        rind = 'http://www.tomshardware.com/news_print.php?p1='
         if tsep:
-            return 'http://www.tomshardware.com/review_print.php?p1=' + article_id
+           rind = 'http://www.tomshardware.com/review_print.php?p1='
-        return 'http://www.tomshardware.com/news_print.php?p1=' + article_id        
+        return rind + article_id        
+
+    def cleanup_image_tags(self,soup):
+        for item in soup.findAll('img'):
+            for attrib in ['height','width','border','align']:
+                if item.has_key(attrib):
+                   del item[attrib]
+        return soup
 
     def preprocess_html(self, soup):
         del(soup.body['onload'])
-        return soup
+        for item in soup.findAll(style=True):
+            del item['style']
+        for it in soup.findAll('span'):
+            it.name="div"
+        return self.cleanup_image_tags(soup)

src/calibre/web/feeds/recipes/recipe_wash_post.py

@@ -11,13 +11,13 @@ class WashingtonPost(BasicNewsRecipe):
     max_articles_per_feed = 20
     language = _('English')
 
-    
-    remove_javascript = True   
 
-  
+    remove_javascript = True
+
+
     feeds = [ ('Today\'s Highlights', 'http://www.washingtonpost.com/wp-dyn/rss/linkset/2005/03/24/LI2005032400102.xml'),
      	         ('Politics', 'http://www.washingtonpost.com/wp-dyn/rss/politics/index.xml'),
-     	         ('Nation', 'http://www.www.washingtonpost.com/wp-dyn/rss/nation/index.xml'),
+     	         ('Nation', 'http://www.washingtonpost.com/wp-dyn/rss/nation/index.xml'),
      	         ('World', 'http://www.washingtonpost.com/wp-dyn/rss/world/index.xml'),
      	         ('Business', 'http://www.washingtonpost.com/wp-dyn/rss/business/index.xml'),
      	         ('Technology', 'http://www.washingtonpost.com/wp-dyn/rss/technology/index.xml'),
@@ -25,7 +25,7 @@ class WashingtonPost(BasicNewsRecipe):
      	         ('Education', 'http://www.washingtonpost.com/wp-dyn/rss/education/index.xml'),
      	         ('Editorials', 'http://www.washingtonpost.com/wp-dyn/rss/linkset/2005/05/30/LI2005053000331.xml'),
      	]
-    
+
     remove_tags = [{'id':['pfmnav', 'ArticleCommentsWrapper']}]
 
 
@@ -34,7 +34,7 @@ class WashingtonPost(BasicNewsRecipe):
 
     def print_version(self, url):
         return url.rpartition('.')[0] + '_pf.html'
-    
+
     def postprocess_html(self, soup, first):
         for div in soup.findAll(name='div', style=re.compile('margin')):
             div['style'] = ''

src/calibre/web/fetch/simple.py

@@ -7,18 +7,19 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Fetch a webpage and its links recursively. The webpages are saved to disk in
 UTF-8 encoding with any charset declarations removed.
 '''
-import sys, socket, os, urlparse, logging, re, time, copy, urllib2, threading, traceback
+import sys, socket, os, urlparse, re, time, copy, urllib2, threading, traceback
 from urllib import url2pathname, quote
 from threading import RLock
 from httplib import responses
 from PIL import Image
 from cStringIO import StringIO
 
-from calibre import setup_cli_handlers, browser, sanitize_file_name, \
+from calibre import browser, sanitize_file_name, \
                     relpath, unicode_path
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.utils.config import OptionParser
+from calibre.utils.logging import Log
 
 class FetchError(Exception):
     pass
@@ -28,10 +29,10 @@ class closing(object):
 
     def __init__(self, thing):
         self.thing = thing
-    
+
     def __enter__(self):
         return self.thing
-    
+
     def __exit__(self, *exc_info):
         try:
             self.thing.close()
@@ -55,47 +56,48 @@ def save_soup(soup, target):
     for meta in metas:
         if 'charset' in meta.get('content', '').lower():
             meta.replaceWith(nm)
-    
+
     selfdir = os.path.dirname(target)
-    
+
     for tag in soup.findAll(['img', 'link', 'a']):
         for key in ('src', 'href'):
             path = tag.get(key, None)
             if path and os.path.isfile(path) and os.path.exists(path) and os.path.isabs(path):
                 tag[key] = unicode_path(relpath(path, selfdir).replace(os.sep, '/'))
-    
+
     html = unicode(soup)
     with open(target, 'wb') as f:
         f.write(html.encode('utf-8'))
-    
+
 class response(str):
-    
+
     def __new__(cls, *args):
         obj = super(response, cls).__new__(cls, *args)
         obj.newurl = None
         return obj
-        
+
 class DummyLock(object):
-    
+
     def __enter__(self, *args): return self
     def __exit__(self, *args): pass
 
 class RecursiveFetcher(object):
-    LINK_FILTER = tuple(re.compile(i, re.IGNORECASE) for i in 
+    LINK_FILTER = tuple(re.compile(i, re.IGNORECASE) for i in
                 ('.exe\s*$', '.mp3\s*$', '.ogg\s*$', '^\s*mailto:', '^\s*$'))
     #ADBLOCK_FILTER = tuple(re.compile(i, re.IGNORECASE) for it in
     #                       (
-    #                        
+    #
     #                        )
     #                       )
     CSS_IMPORT_PATTERN = re.compile(r'\@import\s+url\((.*?)\)', re.IGNORECASE)
     default_timeout = socket.getdefaulttimeout() # Needed here as it is used in __del__
     DUMMY_LOCK = DummyLock()
-    
+
-    def __init__(self, options, logger, image_map={}, css_map={}, job_info=None):
+    def __init__(self, options, log, image_map={}, css_map={}, job_info=None):
         self.base_dir = os.path.abspath(os.path.expanduser(options.dir))
         if not os.path.exists(self.base_dir):
             os.makedirs(self.base_dir)
+        self.log = log
         self.default_timeout = socket.getdefaulttimeout()
         socket.setdefaulttimeout(options.timeout)
         self.verbose = options.verbose
@@ -122,19 +124,19 @@ class RecursiveFetcher(object):
         self.remove_tags_after   = getattr(options, 'remove_tags_after', None)
         self.remove_tags_before  = getattr(options, 'remove_tags_before', None)
         self.keep_only_tags      = getattr(options, 'keep_only_tags', [])
-        self.preprocess_html_ext = getattr(options, 'preprocess_html', lambda soup: soup) 
+        self.preprocess_html_ext = getattr(options, 'preprocess_html', lambda soup: soup)
         self.postprocess_html_ext= getattr(options, 'postprocess_html', None)
         self.download_stylesheets = not options.no_stylesheets
         self.show_progress = True
         self.failed_links = []
         self.job_info = job_info
-        
+
     def get_soup(self, src):
         nmassage = copy.copy(BeautifulSoup.MARKUP_MASSAGE)
         nmassage.extend(self.preprocess_regexps)
         nmassage += [(re.compile(r'<!DOCTYPE .+?>', re.DOTALL), lambda m: '')] # Some websites have buggy doctype declarations that mess up beautifulsoup
         soup = BeautifulSoup(xml_to_unicode(src, self.verbose, strip_encoding_pats=True)[0], markupMassage=nmassage)
-         
+
         if self.keep_only_tags:
             body = Tag(soup, 'body')
             try:
@@ -146,7 +148,7 @@ class RecursiveFetcher(object):
                 soup.find('body').replaceWith(body)
             except AttributeError: # soup has no body element
                 pass
-            
+
         def remove_beyond(tag, next):
             while tag is not None and tag.name != 'body':
                 after = getattr(tag, next)
@@ -155,31 +157,34 @@ class RecursiveFetcher(object):
                     after.extract()
                     after = ns
                 tag = tag.parent
-        
+
         if self.remove_tags_after is not None:
             rt = [self.remove_tags_after] if isinstance(self.remove_tags_after, dict) else self.remove_tags_after
             for spec in rt:
                 tag = soup.find(**spec)
                 remove_beyond(tag, 'nextSibling')
-            
+
         if self.remove_tags_before is not None:
             tag = soup.find(**self.remove_tags_before)
             remove_beyond(tag, 'previousSibling')
-            
+
         for kwds in self.remove_tags:
             for tag in soup.findAll(**kwds):
                 tag.extract()
         return self.preprocess_html_ext(soup)
-        
+
-    
+
     def fetch_url(self, url):
         data = None
-        self.log_debug('Fetching %s', url)
+        self.log.debug('Fetching', url)
-        delta = time.time() - self.last_fetch_at 
+        delta = time.time() - self.last_fetch_at
         if  delta < self.delay:
             time.sleep(delta)
-        if re.search(r'\s+', url) is not None:
+        if re.search(r'\s+|,', url) is not None:
-            url = quote(url)
+            purl = list(urlparse.urlparse(url))
+            for i in range(2, 6):
+                purl[i] = quote(purl[i])
+            url = urlparse.urlunparse(purl)
         with self.browser_lock:
             try:
                 with closing(self.browser.open(url)) as f:
@@ -190,43 +195,43 @@ class RecursiveFetcher(object):
                     raise FetchError, responses[err.code]
                 if getattr(err, 'reason', [0])[0] == 104 or \
                     getattr(getattr(err, 'args', [None])[0], 'errno', None) == -2: # Connection reset by peer or Name or service not know
-                    self.log_debug('Temporary error, retrying in 1 second')
+                    self.log.debug('Temporary error, retrying in 1 second')
                     time.sleep(1)
                     with closing(self.browser.open(url)) as f:
                         data = response(f.read()+f.read())
                         data.newurl = f.geturl()
-                else: 
+                else:
                     raise err
             finally:
                 self.last_fetch_at = time.time()
             return data
 
-        
+
     def start_fetch(self, url):
         soup = BeautifulSoup(u'<a href="'+url+'" />')
-        self.log_info('Downloading')
+        self.log.debug('Downloading')
         res = self.process_links(soup, url, 0, into_dir='')
-        self.log_info('%s saved to %s', url, res)
+        self.log.debug('%s saved to %s'%( url, res))
         return res
-    
+
     def is_link_ok(self, url):
         for i in self.__class__.LINK_FILTER:
             if i.search(url):
                 return False
         return True
-        
+
     def is_link_wanted(self, url):
         if self.filter_regexps:
             for f in self.filter_regexps:
                 if f.search(url):
-                    return False            
+                    return False
         if self.match_regexps:
             for m in self.match_regexps:
                 if m.search(url):
                     return True
             return False
         return True
-        
+
     def process_stylesheets(self, soup, baseurl):
         diskpath = unicode_path(os.path.join(self.current_dir, 'stylesheets'))
         if not os.path.exists(diskpath):
@@ -243,8 +248,7 @@ class RecursiveFetcher(object):
                 try:
                     data = self.fetch_url(iurl)
                 except Exception, err:
-                    self.log_debug('Could not fetch stylesheet %s', iurl)
+                    self.log.exception('Could not fetch stylesheet %s'% iurl)
-                    self.log_debug('Error: %s', str(err), exc_info=True)
                     continue
                 stylepath = os.path.join(diskpath, 'style'+str(c)+'.css')
                 with self.stylemap_lock:
@@ -253,7 +257,7 @@ class RecursiveFetcher(object):
                     x.write(data)
                 tag['href'] = stylepath
             else:
-                for ns in tag.findAll(text=True):                    
+                for ns in tag.findAll(text=True):
                     src = str(ns)
                     m = self.__class__.CSS_IMPORT_PATTERN.search(src)
                     if m:
@@ -267,8 +271,7 @@ class RecursiveFetcher(object):
                         try:
                             data = self.fetch_url(iurl)
                         except Exception, err:
-                            self.log_warning('Could not fetch stylesheet %s', iurl)
+                            self.log.exception('Could not fetch stylesheet %s'% iurl)
-                            self.log_debug('Error: %s', str(err), exc_info=True)
                             continue
                         c += 1
                         stylepath = os.path.join(diskpath, 'style'+str(c)+'.css')
@@ -277,9 +280,9 @@ class RecursiveFetcher(object):
                         with open(stylepath, 'wb') as x:
                             x.write(data)
                         ns.replaceWith(src.replace(m.group(1), stylepath))
-                        
+
-                        
+
-    
+
     def process_images(self, soup, baseurl):
         diskpath = unicode_path(os.path.join(self.current_dir, 'images'))
         if not os.path.exists(diskpath):
@@ -291,9 +294,6 @@ class RecursiveFetcher(object):
                 iurl = self.image_url_processor(baseurl, iurl)
             ext  = os.path.splitext(iurl)[1]
             ext  = ext[:5]
-            #if not ext:
-            #    self.log_debug('Skipping extensionless image %s', iurl)
-            #    continue
             if not urlparse.urlsplit(iurl).scheme:
                 iurl = urlparse.urljoin(baseurl, iurl, False)
             with self.imagemap_lock:
@@ -303,8 +303,7 @@ class RecursiveFetcher(object):
             try:
                 data = self.fetch_url(iurl)
             except Exception, err:
-                self.log_warning('Could not fetch image %s', iurl)
+                self.log.exception('Could not fetch image %s'% iurl)
-                self.log_debug('Error: %s', str(err), exc_info=True)
                 continue
             c += 1
             fname = sanitize_file_name('img'+str(c)+ext)
@@ -322,7 +321,7 @@ class RecursiveFetcher(object):
                 traceback.print_exc()
                 continue
 
-    def absurl(self, baseurl, tag, key, filter=True): 
+    def absurl(self, baseurl, tag, key, filter=True):
         iurl = tag[key]
         parts = urlparse.urlsplit(iurl)
         if not parts.netloc and not parts.path:
@@ -330,32 +329,32 @@ class RecursiveFetcher(object):
         if not parts.scheme:
             iurl = urlparse.urljoin(baseurl, iurl, False)
         if not self.is_link_ok(iurl):
-            self.log_debug('Skipping invalid link: %s', iurl)
+            self.log.debug('Skipping invalid link:', iurl)
             return None
         if filter and not self.is_link_wanted(iurl):
-            self.log_debug('Filtered link: '+iurl)
+            self.log.debug('Filtered link: '+iurl)
             return None
         return iurl
-    
+
     def normurl(self, url):
         parts = list(urlparse.urlsplit(url))
         parts[4] = ''
         return urlparse.urlunsplit(parts)
-                
+
     def localize_link(self, tag, key, path):
         parts = urlparse.urlsplit(tag[key])
         suffix = '#'+parts.fragment if parts.fragment else ''
         tag[key] = path+suffix
-    
+
     def process_return_links(self, soup, baseurl):
         for tag in soup.findAll(lambda tag: tag.name.lower()=='a' and tag.has_key('href')):
-            iurl = self.absurl(baseurl, tag, 'href')            
+            iurl = self.absurl(baseurl, tag, 'href')
             if not iurl:
                 continue
             nurl = self.normurl(iurl)
             if self.filemap.has_key(nurl):
                 self.localize_link(tag, 'href', self.filemap[nurl])
-    
+
     def process_links(self, soup, baseurl, recursion_level, into_dir='links'):
         res = ''
         diskpath = os.path.join(self.current_dir, into_dir)
@@ -365,7 +364,7 @@ class RecursiveFetcher(object):
         try:
             self.current_dir = diskpath
             tags = list(soup.findAll('a', href=True))
-            
+
             for c, tag in enumerate(tags):
                 if self.show_progress:
                     print '.',
@@ -395,17 +394,17 @@ class RecursiveFetcher(object):
                         dsrc = dsrc.decode(self.encoding, 'ignore')
                     else:
                         dsrc = xml_to_unicode(dsrc, self.verbose)[0]
-                    
+
                     soup = self.get_soup(dsrc)
-                    
+
                     base = soup.find('base', href=True)
                     if base is not None:
                         newbaseurl = base['href']
-                    self.log_debug('Processing images...')
+                    self.log.debug('Processing images...')
                     self.process_images(soup, newbaseurl)
                     if self.download_stylesheets:
                         self.process_stylesheets(soup, newbaseurl)
-                    
+
                     _fname = basename(iurl)
                     if not isinstance(_fname, unicode):
                         _fname.decode('latin1', 'replace')
@@ -416,56 +415,55 @@ class RecursiveFetcher(object):
                     self.downloaded_paths.append(res)
                     self.filemap[nurl] = res
                     if recursion_level < self.max_recursions:
-                        self.log_debug('Processing links...')
+                        self.log.debug('Processing links...')
                         self.process_links(soup, newbaseurl, recursion_level+1)
                     else:
-                        self.process_return_links(soup, newbaseurl) 
+                        self.process_return_links(soup, newbaseurl)
-                        self.log_debug('Recursion limit reached. Skipping links in %s', iurl)
+                        self.log.debug('Recursion limit reached. Skipping links in', iurl)
-                    
+
                     if callable(self.postprocess_html_ext):
-                        soup = self.postprocess_html_ext(soup, 
+                        soup = self.postprocess_html_ext(soup,
                                 c==0 and recursion_level==0 and not getattr(self, 'called_first', False),
                                 self.job_info)
-                        
+
                         if c==0 and recursion_level == 0:
                             self.called_first = True
-                    
+
                     save_soup(soup, res)
                     self.localize_link(tag, 'href', res)
                 except Exception, err:
                     self.failed_links.append((iurl, traceback.format_exc()))
-                    self.log_warning('Could not fetch link %s', iurl)
+                    self.log.exception('Could not fetch link', iurl)
-                    self.log_debug('Error: %s', str(err), exc_info=True)
                 finally:
                     self.current_dir = diskpath
-                    self.files += 1                
+                    self.files += 1
         finally:
             self.current_dir = prev_dir
         if self.show_progress:
             print
         return res
-    
+
     def __del__(self):
         dt = getattr(self, 'default_timeout', None)
         if dt is not None:
             socket.setdefaulttimeout(dt)
-        
+
 def option_parser(usage=_('%prog URL\n\nWhere URL is for example http://google.com')):
     parser = OptionParser(usage=usage)
-    parser.add_option('-d', '--base-dir', 
+    parser.add_option('-d', '--base-dir',
                       help=_('Base directory into which URL is saved. Default is %default'),
                       default='.', type='string', dest='dir')
-    parser.add_option('-t', '--timeout', 
+    parser.add_option('-t', '--timeout',
                       help=_('Timeout in seconds to wait for a response from the server. Default: %default s'),
                       default=10.0, type='float', dest='timeout')
-    parser.add_option('-r', '--max-recursions', default=1, 
+    parser.add_option('-r', '--max-recursions', default=1,
                       help=_('Maximum number of levels to recurse i.e. depth of links to follow. Default %default'),
                       type='int', dest='max_recursions')
     parser.add_option('-n', '--max-files', default=sys.maxint, type='int', dest='max_files',
                       help=_('The maximum number of files to download. This only applies to files from <a href> tags. Default is %default'))
     parser.add_option('--delay', default=0, dest='delay', type='int',
                       help=_('Minimum interval in seconds between consecutive fetches. Default is %default s'))
-    parser.add_option('--encoding', default=None, 
+    parser.add_option('--encoding', default=None,
                       help=_('The character encoding for the websites you are trying to download. The default is to try and guess the encoding.'))
     parser.add_option('--match-regexp', default=[], action='append', dest='match_regexps',
                       help=_('Only links that match this regular expression will be followed. This option can be specified multiple times, in which case as long as a link matches any one regexp, it will be followed. By default all links are followed.'))
@@ -478,23 +476,21 @@ def option_parser(usage=_('%prog URL\n\nWhere URL is for example http://google.c
     return parser
 
 
-def create_fetcher(options, logger=None, image_map={}):
+def create_fetcher(options, image_map={}, log=None):
-    if logger is None:
+    if log is None:
-        level = logging.DEBUG if options.verbose else logging.INFO
+        log = Log()
-        logger = logging.getLogger('web2disk')
+    return RecursiveFetcher(options, log, image_map={})
-        setup_cli_handlers(logger, level)
-    return RecursiveFetcher(options, logger, image_map={})
 
 def main(args=sys.argv):
-    parser = option_parser()    
+    parser = option_parser()
     options, args = parser.parse_args(args)
     if len(args) != 2:
         parser.print_help()
         return 1
-    
-    fetcher = create_fetcher(options) 
-    fetcher.start_fetch(args[1])
-    
 
-if __name__ == '__main__':    
+    fetcher = create_fetcher(options)
+    fetcher.start_fetch(args[1])
+
+
+if __name__ == '__main__':
     sys.exit(main())