Merge from trunk

recipes/aksiyon_derigisi.recipe

@@ -20,6 +20,7 @@ class Aksiyon (BasicNewsRecipe):
     auto_cleanup = True
     cover_img_url = 'http://www.aksiyon.com.tr/aksiyon/images/aksiyon/top-page/aksiyon_top_r2_c1.jpg'
     masthead_url = 'http://aksiyon.com.tr/aksiyon/images/aksiyon/top-page/aksiyon_top_r2_c1.jpg'
+    ignore_duplicate_articles = { 'title', 'url' }
     remove_empty_feeds= True
     feeds          = [
                       ( u'KAPAK', u'http://www.aksiyon.com.tr/aksiyon/rss?sectionId=26'),

recipes/weblogs_sl.recipe

@@ -2,8 +2,8 @@
 __license__     = 'GPL v3'
 __copyright__   = '4 February 2011, desUBIKado'
 __author__      = 'desUBIKado'
-__version__     = 'v0.08'
+__version__     = 'v0.09'
-__date__        = '30, June 2012'
+__date__        = '02, December 2012'
 '''
 http://www.weblogssl.com/
 '''
@@ -37,6 +37,7 @@ class weblogssl(BasicNewsRecipe):
                           ,(u'Xataka Mexico', u'http://feeds.weblogssl.com/xatakamx')
                           ,(u'Xataka M\xf3vil', u'http://feeds.weblogssl.com/xatakamovil')
                           ,(u'Xataka Android', u'http://feeds.weblogssl.com/xatakandroid')
+                          ,(u'Xataka Windows', u'http://feeds.weblogssl.com/xatakawindows')
                           ,(u'Xataka Foto', u'http://feeds.weblogssl.com/xatakafoto')
                           ,(u'Xataka ON', u'http://feeds.weblogssl.com/xatakaon')
                           ,(u'Xataka Ciencia', u'http://feeds.weblogssl.com/xatakaciencia')
@@ -80,19 +81,31 @@ class weblogssl(BasicNewsRecipe):
 
     keep_only_tags     = [dict(name='div', attrs={'id':'infoblock'}),
                           dict(name='div', attrs={'class':'post'}),
-                          dict(name='div', attrs={'id':'blog-comments'})
+                          dict(name='div', attrs={'id':'blog-comments'}),
+                          dict(name='div', attrs={'class':'container'})   #m.xataka.com
                          ]
 
-    remove_tags        = [dict(name='div', attrs={'id':'comment-nav'})]
+    remove_tags        = [dict(name='div', attrs={'id':'comment-nav'}),
+                          dict(name='menu', attrs={'class':'social-sharing'}),    #m.xataka.com
+                          dict(name='section' , attrs={'class':'comments'}),      #m.xataka.com
+                          dict(name='div' , attrs={'class':'article-comments'}),  #m.xataka.com
+                          dict(name='nav' , attrs={'class':'article-taxonomy'})   #m.xataka.com
+			 ]
+
+    remove_tags_after  = dict(name='section' , attrs={'class':'comments'})
 
     def print_version(self, url):
           return url.replace('http://www.', 'http://m.')
 
     preprocess_regexps = [
                             # Para poner una linea en blanco entre un comentario y el siguiente
-                           (re.compile(r'<li id="c', re.DOTALL|re.IGNORECASE), lambda match: '<br><br><li id="c')
+                           (re.compile(r'<li id="c', re.DOTALL|re.IGNORECASE), lambda match: '<br><br><li id="c'),
+                            # Para ver las imágenes en las noticias de m.xataka.com
+                           (re.compile(r'<noscript>', re.DOTALL|re.IGNORECASE), lambda m: ''),
+                           (re.compile(r'</noscript>', re.DOTALL|re.IGNORECASE), lambda m: '')
                          ]
 
+
     # Para sustituir el video incrustado de YouTube por una imagen
 
     def preprocess_html(self, soup):
@@ -108,14 +121,16 @@ class weblogssl(BasicNewsRecipe):
 
     # Para obtener la url original del articulo a partir de la de "feedsportal"
     # El siguiente código es gracias al usuario "bosplans" de www.mobileread.com
-    # http://www.mobileread.com/forums/sho...d.php?t=130297
+    # http://www.mobileread.com/forums/showthread.php?t=130297
 
     def get_article_url(self, article):
        link = article.get('link', None)
        if link is None:
            return article
+       # if link.split('/')[-4]=="xataka2":
+       #     return article.get('feedburner_origlink', article.get('link', article.get('guid')))
        if link.split('/')[-4]=="xataka2":
-           return article.get('feedburner_origlink', article.get('link', article.get('guid')))
+           return article.get('guid', None)
        if link.split('/')[-1]=="story01.htm":
            link=link.split('/')[-2]
            a=['0B','0C','0D','0E','0F','0G','0N'  ,'0L0S','0A']

recipes/zaman.recipe

@@ -9,15 +9,15 @@ class Zaman (BasicNewsRecipe):
     __author__             = u'thomass'
     oldest_article         = 2
     max_articles_per_feed  =50
-   # no_stylesheets         = True
+    no_stylesheets         = True
     #delay                  = 1
-    #use_embedded_content   = False
+    use_embedded_content   = False
-    encoding               = 'ISO 8859-9'
+    encoding               = 'utf-8'
-    publisher              = 'Zaman'
+    publisher              = 'Feza Gazetecilik'
     category               = 'news, haberler,TR,gazete'
     language               = 'tr'
     publication_type = 'newspaper '
-    extra_css              = '.buyukbaslik{font-weight: bold; font-size: 18px;color:#0000FF}'#body{ font-family: Verdana,Helvetica,Arial,sans-serif } .introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} '
+    extra_css              = 'h1{text-transform: capitalize; font-weight: bold; font-size: 22px;color:#0000FF} p{text-align:justify} ' #.introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} '
     conversion_options = {
                             'tags'            : category
                             ,'language'        : language
@@ -26,25 +26,26 @@ class Zaman (BasicNewsRecipe):
                          }
     cover_img_url = 'https://fbcdn-profile-a.akamaihd.net/hprofile-ak-snc4/188140_81722291869_2111820_n.jpg'
     masthead_url = 'http://medya.zaman.com.tr/extentions/zaman.com.tr/img/section/logo-section.png'
+    ignore_duplicate_articles = { 'title', 'url' }
+    auto_cleanup = False
+    remove_empty_feeds= True
 
 
-    #keep_only_tags      = [dict(name='div', attrs={'id':[ 'news-detail-content']}), dict(name='td', attrs={'class':['columnist-detail','columnist_head']})  ]
+    #keep_only_tags      = [dict(name='div', attrs={'id':[ 'contentposition19']})]#,dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'news-detail-content']}), dict(name='td', attrs={'class':['columnist-detail','columnist_head']}),  ]
-    remove_tags = [ dict(name='img', attrs={'src':['http://medya.zaman.com.tr/zamantryeni/pics/zamanonline.gif']})]#,dict(name='div', attrs={'class':['radioEmbedBg','radyoProgramAdi']}),dict(name='a', attrs={'class':['webkit-html-attribute-value webkit-html-external-link']}),dict(name='table', attrs={'id':['yaziYorumTablosu']}),dict(name='img', attrs={'src':['http://medya.zaman.com.tr/pics/paylas.gif','http://medya.zaman.com.tr/extentions/zaman.com.tr/img/columnist/ma-16.png']})
+    remove_tags = [ dict(name='img', attrs={'src':['http://cmsmedya.zaman.com.tr/images/logo/logo.bmp']}),dict(name='hr', attrs={'class':['interactive-hr']})]#    remove_tags = [ dict(name='div', attrs={'class':[ 'detayUyari']}),dict(name='div', attrs={'class':[ 'detayYorum']}),dict(name='div', attrs={'class':[ 'addthis_toolbox addthis_default_style ']}),dict(name='div', attrs={'id':[ 'tumYazi']})]#,dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'xxx']}),dict(name='div', attrs={'id':[ 'xxx']}),dict(name='img', attrs={'src':['http://medya.zaman.com.tr/zamantryeni/pics/zamanonline.gif']}),dict(name='div', attrs={'class':['radioEmbedBg','radyoProgramAdi']}),dict(name='a', attrs={'class':['webkit-html-attribute-value webkit-html-external-link']}),dict(name='table', attrs={'id':['yaziYorumTablosu']}),dict(name='img', attrs={'src':['http://medya.zaman.com.tr/pics/paylas.gif','http://medya.zaman.com.tr/extentions/zaman.com.tr/img/columnist/ma-16.png']}),dict(name='div', attrs={'id':[ 'news-detail-gallery']}),dict(name='div', attrs={'id':[ 'news-detail-title-bottom-part']}),dict(name='div', attrs={'id':[ 'news-detail-news-paging-main']})]#
 
 
     #remove_attributes = ['width','height']
     remove_empty_feeds= True
 
     feeds          = [
-                      ( u'Anasayfa', u'http://www.zaman.com.tr/anasayfa.rss'),
+                      ( u'Manşet', u'http://www.zaman.com.tr/manset.rss'),
-                      ( u'Son Dakika', u'http://www.zaman.com.tr/sondakika.rss'),
-                      #( u'En çok Okunanlar', u'http://www.zaman.com.tr/max_all.rss'),
-                      #( u'Manşet', u'http://www.zaman.com.tr/manset.rss'),
-                      ( u'Gündem', u'http://www.zaman.com.tr/gundem.rss'),
                       ( u'Yazarlar', u'http://www.zaman.com.tr/yazarlar.rss'),
                       ( u'Politika', u'http://www.zaman.com.tr/politika.rss'),
                       ( u'Ekonomi', u'http://www.zaman.com.tr/ekonomi.rss'),
                       ( u'Dış Haberler', u'http://www.zaman.com.tr/dishaberler.rss'),
+                      ( u'Son Dakika', u'http://www.zaman.com.tr/sondakika.rss'),
+                      ( u'Gündem', u'http://www.zaman.com.tr/gundem.rss'),
                       ( u'Yorumlar', u'http://www.zaman.com.tr/yorumlar.rss'),
                       ( u'Röportaj', u'http://www.zaman.com.tr/roportaj.rss'),
                       ( u'Dizi Yazı', u'http://www.zaman.com.tr/dizi.rss'),
@@ -59,8 +60,9 @@ class Zaman (BasicNewsRecipe):
                       ( u'Cuma Eki', u'http://www.zaman.com.tr/cuma.rss'),
                       ( u'Cumaertesi Eki', u'http://www.zaman.com.tr/cumaertesi.rss'),
                       ( u'Pazar Eki', u'http://www.zaman.com.tr/pazar.rss'),
+                      ( u'En çok Okunanlar', u'http://www.zaman.com.tr/max_all.rss'),
+                      ( u'Anasayfa', u'http://www.zaman.com.tr/anasayfa.rss'),
 
                         ]
     def print_version(self, url):
-     return url.replace('http://www.zaman.com.tr/haber.do?haberno=', 'http://www.zaman.com.tr/yazdir.do?haberno=')
+     return url.replace('http://www.zaman.com.tr/newsDetail_getNewsById.action?newsId=', 'http://www.zaman.com.tr/newsDetail_openPrintPage.action?newsId=')
-

setup/installer/windows/__init__.py

@@ -39,18 +39,6 @@ class Win32(WinBase):
     def msi64(self):
         return installer_name('msi', is64bit=True)
 
-    def sign_msi(self):
-        import xattr
-        print ('Signing installers ...')
-        sign64 = False
-        msi64 = self.msi64
-        if os.path.exists(msi64) and 'user.signed' not in xattr.list(msi64):
-            subprocess.check_call(['scp', msi64, self.VM_NAME +
-                    ':build/%s/%s'%(__appname__, msi64)])
-            sign64 = True
-        subprocess.check_call(['ssh', self.VM_NAME, '~/sign.sh'], shell=False)
-        return sign64
-
     def do_dl(self, installer, errmsg):
         subprocess.check_call(('scp',
             '%s:build/%s/%s'%(self.VM_NAME, __appname__, installer), 'dist'))
@@ -62,14 +50,8 @@ class Win32(WinBase):
         installer = self.installer()
         if os.path.exists('build/winfrozen'):
             shutil.rmtree('build/winfrozen')
-        sign64 = self.sign_msi()
-        if sign64:
-            self.do_dl(self.msi64, 'Failed to d/l signed 64 bit installer')
-            import xattr
-            xattr.set(self.msi64, 'user.signed', 'true')
 
         self.do_dl(installer, 'Failed to freeze')
-
         installer = 'dist/%s-portable-installer-%s.exe'%(__appname__, __version__)
         self.do_dl(installer, 'Failed to get portable installer')
 

setup/installer/windows/freeze.py

@@ -91,6 +91,7 @@ class Win32Freeze(Command, WixMixIn):
         if not is64bit:
             self.build_portable()
             self.build_portable_installer()
+        self.sign_installers()
 
     def remove_CRT_from_manifests(self):
         '''
@@ -488,6 +489,17 @@ class Win32Freeze(Command, WixMixIn):
 
         subprocess.check_call([LZMA + r'\bin\elzma.exe', '-9', '--lzip', name])
 
+    def sign_installers(self):
+        self.info('Signing installers...')
+        files = glob.glob(self.j('dist', '*.msi')) + glob.glob(self.j('dist',
+                                                                      '*.exe'))
+        if not files:
+            raise ValueError('No installers found')
+        subprocess.check_call(['signtool.exe', 'sign', '/a', '/d',
+            'calibre - E-book management', '/du',
+            'http://calibre-ebook.com', '/t',
+            'http://timestamp.verisign.com/scripts/timstamp.dll'] + files)
+
     def add_dir_to_zip(self, zf, path, prefix=''):
         '''
         Add a directory recursively to the zip file with an optional prefix.

src/calibre/devices/kobo/driver.py

@@ -2357,6 +2357,8 @@ class KOBOTOUCH(KOBO):
         update_query = 'UPDATE content SET Series=?, SeriesNumber==? where BookID is Null and ContentID = ?'
         if book.series is None:
             update_values = (None, None, book.contentID, )
+        elif book.series_index is None:         # This should never happen, but...
+            update_values = (book.series, None, book.contentID, )
         else:
             update_values = (book.series, "%g"%book.series_index, book.contentID, )
 

src/calibre/ebooks/mobi/reader/markup.py

@@ -74,11 +74,12 @@ def remove_kindlegen_markup(parts):
         part = "".join(srcpieces)
         parts[i] = part
 
-    # we can safely remove all of the Kindlegen generated data-AmznPageBreak tags
+    # we can safely remove all of the Kindlegen generated data-AmznPageBreak
+    # attributes
     find_tag_with_AmznPageBreak_pattern = re.compile(
             r'''(<[^>]*\sdata-AmznPageBreak=[^>]*>)''', re.IGNORECASE)
     within_tag_AmznPageBreak_position_pattern = re.compile(
-            r'''\sdata-AmznPageBreak=['"][^'"]*['"]''')
+            r'''\sdata-AmznPageBreak=['"]([^'"]*)['"]''')
 
     for i in xrange(len(parts)):
         part = parts[i]
@@ -86,10 +87,8 @@ def remove_kindlegen_markup(parts):
         for j in range(len(srcpieces)):
             tag = srcpieces[j]
             if tag.startswith('<'):
-                for m in within_tag_AmznPageBreak_position_pattern.finditer(tag):
+                srcpieces[j] = within_tag_AmznPageBreak_position_pattern.sub(
-                    replacement = ''
+                    lambda m:' style="page-break-after:%s"'%m.group(1), tag)
-                    tag = within_tag_AmznPageBreak_position_pattern.sub(replacement, tag, 1)
-                srcpieces[j] = tag
         part = "".join(srcpieces)
         parts[i] = part
 

src/calibre/ebooks/mobi/reader/mobi8.py

@@ -44,6 +44,18 @@ def locate_beg_end_of_tag(ml, aid):
         return plt, pgt
     return 0, 0
 
+def reverse_tag_iter(block):
+    ''' Iterate over all tags in block in reverse order, i.e. last tag
+    to first tag. '''
+    end = len(block)
+    while True:
+        pgt = block.rfind(b'>', 0, end)
+        if pgt == -1: break
+        plt = block.rfind(b'<', 0, pgt)
+        if plt == -1: break
+        yield block[plt:pgt+1]
+        end = plt
+
 class Mobi8Reader(object):
 
     def __init__(self, mobi6_reader, log):
@@ -275,13 +287,12 @@ class Mobi8Reader(object):
         return '%s/%s'%(fi.type, fi.filename), idtext
 
     def get_id_tag(self, pos):
-        # find the correct tag by actually searching in the destination
+        # Find the first tag with a named anchor (name or id attribute) before
-        # textblock at position
+        # pos
         fi = self.get_file_info(pos)
         if fi.num is None and fi.start is None:
             raise ValueError('No file contains pos: %d'%pos)
         textblock = self.parts[fi.num]
-        id_map = []
         npos = pos - fi.start
         pgt = textblock.find(b'>', npos)
         plt = textblock.find(b'<', npos)
@@ -290,28 +301,15 @@ class Mobi8Reader(object):
         if plt == npos or pgt < plt:
             npos = pgt + 1
         textblock = textblock[0:npos]
-        # find id links only inside of tags
+        id_re = re.compile(br'''<[^>]+\sid\s*=\s*['"]([^'"]+)['"]''')
-        #    inside any < > pair find all "id=' and return whatever is inside
+        name_re = re.compile(br'''<\s*a\s*\sname\s*=\s*['"]([^'"]+)['"]''')
-        #    the quotes
+        for tag in reverse_tag_iter(textblock):
-        id_pattern = re.compile(br'''<[^>]*\sid\s*=\s*['"]([^'"]*)['"][^>]*>''',
+            m = id_re.match(tag) or name_re.match(tag)
-                re.IGNORECASE)
+            if m is not None:
-        for m in re.finditer(id_pattern, textblock):
+                return m.group(1)
-            id_map.append((m.start(), m.group(1)))
 
-        if not id_map:
+        # No tag found, link to start of file
-            # Found no id in the textblock, link must be to top of file
+        return b''
-            return b''
-        # if npos is before first id= inside a tag, return the first
-        if npos < id_map[0][0]:
-            return id_map[0][1]
-        # if npos is after the last id= inside a tag, return the last
-        if npos > id_map[-1][0]:
-            return id_map[-1][1]
-        # otherwise find last id before npos
-        for i, item in enumerate(id_map):
-            if npos < item[0]:
-                return id_map[i-1][1]
-        return id_map[0][1]
 
     def create_guide(self):
         guide = Guide()

src/calibre/ebooks/oeb/reader.py

@@ -320,13 +320,11 @@ class OEBReader(object):
                 self.logger.warn(u'Spine item %r not found' % idref)
                 continue
             item = manifest.ids[idref]
-            spine.add(item, elem.get('linear'))
+            if item.media_type.lower() in OEB_DOCS and hasattr(item.data, 'xpath'):
-        for item in spine:
+                spine.add(item, elem.get('linear'))
-            if item.media_type.lower() not in OEB_DOCS:
+            else:
-                if not hasattr(item.data, 'xpath'):
+                self.oeb.log.warn('The item %s is not a XML document.'
-                    self.oeb.log.warn('The item %s is not a XML document.'
+                        ' Removing it from spine.'%item.href)
-                            ' Removing it from spine.'%item.href)
-                    spine.remove(item)
         if len(spine) == 0:
             raise OEBError("Spine is empty")
         self._spine_add_extra()

src/calibre/ebooks/pdf/writer.py

@@ -27,10 +27,10 @@ def get_custom_size(opts):
     custom_size = None
     if opts.custom_size != None:
         width, sep, height = opts.custom_size.partition('x')
-        if height != '':
+        if height:
             try:
-                width = int(width)
+                width = float(width)
-                height = int(height)
+                height = float(height)
                 custom_size = (width, height)
             except:
                 custom_size = None

src/calibre/library/catalogs/csv_xml.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 
-__license__   = 'GPL v3'
+__license__ = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
@@ -12,6 +12,7 @@ from calibre.customize import CatalogPlugin
 from calibre.library.catalogs import FIELDS
 from calibre.customize.conversion import DummyReporter
 
+
 class CSV_XML(CatalogPlugin):
     'CSV/XML catalog generator'
 
@@ -22,27 +23,27 @@ class CSV_XML(CatalogPlugin):
     supported_platforms = ['windows', 'osx', 'linux']
     author = 'Greg Riker'
     version = (1, 0, 0)
-    file_types = set(['csv','xml'])
+    file_types = set(['csv', 'xml'])
 
     cli_options = [
             Option('--fields',
-                default = 'all',
+                default='all',
-                dest = 'fields',
+                dest='fields',
-                action = None,
+                action=None,
-                help = _('The fields to output when cataloging books in the '
+                help=_('The fields to output when cataloging books in the '
                     'database.  Should be a comma-separated list of fields.\n'
                     'Available fields: %(fields)s,\n'
                     'plus user-created custom fields.\n'
                     'Example: %(opt)s=title,authors,tags\n'
                     "Default: '%%default'\n"
-                    "Applies to: CSV, XML output formats")%dict(
+                    "Applies to: CSV, XML output formats") % dict(
                         fields=', '.join(FIELDS), opt='--fields')),
 
             Option('--sort-by',
-                default = 'id',
+                default='id',
-                dest = 'sort_by',
+                dest='sort_by',
-                action = None,
+                action=None,
-                help = _('Output field to sort on.\n'
+                help=_('Output field to sort on.\n'
                 'Available fields: author_sort, id, rating, size, timestamp, title_sort\n'
                 "Default: '%default'\n"
                 "Applies to: CSV, XML output formats"))]
@@ -97,7 +98,7 @@ class CSV_XML(CatalogPlugin):
             for entry in data:
                 entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice']
 
-        fm = {x:db.field_metadata.get(x, {}) for x in fields}
+        fm = {x: db.field_metadata.get(x, {}) for x in fields}
 
         if self.fmt == 'csv':
             outfile = codecs.open(path_to_output, 'w', 'utf8')
@@ -113,7 +114,7 @@ class CSV_XML(CatalogPlugin):
                 outstr = []
                 for field in fields:
                     if field.startswith('#'):
-                        item = db.get_field(entry['id'],field,index_is_id=True)
+                        item = db.get_field(entry['id'], field, index_is_id=True)
                     elif field == 'library_name':
                         item = current_library
                     elif field == 'title_sort':
@@ -129,7 +130,7 @@ class CSV_XML(CatalogPlugin):
                         for format in item:
                             fmt_list.append(format.rpartition('.')[2].lower())
                         item = ', '.join(fmt_list)
-                    elif field in ['authors','tags']:
+                    elif field in ['authors', 'tags']:
                         item = ', '.join(item)
                     elif field == 'isbn':
                         # Could be 9, 10 or 13 digits
@@ -137,20 +138,20 @@ class CSV_XML(CatalogPlugin):
                     elif field in ['pubdate', 'timestamp']:
                         item = isoformat(item)
                     elif field == 'comments':
-                        item = item.replace(u'\r\n',u' ')
+                        item = item.replace(u'\r\n', u' ')
-                        item = item.replace(u'\n',u' ')
+                        item = item.replace(u'\n', u' ')
                     elif fm.get(field, {}).get('datatype', None) == 'rating' and item:
-                        item = u'%.2g'%(item/2.0)
+                        item = u'%.2g' % (item / 2.0)
 
                     # Convert HTML to markdown text
                     if type(item) is unicode:
-                        opening_tag = re.search('<(\w+)(\x20|>)',item)
+                        opening_tag = re.search('<(\w+)(\x20|>)', item)
                         if opening_tag:
                             closing_tag = re.search('<\/%s>$' % opening_tag.group(1), item)
                             if closing_tag:
                                 item = html2text(item)
 
-                    outstr.append(u'"%s"' % unicode(item).replace('"','""'))
+                    outstr.append(u'"%s"' % unicode(item).replace('"', '""'))
 
                 outfile.write(u','.join(outstr) + u'\n')
             outfile.close()
@@ -165,14 +166,14 @@ class CSV_XML(CatalogPlugin):
 
                 for field in fields:
                     if field.startswith('#'):
-                        val = db.get_field(r['id'],field,index_is_id=True)
+                        val = db.get_field(r['id'], field, index_is_id=True)
                         if not isinstance(val, (str, unicode)):
                             val = unicode(val)
-                        item = getattr(E, field.replace('#','_'))(val)
+                        item = getattr(E, field.replace('#', '_'))(val)
                         record.append(item)
 
                 for field in ('id', 'uuid', 'publisher', 'rating', 'size',
-                              'isbn','ondevice', 'identifiers'):
+                              'isbn', 'ondevice', 'identifiers'):
                     if field in fields:
                         val = r[field]
                         if not val:
@@ -180,7 +181,7 @@ class CSV_XML(CatalogPlugin):
                         if not isinstance(val, (str, unicode)):
                             if (fm.get(field, {}).get('datatype', None) ==
                                     'rating' and val):
-                                val = u'%.2g'%(val/2.0)
+                                val = u'%.2g' % (val / 2.0)
                             val = unicode(val)
                         item = getattr(E, field)(val)
                         record.append(item)
@@ -227,4 +228,3 @@ class CSV_XML(CatalogPlugin):
             with open(path_to_output, 'w') as f:
                 f.write(etree.tostring(root, encoding='utf-8',
                     xml_declaration=True, pretty_print=True))
-

src/calibre/library/catalogs/epub_mobi.py

@@ -3,7 +3,7 @@
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
-__license__   = 'GPL v3'
+__license__ = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
@@ -21,6 +21,7 @@ from calibre.utils.localization import get_lang
 
 Option = namedtuple('Option', 'option, default, dest, action, help')
 
+
 class EPUB_MOBI(CatalogPlugin):
     'ePub catalog generator'
 
@@ -30,29 +31,29 @@ class EPUB_MOBI(CatalogPlugin):
     minimum_calibre_version = (0, 7, 40)
     author = 'Greg Riker'
     version = (1, 0, 0)
-    file_types = set(['azw3','epub','mobi'])
+    file_types = set(['azw3', 'epub', 'mobi'])
 
     THUMB_SMALLEST = "1.0"
     THUMB_LARGEST = "2.0"
 
-    cli_options = [Option('--catalog-title', # {{{
+    cli_options = [Option('--catalog-title',  # {{{
-                          default = 'My Books',
+                          default='My Books',
-                          dest = 'catalog_title',
+                          dest='catalog_title',
-                          action = None,
+                          action=None,
-                          help = _('Title of generated catalog used as title in metadata.\n'
+                          help=_('Title of generated catalog used as title in metadata.\n'
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--cross-reference-authors',
                           default=False,
                           dest='cross_reference_authors',
-                          action = 'store_true',
+                          action='store_true',
                           help=_("Create cross-references in Authors section for books with multiple authors.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--debug-pipeline',
                            default=None,
                            dest='debug_pipeline',
-                           action = None,
+                           action=None,
                            help=_("Save the output from different stages of the conversion "
                            "pipeline to the specified "
                            "directory. Useful if you are unsure at which stage "
@@ -62,7 +63,7 @@ class EPUB_MOBI(CatalogPlugin):
                    Option('--exclude-genre',
                           default='\[.+\]|^\+$',
                           dest='exclude_genre',
-                          action = None,
+                          action=None,
                           help=_("Regex describing tags to exclude as genres.\n"
                           "Default: '%default' excludes bracketed tags, e.g. '[Project Gutenberg]', and '+', the default tag for read books.\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
@@ -82,63 +83,63 @@ class EPUB_MOBI(CatalogPlugin):
                    Option('--generate-authors',
                           default=False,
                           dest='generate_authors',
-                          action = 'store_true',
+                          action='store_true',
                           help=_("Include 'Authors' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-descriptions',
                           default=False,
                           dest='generate_descriptions',
-                          action = 'store_true',
+                          action='store_true',
                           help=_("Include 'Descriptions' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-genres',
                           default=False,
                           dest='generate_genres',
-                          action = 'store_true',
+                          action='store_true',
                           help=_("Include 'Genres' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-titles',
                           default=False,
                           dest='generate_titles',
-                          action = 'store_true',
+                          action='store_true',
                           help=_("Include 'Titles' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-series',
                           default=False,
                           dest='generate_series',
-                          action = 'store_true',
+                          action='store_true',
                           help=_("Include 'Series' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--generate-recently-added',
                           default=False,
                           dest='generate_recently_added',
-                          action = 'store_true',
+                          action='store_true',
                           help=_("Include 'Recently Added' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--genre-source-field',
                           default='Tags',
                           dest='genre_source_field',
-                          action = None,
+                          action=None,
                           help=_("Source field for Genres section.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--header-note-source-field',
                           default='',
                           dest='header_note_source_field',
-                          action = None,
+                          action=None,
                           help=_("Custom field containing note text to insert in Description header.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--merge-comments-rule',
                           default='::',
                           dest='merge_comments_rule',
-                          action = None,
+                          action=None,
                           help=_("#<custom field>:[before|after]:[True|False] specifying:\n"
                           " <custom field> Custom field containing notes to merge with Comments\n"
                           " [before|after] Placement of notes with respect to Comments\n"
@@ -148,7 +149,7 @@ class EPUB_MOBI(CatalogPlugin):
                    Option('--output-profile',
                           default=None,
                           dest='output_profile',
-                          action = None,
+                          action=None,
                           help=_("Specifies the output profile.  In some cases, an output profile is required to optimize the catalog for the device.  For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
@@ -164,14 +165,14 @@ class EPUB_MOBI(CatalogPlugin):
                    Option('--use-existing-cover',
                           default=False,
                           dest='use_existing_cover',
-                          action = 'store_true',
+                          action='store_true',
                           help=_("Replace existing cover when generating the catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--thumb-width',
                           default='1.0',
                           dest='thumb_width',
-                          action = None,
+                          action=None,
                           help=_("Size hint (in inches) for book covers in catalog.\n"
                           "Range: 1.0 - 2.0\n"
                           "Default: '%default'\n"
@@ -199,7 +200,7 @@ class EPUB_MOBI(CatalogPlugin):
         if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower():
             opts.connected_kindle = True
             if opts.connected_device['serial'] and \
-               opts.connected_device['serial'][:4] in ['B004','B005']:
+               opts.connected_device['serial'][:4] in ['B004', 'B005']:
                 op = "kindle_dx"
             else:
                 op = "kindle"
@@ -209,7 +210,7 @@ class EPUB_MOBI(CatalogPlugin):
         opts.output_profile = op
 
         opts.basename = "Catalog"
-        opts.cli_environment = not hasattr(opts,'sync')
+        opts.cli_environment = not hasattr(opts, 'sync')
 
         # Hard-wired to always sort descriptions by author, with series after non-series
         opts.sort_descriptions_by_author = True
@@ -278,14 +279,14 @@ class EPUB_MOBI(CatalogPlugin):
                 opts.generate_genres = True
                 opts.generate_recently_added = True
                 opts.generate_descriptions = True
-                sections_list = ['Authors','Titles','Series','Genres','Recently Added','Descriptions']
+                sections_list = ['Authors', 'Titles', 'Series', 'Genres', 'Recently Added', 'Descriptions']
             else:
                 opts.log.warn('\n*** No enabled Sections, terminating catalog generation ***')
-                return ["No Included Sections","No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"]
+                return ["No Included Sections", "No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"]
         if opts.fmt == 'mobi' and sections_list == ['Descriptions']:
                 warning = _("\n*** Adding 'By Authors' Section required for MOBI output ***")
                 opts.log.warn(warning)
-                sections_list.insert(0,'Authors')
+                sections_list.insert(0, 'Authors')
                 opts.generate_authors = True
 
         opts.log(u" Sections: %s" % ', '.join(sections_list))
@@ -294,14 +295,14 @@ class EPUB_MOBI(CatalogPlugin):
         # Limit thumb_width to 1.0" - 2.0"
         try:
             if float(opts.thumb_width) < float(self.THUMB_SMALLEST):
-                log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST))
+                log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_SMALLEST))
                 opts.thumb_width = self.THUMB_SMALLEST
             if float(opts.thumb_width) > float(self.THUMB_LARGEST):
-                log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_LARGEST))
+                log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_LARGEST))
                 opts.thumb_width = self.THUMB_LARGEST
             opts.thumb_width = "%.2f" % float(opts.thumb_width)
         except:
-            log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST))
+            log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_SMALLEST))
             opts.thumb_width = "1.0"
 
         # eval prefix_rules if passed from command line
@@ -331,13 +332,13 @@ class EPUB_MOBI(CatalogPlugin):
         keys.sort()
         build_log.append(" opts:")
         for key in keys:
-            if key in ['catalog_title','author_clip','connected_kindle','creator',
+            if key in ['catalog_title', 'author_clip', 'connected_kindle', 'creator',
-                       'cross_reference_authors','description_clip','exclude_book_marker',
+                       'cross_reference_authors', 'description_clip', 'exclude_book_marker',
-                       'exclude_genre','exclude_tags','exclusion_rules', 'fmt',
+                       'exclude_genre', 'exclude_tags', 'exclusion_rules', 'fmt',
-                       'genre_source_field', 'header_note_source_field','merge_comments_rule',
+                       'genre_source_field', 'header_note_source_field', 'merge_comments_rule',
-                       'output_profile','prefix_rules','read_book_marker',
+                       'output_profile', 'prefix_rules', 'read_book_marker',
-                       'search_text','sort_by','sort_descriptions_by_author','sync',
+                       'search_text', 'sort_by', 'sort_descriptions_by_author', 'sync',
-                       'thumb_width','use_existing_cover','wishlist_tag']:
+                       'thumb_width', 'use_existing_cover', 'wishlist_tag']:
                 build_log.append("  %s: %s" % (key, repr(opts_dict[key])))
         if opts.verbose:
             log('\n'.join(line for line in build_log))
@@ -370,8 +371,8 @@ class EPUB_MOBI(CatalogPlugin):
             """
             GENERATE_DEBUG_EPUB = False
             if GENERATE_DEBUG_EPUB:
-                catalog_debug_path = os.path.join(os.path.expanduser('~'),'Desktop','Catalog debug')
+                catalog_debug_path = os.path.join(os.path.expanduser('~'), 'Desktop', 'Catalog debug')
-                setattr(opts,'debug_pipeline',os.path.expanduser(catalog_debug_path))
+                setattr(opts, 'debug_pipeline', os.path.expanduser(catalog_debug_path))
 
             dp = getattr(opts, 'debug_pipeline', None)
             if dp is not None:
@@ -381,11 +382,13 @@ class EPUB_MOBI(CatalogPlugin):
             if opts.output_profile and opts.output_profile.startswith("kindle"):
                 recommendations.append(('output_profile', opts.output_profile,
                     OptionRecommendation.HIGH))
-                recommendations.append(('book_producer',opts.output_profile,
+                recommendations.append(('book_producer', opts.output_profile,
                     OptionRecommendation.HIGH))
                 if opts.fmt == 'mobi':
                     recommendations.append(('no_inline_toc', True,
                         OptionRecommendation.HIGH))
+                    recommendations.append(('verbose', 2,
+                        OptionRecommendation.HIGH))
 
             # Use existing cover or generate new cover
             cpath = None
@@ -432,14 +435,13 @@ class EPUB_MOBI(CatalogPlugin):
                 from calibre.ebooks.epub import initialize_container
                 from calibre.ebooks.tweak import zip_rebuilder
                 from calibre.utils.zipfile import ZipFile
-                input_path = os.path.join(catalog_debug_path,'input')
+                input_path = os.path.join(catalog_debug_path, 'input')
-                epub_shell = os.path.join(catalog_debug_path,'epub_shell.zip')
+                epub_shell = os.path.join(catalog_debug_path, 'epub_shell.zip')
                 initialize_container(epub_shell, opf_name='content.opf')
                 with ZipFile(epub_shell, 'r') as zf:
                     zf.extractall(path=input_path)
                 os.remove(epub_shell)
-                zip_rebuilder(input_path, os.path.join(catalog_debug_path,'input.epub'))
+                zip_rebuilder(input_path, os.path.join(catalog_debug_path, 'input.epub'))
 
         # returns to gui2.actions.catalog:catalog_generated()
         return catalog.error
-