Merge from main branch

recipes/estadao.recipe

@@ -1,4 +1,3 @@
-#!/usr/bin/env  python
 from calibre.web.feeds.news import BasicNewsRecipe
 from datetime import datetime, timedelta
 from calibre.ebooks.BeautifulSoup import Tag,BeautifulSoup
@@ -6,18 +5,17 @@ from calibre.utils.magick import Image, PixelWand
 from urllib2 import Request, urlopen, URLError
 
 class Estadao(BasicNewsRecipe):
-    THUMBALIZR_API        = "0123456789abcdef01234567890" # ---->Get your at http://www.thumbalizr.com/
+    THUMBALIZR_API        = '' # ---->Get your at http://www.thumbalizr.com/ and put here
     LANGUAGE              = 'pt_br'
     language = 'pt'
     LANGHTM               = 'pt-br'
     ENCODING              = 'utf'
     ENCHTM                = 'utf-8'
     directionhtm          = 'ltr'
-    requires_version      = (0,8,47)
+    requires_version      = (0,7,47)
     news                  = True
-    publication_type      = 'newsportal'
 
-    title                 = u'Estadao'
+    title                 = u'Estad\xe3o'
     __author__            = 'Euler Alves'
     description           = u'Brazilian news from Estad\xe3o'
     publisher             = u'Estad\xe3o'
@@ -33,14 +31,6 @@ class Estadao(BasicNewsRecipe):
     remove_empty_feeds    = True
     timefmt               = ' [%d %b %Y (%a)]'
 
-    html2lrf_options      = [
-                            '--comment', description
-                            ,'--category', category
-                            ,'--publisher', publisher
-    ]
-
-    html2epub_options     = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
-
     hoje                  = datetime.now()-timedelta(days=2)
     pubdate               = hoje.strftime('%a, %d %b')
     if hoje.hour<10:
@@ -69,6 +59,7 @@ class Estadao(BasicNewsRecipe):
                     ,dict(name='script')
     ]
 
+
     feeds = [
     (u'\xDAltimas Not\xEDcias', u'http://www.estadao.com.br/rss/ultimas.xml')
     ,(u'Manchetes', u'http://www.estadao.com.br/rss/manchetes.xml')
@@ -109,6 +100,8 @@ class Estadao(BasicNewsRecipe):
             img.open(iurl)
             width, height = img.size
             print 'img is: ', iurl, 'width is: ', width, 'height is: ', height
+            if img < 0:
+                raise RuntimeError('Out of memory')
             pw = PixelWand()
             if( width > height and width > 590) :
                 print 'Rotate image'
@@ -117,6 +110,7 @@ class Estadao(BasicNewsRecipe):
         return soup
 
     def get_cover_url(self):
+        if self.THUMBALIZR_API:
             cover_url      = self.CAPA
             pedido         = Request(self.CAPA)
             pedido.add_header('User-agent','Mozilla/5.0 (Windows; U; Windows NT 5.1; '+self.LANGHTM+'; userid='+self.THUMBALIZR_API+') Calibre/0.8.47 (like Gecko)')
@@ -132,3 +126,4 @@ class Estadao(BasicNewsRecipe):
             except URLError:
                 cover_url='http://api.thumbalizr.com/?api_key='+self.THUMBALIZR_API+'&url='+self.SCREENSHOT+'&width=600&quality=90'
                 return cover_url
+

recipes/folhadesaopaulo.recipe

@@ -5,16 +5,15 @@ from calibre.utils.magick import Image, PixelWand
 from urllib2 import Request, urlopen, URLError
 
 class FolhaOnline(BasicNewsRecipe):
-    THUMBALIZR_API        = "0123456789abcdef01234567890" # ---->Get your at http://www.thumbalizr.com/
+    THUMBALIZR_API        = '' # ---->Get your at http://www.thumbalizr.com/ and put here
     LANGUAGE              = 'pt_br'
     language = 'pt'
     LANGHTM               = 'pt-br'
     ENCODING              = 'cp1252'
     ENCHTM                = 'iso-8859-1'
     directionhtm          = 'ltr'
-    requires_version      = (0,8,47)
+    requires_version      = (0,7,47)
     news                  = True
-    publication_type      = 'newsportal'
 
     title                 = u'Folha de S\xE3o Paulo'
     __author__            = 'Euler Alves'
@@ -96,6 +95,7 @@ class FolhaOnline(BasicNewsRecipe):
     ,(u'Valdo Cruz', u'http://http://feeds.folha.uol.com.br/folha/pensata/valdocruz/rss091.xml')
     ]
 
+
     conversion_options = {
     'title'            : title
     ,'comments'        : description
@@ -124,6 +124,8 @@ class FolhaOnline(BasicNewsRecipe):
             img.open(iurl)
             width, height = img.size
             print 'img is: ', iurl, 'width is: ', width, 'height is: ', height
+            if img < 0:
+                raise RuntimeError('Out of memory')
             pw = PixelWand()
             if( width > height and width > 590) :
                 print 'Rotate image'

recipes/lifehacker.recipe

@@ -1,37 +1,100 @@
-__license__   = 'GPL v3'
-__copyright__ = '2010, NA'
-'''
-lifehacker.com
-'''
-
 from calibre.web.feeds.news import BasicNewsRecipe
+from datetime import datetime
+from calibre.ebooks.BeautifulSoup import Tag
+from calibre.utils.magick import Image, PixelWand
 
-class Lifehacker(BasicNewsRecipe):
-    title                 = 'Lifehacker'
-    __author__            = 'Kovid Goyal'
-    description           = "Computers make us more productive. Yeah, right. Lifehacker recommends the software downloads and web sites that actually save time. Don't live to geek; geek to live."
-    publisher             = 'lifehacker.com'
-    category              = 'news, IT, Internet, gadgets, tips and tricks, howto, diy'
-    oldest_article        = 2
-    max_articles_per_feed = 100
-    no_stylesheets        = True
-    encoding              = 'utf-8'
-    use_embedded_content  = True
+class LifeHacker(BasicNewsRecipe):
+    THUMBALIZR_API        = '' # ---->Get your at http://www.thumbalizr.com/ and put here
+    LANGUAGE              = 'en'
+    LANGHTM               = 'en'
     language = 'en'
+    ENCODING              = 'utf'
+    ENCHTM                = 'utf-8'
+    requires_version      = (0,7,47)
+    news                  = True
+
+    title                 = u'LifeHacker'
+    __author__            = 'Euler Alves'
+    description           = u'Tips, tricks, and downloads for getting things done.'
+    publisher             = u'lifehacker.com'
+    author                = u'Adam Pash & Kevin Purdy & Adam Dachis & Whitson Gordon & Gina Trapani'
+    category              = 'news, rss'
+
+    oldest_article        = 4
+    max_articles_per_feed = 20
+    summary_length        = 1000
+
+    remove_javascript     = True
+    no_stylesheets        = True
+    use_embedded_content  = True
+    remove_empty_feeds    = True
+    timefmt               = ' [%d %b %Y (%a)]'
+
+    hoje                  = datetime.now()
+    pubdate               = hoje.strftime('%a, %d %b')
+    cover_url             = 'http://api.thumbalizr.com/?api_key='+THUMBALIZR_API+'&url=http://lifehacker.com&width=600&quality=90'
+    cover_margins         = (0,0,'white')
     masthead_url          = 'http://cache.gawkerassets.com/assets/lifehacker.com/img/logo.png'
-    conversion_options = {
-                          'comment'   : description
-                        , 'tags'      : category
-                        , 'publisher' : publisher
-                        , 'language'  : language
-                        }
 
     remove_tags = [
                     {'class': 'feedflare'},
+                    dict(name='div',
+                        attrs={'class':[
+                            'ad_container'
+                            ,'ad_300x250'
+                            ,'ad_interstitial'
+                            ,'share-wrap'
+                            ,'ad_300x600'
+                            ,'ad_perma-footer-adsense'
+                            ,'ad_perma-panorama'
+                            ,'ad panorama'
+                            ,'ad_container'
+                        ]})
+                    ,dict(name='div',
+                        attrs={'id':[
+                            'agegate_container'
+                            ,'agegate_container_rejected'
+                            ,'sharemenu-wrap'
+                        ]})
     ]
 
     feeds = [(u'Articles', u'http://feeds.gawker.com/lifehacker/vip?format=xml')]
 
-    def preprocess_html(self, soup):
-        return self.adeify_images(soup)
+    conversion_options = {
+    'title'            : title
+    ,'comments'        : description
+    ,'publisher'       : publisher
+    ,'tags'            : category
+    ,'language'        : LANGUAGE
+    ,'linearize_tables': True
+    }
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        if not soup.find(attrs={'http-equiv':'Content-Language'}):
+            meta0 = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.LANGHTM)])
+            soup.head.insert(0,meta0)
+        if not soup.find(attrs={'http-equiv':'Content-Type'}):
+            meta1 = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset="+self.ENCHTM)])
+            soup.head.insert(0,meta1)
+        return soup
+
+    def postprocess_html(self, soup, first):
+        #process all the images. assumes that the new html has the correct path
+        for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')):
+            iurl = tag['src']
+            img = Image()
+            img.open(iurl)
+            width, height = img.size
+            print 'img is: ', iurl, 'width is: ', width, 'height is: ', height
+            if img < 0:
+                raise RuntimeError('Out of memory')
+            pw = PixelWand()
+            if( width > height and width > 590) :
+                print 'Rotate image'
+                img.rotate(pw, -90)
+                img.save(iurl)
+        return soup
+
 

setup/upload.py

@@ -5,7 +5,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, re, cStringIO, base64, httplib, subprocess, hashlib, shutil, time, glob
+import os, re, cStringIO, base64, httplib, subprocess, hashlib, shutil, time, \
+    glob, stat
 from subprocess import check_call
 from tempfile import NamedTemporaryFile, mkdtemp
 from zipfile import ZipFile
@@ -344,6 +345,8 @@ class UploadUserManual(Command): # {{{
     def build_plugin_example(self, path):
         from calibre import CurrentDir
         with NamedTemporaryFile(suffix='.zip') as f:
+            os.fchmod(f.fileno(),
+                stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH|stat.S_IWRITE)
             with CurrentDir(self.d(path)):
                 with ZipFile(f, 'w') as zf:
                     for x in os.listdir('.'):
@@ -352,8 +355,8 @@ class UploadUserManual(Command): # {{{
                             for y in os.listdir(x):
                                 zf.write(os.path.join(x, y))
             bname = self.b(path) + '_plugin.zip'
-            subprocess.check_call(['scp', f.name, 'divok:%s/%s'%(DOWNLOADS,
-                bname)])
+            dest = '%s/%s'%(DOWNLOADS, bname)
+            subprocess.check_call(['scp', f.name, dest])
 
     def run(self, opts):
         path = self.j(self.SRC, 'calibre', 'manual', 'plugin_examples')

src/calibre/ebooks/chardet/__init__.py

@@ -110,4 +110,11 @@ def xml_to_unicode(raw, verbose=False, strip_encoding_pats=False,
     if resolve_entities:
         raw = substitute_entites(raw)
 
+    if encoding and encoding.lower().replace('_', '-').strip() in (
+            'gb2312', 'chinese', 'csiso58gb231280', 'euc-cn', 'euccn',
+            'eucgb2312-cn', 'gb2312-1980', 'gb2312-80', 'iso-ir-58'):
+        # Microsoft Word exports to HTML with encoding incorrectly set to
+        # gb2312 instead of gbk. gbk is a superset of gb2312, anyway.
+        encoding = 'gbk'
+
     return raw, encoding

src/calibre/ebooks/metadata/book/base.py

@@ -198,7 +198,9 @@ class Metadata(object):
         return copy.deepcopy(ans)
 
     def _clean_identifier(self, typ, val):
+        if typ:
             typ = icu_lower(typ).strip().replace(':', '').replace(',', '')
+        if val:
             val = val.strip().replace(',', '|').replace(':', '|')
         return typ, val
 

src/calibre/ebooks/metadata/sources/amazon.py

@@ -503,7 +503,7 @@ if __name__ == '__main__': # tests {{{
             (  # This isbn not on amazon
                 {'identifiers':{'isbn': '8324616489'}, 'title':'Learning Python',
                     'authors':['Lutz']},
-                [title_test('Learning Python: Powerful Object-Oriented Programming',
+                [title_test('Learning Python, 3rd Edition',
                     exact=True), authors_test(['Mark Lutz'])
                  ]
 

src/calibre/gui2/library/models.py

@@ -1132,7 +1132,7 @@ class DeviceBooksModel(BooksModel): # {{{
             self.sorted_map = list(self.map)
         else:
             self.sorted_map = list(range(len(self.db)))
-            self.sorted_map.sort(cmp=keygen, reverse=descending)
+            self.sorted_map.sort(key=keygen, reverse=descending)
         self.sorted_on = (self.column_map[col], order)
         self.sort_history.insert(0, self.sorted_on)
         if hasattr(keygen, 'db'):

src/calibre/utils/html2text.py

@@ -1,8 +1,14 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 """html2text: Turn HTML into equivalent Markdown-structured text."""
-__version__ = "2.39"
-__author__ = "Aaron Swartz (me@aaronsw.com)"
-__copyright__ = "(C) 2004-2008 Aaron Swartz. GNU GPL 3."
+# Last upstream version before changes
+#__version__ = "2.39"
+__license__ = 'GPL 3'
+__copyright__ = '''
+Copyright (c) 2011, John Schember <john@nachtimwald.com>
+(C) 2004-2008 Aaron Swartz <me@aaronsw.com>
+'''
 __contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"]
 
 # TODO:
@@ -11,7 +17,6 @@ __contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"]
 if not hasattr(__builtins__, 'True'): True, False = 1, 0
 import re, sys, urllib, htmlentitydefs, codecs
 import sgmllib
-import urlparse
 sgmllib.charref = re.compile('&#([xX]?[0-9a-fA-F]+)[^0-9a-fA-F]')
 
 try: from textwrap import wrap
@@ -145,9 +150,7 @@ class _html2text(sgmllib.SGMLParser):
         self.outcount = 0
         self.start = 1
         self.space = 0
-        self.a = []
         self.astack = []
-        self.acount = 0
         self.list = []
         self.blockquote = 0
         self.pre = 0
@@ -181,29 +184,6 @@ class _html2text(sgmllib.SGMLParser):
     def unknown_endtag(self, tag):
         self.handle_tag(tag, None, 0)
 
-    def previousIndex(self, attrs):
-        """ returns the index of certain set of attributes (of a link) in the
-            self.a list
-
-            If the set of attributes is not found, returns None
-        """
-        if not attrs.has_key('href'): return None
-
-        i = -1
-        for a in self.a:
-            i += 1
-            match = 0
-
-            if a.has_key('href') and a['href'] == attrs['href']:
-                if a.has_key('title') or attrs.has_key('title'):
-                        if (a.has_key('title') and attrs.has_key('title') and
-                            a['title'] == attrs['title']):
-                            match = True
-                else:
-                    match = True
-
-            if match: return i
-
     def handle_tag(self, tag, attrs, start):
         attrs = fixattrs(attrs)
 
@@ -268,34 +248,23 @@ class _html2text(sgmllib.SGMLParser):
                 if self.astack:
                     a = self.astack.pop()
                     if a:
-                        i = self.previousIndex(a)
-                        if i is not None:
-                            a = self.a[i]
-                        else:
-                            self.acount += 1
-                            a['count'] = self.acount
-                            a['outcount'] = self.outcount
-                            self.a.append(a)
-                        self.o("][" + `a['count']` + "]")
+                        title = ''
+                        if a.has_key('title'):
+                            title = ' "%s"' % a['title']
+                        self.o('](%s%s)' % (a['href'], title))
 
         if tag == "img" and start:
             attrsD = {}
             for (x, y) in attrs: attrsD[x] = y
             attrs = attrsD
             if attrs.has_key('src'):
-                attrs['href'] = attrs['src']
                 alt = attrs.get('alt', '')
-                i = self.previousIndex(attrs)
-                if i is not None:
-                    attrs = self.a[i]
-                else:
-                    self.acount += 1
-                    attrs['count'] = self.acount
-                    attrs['outcount'] = self.outcount
-                    self.a.append(attrs)
                 self.o("![")
                 self.o(alt)
-                self.o("]["+`attrs['count']`+"]")
+                title = ''
+                if attrs.has_key('title'):
+                    title = ' "%s"' % attrs['title']
+                self.o('](%s%s)' % (attrs['src'], title))
 
         if tag == 'dl' and start: self.p()
         if tag == 'dt' and not start: self.pbr()
@@ -373,7 +342,6 @@ class _html2text(sgmllib.SGMLParser):
                 self.out("\n")
                 self.space = 0
 
-
             if self.p_p:
                 self.out(('\n'+bq)*self.p_p)
                 self.space = 0
@@ -382,22 +350,6 @@ class _html2text(sgmllib.SGMLParser):
                 if not self.lastWasNL: self.out(' ')
                 self.space = 0
 
-            if self.a and ((self.p_p == 2 and LINKS_EACH_PARAGRAPH) or force == "end"):
-                if force == "end": self.out("\n")
-
-                newa = []
-                for link in self.a:
-                    if self.outcount > link['outcount']:
-                        self.out("   ["+`link['count']`+"]: " + urlparse.urljoin(self.baseurl, link['href']))
-                        if link.has_key('title'): self.out(" ("+link['title']+")")
-                        self.out("\n")
-                    else:
-                        newa.append(link)
-
-                if self.a != newa: self.out("\n") # Don't need an extra line when nothing was done.
-
-                self.a = newa
-
             if self.abbr_list and force == "end":
                 for abbr, definition in self.abbr_list.items():
                     self.out("  *[" + abbr + "]: " + definition + "\n")