Sync to trunk.

resources/recipes/entrepeneur.recipe

@@ -1,23 +1,19 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
 
-class EntrepeneurRecipe(BasicNewsRecipe):
+class EntrepeneurMagRecipe(BasicNewsRecipe):
     __license__  = 'GPL v3'
     __author__ = 'kwetal'
     language = 'en'
     version = 1
 
-    title = u'Entrepeneur'
+    title = u'Entrepeneur Magazine'
     publisher = u'Entrepreneur Media, Inc'
     category = u'Business'
     description = u'Online magazine on business, small business, management and economy'
 
-    oldest_article = 21
-    max_articles_per_feed = 100
-    use_embedded_content = False
     encoding = 'utf-8'
 
-    remove_empty_feeds = True
     no_stylesheets = True
     remove_javascript = True
 
@@ -30,15 +26,7 @@ class EntrepeneurRecipe(BasicNewsRecipe):
 
     remove_attributes = ['style']
 
-    # feeds from http://www.entrepreneur.com/feeds/index.html
+    INDEX = 'http://www.entrepeneur.com'
-    feeds = []
-    feeds.append((u'The Latest Headlines', u'http://feeds.feedburner.com/entrepreneur/latest'))
-    feeds.append((u'Starting a Business', u'http://feeds.feedburner.com/entrepreneur/startingabusiness.rss'))
-    feeds.append((u'Grow Your Business', u'http://feeds.feedburner.com/entrepreneur/growingyourbusiness.rss'))
-    feeds.append((u'Sales and Marketing', u'http://feeds.feedburner.com/entrepreneur/salesandmarketing'))
-    feeds.append((u'Online Business', u'http://feeds.feedburner.com/entrepreneur/ebiz'))
-    feeds.append((u'Franchises', u'http://feeds.feedburner.com/entrepreneur/franchises'))
-    #feeds.append((u'', u''))
 
     extra_css = '''
                 body{font-family:verdana,arial,helvetica,geneva,sans-serif;}
@@ -52,11 +40,41 @@ class EntrepeneurRecipe(BasicNewsRecipe):
                     margin-bottom: 0em;}
                 '''
 
-    def get_article_url(self, article):
+    conversion_options = {'comments': description, 'language': 'en',
-        return article.feedburner_origlink
+                          'publisher': publisher}
+
+    def parse_index(self):
+        soup = self.index_to_soup('http://www.entrepreneur.com/magazine/entrepreneur/index.html')
+        answer = []
+
+        div = soup.find('div', attrs = {'id': 'magfeature'})
+        if div:
+            a = div.find('a', attrs = {'class': 'headline'})
+            if a:
+                title = self.tag_to_string(a)
+                url = self.INDEX + a['href']
+                description = a.findNextSibling(text = True)
+
+                articles = [{'title': title, 'date': None, 'url': url, 'description': description}]
+                answer.append(('Cover Story', articles))
+
+        for div in soup.findAll('div', attrs = {'class': 'subhead-special'}):
+            articles = []
+            for tag in div.findNextSiblings(['a', 'div'], attrs = {'class': ['h2', 'subhead-special']}):
+                if tag.name == 'div' and tag['class'] == 'subhead-special':
+                    break
+                if tag.name == 'a' and tag['class'] == 'h2':
+                    title = self.tag_to_string(tag)
+                    url = tag['href']
+                    description = tag.findNextSibling(text = True)
+                    articles.append({'title': title, 'date': None, 'url': url, 'description': description})
+            answer.append((self.tag_to_string(div), articles))
+
+        return answer
+
 
     def print_version(self, url):
-        id = url.rpartition('/')[2].replace('article', '')
+        id = url.rpartition('/')[2]
 
         return 'http://www.entrepreneur.com/article/printthis/' + id
 

resources/recipes/freenature.recipe

@@ -0,0 +1,44 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
+import re
+
+class NatureNews(BasicNewsRecipe):
+    title          = u'Nature News'
+    language       = 'en'
+    __author__     = 'Krittika Goyal'
+    oldest_article = 31 #days
+    max_articles_per_feed = 50
+    #encoding = 'latin1'
+
+    no_stylesheets = True
+    remove_tags_before = dict(name='h1', attrs={'class':'heading entry-title'})
+    remove_tags_after  = dict(name='h2', attrs={'id':'comments'})
+    remove_tags = [
+       #dict(name='iframe'),
+       #dict(name='div', attrs={'class':['pt-box-title', 'pt-box-content']}),
+       #dict(name='div', attrs={'id':['block-td_search_160', 'block-cam_search_160']}),
+       dict(name='h2', attrs={'id':'comments'}),
+       dict(name='ul', attrs={'class':'toolsmenu xoxo'}),	
+    ]
+
+    preprocess_regexps = [
+      (re.compile(r'<script.*?</script>', re.DOTALL), lambda m: '')
+    ]
+
+    feeds = [('Nature News', 'http://feeds.nature.com/news/rss/most_recent')]
+        
+    def get_article_url(self, article):
+        return article.get('id')
+
+    #def preprocess_html(self, soup):
+        #story = soup.find(name='div', attrs={'id':'contentColumn'})
+        #td = heading.findParent(name='td')
+        #td.extract()
+        #soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
+        #body = soup.find(name='body')
+        #body.insert(0, story)
+        #for x in soup.findAll(name='p', text=lambda x:x and '--&gt;' in x):
+             #p = x.findParent('p')
+             #if p is not None:
+                  #p.extract()
+        #return soup

src/calibre/ebooks/pdf/reflow.py

@@ -32,8 +32,31 @@ class Text(object):
         self.color = self.font.color
         self.font_family = self.font.family
 
+        text.tail = ''
         self.text_as_string = etree.tostring(text, method='text',
                 encoding=unicode)
+        self.raw = text.text if text.text else u''
+        for x in text.iterchildren():
+            self.raw += etree.tostring(x, method='xml', encoding=unicode)
+            if x.tail:
+                self.raw += x.tail
+        self.average_character_width = self.width/len(self.text_as_string)
+
+    def coalesce(self, other, page_number):
+        if self.opts.verbose > 2:
+            self.log.debug('Coalescing %r with %r on page %d'%(self.text_as_string,
+                other.text_as_string, page_number))
+        self.top = min(self.top, other.top)
+        self.right = other.right
+        self.width = self.right - self.left
+        self.bottom = max(self.bottom, other.bottom)
+        self.height = self.bottom - self.top
+        self.font_size = max(self.font_size, other.font_size)
+        self.font = other.font if self.font_size == other.font_size else other.font
+        self.text_as_string += other.text_as_string
+        self.raw += other.raw
+        self.average_character_width = (self.average_character_width +
+                other.average_character_width)/2.0
 
 class FontSizeStats(dict):
 
@@ -110,6 +133,15 @@ class HorizontalBox(object):
 
 class Page(object):
 
+    # Fraction of a character width that two strings have to be apart,
+    # for them to be considered part of the same text fragment
+    COALESCE_FACTOR = 0.5
+
+    # Fraction of text height that two strings' bottoms can differ by
+    # for them to be considered to be part of the same text fragment
+    LINE_FACTOR = 0.4
+
+
     def __init__(self, page, font_map, opts, log):
         self.opts, self.log = opts, log
         self.font_map = font_map
@@ -123,6 +155,7 @@ class Page(object):
 
         for text in page.xpath('descendant::text'):
             self.texts.append(Text(text, self.font_map, self.opts, self.log))
+            text = self.texts[-1]
             self.left_margin = min(text.left, self.left_margin)
             self.right_margin = max(text.right, self.right_margin)
 
@@ -136,7 +169,32 @@ class Page(object):
 
         self.font_size_stats = FontSizeStats(self.font_size_stats)
 
-        self.identify_columns()
+        self.coalesce_fragments()
+
+        #self.identify_columns()
+
+    def coalesce_fragments(self):
+
+        def find_match(frag):
+            for t in self.texts:
+                hdelta = t.left - frag.right
+                hoverlap = self.COALESCE_FACTOR * frag.average_character_width
+                if t is not frag and hdelta > -hoverlap and \
+                    hdelta < hoverlap and \
+                    abs(t.bottom - frag.bottom) < self.LINE_FACTOR*frag.height:
+                        return t
+
+        match_found = True
+        while match_found:
+            match_found, match = False, None
+            for frag in self.texts:
+                match = find_match(frag)
+                if match is not None:
+                    match_found = True
+                    frag.coalesce(match, self.number)
+                    break
+            if match is not None:
+                self.texts.remove(match)
 
     def sort_into_horizontal_boxes(self, document_font_size_stats):
         self.horizontal_boxes = []
@@ -190,7 +248,7 @@ class PDFDocument(object):
         self.fonts = []
         self.font_map = {}
 
-        for spec in self.root.xpath('//fonts'):
+        for spec in self.root.xpath('//font'):
             self.fonts.append(Font(spec))
             self.font_map[self.fonts[-1].id] = self.fonts[-1]
 
@@ -210,7 +268,8 @@ class PDFDocument(object):
     def collect_font_statistics(self):
         self.font_size_stats = {}
         for p in self.pages:
-            for sz, chars in p.font_size_stats:
+            for sz in p.font_size_stats:
+                chars = p.font_size_stats[sz]
                 if sz not in self.font_size_stats:
                     self.font_size_stats[sz] = 0
                 self.font_size_stats[sz] += chars