Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

New recipe for Nature News by Krittika Goyal

Browse Source
This commit is contained in:
Kovid Goyal 2010-01-06 19:17:01 -07:00
parent 5a2c64d922
commit c395dc8be7
3 changed files with 401 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
resources/recipes/freenature.recipe Normal file
View File

@ -0,0 +1,44 @@
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup
import re
class NatureNews(BasicNewsRecipe):
title = u'Nature News'
language = 'en'
__author__ = 'Krittika Goyal'
oldest_article = 31 #days
max_articles_per_feed = 50
#encoding = 'latin1'
no_stylesheets = True
remove_tags_before = dict(name='h1', attrs={'class':'heading entry-title'})
remove_tags_after = dict(name='h2', attrs={'id':'comments'})
remove_tags = [
#dict(name='iframe'),
#dict(name='div', attrs={'class':['pt-box-title', 'pt-box-content']}),
#dict(name='div', attrs={'id':['block-td_search_160', 'block-cam_search_160']}),
dict(name='h2', attrs={'id':'comments'}),
dict(name='ul', attrs={'class':'toolsmenu xoxo'}),
]
preprocess_regexps = [
(re.compile(r'<script.*?</script>', re.DOTALL), lambda m: '')
]
feeds = [('Nature News', 'http://feeds.nature.com/news/rss/most_recent')]
def get_article_url(self, article):
return article.get('id')
#def preprocess_html(self, soup):
#story = soup.find(name='div', attrs={'id':'contentColumn'})
#td = heading.findParent(name='td')
#td.extract()
#soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
#body = soup.find(name='body')
#body.insert(0, story)
#for x in soup.findAll(name='p', text=lambda x:x and '--&gt;' in x):
#p = x.findParent('p')
#if p is not None:
#p.extract()
#return soup

65
src/calibre/ebooks/pdf/reflow.py
View File

@ -32,8 +32,31 @@ class Text(object):
self.color = self.font.color self.color = self.font.color
self.font_family = self.font.family self.font_family = self.font.family
text.tail = ''
self.text_as_string = etree.tostring(text, method='text', self.text_as_string = etree.tostring(text, method='text',
encoding=unicode) encoding=unicode)
self.raw = text.text if text.text else u''
for x in text.iterchildren():
self.raw += etree.tostring(x, method='xml', encoding=unicode)
if x.tail:
self.raw += x.tail
self.average_character_width = self.width/len(self.text_as_string)
def coalesce(self, other, page_number):
if self.opts.verbose > 2:
self.log.debug('Coalescing %r with %r on page %d'%(self.text_as_string,
other.text_as_string, page_number))
self.top = min(self.top, other.top)
self.right = other.right
self.width = self.right - self.left
self.bottom = max(self.bottom, other.bottom)
self.height = self.bottom - self.top
self.font_size = max(self.font_size, other.font_size)
self.font = other.font if self.font_size == other.font_size else other.font
self.text_as_string += other.text_as_string
self.raw += other.raw
self.average_character_width = (self.average_character_width +
other.average_character_width)/2.0
class FontSizeStats(dict): class FontSizeStats(dict):
@ -110,6 +133,15 @@ class HorizontalBox(object):
class Page(object): class Page(object):
# Fraction of a character width that two strings have to be apart,
# for them to be considered part of the same text fragment
COALESCE_FACTOR = 0.5
# Fraction of text height that two strings' bottoms can differ by
# for them to be considered to be part of the same text fragment
LINE_FACTOR = 0.4
def __init__(self, page, font_map, opts, log): def __init__(self, page, font_map, opts, log):
self.opts, self.log = opts, log self.opts, self.log = opts, log
self.font_map = font_map self.font_map = font_map
@ -123,6 +155,7 @@ class Page(object):
for text in page.xpath('descendant::text'): for text in page.xpath('descendant::text'):
self.texts.append(Text(text, self.font_map, self.opts, self.log)) self.texts.append(Text(text, self.font_map, self.opts, self.log))
text = self.texts[-1]
self.left_margin = min(text.left, self.left_margin) self.left_margin = min(text.left, self.left_margin)
self.right_margin = max(text.right, self.right_margin) self.right_margin = max(text.right, self.right_margin)
@ -136,7 +169,32 @@ class Page(object):
self.font_size_stats = FontSizeStats(self.font_size_stats) self.font_size_stats = FontSizeStats(self.font_size_stats)
self.identify_columns() self.coalesce_fragments()
#self.identify_columns()
def coalesce_fragments(self):
def find_match(frag):
for t in self.texts:
hdelta = t.left - frag.right
hoverlap = self.COALESCE_FACTOR * frag.average_character_width
if t is not frag and hdelta > -hoverlap and \
hdelta < hoverlap and \
abs(t.bottom - frag.bottom) < self.LINE_FACTOR*frag.height:
return t
match_found = True
while match_found:
match_found, match = False, None
for frag in self.texts:
match = find_match(frag)
if match is not None:
match_found = True
frag.coalesce(match, self.number)
break
if match is not None:
self.texts.remove(match)
def sort_into_horizontal_boxes(self, document_font_size_stats): def sort_into_horizontal_boxes(self, document_font_size_stats):
self.horizontal_boxes = [] self.horizontal_boxes = []
@ -190,7 +248,7 @@ class PDFDocument(object):
self.fonts = [] self.fonts = []
self.font_map = {} self.font_map = {}
for spec in self.root.xpath('//fonts'): for spec in self.root.xpath('//font'):
self.fonts.append(Font(spec)) self.fonts.append(Font(spec))
self.font_map[self.fonts[-1].id] = self.fonts[-1] self.font_map[self.fonts[-1].id] = self.fonts[-1]
@ -210,7 +268,8 @@ class PDFDocument(object):
def collect_font_statistics(self): def collect_font_statistics(self):
self.font_size_stats = {} self.font_size_stats = {}
for p in self.pages: for p in self.pages:
for sz, chars in p.font_size_stats: for sz in p.font_size_stats:
chars = p.font_size_stats[sz]
if sz not in self.font_size_stats: if sz not in self.font_size_stats:
self.font_size_stats[sz] = 0 self.font_size_stats[sz] = 0
self.font_size_stats[sz] += chars self.font_size_stats[sz] += chars

559
src/calibre/translations/calibre.pot
View File

File diff suppressed because it is too large Load Diff