Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-08 02:34:06 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge

Browse Source
This commit is contained in:
Kolenka 2012-06-04 08:36:01 -07:00
commit a46213b696
4 changed files with 188 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
recipes/chr_mon.recipe
View File

@ -1,152 +1,111 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__author__ = 'Kovid Goyal and Sujata Raman, Lorenzo Vigentini'
__copyright__ = '2009, Kovid Goyal and Sujata Raman'
__version__ = 'v1.02'
__date__ = '10, January 2010'
__description__ = 'Providing context and clarity on national and international news, peoples and cultures'
__copyright__ = '2012, Darko Miletic <darko.miletic at gmail.com>'
'''
www.csmonitor.com
'''
'''csmonitor.com'''
import re
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup
class ChristianScienceMonitor(BasicNewsRecipe):
__author__ = 'Kovid Goyal'
description = 'Providing context and clarity on national and international news, peoples and cultures'
cover_url = 'http://www.csmonitor.com/extension/csm_base/design/csm_design/images/csmlogo_179x46.gif'
title = 'Christian Science Monitor'
class CSMonitor(BasicNewsRecipe):
title = 'The Christian Science Monitor - daily'
__author__ = 'Darko Miletic'
description = 'The Christian Science Monitor is an international news organization that delivers thoughtful, global coverage via its website, weekly magazine, daily news briefing, and email newsletters.'
publisher = 'The Christian Science Monitor'
category = 'News, politics, culture, economy, general interest'
language = 'en'
encoding = 'utf-8'
timefmt = '[%a, %d %b, %Y]'
oldest_article = 16
max_articles_per_feed = 20
use_embedded_content = False
recursion = 10
remove_javascript = True
category = 'news, politics, USA'
oldest_article = 2
max_articles_per_feed = 200
no_stylesheets = True
requires_version = (0, 8, 39)
encoding = 'utf8'
use_embedded_content = False
language = 'en'
remove_empty_feeds = True
publication_type = 'newspaper'
masthead_url = 'http://www.csmonitor.com/extension/csm_base/design/csm_design/images/csmlogo_179x46.gif'
extra_css = """
body{font-family: Arial,Tahoma,Verdana,Helvetica,sans-serif }
img{margin-bottom: 0.4em; display:block}
.head {font-family: Georgia,"Times New Roman",Times,serif}
.sByline,.caption{font-size: x-small}
.hide{display: none}
.sLoc{font-weight: bold}
ul{list-style-type: none}
"""
def preprocess_raw_html(self, raw, url):
try:
from html5lib import parse
root = parse(raw, namespaceHTMLElements=False,
treebuilder='lxml').getroot()
from lxml import etree
for tag in root.xpath(
'//script|//style|//noscript|//meta|//link|//object'):
tag.getparent().remove(tag)
for elem in list(root.iterdescendants(tag=etree.Comment)):
elem.getparent().remove(elem)
ans = etree.tostring(root, encoding=unicode)
ans = re.sub('.*<html', '<html', ans, flags=re.DOTALL)
return ans
except:
import traceback
traceback.print_exc()
raise
def index_to_soup(self, url):
raw = BasicNewsRecipe.index_to_soup(self, url,
raw=True).decode('utf-8')
raw = self.preprocess_raw_html(raw, url)
return BasicNewsRecipe.index_to_soup(self, raw)
def append_page(self, soup, appendtag, position):
nav = soup.find('div',attrs={'class':'navigation'})
if nav:
pager = nav.findAll('a')
for part in pager:
if 'Next' in part:
nexturl = ('http://www.csmonitor.com' +
re.findall(r'href="(.*?)"', str(part))[0])
soup2 = self.index_to_soup(nexturl)
texttag = soup2.find('div',
attrs={'class': re.compile('list-article-.*')})
trash_c = soup2.findAll(attrs={'class': 'list-description'})
trash_h = soup2.h1
for tc in trash_c: tc.extract()
trash_h.extract()
newpos = len(texttag.contents)
self.append_page(soup2, texttag, newpos)
texttag.extract()
appendtag.insert(position, texttag)
def preprocess_html(self, soup):
PRINT_RE = re.compile(r'/layout/set/print/content/view/print/[0-9]*')
html = str(soup)
try:
print_found = PRINT_RE.findall(html)
except Exception:
pass
if print_found:
print_url = 'http://www.csmonitor.com' + print_found[0]
print_soup = self.index_to_soup(print_url)
else:
self.append_page(soup, soup.body, 3)
trash_a = soup.findAll(attrs={'class': re.compile('navigation.*')})
trash_b = soup.findAll(attrs={'style': re.compile('.*')})
trash_d = soup.findAll(attrs={'class': 'sByline'})
for ta in trash_a: ta.extract()
for tb in trash_b: tb.extract()
for td in trash_d: td.extract()
print_soup = soup
return print_soup
extra_css = '''
h1{ color:#000000;font-family: Georgia,Times,"Times New Roman",serif; font-size: large}
.sub{ color:#000000;font-family: Georgia,Times,"Times New Roman",serif; font-size: small;}
.byline{ font-family:Arial,Helvetica,sans-serif ; color:#999999; font-size: x-small;}
.postdate{color:#999999 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
h3{color:#999999 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
.photoCutline{ color:#333333 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
.photoCredit{ color:#999999 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
#story{font-family:Arial,Tahoma,Verdana,Helvetica,sans-serif ; font-size: small; }
#main{font-family:Arial,Tahoma,Verdana,Helvetica,sans-serif ; font-size: small; }
#photo-details{ font-family:Arial,Helvetica,sans-serif ; color:#999999; font-size: x-small;}
span.name{color:#205B87;font-family: Georgia,Times,"Times New Roman",serif; font-size: x-small}
p#dateline{color:#444444 ; font-family:Arial,Helvetica,sans-serif ; font-style:italic;} '''
feeds = [(u'Top Stories', u'http://rss.csmonitor.com/feeds/top'),
(u'World' , u'http://rss.csmonitor.com/feeds/world'),
(u'USA' , u'http://rss.csmonitor.com/feeds/usa'),
(u'Commentary' , u'http://rss.csmonitor.com/feeds/commentary'),
(u'Money' , u'http://rss.csmonitor.com/feeds/wam'),
(u'Learning' , u'http://rss.csmonitor.com/feeds/learning'),
(u'Living', u'http://rss.csmonitor.com/feeds/living'),
(u'Innovation', u'http://rss.csmonitor.com/feeds/scitech'),
(u'Gardening', u'http://rss.csmonitor.com/feeds/gardening'),
(u'Environment',u'http://rss.csmonitor.com/feeds/environment'),
(u'Arts', u'http://rss.csmonitor.com/feeds/arts'),
(u'Books', u'http://rss.csmonitor.com/feeds/books'),
(u'Home Forum' , u'http://rss.csmonitor.com/feeds/homeforum')
]
keep_only_tags = [dict(name='div', attrs={'id':'mainColumn'}), ]
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
}
remove_tags = [
dict(name='div', attrs={'id':['story-tools','videoPlayer','storyRelatedBottom','enlarge-photo','photo-paginate']}),
dict(name=['div','a'], attrs={'class':
['storyToolbar cfx','podStoryRel','spacer3',
'divvy spacer7','comment','storyIncludeBottom',
'hide', 'podBrdr']}),
dict(name='ul', attrs={'class':[ 'centerliststories']}) ,
dict(name='form', attrs={'id':[ 'commentform']}) ,
dict(name='div', attrs={'class': ['ui-comments']})
dict(name=['meta','link','iframe','object','embed'])
,dict(attrs={'class':['podStoryRel','bottom-rel','hide']})
,dict(attrs={'id':['pgallerycarousel_enlarge','pgallerycarousel_related']})
]
keep_only_tags = [
dict(name='h1', attrs={'class':'head'})
,dict(name='h2', attrs={'class':'subhead'})
,dict(attrs={'class':['sByline','podStoryGal','ui-body-header','sBody']})
]
remove_attributes=['xmlns:fb']
feeds = [
(u'USA' , u'http://rss.csmonitor.com/feeds/usa' )
,(u'World' , u'http://rss.csmonitor.com/feeds/world' )
,(u'Politics' , u'http://rss.csmonitor.com/feeds/politics' )
,(u'Business' , u'http://rss.csmonitor.com/feeds/wam' )
,(u'Commentary' , u'http://rss.csmonitor.com/feeds/commentary' )
,(u'Books' , u'http://rss.csmonitor.com/feeds/books' )
,(u'Arts' , u'http://rss.csmonitor.com/feeds/arts' )
,(u'Environment' , u'http://rss.csmonitor.com/feeds/environment')
,(u'Innovation' , u'http://rss.csmonitor.com/feeds/scitech' )
,(u'Living' , u'http://rss.csmonitor.com/feeds/living' )
,(u'Science' , u'http://rss.csmonitor.com/feeds/science' )
,(u'The Culture' , u'http://rss.csmonitor.com/feeds/theculture' )
,(u'The Home Forum', u'http://rss.csmonitor.com/feeds/homeforum' )
,(u'Articles' , u'http://rss.csmonitor.com/feeds/csarticles' )
]
remove_tags_after = [ dict(name='div', attrs={'class':[ 'ad csmAd']}),
dict(name='div', attrs={'class': [re.compile('navigation.*')]}),
dict(name='div', attrs={'style': [re.compile('.*')]})
]
def append_page(self, soup):
pager = soup.find('div', attrs={'class':'navigation'})
if pager:
nexttag = pager.find(attrs={'id':'next-button'})
if nexttag:
nurl = 'http://www.csmonitor.com' + nexttag['href']
soup2 = self.index_to_soup(nurl)
texttag = soup2.find(attrs={'class':'sBody'})
if texttag:
appendtag = soup.find(attrs={'class':'sBody'})
for citem in texttag.findAll(attrs={'class':['podStoryRel','bottom-rel','hide']}):
citem.extract()
self.append_page(soup2)
texttag.extract()
pager.extract()
appendtag.append(texttag)
def preprocess_html(self, soup):
self.append_page(soup)
pager = soup.find('div', attrs={'class':'navigation'})
if pager:
pager.extract()
for item in soup.findAll('a'):
limg = item.find('img')
if item.string is not None:
str = item.string
item.replaceWith(str)
else:
if limg:
item.name = 'div'
item.attrs = []
else:
str = self.tag_to_string(item)
item.replaceWith(str)
for item in soup.findAll('img'):
if 'scorecardresearch' in item['src']:
item.extract()
else:
if not item.has_key('alt'):
item['alt'] = 'image'
return soup

100
recipes/elpais_impreso.recipe
View File

@ -1,5 +1,6 @@
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
__license__ = 'GPL v3'
__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2010-2012, Darko Miletic <darko.miletic at gmail.com>'
'''
www.elpais.com
'''
@ -7,23 +8,24 @@ www.elpais.com
from calibre.web.feeds.news import BasicNewsRecipe
class ElPais_RSS(BasicNewsRecipe):
title = 'El Pais'
title = u'El País'
__author__ = 'Darko Miletic'
description = 'el periodico global en Castellano'
description = u'Noticias de última hora sobre la actualidad en España y el mundo: política, economía, deportes, cultura, sociedad, tecnología, gente, opinión, viajes, moda, televisión, los blogs y las firmas de EL PAÍS. Además especiales, vídeos, fotos, audios, gráficos, entrevistas, promociones y todos los servicios de EL PAÍS.'
publisher = 'EDICIONES EL PAIS, S.L.'
category = 'news, politics, finances, world, spain'
oldest_article = 2
max_articles_per_feed = 200
no_stylesheets = True
encoding = 'cp1252'
encoding = 'utf8'
use_embedded_content = False
language = 'es'
remove_empty_feeds = True
publication_type = 'newspaper'
masthead_url = 'http://www.elpais.com/im/tit_logo.gif'
masthead_url = 'http://ep01.epimg.net/iconos/v1.x/v1.0/logos/cabecera_portada.png'
extra_css = """
body{font-family: Georgia,"Times New Roman",Times,serif }
h3{font-family: Arial,Helvetica,sans-serif}
h1{font-family: Georgia,"Times New Roman",Times,serif }
#subtitulo_noticia, .firma, .figcaption{font-size: small}
body{font-family: Arial,Helvetica,Garuda,sans-serif}
img{margin-bottom: 0.4em; display:block}
"""
@ -34,49 +36,61 @@ class ElPais_RSS(BasicNewsRecipe):
, 'language' : language
}
keep_only_tags = [dict(attrs={'class':['cabecera_noticia estirar','cabecera_noticia','','contenido_noticia']})]
keep_only_tags = [
dict(attrs={'id':['titulo_noticia','subtitulo_noticia']})
,dict(attrs={'class':['firma','columna_texto','entrevista_p_r']})
]
remove_tags = [
dict(name=['meta','link','base','iframe','embed','object'])
,dict(attrs={'class':['info_complementa','estructura_2col_der','votos estirar','votos']})
,dict(attrs={'id':'utilidades'})
,dict(attrs={'class':'disposicion_vertical'})
]
remove_tags_after = dict(attrs={'id':'utilidades'})
remove_attributes = ['lang','border','width','height']
feeds = [
(u'Lo ultimo' , u'http://www.elpais.com/rss/feed.html?feedId=17046')
,(u'America Latina' , u'http://www.elpais.com/rss/feed.html?feedId=17041')
,(u'Mexico' , u'http://www.elpais.com/rss/feed.html?feedId=17042')
,(u'Europa' , u'http://www.elpais.com/rss/feed.html?feedId=17043')
,(u'Estados Unidos' , u'http://www.elpais.com/rss/feed.html?feedId=17044')
,(u'Oriente proximo' , u'http://www.elpais.com/rss/feed.html?feedId=17045')
,(u'Espana' , u'http://www.elpais.com/rss/feed.html?feedId=1002' )
,(u'Andalucia' , u'http://www.elpais.com/rss/feed.html?feedId=17057')
,(u'Catalunia' , u'http://www.elpais.com/rss/feed.html?feedId=17059')
,(u'Comunidad Valenciana' , u'http://www.elpais.com/rss/feed.html?feedId=17061')
,(u'Madrid' , u'http://www.elpais.com/rss/feed.html?feedId=1016' )
,(u'Pais Vasco' , u'http://www.elpais.com/rss/feed.html?feedId=17062')
,(u'Galicia' , u'http://www.elpais.com/rss/feed.html?feedId=17063')
,(u'Opinion' , u'http://www.elpais.com/rss/feed.html?feedId=1003' )
,(u'Sociedad' , u'http://www.elpais.com/rss/feed.html?feedId=1004' )
,(u'Deportes' , u'http://www.elpais.com/rss/feed.html?feedId=1007' )
,(u'Cultura' , u'http://www.elpais.com/rss/feed.html?feedId=1008' )
,(u'Cine' , u'http://www.elpais.com/rss/feed.html?feedId=17052')
,(u'Literatura' , u'http://www.elpais.com/rss/feed.html?feedId=17053')
,(u'Musica' , u'http://www.elpais.com/rss/feed.html?feedId=17051')
,(u'Arte' , u'http://www.elpais.com/rss/feed.html?feedId=17060')
,(u'Tecnologia' , u'http://www.elpais.com/rss/feed.html?feedId=1005' )
,(u'Economia' , u'http://www.elpais.com/rss/feed.html?feedId=1006' )
,(u'Ciencia' , u'http://www.elpais.com/rss/feed.html?feedId=17068')
,(u'Salud' , u'http://www.elpais.com/rss/feed.html?feedId=17074')
,(u'Ocio' , u'http://www.elpais.com/rss/feed.html?feedId=17075')
,(u'Justicia y Leyes' , u'http://www.elpais.com/rss/feed.html?feedId=17069')
,(u'Guerras y conflictos' , u'http://www.elpais.com/rss/feed.html?feedId=17070')
,(u'Politica' , u'http://www.elpais.com/rss/feed.html?feedId=17073')
(u'Lo ultimo' , u'http://ep00.epimg.net/rss/tags/ultimas_noticias.xml')
,(u'America Latina' , u'http://elpais.com/tag/rss/latinoamerica/a/' )
,(u'Mexico' , u'http://elpais.com/tag/rss/mexico/a/' )
,(u'Europa' , u'http://elpais.com/tag/rss/europa/a/' )
,(u'Estados Unidos' , u'http://elpais.com/tag/rss/estados_unidos/a/' )
,(u'Oriente proximo' , u'http://elpais.com/tag/rss/oriente_proximo/a/' )
,(u'Andalucia' , u'http://ep00.epimg.net/rss/ccaa/andalucia.xml' )
,(u'Catalunia' , u'http://ep00.epimg.net/rss/ccaa/catalunya.xml' )
,(u'Comunidad Valenciana' , u'http://ep00.epimg.net/rss/ccaa/valencia.xml' )
,(u'Madrid' , u'http://ep00.epimg.net/rss/ccaa/madrid.xml' )
,(u'Pais Vasco' , u'http://ep00.epimg.net/rss/ccaa/paisvasco.xml' )
,(u'Galicia' , u'http://ep00.epimg.net/rss/ccaa/galicia.xml' )
,(u'Sociedad' , u'http://ep00.epimg.net/rss/sociedad/portada.xml' )
,(u'Deportes' , u'http://ep00.epimg.net/rss/deportes/portada.xml' )
,(u'Cultura' , u'http://ep00.epimg.net/rss/cultura/portada.xml' )
,(u'Cine' , u'http://elpais.com/tag/rss/cine/a/' )
,(u'Economía' , u'http://elpais.com/tag/rss/economia/a/' )
,(u'Literatura' , u'http://elpais.com/tag/rss/libros/a/' )
,(u'Musica' , u'http://elpais.com/tag/rss/musica/a/' )
,(u'Arte' , u'http://elpais.com/tag/rss/arte/a/' )
,(u'Medio Ambiente' , u'http://elpais.com/tag/rss/medio_ambiente/a/' )
,(u'Tecnologia' , u'http://ep01.epimg.net/rss/tecnologia/portada.xml' )
,(u'Ciencia' , u'http://ep00.epimg.net/rss/tags/c_ciencia.xml' )
,(u'Salud' , u'http://elpais.com/tag/rss/salud/a/' )
,(u'Ocio' , u'http://elpais.com/tag/rss/ocio/a/' )
,(u'Justicia y Leyes' , u'http://elpais.com/tag/rss/justicia/a/' )
,(u'Guerras y conflictos' , u'http://elpais.com/tag/rss/conflictos/a/' )
,(u'Politica' , u'http://ep00.epimg.net/rss/politica/portada.xml' )
,(u'Opinion' , u'http://ep01.epimg.net/rss/politica/opinion.xml' )
]
def print_version(self, url):
return url + '?print=1'
def get_article_url(self, article):
url = BasicNewsRecipe.get_article_url(self, article)
if url and (not('/album/' in url) and not('/futbol/partido/' in url)):
return url
self.log('Skipping non-article', url)
return None
def get_cover_url(self):
soup = self.index_to_soup('http://elpais.com/')
for image in soup.findAll('img'):
if image['src'].endswith('elpaisTodayMiddle.jpg'):
sstr = image['src']
return sstr.replace('elpaisTodayMiddle.jpg', 'elpaisToday.jpg')
return None
def preprocess_html(self, soup):
for item in soup.findAll(style=True):

3
src/calibre/devices/usbms/driver.py
View File

@ -382,7 +382,8 @@ class USBMS(CLI, Device):
os.makedirs(self.normalize_path(self._main_prefix))
def write_prefix(prefix, listid):
if prefix is not None and isinstance(booklists[listid], self.booklist_class):
if (prefix is not None and len(booklists) > listid and
isinstance(booklists[listid], self.booklist_class)):
if not os.path.exists(prefix):
os.makedirs(self.normalize_path(prefix))
with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f: