Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

merge with John's branch

Browse Source
This commit is contained in:
Tomasz Długosz 2011-07-10 12:43:00 +02:00
commit d09610d89e
180 changed files with 70634 additions and 60242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
Changelog.yaml
View File

@ -19,12 +19,92 @@
# new recipes:
# - title:
- version: 0.8.8
date: 2011-07-01
new features:
- title: "Make author names in the Book Details panel clickable. Clicking them takes you to the wikipedia page for the author by default. You may have to tell calibre to display author names in the Book details panel first via Preferences->Look & Feel->Book details. You can change the link for individual authors by right clicking on the author's name in the Tag Browser and selecting Manage Authors."
- title: "Get Books: Add 'Open Books' as an available book source"
- title: "Get Books: When a free download is available for a search result, for example, for public domain books, allow direct download of the book into your calibre library."
- title: "Support for detecting and mounting reader devices on FreeBSD."
tickets: [802708]
- title: "When creating a composite custom column, allow the use of HTML to create links and other markup that display in the Book details panel"
- title: "Add the swap_around_comma function to the template language."
- title: "Drivers for HTC G2, Advent Vega, iRiver Story HD, Lark FreeMe and Moovyman mp7"
- title: "Quick View: Survives changing libraries. Also allow sorting by series index as well as name."
- title: "Connect to iTunes: Add an option to control how the driver works depending on whether you have iTunes setup to copy files to its media directory or not. Set this option by customizing the Apple driver in Preferences->Plugins. Having iTunes copy media to its storage folder is no longer neccessary. See http://www.mobileread.com/forums/showthread.php?t=118559 for details"
- title: "Remove the delete library functionality from calibre, instead you can now remove a library, so calibre will forget about it, but you have to delete the files manually"
bug fixes:
- title: "Fix a regression introduced in 0.8.7 in the Tag Browser that could cause calibre to crash after performing various actions"
- title: "Fix an unhandled error when deleting all saved searches"
tickets: [804383]
- title: "Fix row numbers in a previous selection being incorrect after a sort operation."
- title: "Fix ISBN identifier type not recognized if it is in upper case"
tickets: [802288]
- title: "Fix a regression in 0.8.7 that broke reading metadata from MOBI files in the Edit metadata dialog."
tickets: [801981]
- title: "Fix handling of filenames that have an even number of periods before the file extension."
tickets: [801939]
- title: "Fix lack of thread saefty in template format system, that could lead to incorrect template evaluation in some cases."
tickets: [801944]
- title: "Fix conversion to PDB when the input document has no text"
tickets: [801888]
- title: "Fix clicking on first letter of author names generating incorrect search."
- title: "Also fix updating bulk metadata in custom column causing unnneccessary Tag Browser refreshes."
- title: "Fix a regression in 0.8.7 that broke renaming items via the Tag Browser"
- title: "Fix a regression in 0.8.7 that caused the regex builder wizard to fail with LIT files as the input"
improved recipes:
- Zaman Gazetesi
- Infobae
- El Cronista
- Critica de la Argentina
- Buenos Aires Economico
- El Universal (Venezuela)
- wprost
- Financial Times UK
new recipes:
- title: "Today's Zaman by thomass"
- title: "Athens News by Darko Miletic"
- title: "Catholic News Agency"
author: Jetkey
- title: "Arizona Republic"
author: Jim Olo
- title: "Add Ming Pao Vancouver and Toronto"
author: Eddie Lau
- version: 0.8.7
date: 2011-06-24
new features:
- title: "Connect to iTunes: You now need to tell iTunes to keep its own copy of every ebook. Do this in iTunes by going to Preferences->Advanced and setting the 'Copy files to iTunes Media folder when adding to library' option. To learn about why this is necessary, see: http://www.mobileread.com/forums/showthread.php?t=140260"
type: major
- title: "Add a couple of date related functions to the calibre template langauge to get 'todays' date and create text based on the value of a date type field"

68
recipes/arizona_republic.recipe Normal file
View File

@ -0,0 +1,68 @@
__license__ = 'GPL v3'
__copyright__ = '2010, jolo'
'''
azrepublic.com
'''
from calibre.web.feeds.recipes import BasicNewsRecipe
class AdvancedUserRecipe1307301031(BasicNewsRecipe):
title = u'AZRepublic'
__author__ = 'Jim Olo'
language = 'en'
description = "The Arizona Republic is Arizona's leading provider of news and information, and has published a daily newspaper in Phoenix for more than 110 years"
publisher = 'AZRepublic/AZCentral'
masthead_url = 'http://freedom2t.com/wp-content/uploads/press_az_republic_v2.gif'
cover_url = 'http://www.valleyleadership.org/Common/Img/2line4c_AZRepublic%20with%20azcentral%20logo.jpg'
category = 'news, politics, USA, AZ, Arizona'
oldest_article = 7
max_articles_per_feed = 100
remove_empty_feeds = True
no_stylesheets = True
remove_javascript = True
# extra_css = '.headline {font-size: medium;} \n .fact { padding-top: 10pt }'
extra_css = ' body{ font-family: Verdana,Helvetica,Arial,sans-serif } .headline {font-size: medium} .introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} '
remove_attributes = ['width','height','h2','subHeadline','style']
remove_tags = [
dict(name='div', attrs={'id':['slidingBillboard', 'top728x90', 'subindex-header', 'topSearch']}),
dict(name='div', attrs={'id':['simplesearch', 'azcLoginBox', 'azcLoginBoxInner', 'topNav']}),
dict(name='div', attrs={'id':['carsDrop', 'homesDrop', 'rentalsDrop', 'classifiedDrop']}),
dict(name='div', attrs={'id':['nav', 'mp', 'subnav', 'jobsDrop']}),
dict(name='h6', attrs={'class':['section-header']}),
dict(name='a', attrs={'href':['#comments']}),
dict(name='div', attrs={'class':['articletools clearfix', 'floatRight']}),
dict(name='div', attrs={'id':['fbFrame', 'ob', 'storyComments', 'storyGoogleAdBox']}),
dict(name='div', attrs={'id':['storyTopHomes', 'openRight', 'footerwrap', 'copyright']}),
dict(name='div', attrs={'id':['blogsHed', 'blog_comments', 'blogByline','blogTopics']}),
dict(name='div', attrs={'id':['membersRightMain', 'dealsfooter', 'azrTopHed', 'azrRightCol']}),
dict(name='div', attrs={'id':['ttdHeader', 'ttdTimeWeather']}),
dict(name='div', attrs={'id':['membersRightMain', 'deals-header-wrap']}),
dict(name='div', attrs={'id':['todoTopSearchBar', 'byline clearfix', 'subdex-topnav']}),
dict(name='h1', attrs={'id':['SEOtext']}),
dict(name='table', attrs={'class':['ap-mediabox-table']}),
dict(name='p', attrs={'class':['ap_para']}),
dict(name='span', attrs={'class':['source-org vcard', 'org fn']}),
dict(name='a', attrs={'href':['http://hosted2.ap.org/APDEFAULT/privacy']}),
dict(name='a', attrs={'href':['http://hosted2.ap.org/APDEFAULT/terms']}),
dict(name='div', attrs={'id':['onespot_nextclick']}),
]
feeds = [
(u'FrontPage', u'http://www.azcentral.com/rss/feeds/republicfront.xml'),
(u'TopUS-News', u'http://hosted.ap.org/lineups/USHEADS.rss?SITE=AZPHG&SECTION=HOME'),
(u'WorldNews', u'http://hosted.ap.org/lineups/WORLDHEADS.rss?SITE=AZPHG&SECTION=HOME'),
(u'TopBusiness', u'http://hosted.ap.org/lineups/BUSINESSHEADS.rss?SITE=AZPHG&SECTION=HOME'),
(u'Entertainment', u'http://hosted.ap.org/lineups/ENTERTAINMENT.rss?SITE=AZPHG&SECTION=HOME'),
(u'ArizonaNews', u'http://www.azcentral.com/rss/feeds/news.xml'),
(u'Gilbert', u'http://www.azcentral.com/rss/feeds/gilbert.xml'),
(u'Chandler', u'http://www.azcentral.com/rss/feeds/chandler.xml'),
(u'DiningReviews', u'http://www.azcentral.com/rss/feeds/diningreviews.xml'),
(u'AZBusiness', u'http://www.azcentral.com/rss/feeds/business.xml'),
(u'ArizonaDeals', u'http://www.azcentral.com/members/Blog%7E/RealDealsblog'),
(u'GroceryDeals', u'http://www.azcentral.com/members/Blog%7E/RealDealsblog/tag/2646')
]

70
recipes/athens_news.recipe Normal file
View File

@ -0,0 +1,70 @@
__license__ = 'GPL v3'
__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
'''
www.athensnews.gr
'''
from calibre.web.feeds.news import BasicNewsRecipe
class AthensNews(BasicNewsRecipe):
title = 'Athens News'
__author__ = 'Darko Miletic'
description = 'Greece in English since 1952'
publisher = 'NEP Publishing Company SA'
category = 'news, politics, Greece, Athens'
oldest_article = 1
max_articles_per_feed = 200
no_stylesheets = True
encoding = 'utf8'
use_embedded_content = False
language = 'en_GR'
remove_empty_feeds = True
publication_type = 'newspaper'
masthead_url = 'http://www.athensnews.gr/sites/athensnews/themes/athensnewsv3/images/logo.jpg'
extra_css = """
body{font-family: Arial,Helvetica,sans-serif }
img{margin-bottom: 0.4em; display:block}
.big{font-size: xx-large; font-family: Georgia,serif}
.articlepubdate{font-size: small; color: gray; font-family: Georgia,serif}
.lezanta{font-size: x-small; font-weight: bold; text-align: left; margin-bottom: 1em; display: block}
"""
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
, 'linearize_tables' : True
}
remove_tags = [
dict(name=['meta','link'])
]
keep_only_tags=[
dict(name='span',attrs={'class':'big'})
,dict(name='td', attrs={'class':['articlepubdate','text']})
]
remove_attributes=['lang']
feeds = [
(u'News' , u'http://www.athensnews.gr/category/1/feed' )
,(u'Politics' , u'http://www.athensnews.gr/category/8/feed' )
,(u'Business' , u'http://www.athensnews.gr/category/2/feed' )
,(u'Economy' , u'http://www.athensnews.gr/category/11/feed')
,(u'Community' , u'http://www.athensnews.gr/category/5/feed' )
,(u'Arts' , u'http://www.athensnews.gr/category/3/feed' )
,(u'Living in Athens', u'http://www.athensnews.gr/category/7/feed' )
,(u'Sports' , u'http://www.athensnews.gr/category/4/feed' )
,(u'Travel' , u'http://www.athensnews.gr/category/6/feed' )
,(u'Letters' , u'http://www.athensnews.gr/category/44/feed')
,(u'Media' , u'http://www.athensnews.gr/multimedia/feed' )
]
def print_version(self, url):
return url + '?action=print'
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
return soup

87
recipes/buenosaireseconomico.recipe
View File

@ -1,72 +1,59 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2009-2011, Darko Miletic <darko.miletic at gmail.com>'
'''
elargentino.com
www.diariobae.com
'''
from calibre import strftime
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import Tag
class BsAsEconomico(BasicNewsRecipe):
title = 'Buenos Aires Economico'
__author__ = 'Darko Miletic'
description = 'Revista Argentina'
publisher = 'ElArgentino.com'
description = 'Diario BAE es el diario economico-politico con mas influencia en la Argentina. Fuente de empresarios y politicos del pais y el exterior. El pozo estaria aportando en periodos breves un volumen equivalente a 800m3 diarios. Pero todavia deben efectuarse otras perforaciones adicionales.'
publisher = 'Diario BAE'
category = 'news, politics, economy, Argentina'
oldest_article = 2
max_articles_per_feed = 100
no_stylesheets = True
use_embedded_content = False
encoding = 'utf-8'
language = 'es_AR'
language = 'es_AR'
cover_url = strftime('http://www.diariobae.com/imgs_portadas/%Y%m%d_portadasBAE.jpg')
masthead_url = 'http://www.diariobae.com/img/logo_bae.png'
remove_empty_feeds = True
publication_type = 'newspaper'
extra_css = """
body{font-family: Georgia,"Times New Roman",Times,serif}
#titulo{font-size: x-large}
#epi{font-size: small; font-style: italic; font-weight: bold}
img{display: block; margin-top: 1em}
"""
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
}
lang = 'es-AR'
direction = 'ltr'
INDEX = 'http://www.elargentino.com/medios/121/Buenos-Aires-Economico.html'
extra_css = ' .titulo{font-size: x-large; font-weight: bold} .volantaImp{font-size: small; font-weight: bold} '
html2lrf_options = [
'--comment' , description
, '--category' , category
, '--publisher', publisher
remove_tags_before= dict(attrs={'id':'titulo'})
remove_tags_after = dict(attrs={'id':'autor' })
remove_tags = [
dict(name=['meta','base','iframe','link','lang'])
,dict(attrs={'id':'barra_tw'})
]
remove_attributes = ['data-count','data-via']
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0cm; margin-top: 0em; margin-bottom: 0.5em} "'
keep_only_tags = [dict(name='div', attrs={'class':'ContainerPop'})]
remove_tags = [dict(name='link')]
feeds = [(u'Articulos', u'http://www.elargentino.com/Highlights.aspx?ParentType=Section&ParentId=121&Content-Type=text/xml&ChannelDesc=Buenos%20Aires%20Econ%C3%B3mico')]
def print_version(self, url):
main, sep, article_part = url.partition('/nota-')
article_id, rsep, rrest = article_part.partition('-')
return u'http://www.elargentino.com/Impresion.aspx?Id=' + article_id
feeds = [
(u'Argentina' , u'http://www.diariobae.com/rss/argentina.xml' )
,(u'Valores' , u'http://www.diariobae.com/rss/valores.xml' )
,(u'Finanzas' , u'http://www.diariobae.com/rss/finanzas.xml' )
,(u'Negocios' , u'http://www.diariobae.com/rss/negocios.xml' )
,(u'Mundo' , u'http://www.diariobae.com/rss/mundo.xml' )
,(u'5 dias' , u'http://www.diariobae.com/rss/5dias.xml' )
,(u'Espectaculos', u'http://www.diariobae.com/rss/espectaculos.xml')
]
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
soup.html['lang'] = self.lang
soup.html['dir' ] = self.direction
mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)])
mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=utf-8")])
soup.head.insert(0,mlang)
soup.head.insert(1,mcharset)
return soup
def get_cover_url(self):
cover_url = None
soup = self.index_to_soup(self.INDEX)
cover_item = soup.find('div',attrs={'class':'colder'})
if cover_item:
clean_url = self.image_url_processor(None,cover_item.div.img['src'])
cover_url = 'http://www.elargentino.com' + clean_url + '&height=600'
return cover_url
def image_url_processor(self, baseurl, url):
base, sep, rest = url.rpartition('?Id=')
img, sep2, rrest = rest.partition('&')
return base + sep + img

13
recipes/catholic_news_agency.recipe Normal file
View File

@ -0,0 +1,13 @@
from calibre.web.feeds.news import BasicNewsRecipe
class AdvancedUserRecipe1301972345(BasicNewsRecipe):
title = u'Catholic News Agency'
language = 'en'
__author__ = 'Jetkey'
oldest_article = 5
max_articles_per_feed = 20
feeds = [(u'U.S. News', u'http://feeds.feedburner.com/catholicnewsagency/dailynews-us'),
(u'Vatican', u'http://feeds.feedburner.com/catholicnewsagency/dailynews-vatican'),
(u'Bishops Corner', u'http://feeds.feedburner.com/catholicnewsagency/columns/bishopscorner'),
(u'Saint of the Day', u'http://feeds.feedburner.com/catholicnewsagency/saintoftheday')]

69
recipes/criticadigital.recipe
View File

@ -1,69 +0,0 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
'''
criticadigital.com
'''
from calibre.web.feeds.news import BasicNewsRecipe
class CriticaDigital(BasicNewsRecipe):
title = 'Critica de la Argentina'
__author__ = 'Darko Miletic and Sujata Raman'
description = 'Noticias de Argentina'
oldest_article = 2
max_articles_per_feed = 100
language = 'es_AR'
no_stylesheets = True
use_embedded_content = False
encoding = 'cp1252'
extra_css = '''
h1{font-family:"Trebuchet MS";}
h3{color:#9A0000; font-family:Tahoma; font-size:x-small;}
h2{color:#504E53; font-family:Arial,Helvetica,sans-serif ;font-size:small;}
#epigrafe{font-family:Arial,Helvetica,sans-serif ;color:#666666 ; font-size:x-small;}
p {font-family:Arial,Helvetica,sans-serif;}
#fecha{color:#858585; font-family:Tahoma; font-size:x-small;}
#autor{color:#858585; font-family:Tahoma; font-size:x-small;}
#hora{color:#F00000;font-family:Tahoma; font-size:x-small;}
'''
keep_only_tags = [
dict(name='div', attrs={'class':['bloqueTitulosNoticia','cfotonota']})
,dict(name='div', attrs={'id':'boxautor'})
,dict(name='p', attrs={'id':'textoNota'})
]
remove_tags = [
dict(name='div', attrs={'class':'box300' })
,dict(name='div', style=True )
,dict(name='div', attrs={'class':'titcomentario'})
,dict(name='div', attrs={'class':'comentario' })
,dict(name='div', attrs={'class':'paginador' })
]
feeds = [
(u'Politica', u'http://www.criticadigital.com/herramientas/rss.php?ch=politica' )
,(u'Economia', u'http://www.criticadigital.com/herramientas/rss.php?ch=economia' )
,(u'Deportes', u'http://www.criticadigital.com/herramientas/rss.php?ch=deportes' )
,(u'Espectaculos', u'http://www.criticadigital.com/herramientas/rss.php?ch=espectaculos')
,(u'Mundo', u'http://www.criticadigital.com/herramientas/rss.php?ch=mundo' )
,(u'Policiales', u'http://www.criticadigital.com/herramientas/rss.php?ch=policiales' )
,(u'Sociedad', u'http://www.criticadigital.com/herramientas/rss.php?ch=sociedad' )
,(u'Salud', u'http://www.criticadigital.com/herramientas/rss.php?ch=salud' )
,(u'Tecnologia', u'http://www.criticadigital.com/herramientas/rss.php?ch=tecnologia' )
,(u'Santa Fe', u'http://www.criticadigital.com/herramientas/rss.php?ch=santa_fe' )
]
def get_cover_url(self):
cover_url = None
index = 'http://www.criticadigital.com/impresa/'
soup = self.index_to_soup(index)
link_item = soup.find('div',attrs={'class':'tapa'})
if link_item:
cover_url = index + link_item.img['src']
return cover_url

99
recipes/elcronista.recipe
View File

@ -1,72 +1,59 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
'''
cronista.com
www.cronista.com
'''
from calibre.web.feeds.news import BasicNewsRecipe
class ElCronista(BasicNewsRecipe):
title = 'El Cronista'
class Pagina12(BasicNewsRecipe):
title = 'El Cronista Comercial'
__author__ = 'Darko Miletic'
description = 'Noticias de Argentina'
description = 'El Cronista Comercial es el Diario economico-politico mas valorado. Es la fuente mas confiable de informacion en temas de economia, finanzas y negocios enmarcados politicamente.'
publisher = 'Cronista.com'
category = 'news, politics, economy, finances, Argentina'
oldest_article = 2
language = 'es_AR'
max_articles_per_feed = 100
max_articles_per_feed = 200
no_stylesheets = True
encoding = 'utf8'
use_embedded_content = False
encoding = 'cp1252'
language = 'es_AR'
remove_empty_feeds = True
publication_type = 'newspaper'
masthead_url = 'http://www.cronista.com/export/sites/diarioelcronista/arte/header-logo.gif'
extra_css = """
body{font-family: Arial,Helvetica,sans-serif }
h2{font-family: Georgia,"Times New Roman",Times,serif }
img{margin-bottom: 0.4em; display:block}
.nom{font-weight: bold; vertical-align: baseline}
.autor-cfoto{border-bottom: 1px solid #D2D2D2;
border-top: 1px solid #D2D2D2;
display: inline-block;
margin: 0 10px 10px 0;
padding: 10px;
width: 210px}
.under{font-weight: bold}
.time{font-size: small}
"""
html2lrf_options = [
'--comment' , description
, '--category' , 'news, Argentina'
, '--publisher' , title
]
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
}
keep_only_tags = [
dict(name='table', attrs={'width':'100%' })
,dict(name='h1' , attrs={'class':'Arialgris16normal'})
]
remove_tags = [
dict(name=['meta','link','base','iframe','object','embed'])
,dict(attrs={'class':['user-tools','tabsmedia']})
]
remove_attributes = ['lang']
remove_tags_before = dict(attrs={'class':'top'})
remove_tags_after = dict(attrs={'class':'content-nota'})
feeds = [(u'Ultimas noticias', u'http://www.cronista.com/rss.html')]
remove_tags = [dict(name='a', attrs={'class':'Arialazul12'})]
feeds = [
(u'Economia' , u'http://www.cronista.com/adjuntos/8/rss/Economia_EI.xml' )
,(u'Negocios' , u'http://www.cronista.com/adjuntos/8/rss/negocios_EI.xml' )
,(u'Ultimo momento' , u'http://www.cronista.com/adjuntos/8/rss/ultimo_momento.xml' )
,(u'Finanzas y Mercados' , u'http://www.cronista.com/adjuntos/8/rss/Finanzas_Mercados_EI.xml' )
,(u'Financial Times' , u'http://www.cronista.com/adjuntos/8/rss/FT_EI.xml' )
,(u'Opinion edicion impresa' , u'http://www.cronista.com/adjuntos/8/rss/opinion_edicion_impresa.xml' )
,(u'Socialmente Responsables', u'http://www.cronista.com/adjuntos/8/rss/Socialmente_Responsables.xml')
,(u'Asuntos Legales' , u'http://www.cronista.com/adjuntos/8/rss/asuntoslegales.xml' )
,(u'IT Business' , u'http://www.cronista.com/adjuntos/8/rss/itbusiness.xml' )
,(u'Management y RR.HH.' , u'http://www.cronista.com/adjuntos/8/rss/management.xml' )
,(u'Inversiones Personales' , u'http://www.cronista.com/adjuntos/8/rss/inversionespersonales.xml' )
]
def print_version(self, url):
main, sep, rest = url.partition('.com/notas/')
article_id, lsep, rrest = rest.partition('-')
return 'http://www.cronista.com/interior/index.php?p=imprimir_nota&idNota=' + article_id
def preprocess_html(self, soup):
mtag = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
soup.head.insert(0,mtag)
soup.head.base.extract()
htext = soup.find('h1',attrs={'class':'Arialgris16normal'})
htext.name = 'p'
soup.prettify()
for item in soup.findAll(style=True):
del item['style']
return soup
def get_cover_url(self):
cover_url = None
index = 'http://www.cronista.com/contenidos/'
soup = self.index_to_soup(index + 'ee.html')
link_item = soup.find('a',attrs={'href':"javascript:Close()"})
if link_item:
cover_url = index + link_item.img['src']
return cover_url

19
recipes/eluniversal_ve.recipe
View File

@ -1,5 +1,5 @@
__license__ = 'GPL v3'
__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2010-2011, Darko Miletic <darko.miletic at gmail.com>'
'''
www.eluniversal.com
'''
@ -15,12 +15,20 @@ class ElUniversal(BasicNewsRecipe):
max_articles_per_feed = 100
no_stylesheets = True
use_embedded_content = False
remove_empty_feeds = True
encoding = 'cp1252'
publisher = 'El Universal'
category = 'news, Caracas, Venezuela, world'
language = 'es_VE'
publication_type = 'newspaper'
cover_url = strftime('http://static.eluniversal.com/%Y/%m/%d/portada.jpg')
extra_css = """
.txt60{font-family: Tahoma,Geneva,sans-serif; font-size: small}
.txt29{font-family: Tahoma,Geneva,sans-serif; font-size: small; color: gray}
.txt38{font-family: Georgia,"Times New Roman",Times,serif; font-size: xx-large}
.txt35{font-family: Georgia,"Times New Roman",Times,serif; font-size: large}
body{font-family: Verdana,Arial,Helvetica,sans-serif}
"""
conversion_options = {
'comments' : description
,'tags' : category
@ -28,10 +36,11 @@ class ElUniversal(BasicNewsRecipe):
,'publisher' : publisher
}
keep_only_tags = [dict(name='div', attrs={'class':'Nota'})]
remove_tags_before=dict(attrs={'class':'header-print MB10'})
remove_tags_after= dict(attrs={'id':'SizeText'})
remove_tags = [
dict(name=['object','link','script','iframe'])
,dict(name='div',attrs={'class':'Herramientas'})
dict(name=['object','link','script','iframe','meta'])
,dict(attrs={'class':'header-print MB10'})
]
feeds = [

25
recipes/endgadget.recipe
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2008 - 2009, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = 'Copyright 2011 Starson17'
'''
engadget.com
'''
@ -9,14 +9,29 @@ engadget.com
from calibre.web.feeds.news import BasicNewsRecipe
class Engadget(BasicNewsRecipe):
title = u'Engadget'
__author__ = 'Darko Miletic'
title = u'Engadget_Full'
__author__ = 'Starson17'
__version__ = 'v1.00'
__date__ = '02, July 2011'
description = 'Tech news'
language = 'en'
oldest_article = 7
max_articles_per_feed = 100
no_stylesheets = True
use_embedded_content = True
use_embedded_content = False
remove_javascript = True
remove_empty_feeds = True
feeds = [ (u'Posts', u'http://www.engadget.com/rss.xml')]
keep_only_tags = [dict(name='div', attrs={'class':['post_content permalink ','post_content permalink alt-post-full']})]
remove_tags = [dict(name='div', attrs={'class':['filed_under','post_footer']})]
remove_tags_after = [dict(name='div', attrs={'class':['post_footer']})]
feeds = [(u'Posts', u'http://www.engadget.com/rss.xml')]
extra_css = '''
h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
'''

99
recipes/financial_times.recipe
View File

@ -1,32 +1,41 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2010-2011, Darko Miletic <darko.miletic at gmail.com>'
'''
ft.com
www.ft.com
'''
import datetime
from calibre.web.feeds.news import BasicNewsRecipe
class FinancialTimes(BasicNewsRecipe):
title = u'Financial Times'
__author__ = 'Darko Miletic and Sujata Raman'
description = ('Financial world news. Available after 5AM '
'GMT, daily.')
class FinancialTimes_rss(BasicNewsRecipe):
title = 'Financial Times'
__author__ = 'Darko Miletic'
description = "The Financial Times (FT) is one of the world's leading business news and information organisations, recognised internationally for its authority, integrity and accuracy."
publisher = 'The Financial Times Ltd.'
category = 'news, finances, politics, World'
oldest_article = 2
language = 'en'
max_articles_per_feed = 100
language = 'en'
max_articles_per_feed = 250
no_stylesheets = True
use_embedded_content = False
needs_subscription = True
simultaneous_downloads= 1
delay = 1
encoding = 'utf8'
publication_type = 'newspaper'
masthead_url = 'http://im.media.ft.com/m/img/masthead_main.jpg'
LOGIN = 'https://registration.ft.com/registration/barrier/login'
INDEX = 'http://www.ft.com'
LOGIN = 'https://registration.ft.com/registration/barrier/login'
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
, 'linearize_tables' : True
}
def get_browser(self):
br = BasicNewsRecipe.get_browser()
br.open(self.INDEX)
if self.username is not None and self.password is not None:
br.open(self.LOGIN)
br.select_form(name='loginForm')
@ -35,31 +44,63 @@ class FinancialTimes(BasicNewsRecipe):
br.submit()
return br
keep_only_tags = [ dict(name='div', attrs={'id':'cont'}) ]
remove_tags_after = dict(name='p', attrs={'class':'copyright'})
keep_only_tags = [dict(name='div', attrs={'class':['fullstory fullstoryHeader','fullstory fullstoryBody','ft-story-header','ft-story-body','index-detail']})]
remove_tags = [
dict(name='div', attrs={'id':'floating-con'})
dict(name='div', attrs={'id':'floating-con'})
,dict(name=['meta','iframe','base','object','embed','link'])
,dict(attrs={'class':['storyTools','story-package','screen-copy','story-package separator','expandable-image']})
]
remove_attributes = ['width','height','lang']
extra_css = '''
body{font-family:Arial,Helvetica,sans-serif;}
h2(font-size:large;}
.ft-story-header(font-size:xx-small;}
.ft-story-body(font-size:small;}
a{color:#003399;}
extra_css = """
body{font-family: Georgia,Times,"Times New Roman",serif}
h2{font-size:large}
.ft-story-header{font-size: x-small}
.container{font-size:x-small;}
h3{font-size:x-small;color:#003399;}
'''
.copyright{font-size: x-small}
img{margin-top: 0.8em; display: block}
.lastUpdated{font-family: Arial,Helvetica,sans-serif; font-size: x-small}
.byline,.ft-story-body,.ft-story-header{font-family: Arial,Helvetica,sans-serif}
"""
feeds = [
(u'UK' , u'http://www.ft.com/rss/home/uk' )
,(u'US' , u'http://www.ft.com/rss/home/us' )
,(u'Europe' , u'http://www.ft.com/rss/home/europe' )
,(u'Asia' , u'http://www.ft.com/rss/home/asia' )
,(u'Middle East', u'http://www.ft.com/rss/home/middleeast')
]
def preprocess_html(self, soup):
content_type = soup.find('meta', {'http-equiv':'Content-Type'})
if content_type:
content_type['content'] = 'text/html; charset=utf-8'
items = ['promo-box','promo-title',
'promo-headline','promo-image',
'promo-intro','promo-link','subhead']
for item in items:
for it in soup.findAll(item):
it.name = 'div'
it.attrs = []
for item in soup.findAll(style=True):
del item['style']
for item in soup.findAll('a'):
limg = item.find('img')
if item.string is not None:
str = item.string
item.replaceWith(str)
else:
if limg:
item.name = 'div'
item.attrs = []
else:
str = self.tag_to_string(item)
item.replaceWith(str)
for item in soup.findAll('img'):
if not item.has_key('alt'):
item['alt'] = 'image'
return soup
def get_cover_url(self):
cdate = datetime.date.today()
if cdate.isoweekday() == 7:
cdate -= datetime.timedelta(days=1)
return cdate.strftime('http://specials.ft.com/vtf_pdf/%d%m%y_FRONT1_USA.pdf')

10
recipes/financial_times_uk.recipe
View File

@ -3,6 +3,8 @@ __copyright__ = '2010-2011, Darko Miletic <darko.miletic at gmail.com>'
'''
www.ft.com/uk-edition
'''
import datetime
from calibre import strftime
from calibre.web.feeds.news import BasicNewsRecipe
@ -20,7 +22,6 @@ class FinancialTimes(BasicNewsRecipe):
needs_subscription = True
encoding = 'utf8'
publication_type = 'newspaper'
cover_url = strftime('http://specials.ft.com/vtf_pdf/%d%m%y_FRONT1_LON.pdf')
masthead_url = 'http://im.media.ft.com/m/img/masthead_main.jpg'
LOGIN = 'https://registration.ft.com/registration/barrier/login'
INDEX = 'http://www.ft.com/uk-edition'
@ -128,3 +129,10 @@ class FinancialTimes(BasicNewsRecipe):
if not item.has_key('alt'):
item['alt'] = 'image'
return soup
def get_cover_url(self):
cdate = datetime.date.today()
if cdate.isoweekday() == 7:
cdate -= datetime.timedelta(days=1)
return cdate.strftime('http://specials.ft.com/vtf_pdf/%d%m%y_FRONT1_LON.pdf')

15
recipes/hbr.recipe
View File

@ -1,5 +1,6 @@
from calibre.web.feeds.news import BasicNewsRecipe
import re
from datetime import date, timedelta
class HBR(BasicNewsRecipe):
@ -12,13 +13,14 @@ class HBR(BasicNewsRecipe):
no_stylesheets = True
LOGIN_URL = 'http://hbr.org/login?request_url=/'
INDEX = 'http://hbr.org/current'
INDEX = 'http://hbr.org/archive-toc/BR'
keep_only_tags = [dict(name='div', id='pageContainer')]
remove_tags = [dict(id=['mastheadContainer', 'magazineHeadline',
'articleToolbarTopRD', 'pageRightSubColumn', 'pageRightColumn',
'todayOnHBRListWidget', 'mostWidget', 'keepUpWithHBR',
'mailingListTout', 'partnerCenter', 'pageFooter',
'superNavHeadContainer', 'hbrDisqus',
'articleToolbarTop', 'articleToolbarBottom', 'articleToolbarRD']),
dict(name='iframe')]
extra_css = '''
@ -55,9 +57,14 @@ class HBR(BasicNewsRecipe):
def hbr_get_toc(self):
soup = self.index_to_soup(self.INDEX)
url = soup.find('a', text=lambda t:'Full Table of Contents' in t).parent.get('href')
return self.index_to_soup('http://hbr.org'+url)
today = date.today()
future = today + timedelta(days=30)
for x in [x.strftime('%y%m') for x in (future, today)]:
url = self.INDEX + x
soup = self.index_to_soup(url)
if not soup.find(text='Issue Not Found'):
return soup
raise Exception('Could not find current issue')
def hbr_parse_section(self, container, feeds):
current_section = None

BIN
recipes/icons/athens_news.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 514 B

BIN
recipes/icons/buenosaireseconomico.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 400 B

BIN
recipes/icons/elcronista.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 770 B

After

Width:  |  Height:  |  Size: 1.1 KiB

BIN
recipes/icons/financial_times.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.4 KiB

37
recipes/independent.recipe
View File

@ -6,7 +6,7 @@ class TheIndependent(BasicNewsRecipe):
language = 'en_GB'
__author__ = 'Krittika Goyal'
oldest_article = 1 #days
max_articles_per_feed = 25
max_articles_per_feed = 30
encoding = 'latin1'
no_stylesheets = True
@ -25,24 +25,39 @@ class TheIndependent(BasicNewsRecipe):
'http://www.independent.co.uk/news/uk/rss'),
('World',
'http://www.independent.co.uk/news/world/rss'),
('Sport',
'http://www.independent.co.uk/sport/rss'),
('Arts and Entertainment',
'http://www.independent.co.uk/arts-entertainment/rss'),
('Business',
'http://www.independent.co.uk/news/business/rss'),
('Life and Style',
'http://www.independent.co.uk/life-style/gadgets-and-tech/news/rss'),
('Science',
'http://www.independent.co.uk/news/science/rss'),
('People',
'http://www.independent.co.uk/news/people/rss'),
('Science',
'http://www.independent.co.uk/news/science/rss'),
('Media',
'http://www.independent.co.uk/news/media/rss'),
('Health and Families',
'http://www.independent.co.uk/life-style/health-and-families/rss'),
('Education',
'http://www.independent.co.uk/news/education/rss'),
('Obituaries',
'http://www.independent.co.uk/news/obituaries/rss'),
('Opinion',
'http://www.independent.co.uk/opinion/rss'),
('Environment',
'http://www.independent.co.uk/environment/rss'),
('Sport',
'http://www.independent.co.uk/sport/rss'),
('Life and Style',
'http://www.independent.co.uk/life-style/rss'),
('Arts and Entertainment',
'http://www.independent.co.uk/arts-entertainment/rss'),
('Travel',
'http://www.independent.co.uk/travel/rss'),
('Money',
'http://www.independent.co.uk/money/rss'),
]
def preprocess_html(self, soup):

64
recipes/infobae.recipe
View File

@ -1,5 +1,5 @@
__license__ = 'GPL v3'
__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
'''
infobae.com
'''
@ -9,7 +9,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
class Infobae(BasicNewsRecipe):
title = 'Infobae.com'
__author__ = 'Darko Miletic and Sujata Raman'
description = 'Informacion Libre las 24 horas'
description = 'Infobae.com es el sitio de noticias con mayor actualizacion de Latinoamérica. Noticias actualizadas las 24 horas, los 365 días del año.'
publisher = 'Infobae.com'
category = 'news, politics, Argentina'
oldest_article = 1
@ -17,13 +17,13 @@ class Infobae(BasicNewsRecipe):
no_stylesheets = True
use_embedded_content = False
language = 'es_AR'
encoding = 'cp1252'
masthead_url = 'http://www.infobae.com/imgs/header/header.gif'
remove_javascript = True
encoding = 'utf8'
masthead_url = 'http://www.infobae.com/media/img/static/logo-infobae.gif'
remove_empty_feeds = True
extra_css = '''
body{font-family:Arial,Helvetica,sans-serif;}
.popUpTitulo{color:#0D4261; font-size: xx-large}
body{font-family: Arial,Helvetica,sans-serif}
img{display: block}
.categoria{font-size: small; text-transform: uppercase}
'''
conversion_options = {
@ -31,26 +31,44 @@ class Infobae(BasicNewsRecipe):
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
, 'linearize_tables' : True
}
keep_only_tags = [dict(attrs={'class':['titularnota','nota','post-title','post-entry','entry-title','entry-info','entry-content']})]
remove_tags_after = dict(attrs={'class':['interior-noticia','nota-desc','tags']})
remove_tags = [
dict(name=['base','meta','link','iframe','object','embed','ins'])
,dict(attrs={'class':['barranota','tags']})
]
feeds = [
(u'Noticias' , u'http://www.infobae.com/adjuntos/html/RSS/hoy.xml' )
,(u'Salud' , u'http://www.infobae.com/adjuntos/html/RSS/salud.xml' )
,(u'Tecnologia', u'http://www.infobae.com/adjuntos/html/RSS/tecnologia.xml')
,(u'Deportes' , u'http://www.infobae.com/adjuntos/html/RSS/deportes.xml' )
(u'Saludable' , u'http://www.infobae.com/rss/saludable.xml')
,(u'Economia' , u'http://www.infobae.com/rss/economia.xml' )
,(u'En Numeros', u'http://www.infobae.com/rss/rating.xml' )
,(u'Finanzas' , u'http://www.infobae.com/rss/finanzas.xml' )
,(u'Mundo' , u'http://www.infobae.com/rss/mundo.xml' )
,(u'Sociedad' , u'http://www.infobae.com/rss/sociedad.xml' )
,(u'Politica' , u'http://www.infobae.com/rss/politica.xml' )
,(u'Deportes' , u'http://www.infobae.com/rss/deportes.xml' )
]
def print_version(self, url):
article_part = url.rpartition('/')[2]
article_id= article_part.partition('-')[0]
return 'http://www.infobae.com/notas/nota_imprimir.php?Idx=' + article_id
def postprocess_html(self, soup, first):
for tag in soup.findAll(name='strong'):
tag.name = 'b'
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
for item in soup.findAll('a'):
limg = item.find('img')
if item.string is not None:
str = item.string
item.replaceWith(str)
else:
if limg:
item.name = 'div'
item.attrs = []
else:
str = self.tag_to_string(item)
item.replaceWith(str)
for item in soup.findAll('img'):
if not item.has_key('alt'):
item['alt'] = 'image'
return soup

2
recipes/le_monde.recipe
View File

@ -99,7 +99,7 @@ class LeMonde(BasicNewsRecipe):
keep_only_tags = [
dict(name='div', attrs={'class':['contenu']})
]
remove_tags = [dict(name='div', attrs={'class':['LM_atome']})]
remove_tags_after = [dict(id='appel_temoignage')]
def get_article_url(self, article):

334
recipes/ming_pao.recipe
View File

@ -1,17 +1,23 @@
# -*- coding: utf-8 -*-
__license__ = 'GPL v3'
__copyright__ = '2010-2011, Eddie Lau'
# Region - Hong Kong, Vancouver, Toronto
__Region__ = 'Hong Kong'
# Users of Kindle 3 with limited system-level CJK support
# please replace the following "True" with "False".
__MakePeriodical__ = True
# Turn below to true if your device supports display of CJK titles
__UseChineseTitle__ = False
# Trun below to true if you wish to use life.mingpao.com as the main article source
# Set it to False if you want to skip images
__KeepImages__ = True
# (HK only) Turn below to true if you wish to use life.mingpao.com as the main article source
__UseLife__ = True
'''
Change Log:
2011/06/26: add fetching Vancouver and Toronto versions of the paper, also provide captions for images using life.mingpao fetch source
provide options to remove all images in the file
2011/05/12: switch the main parse source to life.mingpao.com, which has more photos on the article pages
2011/03/06: add new articles for finance section, also a new section "Columns"
2011/02/28: rearrange the sections
@ -34,21 +40,96 @@ Change Log:
import os, datetime, re
from calibre.web.feeds.recipes import BasicNewsRecipe
from contextlib import nested
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.ebooks.metadata.opf2 import OPFCreator
from calibre.ebooks.metadata.toc import TOC
from calibre.ebooks.metadata import MetaInformation
class MPHKRecipe(BasicNewsRecipe):
title = 'Ming Pao - Hong Kong'
# MAIN CLASS
class MPRecipe(BasicNewsRecipe):
if __Region__ == 'Hong Kong':
title = 'Ming Pao - Hong Kong'
description = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)'
category = 'Chinese, News, Hong Kong'
extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}'
masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif'
keep_only_tags = [dict(name='h1'),
dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page title
dict(name='font', attrs={'color':['AA0000']}), # for column articles title
dict(attrs={'id':['newscontent']}), # entertainment and column page content
dict(attrs={'id':['newscontent01','newscontent02']}),
dict(attrs={'class':['photo']}),
dict(name='table', attrs={'width':['100%'], 'border':['0'], 'cellspacing':['5'], 'cellpadding':['0']}), # content in printed version of life.mingpao.com
dict(name='img', attrs={'width':['180'], 'alt':['按圖放大']}) # images for source from life.mingpao.com
]
if __KeepImages__:
remove_tags = [dict(name='style'),
dict(attrs={'id':['newscontent135']}), # for the finance page from mpfinance.com
dict(name='font', attrs={'size':['2'], 'color':['666666']}), # article date in life.mingpao.com article
#dict(name='table') # for content fetched from life.mingpao.com
]
else:
remove_tags = [dict(name='style'),
dict(attrs={'id':['newscontent135']}), # for the finance page from mpfinance.com
dict(name='font', attrs={'size':['2'], 'color':['666666']}), # article date in life.mingpao.com article
dict(name='img'),
#dict(name='table') # for content fetched from life.mingpao.com
]
remove_attributes = ['width']
preprocess_regexps = [
(re.compile(r'<h5>', re.DOTALL|re.IGNORECASE),
lambda match: '<h1>'),
(re.compile(r'</h5>', re.DOTALL|re.IGNORECASE),
lambda match: '</h1>'),
(re.compile(r'<p><a href=.+?</a></p>', re.DOTALL|re.IGNORECASE), # for entertainment page
lambda match: ''),
# skip <br> after title in life.mingpao.com fetched article
(re.compile(r"<div id='newscontent'><br>", re.DOTALL|re.IGNORECASE),
lambda match: "<div id='newscontent'>"),
(re.compile(r"<br><br></b>", re.DOTALL|re.IGNORECASE),
lambda match: "</b>")
]
elif __Region__ == 'Vancouver':
title = 'Ming Pao - Vancouver'
description = 'Vancouver Chinese Newspaper (http://www.mingpaovan.com)'
category = 'Chinese, News, Vancouver'
extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} b>font {font-size:200%; font-weight:bold;}'
masthead_url = 'http://www.mingpaovan.com/image/mainlogo2_VAN2.gif'
keep_only_tags = [dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['1']}),
dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['3'], 'cellpadding':['3'], 'id':['tblContent3']}),
dict(name='table', attrs={'width':['180'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['0'], 'bgcolor':['F0F0F0']}),
]
if __KeepImages__:
remove_tags = [dict(name='img', attrs={'src':['../../../image/magnifier.gif']})] # the magnifier icon
else:
remove_tags = [dict(name='img')]
remove_attributes = ['width']
preprocess_regexps = [(re.compile(r'&nbsp;', re.DOTALL|re.IGNORECASE),
lambda match: ''),
]
elif __Region__ == 'Toronto':
title = 'Ming Pao - Toronto'
description = 'Toronto Chinese Newspaper (http://www.mingpaotor.com)'
category = 'Chinese, News, Toronto'
extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} b>font {font-size:200%; font-weight:bold;}'
masthead_url = 'http://www.mingpaotor.com/image/mainlogo2_TOR2.gif'
keep_only_tags = [dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['1']}),
dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['3'], 'cellpadding':['3'], 'id':['tblContent3']}),
dict(name='table', attrs={'width':['180'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['0'], 'bgcolor':['F0F0F0']}),
]
if __KeepImages__:
remove_tags = [dict(name='img', attrs={'src':['../../../image/magnifier.gif']})] # the magnifier icon
else:
remove_tags = [dict(name='img')]
remove_attributes = ['width']
preprocess_regexps = [(re.compile(r'&nbsp;', re.DOTALL|re.IGNORECASE),
lambda match: ''),
]
oldest_article = 1
max_articles_per_feed = 100
__author__ = 'Eddie Lau'
description = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)'
publisher = 'MingPao'
category = 'Chinese, News, Hong Kong'
remove_javascript = True
use_embedded_content = False
no_stylesheets = True
@ -57,33 +138,6 @@ class MPHKRecipe(BasicNewsRecipe):
recursions = 0
conversion_options = {'linearize_tables':True}
timefmt = ''
extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}'
masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif'
keep_only_tags = [dict(name='h1'),
dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page title
dict(name='font', attrs={'color':['AA0000']}), # for column articles title
dict(attrs={'id':['newscontent']}), # entertainment and column page content
dict(attrs={'id':['newscontent01','newscontent02']}),
dict(attrs={'class':['photo']}),
dict(name='img', attrs={'width':['180'], 'alt':['按圖放大']}) # images for source from life.mingpao.com
]
remove_tags = [dict(name='style'),
dict(attrs={'id':['newscontent135']}), # for the finance page from mpfinance.com
dict(name='table')] # for content fetched from life.mingpao.com
remove_attributes = ['width']
preprocess_regexps = [
(re.compile(r'<h5>', re.DOTALL|re.IGNORECASE),
lambda match: '<h1>'),
(re.compile(r'</h5>', re.DOTALL|re.IGNORECASE),
lambda match: '</h1>'),
(re.compile(r'<p><a href=.+?</a></p>', re.DOTALL|re.IGNORECASE), # for entertainment page
lambda match: ''),
# skip <br> after title in life.mingpao.com fetched article
(re.compile(r"<div id='newscontent'><br>", re.DOTALL|re.IGNORECASE),
lambda match: "<div id='newscontent'>"),
(re.compile(r"<br><br></b>", re.DOTALL|re.IGNORECASE),
lambda match: "</b>")
]
def image_url_processor(cls, baseurl, url):
# trick: break the url at the first occurance of digit, add an additional
@ -124,8 +178,18 @@ class MPHKRecipe(BasicNewsRecipe):
def get_dtlocal(self):
dt_utc = datetime.datetime.utcnow()
# convert UTC to local hk time - at around HKT 6.00am, all news are available
dt_local = dt_utc - datetime.timedelta(-2.0/24)
if __Region__ == 'Hong Kong':
# convert UTC to local hk time - at HKT 5.30am, all news are available
dt_local = dt_utc + datetime.timedelta(8.0/24) - datetime.timedelta(5.5/24)
# dt_local = dt_utc.astimezone(pytz.timezone('Asia/Hong_Kong')) - datetime.timedelta(5.5/24)
elif __Region__ == 'Vancouver':
# convert UTC to local Vancouver time - at PST time 5.30am, all news are available
dt_local = dt_utc + datetime.timedelta(-8.0/24) - datetime.timedelta(5.5/24)
#dt_local = dt_utc.astimezone(pytz.timezone('America/Vancouver')) - datetime.timedelta(5.5/24)
elif __Region__ == 'Toronto':
# convert UTC to local Toronto time - at EST time 8.30am, all news are available
dt_local = dt_utc + datetime.timedelta(-5.0/24) - datetime.timedelta(8.5/24)
#dt_local = dt_utc.astimezone(pytz.timezone('America/Toronto')) - datetime.timedelta(8.5/24)
return dt_local
def get_fetchdate(self):
@ -135,13 +199,15 @@ class MPHKRecipe(BasicNewsRecipe):
return self.get_dtlocal().strftime("%Y-%m-%d")
def get_fetchday(self):
# dt_utc = datetime.datetime.utcnow()
# convert UTC to local hk time - at around HKT 6.00am, all news are available
# dt_local = dt_utc - datetime.timedelta(-2.0/24)
return self.get_dtlocal().strftime("%d")
def get_cover_url(self):
cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg'
if __Region__ == 'Hong Kong':
cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg'
elif __Region__ == 'Vancouver':
cover = 'http://www.mingpaovan.com/ftp/News/' + self.get_fetchdate() + '/' + self.get_fetchday() + 'pgva1s.jpg'
elif __Region__ == 'Toronto':
cover = 'http://www.mingpaotor.com/ftp/News/' + self.get_fetchdate() + '/' + self.get_fetchday() + 'pgtas.jpg'
br = BasicNewsRecipe.get_browser()
try:
br.open(cover)
@ -153,76 +219,104 @@ class MPHKRecipe(BasicNewsRecipe):
feeds = []
dateStr = self.get_fetchdate()
if __UseLife__:
for title, url, keystr in [(u'\u8981\u805e Headline', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalga', 'nal'),
(u'\u6e2f\u805e Local', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgb', 'nal'),
(u'\u6559\u80b2 Education', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgf', 'nal'),
(u'\u793e\u8a55/\u7b46\u9663 Editorial', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr', 'nal'),
(u'\u8ad6\u58c7 Forum', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalfa', 'nal'),
(u'\u4e2d\u570b China', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalca', 'nal'),
(u'\u570b\u969b World', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalta', 'nal'),
(u'\u7d93\u6fdf Finance', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea', 'nal'),
(u'\u9ad4\u80b2 Sport', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalsp', 'nal'),
(u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal'),
(u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl')]:
articles = self.parse_section2(url, keystr)
if __Region__ == 'Hong Kong':
if __UseLife__:
for title, url, keystr in [(u'\u8981\u805e Headline', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalga', 'nal'),
(u'\u6e2f\u805e Local', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgb', 'nal'),
(u'\u6559\u80b2 Education', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgf', 'nal'),
(u'\u793e\u8a55/\u7b46\u9663 Editorial', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr', 'nal'),
(u'\u8ad6\u58c7 Forum', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalfa', 'nal'),
(u'\u4e2d\u570b China', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalca', 'nal'),
(u'\u570b\u969b World', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalta', 'nal'),
(u'\u7d93\u6fdf Finance', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea', 'nal'),
(u'\u9ad4\u80b2 Sport', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalsp', 'nal'),
(u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal'),
(u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl')]:
articles = self.parse_section2(url, keystr)
if articles:
feeds.append((title, articles))
for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
(u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
else:
for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'),
(u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'),
(u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special- editorial
ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr')
if ed_articles:
feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles))
for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'),
(u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'),
(u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special - finance
#fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm')
fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea')
if fin_articles:
feeds.append((u'\u7d93\u6fdf Finance', fin_articles))
for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'),
(u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special - entertainment
ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm')
if ent_articles:
feeds.append((u'\u5f71\u8996 Film/TV', ent_articles))
for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
(u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special- columns
col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn')
if col_articles:
feeds.append((u'\u5c08\u6b04 Columns', col_articles))
elif __Region__ == 'Vancouver':
for title, url in [(u'\u8981\u805e Headline', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VAindex.htm'),
(u'\u52a0\u570b Canada', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VBindex.htm'),
(u'\u793e\u5340 Local', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VDindex.htm'),
(u'\u6e2f\u805e Hong Kong', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/HK-VGindex.htm'),
(u'\u570b\u969b World', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VTindex.htm'),
(u'\u4e2d\u570b China', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VCindex.htm'),
(u'\u7d93\u6fdf Economics', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VEindex.htm'),
(u'\u9ad4\u80b2 Sports', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VSindex.htm'),
(u'\u5f71\u8996 Film/TV', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/HK-MAindex.htm'),
(u'\u526f\u520a Supplements', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/WWindex.htm'),]:
articles = self.parse_section3(url, 'http://www.mingpaovan.com/')
if articles:
feeds.append((title, articles))
for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
(u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
articles = self.parse_section(url)
elif __Region__ == 'Toronto':
for title, url in [(u'\u8981\u805e Headline', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TAindex.htm'),
(u'\u52a0\u570b Canada', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TDindex.htm'),
(u'\u793e\u5340 Local', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TFindex.htm'),
(u'\u4e2d\u570b China', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TCAindex.htm'),
(u'\u570b\u969b World', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TTAindex.htm'),
(u'\u6e2f\u805e Hong Kong', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/HK-GAindex.htm'),
(u'\u7d93\u6fdf Economics', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/THindex.htm'),
(u'\u9ad4\u80b2 Sports', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TSindex.htm'),
(u'\u5f71\u8996 Film/TV', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/HK-MAindex.htm'),
(u'\u526f\u520a Supplements', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/WWindex.htm'),]:
articles = self.parse_section3(url, 'http://www.mingpaotor.com/')
if articles:
feeds.append((title, articles))
else:
for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'),
(u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'),
(u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special- editorial
ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr')
if ed_articles:
feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles))
for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'),
(u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'),
(u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special - finance
#fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm')
fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea')
if fin_articles:
feeds.append((u'\u7d93\u6fdf Finance', fin_articles))
for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'),
(u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special - entertainment
ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm')
if ent_articles:
feeds.append((u'\u5f71\u8996 Film/TV', ent_articles))
for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
(u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special- columns
col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn')
if col_articles:
feeds.append((u'\u5c08\u6b04 Columns', col_articles))
return feeds
# parse from news.mingpao.com
@ -256,11 +350,30 @@ class MPHKRecipe(BasicNewsRecipe):
title = self.tag_to_string(i)
url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind(keystr) == -1):
url = url.replace('dailynews3.cfm', 'dailynews3a.cfm') # use printed version of the article
current_articles.append({'title': title, 'url': url, 'description': ''})
included_urls.append(url)
current_articles.reverse()
return current_articles
# parse from www.mingpaovan.com
def parse_section3(self, url, baseUrl):
self.get_fetchdate()
soup = self.index_to_soup(url)
divs = soup.findAll(attrs={'class': ['ListContentLargeLink']})
current_articles = []
included_urls = []
divs.reverse()
for i in divs:
title = self.tag_to_string(i)
urlstr = i.get('href', False)
urlstr = baseUrl + '/' + urlstr.replace('../../../', '')
if urlstr not in included_urls:
current_articles.append({'title': title, 'url': urlstr, 'description': '', 'date': ''})
included_urls.append(urlstr)
current_articles.reverse()
return current_articles
def parse_ed_section(self, url):
self.get_fetchdate()
soup = self.index_to_soup(url)
@ -338,7 +451,12 @@ class MPHKRecipe(BasicNewsRecipe):
if dir is None:
dir = self.output_dir
if __UseChineseTitle__ == True:
title = u'\u660e\u5831 (\u9999\u6e2f)'
if __Region__ == 'Hong Kong':
title = u'\u660e\u5831 (\u9999\u6e2f)'
elif __Region__ == 'Vancouver':
title = u'\u660e\u5831 (\u6eab\u54e5\u83ef)'
elif __Region__ == 'Toronto':
title = u'\u660e\u5831 (\u591a\u502b\u591a)'
else:
title = self.short_title()
# if not generating a periodical, force date to apply in title

594
recipes/ming_pao_toronto.recipe Normal file
View File

@ -0,0 +1,594 @@
__license__ = 'GPL v3'
__copyright__ = '2010-2011, Eddie Lau'
# Region - Hong Kong, Vancouver, Toronto
__Region__ = 'Toronto'
# Users of Kindle 3 with limited system-level CJK support
# please replace the following "True" with "False".
__MakePeriodical__ = True
# Turn below to true if your device supports display of CJK titles
__UseChineseTitle__ = False
# Set it to False if you want to skip images
__KeepImages__ = True
# (HK only) Turn below to true if you wish to use life.mingpao.com as the main article source
__UseLife__ = True
'''
Change Log:
2011/06/26: add fetching Vancouver and Toronto versions of the paper, also provide captions for images using life.mingpao fetch source
provide options to remove all images in the file
2011/05/12: switch the main parse source to life.mingpao.com, which has more photos on the article pages
2011/03/06: add new articles for finance section, also a new section "Columns"
2011/02/28: rearrange the sections
[Disabled until Kindle has better CJK support and can remember last (section,article) read in Sections & Articles
View] make it the same title if generating a periodical, so past issue will be automatically put into "Past Issues"
folder in Kindle 3
2011/02/20: skip duplicated links in finance section, put photos which may extend a whole page to the back of the articles
clean up the indentation
2010/12/07: add entertainment section, use newspaper front page as ebook cover, suppress date display in section list
(to avoid wrong date display in case the user generates the ebook in a time zone different from HKT)
2010/11/22: add English section, remove eco-news section which is not updated daily, correct
ordering of articles
2010/11/12: add news image and eco-news section
2010/11/08: add parsing of finance section
2010/11/06: temporary work-around for Kindle device having no capability to display unicode
in section/article list.
2010/10/31: skip repeated articles in section pages
'''
import os, datetime, re
from calibre.web.feeds.recipes import BasicNewsRecipe
from contextlib import nested
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.ebooks.metadata.opf2 import OPFCreator
from calibre.ebooks.metadata.toc import TOC
from calibre.ebooks.metadata import MetaInformation
# MAIN CLASS
class MPRecipe(BasicNewsRecipe):
if __Region__ == 'Hong Kong':
title = 'Ming Pao - Hong Kong'
description = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)'
category = 'Chinese, News, Hong Kong'
extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}'
masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif'
keep_only_tags = [dict(name='h1'),
dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page title
dict(name='font', attrs={'color':['AA0000']}), # for column articles title
dict(attrs={'id':['newscontent']}), # entertainment and column page content
dict(attrs={'id':['newscontent01','newscontent02']}),
dict(attrs={'class':['photo']}),
dict(name='table', attrs={'width':['100%'], 'border':['0'], 'cellspacing':['5'], 'cellpadding':['0']}), # content in printed version of life.mingpao.com
dict(name='img', attrs={'width':['180'], 'alt':['按圖放大']}) # images for source from life.mingpao.com
]
if __KeepImages__:
remove_tags = [dict(name='style'),
dict(attrs={'id':['newscontent135']}), # for the finance page from mpfinance.com
dict(name='font', attrs={'size':['2'], 'color':['666666']}), # article date in life.mingpao.com article
#dict(name='table') # for content fetched from life.mingpao.com
]
else:
remove_tags = [dict(name='style'),
dict(attrs={'id':['newscontent135']}), # for the finance page from mpfinance.com
dict(name='font', attrs={'size':['2'], 'color':['666666']}), # article date in life.mingpao.com article
dict(name='img'),
#dict(name='table') # for content fetched from life.mingpao.com
]
remove_attributes = ['width']
preprocess_regexps = [
(re.compile(r'<h5>', re.DOTALL|re.IGNORECASE),
lambda match: '<h1>'),
(re.compile(r'</h5>', re.DOTALL|re.IGNORECASE),
lambda match: '</h1>'),
(re.compile(r'<p><a href=.+?</a></p>', re.DOTALL|re.IGNORECASE), # for entertainment page
lambda match: ''),
# skip <br> after title in life.mingpao.com fetched article
(re.compile(r"<div id='newscontent'><br>", re.DOTALL|re.IGNORECASE),
lambda match: "<div id='newscontent'>"),
(re.compile(r"<br><br></b>", re.DOTALL|re.IGNORECASE),
lambda match: "</b>")
]
elif __Region__ == 'Vancouver':
title = 'Ming Pao - Vancouver'
description = 'Vancouver Chinese Newspaper (http://www.mingpaovan.com)'
category = 'Chinese, News, Vancouver'
extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} b>font {font-size:200%; font-weight:bold;}'
masthead_url = 'http://www.mingpaovan.com/image/mainlogo2_VAN2.gif'
keep_only_tags = [dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['1']}),
dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['3'], 'cellpadding':['3'], 'id':['tblContent3']}),
dict(name='table', attrs={'width':['180'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['0'], 'bgcolor':['F0F0F0']}),
]
if __KeepImages__:
remove_tags = [dict(name='img', attrs={'src':['../../../image/magnifier.gif']})] # the magnifier icon
else:
remove_tags = [dict(name='img')]
remove_attributes = ['width']
preprocess_regexps = [(re.compile(r'&nbsp;', re.DOTALL|re.IGNORECASE),
lambda match: ''),
]
elif __Region__ == 'Toronto':
title = 'Ming Pao - Toronto'
description = 'Toronto Chinese Newspaper (http://www.mingpaotor.com)'
category = 'Chinese, News, Toronto'
extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} b>font {font-size:200%; font-weight:bold;}'
masthead_url = 'http://www.mingpaotor.com/image/mainlogo2_TOR2.gif'
keep_only_tags = [dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['1']}),
dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['3'], 'cellpadding':['3'], 'id':['tblContent3']}),
dict(name='table', attrs={'width':['180'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['0'], 'bgcolor':['F0F0F0']}),
]
if __KeepImages__:
remove_tags = [dict(name='img', attrs={'src':['../../../image/magnifier.gif']})] # the magnifier icon
else:
remove_tags = [dict(name='img')]
remove_attributes = ['width']
preprocess_regexps = [(re.compile(r'&nbsp;', re.DOTALL|re.IGNORECASE),
lambda match: ''),
]
oldest_article = 1
max_articles_per_feed = 100
__author__ = 'Eddie Lau'
publisher = 'MingPao'
remove_javascript = True
use_embedded_content = False
no_stylesheets = True
language = 'zh'
encoding = 'Big5-HKSCS'
recursions = 0
conversion_options = {'linearize_tables':True}
timefmt = ''
def image_url_processor(cls, baseurl, url):
# trick: break the url at the first occurance of digit, add an additional
# '_' at the front
# not working, may need to move this to preprocess_html() method
# minIdx = 10000
# i0 = url.find('0')
# if i0 >= 0 and i0 < minIdx:
# minIdx = i0
# i1 = url.find('1')
# if i1 >= 0 and i1 < minIdx:
# minIdx = i1
# i2 = url.find('2')
# if i2 >= 0 and i2 < minIdx:
# minIdx = i2
# i3 = url.find('3')
# if i3 >= 0 and i0 < minIdx:
# minIdx = i3
# i4 = url.find('4')
# if i4 >= 0 and i4 < minIdx:
# minIdx = i4
# i5 = url.find('5')
# if i5 >= 0 and i5 < minIdx:
# minIdx = i5
# i6 = url.find('6')
# if i6 >= 0 and i6 < minIdx:
# minIdx = i6
# i7 = url.find('7')
# if i7 >= 0 and i7 < minIdx:
# minIdx = i7
# i8 = url.find('8')
# if i8 >= 0 and i8 < minIdx:
# minIdx = i8
# i9 = url.find('9')
# if i9 >= 0 and i9 < minIdx:
# minIdx = i9
return url
def get_dtlocal(self):
dt_utc = datetime.datetime.utcnow()
if __Region__ == 'Hong Kong':
# convert UTC to local hk time - at HKT 5.30am, all news are available
dt_local = dt_utc + datetime.timedelta(8.0/24) - datetime.timedelta(5.5/24)
# dt_local = dt_utc.astimezone(pytz.timezone('Asia/Hong_Kong')) - datetime.timedelta(5.5/24)
elif __Region__ == 'Vancouver':
# convert UTC to local Vancouver time - at PST time 5.30am, all news are available
dt_local = dt_utc + datetime.timedelta(-8.0/24) - datetime.timedelta(5.5/24)
#dt_local = dt_utc.astimezone(pytz.timezone('America/Vancouver')) - datetime.timedelta(5.5/24)
elif __Region__ == 'Toronto':
# convert UTC to local Toronto time - at EST time 8.30am, all news are available
dt_local = dt_utc + datetime.timedelta(-5.0/24) - datetime.timedelta(8.5/24)
#dt_local = dt_utc.astimezone(pytz.timezone('America/Toronto')) - datetime.timedelta(8.5/24)
return dt_local
def get_fetchdate(self):
return self.get_dtlocal().strftime("%Y%m%d")
def get_fetchformatteddate(self):
return self.get_dtlocal().strftime("%Y-%m-%d")
def get_fetchday(self):
return self.get_dtlocal().strftime("%d")
def get_cover_url(self):
if __Region__ == 'Hong Kong':
cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg'
elif __Region__ == 'Vancouver':
cover = 'http://www.mingpaovan.com/ftp/News/' + self.get_fetchdate() + '/' + self.get_fetchday() + 'pgva1s.jpg'
elif __Region__ == 'Toronto':
cover = 'http://www.mingpaotor.com/ftp/News/' + self.get_fetchdate() + '/' + self.get_fetchday() + 'pgtas.jpg'
br = BasicNewsRecipe.get_browser()
try:
br.open(cover)
except:
cover = None
return cover
def parse_index(self):
feeds = []
dateStr = self.get_fetchdate()
if __Region__ == 'Hong Kong':
if __UseLife__:
for title, url, keystr in [(u'\u8981\u805e Headline', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalga', 'nal'),
(u'\u6e2f\u805e Local', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgb', 'nal'),
(u'\u6559\u80b2 Education', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgf', 'nal'),
(u'\u793e\u8a55/\u7b46\u9663 Editorial', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr', 'nal'),
(u'\u8ad6\u58c7 Forum', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalfa', 'nal'),
(u'\u4e2d\u570b China', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalca', 'nal'),
(u'\u570b\u969b World', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalta', 'nal'),
(u'\u7d93\u6fdf Finance', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea', 'nal'),
(u'\u9ad4\u80b2 Sport', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalsp', 'nal'),
(u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal'),
(u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl')]:
articles = self.parse_section2(url, keystr)
if articles:
feeds.append((title, articles))
for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
(u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
else:
for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'),
(u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'),
(u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special- editorial
ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr')
if ed_articles:
feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles))
for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'),
(u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'),
(u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special - finance
#fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm')
fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea')
if fin_articles:
feeds.append((u'\u7d93\u6fdf Finance', fin_articles))
for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'),
(u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special - entertainment
ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm')
if ent_articles:
feeds.append((u'\u5f71\u8996 Film/TV', ent_articles))
for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
(u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special- columns
col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn')
if col_articles:
feeds.append((u'\u5c08\u6b04 Columns', col_articles))
elif __Region__ == 'Vancouver':
for title, url in [(u'\u8981\u805e Headline', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VAindex.htm'),
(u'\u52a0\u570b Canada', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VBindex.htm'),
(u'\u793e\u5340 Local', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VDindex.htm'),
(u'\u6e2f\u805e Hong Kong', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/HK-VGindex.htm'),
(u'\u570b\u969b World', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VTindex.htm'),
(u'\u4e2d\u570b China', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VCindex.htm'),
(u'\u7d93\u6fdf Economics', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VEindex.htm'),
(u'\u9ad4\u80b2 Sports', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VSindex.htm'),
(u'\u5f71\u8996 Film/TV', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/HK-MAindex.htm'),
(u'\u526f\u520a Supplements', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/WWindex.htm'),]:
articles = self.parse_section3(url, 'http://www.mingpaovan.com/')
if articles:
feeds.append((title, articles))
elif __Region__ == 'Toronto':
for title, url in [(u'\u8981\u805e Headline', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TAindex.htm'),
(u'\u52a0\u570b Canada', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TDindex.htm'),
(u'\u793e\u5340 Local', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TFindex.htm'),
(u'\u4e2d\u570b China', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TCAindex.htm'),
(u'\u570b\u969b World', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TTAindex.htm'),
(u'\u6e2f\u805e Hong Kong', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/HK-GAindex.htm'),
(u'\u7d93\u6fdf Economics', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/THindex.htm'),
(u'\u9ad4\u80b2 Sports', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TSindex.htm'),
(u'\u5f71\u8996 Film/TV', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/HK-MAindex.htm'),
(u'\u526f\u520a Supplements', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/WWindex.htm'),]:
articles = self.parse_section3(url, 'http://www.mingpaotor.com/')
if articles:
feeds.append((title, articles))
return feeds
# parse from news.mingpao.com
def parse_section(self, url):
dateStr = self.get_fetchdate()
soup = self.index_to_soup(url)
divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']})
current_articles = []
included_urls = []
divs.reverse()
for i in divs:
a = i.find('a', href = True)
title = self.tag_to_string(a)
url = a.get('href', False)
url = 'http://news.mingpao.com/' + dateStr + '/' +url
if url not in included_urls and url.rfind('Redirect') == -1:
current_articles.append({'title': title, 'url': url, 'description':'', 'date':''})
included_urls.append(url)
current_articles.reverse()
return current_articles
# parse from life.mingpao.com
def parse_section2(self, url, keystr):
self.get_fetchdate()
soup = self.index_to_soup(url)
a = soup.findAll('a', href=True)
a.reverse()
current_articles = []
included_urls = []
for i in a:
title = self.tag_to_string(i)
url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind(keystr) == -1):
url = url.replace('dailynews3.cfm', 'dailynews3a.cfm') # use printed version of the article
current_articles.append({'title': title, 'url': url, 'description': ''})
included_urls.append(url)
current_articles.reverse()
return current_articles
# parse from www.mingpaovan.com
def parse_section3(self, url, baseUrl):
self.get_fetchdate()
soup = self.index_to_soup(url)
divs = soup.findAll(attrs={'class': ['ListContentLargeLink']})
current_articles = []
included_urls = []
divs.reverse()
for i in divs:
title = self.tag_to_string(i)
urlstr = i.get('href', False)
urlstr = baseUrl + '/' + urlstr.replace('../../../', '')
if urlstr not in included_urls:
current_articles.append({'title': title, 'url': urlstr, 'description': '', 'date': ''})
included_urls.append(urlstr)
current_articles.reverse()
return current_articles
def parse_ed_section(self, url):
self.get_fetchdate()
soup = self.index_to_soup(url)
a = soup.findAll('a', href=True)
a.reverse()
current_articles = []
included_urls = []
for i in a:
title = self.tag_to_string(i)
url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('nal') == -1):
current_articles.append({'title': title, 'url': url, 'description': ''})
included_urls.append(url)
current_articles.reverse()
return current_articles
def parse_fin_section(self, url):
self.get_fetchdate()
soup = self.index_to_soup(url)
a = soup.findAll('a', href= True)
current_articles = []
included_urls = []
for i in a:
#url = 'http://www.mpfinance.com/cfm/' + i.get('href', False)
url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
#if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1:
if url not in included_urls and (not url.rfind('txt') == -1) and (not url.rfind('nal') == -1):
title = self.tag_to_string(i)
current_articles.append({'title': title, 'url': url, 'description':''})
included_urls.append(url)
return current_articles
def parse_ent_section(self, url):
self.get_fetchdate()
soup = self.index_to_soup(url)
a = soup.findAll('a', href=True)
a.reverse()
current_articles = []
included_urls = []
for i in a:
title = self.tag_to_string(i)
url = 'http://ol.mingpao.com/cfm/' + i.get('href', False)
if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1):
current_articles.append({'title': title, 'url': url, 'description': ''})
included_urls.append(url)
current_articles.reverse()
return current_articles
def parse_col_section(self, url):
self.get_fetchdate()
soup = self.index_to_soup(url)
a = soup.findAll('a', href=True)
a.reverse()
current_articles = []
included_urls = []
for i in a:
title = self.tag_to_string(i)
url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('ncl') == -1):
current_articles.append({'title': title, 'url': url, 'description': ''})
included_urls.append(url)
current_articles.reverse()
return current_articles
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
for item in soup.findAll(style=True):
del item['width']
for item in soup.findAll(stype=True):
del item['absmiddle']
return soup
def create_opf(self, feeds, dir=None):
if dir is None:
dir = self.output_dir
if __UseChineseTitle__ == True:
if __Region__ == 'Hong Kong':
title = u'\u660e\u5831 (\u9999\u6e2f)'
elif __Region__ == 'Vancouver':
title = u'\u660e\u5831 (\u6eab\u54e5\u83ef)'
elif __Region__ == 'Toronto':
title = u'\u660e\u5831 (\u591a\u502b\u591a)'
else:
title = self.short_title()
# if not generating a periodical, force date to apply in title
if __MakePeriodical__ == False:
title = title + ' ' + self.get_fetchformatteddate()
if True:
mi = MetaInformation(title, [self.publisher])
mi.publisher = self.publisher
mi.author_sort = self.publisher
if __MakePeriodical__ == True:
mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title()
else:
mi.publication_type = self.publication_type+':'+self.short_title()
#mi.timestamp = nowf()
mi.timestamp = self.get_dtlocal()
mi.comments = self.description
if not isinstance(mi.comments, unicode):
mi.comments = mi.comments.decode('utf-8', 'replace')
#mi.pubdate = nowf()
mi.pubdate = self.get_dtlocal()
opf_path = os.path.join(dir, 'index.opf')
ncx_path = os.path.join(dir, 'index.ncx')
opf = OPFCreator(dir, mi)
# Add mastheadImage entry to <guide> section
mp = getattr(self, 'masthead_path', None)
if mp is not None and os.access(mp, os.R_OK):
from calibre.ebooks.metadata.opf2 import Guide
ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu())
ref.type = 'masthead'
ref.title = 'Masthead Image'
opf.guide.append(ref)
manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))]
manifest.append(os.path.join(dir, 'index.html'))
manifest.append(os.path.join(dir, 'index.ncx'))
# Get cover
cpath = getattr(self, 'cover_path', None)
if cpath is None:
pf = open(os.path.join(dir, 'cover.jpg'), 'wb')
if self.default_cover(pf):
cpath = pf.name
if cpath is not None and os.access(cpath, os.R_OK):
opf.cover = cpath
manifest.append(cpath)
# Get masthead
mpath = getattr(self, 'masthead_path', None)
if mpath is not None and os.access(mpath, os.R_OK):
manifest.append(mpath)
opf.create_manifest_from_files_in(manifest)
for mani in opf.manifest:
if mani.path.endswith('.ncx'):
mani.id = 'ncx'
if mani.path.endswith('mastheadImage.jpg'):
mani.id = 'masthead-image'
entries = ['index.html']
toc = TOC(base_path=dir)
self.play_order_counter = 0
self.play_order_map = {}
def feed_index(num, parent):
f = feeds[num]
for j, a in enumerate(f):
if getattr(a, 'downloaded', False):
adir = 'feed_%d/article_%d/'%(num, j)
auth = a.author
if not auth:
auth = None
desc = a.text_summary
if not desc:
desc = None
else:
desc = self.description_limiter(desc)
entries.append('%sindex.html'%adir)
po = self.play_order_map.get(entries[-1], None)
if po is None:
self.play_order_counter += 1
po = self.play_order_counter
parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'),
play_order=po, author=auth, description=desc)
last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep))
for sp in a.sub_pages:
prefix = os.path.commonprefix([opf_path, sp])
relp = sp[len(prefix):]
entries.append(relp.replace(os.sep, '/'))
last = sp
if os.path.exists(last):
with open(last, 'rb') as fi:
src = fi.read().decode('utf-8')
soup = BeautifulSoup(src)
body = soup.find('body')
if body is not None:
prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last))))
templ = self.navbar.generate(True, num, j, len(f),
not self.has_single_feed,
a.orig_url, self.publisher, prefix=prefix,
center=self.center_navbar)
elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div')
body.insert(len(body.contents), elem)
with open(last, 'wb') as fi:
fi.write(unicode(soup).encode('utf-8'))
if len(feeds) == 0:
raise Exception('All feeds are empty, aborting.')
if len(feeds) > 1:
for i, f in enumerate(feeds):
entries.append('feed_%d/index.html'%i)
po = self.play_order_map.get(entries[-1], None)
if po is None:
self.play_order_counter += 1
po = self.play_order_counter
auth = getattr(f, 'author', None)
if not auth:
auth = None
desc = getattr(f, 'description', None)
if not desc:
desc = None
feed_index(i, toc.add_item('feed_%d/index.html'%i, None,
f.title, play_order=po, description=desc, author=auth))
else:
entries.append('feed_%d/index.html'%0)
feed_index(0, toc)
for i, p in enumerate(entries):
entries[i] = os.path.join(dir, p.replace('/', os.sep))
opf.create_spine(entries)
opf.set_toc(toc)
with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file):
opf.render(opf_file, ncx_file)

594
recipes/ming_pao_vancouver.recipe Normal file
View File

@ -0,0 +1,594 @@
__license__ = 'GPL v3'
__copyright__ = '2010-2011, Eddie Lau'
# Region - Hong Kong, Vancouver, Toronto
__Region__ = 'Vancouver'
# Users of Kindle 3 with limited system-level CJK support
# please replace the following "True" with "False".
__MakePeriodical__ = True
# Turn below to true if your device supports display of CJK titles
__UseChineseTitle__ = False
# Set it to False if you want to skip images
__KeepImages__ = True
# (HK only) Turn below to true if you wish to use life.mingpao.com as the main article source
__UseLife__ = True
'''
Change Log:
2011/06/26: add fetching Vancouver and Toronto versions of the paper, also provide captions for images using life.mingpao fetch source
provide options to remove all images in the file
2011/05/12: switch the main parse source to life.mingpao.com, which has more photos on the article pages
2011/03/06: add new articles for finance section, also a new section "Columns"
2011/02/28: rearrange the sections
[Disabled until Kindle has better CJK support and can remember last (section,article) read in Sections & Articles
View] make it the same title if generating a periodical, so past issue will be automatically put into "Past Issues"
folder in Kindle 3
2011/02/20: skip duplicated links in finance section, put photos which may extend a whole page to the back of the articles
clean up the indentation
2010/12/07: add entertainment section, use newspaper front page as ebook cover, suppress date display in section list
(to avoid wrong date display in case the user generates the ebook in a time zone different from HKT)
2010/11/22: add English section, remove eco-news section which is not updated daily, correct
ordering of articles
2010/11/12: add news image and eco-news section
2010/11/08: add parsing of finance section
2010/11/06: temporary work-around for Kindle device having no capability to display unicode
in section/article list.
2010/10/31: skip repeated articles in section pages
'''
import os, datetime, re
from calibre.web.feeds.recipes import BasicNewsRecipe
from contextlib import nested
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.ebooks.metadata.opf2 import OPFCreator
from calibre.ebooks.metadata.toc import TOC
from calibre.ebooks.metadata import MetaInformation
# MAIN CLASS
class MPRecipe(BasicNewsRecipe):
if __Region__ == 'Hong Kong':
title = 'Ming Pao - Hong Kong'
description = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)'
category = 'Chinese, News, Hong Kong'
extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}'
masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif'
keep_only_tags = [dict(name='h1'),
dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page title
dict(name='font', attrs={'color':['AA0000']}), # for column articles title
dict(attrs={'id':['newscontent']}), # entertainment and column page content
dict(attrs={'id':['newscontent01','newscontent02']}),
dict(attrs={'class':['photo']}),
dict(name='table', attrs={'width':['100%'], 'border':['0'], 'cellspacing':['5'], 'cellpadding':['0']}), # content in printed version of life.mingpao.com
dict(name='img', attrs={'width':['180'], 'alt':['按圖放大']}) # images for source from life.mingpao.com
]
if __KeepImages__:
remove_tags = [dict(name='style'),
dict(attrs={'id':['newscontent135']}), # for the finance page from mpfinance.com
dict(name='font', attrs={'size':['2'], 'color':['666666']}), # article date in life.mingpao.com article
#dict(name='table') # for content fetched from life.mingpao.com
]
else:
remove_tags = [dict(name='style'),
dict(attrs={'id':['newscontent135']}), # for the finance page from mpfinance.com
dict(name='font', attrs={'size':['2'], 'color':['666666']}), # article date in life.mingpao.com article
dict(name='img'),
#dict(name='table') # for content fetched from life.mingpao.com
]
remove_attributes = ['width']
preprocess_regexps = [
(re.compile(r'<h5>', re.DOTALL|re.IGNORECASE),
lambda match: '<h1>'),
(re.compile(r'</h5>', re.DOTALL|re.IGNORECASE),
lambda match: '</h1>'),
(re.compile(r'<p><a href=.+?</a></p>', re.DOTALL|re.IGNORECASE), # for entertainment page
lambda match: ''),
# skip <br> after title in life.mingpao.com fetched article
(re.compile(r"<div id='newscontent'><br>", re.DOTALL|re.IGNORECASE),
lambda match: "<div id='newscontent'>"),
(re.compile(r"<br><br></b>", re.DOTALL|re.IGNORECASE),
lambda match: "</b>")
]
elif __Region__ == 'Vancouver':
title = 'Ming Pao - Vancouver'
description = 'Vancouver Chinese Newspaper (http://www.mingpaovan.com)'
category = 'Chinese, News, Vancouver'
extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} b>font {font-size:200%; font-weight:bold;}'
masthead_url = 'http://www.mingpaovan.com/image/mainlogo2_VAN2.gif'
keep_only_tags = [dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['1']}),
dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['3'], 'cellpadding':['3'], 'id':['tblContent3']}),
dict(name='table', attrs={'width':['180'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['0'], 'bgcolor':['F0F0F0']}),
]
if __KeepImages__:
remove_tags = [dict(name='img', attrs={'src':['../../../image/magnifier.gif']})] # the magnifier icon
else:
remove_tags = [dict(name='img')]
remove_attributes = ['width']
preprocess_regexps = [(re.compile(r'&nbsp;', re.DOTALL|re.IGNORECASE),
lambda match: ''),
]
elif __Region__ == 'Toronto':
title = 'Ming Pao - Toronto'
description = 'Toronto Chinese Newspaper (http://www.mingpaotor.com)'
category = 'Chinese, News, Toronto'
extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} b>font {font-size:200%; font-weight:bold;}'
masthead_url = 'http://www.mingpaotor.com/image/mainlogo2_TOR2.gif'
keep_only_tags = [dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['1']}),
dict(name='table', attrs={'width':['450'], 'border':['0'], 'cellspacing':['3'], 'cellpadding':['3'], 'id':['tblContent3']}),
dict(name='table', attrs={'width':['180'], 'border':['0'], 'cellspacing':['0'], 'cellpadding':['0'], 'bgcolor':['F0F0F0']}),
]
if __KeepImages__:
remove_tags = [dict(name='img', attrs={'src':['../../../image/magnifier.gif']})] # the magnifier icon
else:
remove_tags = [dict(name='img')]
remove_attributes = ['width']
preprocess_regexps = [(re.compile(r'&nbsp;', re.DOTALL|re.IGNORECASE),
lambda match: ''),
]
oldest_article = 1
max_articles_per_feed = 100
__author__ = 'Eddie Lau'
publisher = 'MingPao'
remove_javascript = True
use_embedded_content = False
no_stylesheets = True
language = 'zh'
encoding = 'Big5-HKSCS'
recursions = 0
conversion_options = {'linearize_tables':True}
timefmt = ''
def image_url_processor(cls, baseurl, url):
# trick: break the url at the first occurance of digit, add an additional
# '_' at the front
# not working, may need to move this to preprocess_html() method
# minIdx = 10000
# i0 = url.find('0')
# if i0 >= 0 and i0 < minIdx:
# minIdx = i0
# i1 = url.find('1')
# if i1 >= 0 and i1 < minIdx:
# minIdx = i1
# i2 = url.find('2')
# if i2 >= 0 and i2 < minIdx:
# minIdx = i2
# i3 = url.find('3')
# if i3 >= 0 and i0 < minIdx:
# minIdx = i3
# i4 = url.find('4')
# if i4 >= 0 and i4 < minIdx:
# minIdx = i4
# i5 = url.find('5')
# if i5 >= 0 and i5 < minIdx:
# minIdx = i5
# i6 = url.find('6')
# if i6 >= 0 and i6 < minIdx:
# minIdx = i6
# i7 = url.find('7')
# if i7 >= 0 and i7 < minIdx:
# minIdx = i7
# i8 = url.find('8')
# if i8 >= 0 and i8 < minIdx:
# minIdx = i8
# i9 = url.find('9')
# if i9 >= 0 and i9 < minIdx:
# minIdx = i9
return url
def get_dtlocal(self):
dt_utc = datetime.datetime.utcnow()
if __Region__ == 'Hong Kong':
# convert UTC to local hk time - at HKT 5.30am, all news are available
dt_local = dt_utc + datetime.timedelta(8.0/24) - datetime.timedelta(5.5/24)
# dt_local = dt_utc.astimezone(pytz.timezone('Asia/Hong_Kong')) - datetime.timedelta(5.5/24)
elif __Region__ == 'Vancouver':
# convert UTC to local Vancouver time - at PST time 5.30am, all news are available
dt_local = dt_utc + datetime.timedelta(-8.0/24) - datetime.timedelta(5.5/24)
#dt_local = dt_utc.astimezone(pytz.timezone('America/Vancouver')) - datetime.timedelta(5.5/24)
elif __Region__ == 'Toronto':
# convert UTC to local Toronto time - at EST time 8.30am, all news are available
dt_local = dt_utc + datetime.timedelta(-5.0/24) - datetime.timedelta(8.5/24)
#dt_local = dt_utc.astimezone(pytz.timezone('America/Toronto')) - datetime.timedelta(8.5/24)
return dt_local
def get_fetchdate(self):
return self.get_dtlocal().strftime("%Y%m%d")
def get_fetchformatteddate(self):
return self.get_dtlocal().strftime("%Y-%m-%d")
def get_fetchday(self):
return self.get_dtlocal().strftime("%d")
def get_cover_url(self):
if __Region__ == 'Hong Kong':
cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg'
elif __Region__ == 'Vancouver':
cover = 'http://www.mingpaovan.com/ftp/News/' + self.get_fetchdate() + '/' + self.get_fetchday() + 'pgva1s.jpg'
elif __Region__ == 'Toronto':
cover = 'http://www.mingpaotor.com/ftp/News/' + self.get_fetchdate() + '/' + self.get_fetchday() + 'pgtas.jpg'
br = BasicNewsRecipe.get_browser()
try:
br.open(cover)
except:
cover = None
return cover
def parse_index(self):
feeds = []
dateStr = self.get_fetchdate()
if __Region__ == 'Hong Kong':
if __UseLife__:
for title, url, keystr in [(u'\u8981\u805e Headline', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalga', 'nal'),
(u'\u6e2f\u805e Local', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgb', 'nal'),
(u'\u6559\u80b2 Education', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgf', 'nal'),
(u'\u793e\u8a55/\u7b46\u9663 Editorial', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr', 'nal'),
(u'\u8ad6\u58c7 Forum', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalfa', 'nal'),
(u'\u4e2d\u570b China', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalca', 'nal'),
(u'\u570b\u969b World', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalta', 'nal'),
(u'\u7d93\u6fdf Finance', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea', 'nal'),
(u'\u9ad4\u80b2 Sport', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalsp', 'nal'),
(u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal'),
(u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl')]:
articles = self.parse_section2(url, keystr)
if articles:
feeds.append((title, articles))
for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
(u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
else:
for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'),
(u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'),
(u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special- editorial
ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr')
if ed_articles:
feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles))
for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'),
(u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'),
(u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special - finance
#fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm')
fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea')
if fin_articles:
feeds.append((u'\u7d93\u6fdf Finance', fin_articles))
for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'),
(u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special - entertainment
ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm')
if ent_articles:
feeds.append((u'\u5f71\u8996 Film/TV', ent_articles))
for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
(u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
# special- columns
col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn')
if col_articles:
feeds.append((u'\u5c08\u6b04 Columns', col_articles))
elif __Region__ == 'Vancouver':
for title, url in [(u'\u8981\u805e Headline', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VAindex.htm'),
(u'\u52a0\u570b Canada', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VBindex.htm'),
(u'\u793e\u5340 Local', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VDindex.htm'),
(u'\u6e2f\u805e Hong Kong', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/HK-VGindex.htm'),
(u'\u570b\u969b World', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VTindex.htm'),
(u'\u4e2d\u570b China', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VCindex.htm'),
(u'\u7d93\u6fdf Economics', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VEindex.htm'),
(u'\u9ad4\u80b2 Sports', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/VSindex.htm'),
(u'\u5f71\u8996 Film/TV', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/HK-MAindex.htm'),
(u'\u526f\u520a Supplements', 'http://www.mingpaovan.com/htm/News/' + dateStr + '/WWindex.htm'),]:
articles = self.parse_section3(url, 'http://www.mingpaovan.com/')
if articles:
feeds.append((title, articles))
elif __Region__ == 'Toronto':
for title, url in [(u'\u8981\u805e Headline', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TAindex.htm'),
(u'\u52a0\u570b Canada', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TDindex.htm'),
(u'\u793e\u5340 Local', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TFindex.htm'),
(u'\u4e2d\u570b China', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TCAindex.htm'),
(u'\u570b\u969b World', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TTAindex.htm'),
(u'\u6e2f\u805e Hong Kong', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/HK-GAindex.htm'),
(u'\u7d93\u6fdf Economics', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/THindex.htm'),
(u'\u9ad4\u80b2 Sports', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/TSindex.htm'),
(u'\u5f71\u8996 Film/TV', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/HK-MAindex.htm'),
(u'\u526f\u520a Supplements', 'http://www.mingpaotor.com/htm/News/' + dateStr + '/WWindex.htm'),]:
articles = self.parse_section3(url, 'http://www.mingpaotor.com/')
if articles:
feeds.append((title, articles))
return feeds
# parse from news.mingpao.com
def parse_section(self, url):
dateStr = self.get_fetchdate()
soup = self.index_to_soup(url)
divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']})
current_articles = []
included_urls = []
divs.reverse()
for i in divs:
a = i.find('a', href = True)
title = self.tag_to_string(a)
url = a.get('href', False)
url = 'http://news.mingpao.com/' + dateStr + '/' +url
if url not in included_urls and url.rfind('Redirect') == -1:
current_articles.append({'title': title, 'url': url, 'description':'', 'date':''})
included_urls.append(url)
current_articles.reverse()
return current_articles
# parse from life.mingpao.com
def parse_section2(self, url, keystr):
self.get_fetchdate()
soup = self.index_to_soup(url)
a = soup.findAll('a', href=True)
a.reverse()
current_articles = []
included_urls = []
for i in a:
title = self.tag_to_string(i)
url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind(keystr) == -1):
url = url.replace('dailynews3.cfm', 'dailynews3a.cfm') # use printed version of the article
current_articles.append({'title': title, 'url': url, 'description': ''})
included_urls.append(url)
current_articles.reverse()
return current_articles
# parse from www.mingpaovan.com
def parse_section3(self, url, baseUrl):
self.get_fetchdate()
soup = self.index_to_soup(url)
divs = soup.findAll(attrs={'class': ['ListContentLargeLink']})
current_articles = []
included_urls = []
divs.reverse()
for i in divs:
title = self.tag_to_string(i)
urlstr = i.get('href', False)
urlstr = baseUrl + '/' + urlstr.replace('../../../', '')
if urlstr not in included_urls:
current_articles.append({'title': title, 'url': urlstr, 'description': '', 'date': ''})
included_urls.append(urlstr)
current_articles.reverse()
return current_articles
def parse_ed_section(self, url):
self.get_fetchdate()
soup = self.index_to_soup(url)
a = soup.findAll('a', href=True)
a.reverse()
current_articles = []
included_urls = []
for i in a:
title = self.tag_to_string(i)
url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('nal') == -1):
current_articles.append({'title': title, 'url': url, 'description': ''})
included_urls.append(url)
current_articles.reverse()
return current_articles
def parse_fin_section(self, url):
self.get_fetchdate()
soup = self.index_to_soup(url)
a = soup.findAll('a', href= True)
current_articles = []
included_urls = []
for i in a:
#url = 'http://www.mpfinance.com/cfm/' + i.get('href', False)
url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
#if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1:
if url not in included_urls and (not url.rfind('txt') == -1) and (not url.rfind('nal') == -1):
title = self.tag_to_string(i)
current_articles.append({'title': title, 'url': url, 'description':''})
included_urls.append(url)
return current_articles
def parse_ent_section(self, url):
self.get_fetchdate()
soup = self.index_to_soup(url)
a = soup.findAll('a', href=True)
a.reverse()
current_articles = []
included_urls = []
for i in a:
title = self.tag_to_string(i)
url = 'http://ol.mingpao.com/cfm/' + i.get('href', False)
if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1):
current_articles.append({'title': title, 'url': url, 'description': ''})
included_urls.append(url)
current_articles.reverse()
return current_articles
def parse_col_section(self, url):
self.get_fetchdate()
soup = self.index_to_soup(url)
a = soup.findAll('a', href=True)
a.reverse()
current_articles = []
included_urls = []
for i in a:
title = self.tag_to_string(i)
url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('ncl') == -1):
current_articles.append({'title': title, 'url': url, 'description': ''})
included_urls.append(url)
current_articles.reverse()
return current_articles
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
for item in soup.findAll(style=True):
del item['width']
for item in soup.findAll(stype=True):
del item['absmiddle']
return soup
def create_opf(self, feeds, dir=None):
if dir is None:
dir = self.output_dir
if __UseChineseTitle__ == True:
if __Region__ == 'Hong Kong':
title = u'\u660e\u5831 (\u9999\u6e2f)'
elif __Region__ == 'Vancouver':
title = u'\u660e\u5831 (\u6eab\u54e5\u83ef)'
elif __Region__ == 'Toronto':
title = u'\u660e\u5831 (\u591a\u502b\u591a)'
else:
title = self.short_title()
# if not generating a periodical, force date to apply in title
if __MakePeriodical__ == False:
title = title + ' ' + self.get_fetchformatteddate()
if True:
mi = MetaInformation(title, [self.publisher])
mi.publisher = self.publisher
mi.author_sort = self.publisher
if __MakePeriodical__ == True:
mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title()
else:
mi.publication_type = self.publication_type+':'+self.short_title()
#mi.timestamp = nowf()
mi.timestamp = self.get_dtlocal()
mi.comments = self.description
if not isinstance(mi.comments, unicode):
mi.comments = mi.comments.decode('utf-8', 'replace')
#mi.pubdate = nowf()
mi.pubdate = self.get_dtlocal()
opf_path = os.path.join(dir, 'index.opf')
ncx_path = os.path.join(dir, 'index.ncx')
opf = OPFCreator(dir, mi)
# Add mastheadImage entry to <guide> section
mp = getattr(self, 'masthead_path', None)
if mp is not None and os.access(mp, os.R_OK):
from calibre.ebooks.metadata.opf2 import Guide
ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu())
ref.type = 'masthead'
ref.title = 'Masthead Image'
opf.guide.append(ref)
manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))]
manifest.append(os.path.join(dir, 'index.html'))
manifest.append(os.path.join(dir, 'index.ncx'))
# Get cover
cpath = getattr(self, 'cover_path', None)
if cpath is None:
pf = open(os.path.join(dir, 'cover.jpg'), 'wb')
if self.default_cover(pf):
cpath = pf.name
if cpath is not None and os.access(cpath, os.R_OK):
opf.cover = cpath
manifest.append(cpath)
# Get masthead
mpath = getattr(self, 'masthead_path', None)
if mpath is not None and os.access(mpath, os.R_OK):
manifest.append(mpath)
opf.create_manifest_from_files_in(manifest)
for mani in opf.manifest:
if mani.path.endswith('.ncx'):
mani.id = 'ncx'
if mani.path.endswith('mastheadImage.jpg'):
mani.id = 'masthead-image'
entries = ['index.html']
toc = TOC(base_path=dir)
self.play_order_counter = 0
self.play_order_map = {}
def feed_index(num, parent):
f = feeds[num]
for j, a in enumerate(f):
if getattr(a, 'downloaded', False):
adir = 'feed_%d/article_%d/'%(num, j)
auth = a.author
if not auth:
auth = None
desc = a.text_summary
if not desc:
desc = None
else:
desc = self.description_limiter(desc)
entries.append('%sindex.html'%adir)
po = self.play_order_map.get(entries[-1], None)
if po is None:
self.play_order_counter += 1
po = self.play_order_counter
parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'),
play_order=po, author=auth, description=desc)
last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep))
for sp in a.sub_pages:
prefix = os.path.commonprefix([opf_path, sp])
relp = sp[len(prefix):]
entries.append(relp.replace(os.sep, '/'))
last = sp
if os.path.exists(last):
with open(last, 'rb') as fi:
src = fi.read().decode('utf-8')
soup = BeautifulSoup(src)
body = soup.find('body')
if body is not None:
prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last))))
templ = self.navbar.generate(True, num, j, len(f),
not self.has_single_feed,
a.orig_url, self.publisher, prefix=prefix,
center=self.center_navbar)
elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div')
body.insert(len(body.contents), elem)
with open(last, 'wb') as fi:
fi.write(unicode(soup).encode('utf-8'))
if len(feeds) == 0:
raise Exception('All feeds are empty, aborting.')
if len(feeds) > 1:
for i, f in enumerate(feeds):
entries.append('feed_%d/index.html'%i)
po = self.play_order_map.get(entries[-1], None)
if po is None:
self.play_order_counter += 1
po = self.play_order_counter
auth = getattr(f, 'author', None)
if not auth:
auth = None
desc = getattr(f, 'description', None)
if not desc:
desc = None
feed_index(i, toc.add_item('feed_%d/index.html'%i, None,
f.title, play_order=po, description=desc, author=auth))
else:
entries.append('feed_%d/index.html'%0)
feed_index(0, toc)
for i, p in enumerate(entries):
entries[i] = os.path.join(dir, p.replace('/', os.sep))
opf.create_spine(entries)
opf.set_toc(toc)
with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file):
opf.render(opf_file, ncx_file)

80
recipes/scmp.recipe Normal file
View File

@ -0,0 +1,80 @@
__license__ = 'GPL v3'
__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
'''
scmp.com
'''
import re
from calibre.web.feeds.news import BasicNewsRecipe
class SCMP(BasicNewsRecipe):
title = 'South China Morning Post'
__author__ = 'llam'
description = "SCMP.com, Hong Kong's premier online English daily provides exclusive up-to-date news, audio video news, podcasts, RSS Feeds, Blogs, breaking news, top stories, award winning news and analysis on Hong Kong and China."
publisher = 'South China Morning Post Publishers Ltd.'
category = 'SCMP, Online news, Hong Kong News, China news, Business news, English newspaper, daily newspaper, Lifestyle news, Sport news, Audio Video news, Asia news, World news, economy news, investor relations news, RSS Feeds'
oldest_article = 2
delay = 1
max_articles_per_feed = 200
no_stylesheets = True
encoding = 'utf-8'
use_embedded_content = False
language = 'en_CN'
remove_empty_feeds = True
needs_subscription = True
publication_type = 'newspaper'
masthead_url = 'http://www.scmp.com/images/logo_scmp_home.gif'
extra_css = ' body{font-family: Arial,Helvetica,sans-serif } '
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
}
def get_browser(self):
br = BasicNewsRecipe.get_browser()
#br.set_debug_http(True)
#br.set_debug_responses(True)
#br.set_debug_redirects(True)
if self.username is not None and self.password is not None:
br.open('http://www.scmp.com/portal/site/SCMP/')
br.select_form(name='loginForm')
br['Login' ] = self.username
br['Password'] = self.password
br.submit()
return br
remove_attributes=['width','height','border']
keep_only_tags = [
dict(attrs={'id':['ART','photoBox']})
,dict(attrs={'class':['article_label','article_byline','article_body']})
]
preprocess_regexps = [
(re.compile(r'<P><table((?!<table).)*class="embscreen"((?!</table>).)*</table>', re.DOTALL|re.IGNORECASE),
lambda match: ''),
]
feeds = [
(u'Business' , u'http://www.scmp.com/rss/business.xml' )
,(u'Hong Kong' , u'http://www.scmp.com/rss/hong_kong.xml' )
,(u'China' , u'http://www.scmp.com/rss/china.xml' )
,(u'Asia & World' , u'http://www.scmp.com/rss/news_asia_world.xml')
,(u'Opinion' , u'http://www.scmp.com/rss/opinion.xml' )
,(u'LifeSTYLE' , u'http://www.scmp.com/rss/lifestyle.xml' )
,(u'Sport' , u'http://www.scmp.com/rss/sport.xml' )
]
def print_version(self, url):
rpart, sep, rest = url.rpartition('&')
return rpart #+ sep + urllib.quote_plus(rest)
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
items = soup.findAll(src="/images/label_icon.gif")
[item.extract() for item in items]
return self.adeify_images(soup)

40
recipes/sizinti_derigisi.recipe Normal file
View File

@ -0,0 +1,40 @@
# -*- coding: utf-8 -*-
from calibre.web.feeds.news import BasicNewsRecipe
class TodaysZaman_en(BasicNewsRecipe):
title = u'Sızıntı Dergisi'
__author__ = u'thomass'
description = 'a Turkey based daily for national and international news in the fields of business, diplomacy, politics, culture, arts, sports and economics, in addition to commentaries, specials and features'
oldest_article = 30
max_articles_per_feed =80
no_stylesheets = True
#delay = 1
#use_embedded_content = False
encoding = 'utf-8'
#publisher = ' '
category = 'dergi, ilim, kültür, bilim,Türkçe'
language = 'tr'
publication_type = 'magazine'
#extra_css = ' body{ font-family: Verdana,Helvetica,Arial,sans-serif } .introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} '
#keep_only_tags = [dict(name='h1', attrs={'class':['georgia_30']})]
#remove_attributes = ['aria-describedby']
#remove_tags = [dict(name='div', attrs={'id':['renk10']}) ]
cover_img_url = 'http://www.sizinti.com.tr/images/sizintiprint.jpg'
masthead_url = 'http://www.sizinti.com.tr/images/sizintiprint.jpg'
remove_tags_before = dict(id='content-right')
#remove_empty_feeds= True
#remove_attributes = ['width','height']
feeds = [
( u'Sızıntı', u'http://www.sizinti.com.tr/rss'),
]
#def preprocess_html(self, soup):
# return self.adeify_images(soup)
#def print_version(self, url): #there is a probem caused by table format
#return url.replace('http://www.todayszaman.com/newsDetail_getNewsById.action?load=detay&', 'http://www.todayszaman.com/newsDetail_openPrintPage.action?')

1
recipes/telegraph_uk.recipe
View File

@ -56,6 +56,7 @@ class TelegraphUK(BasicNewsRecipe):
,(u'Sport' , u'http://www.telegraph.co.uk/sport/rss' )
,(u'Earth News' , u'http://www.telegraph.co.uk/earth/earthnews/rss' )
,(u'Comment' , u'http://www.telegraph.co.uk/comment/rss' )
,(u'Travel' , u'http://www.telegraph.co.uk/travel/rss' )
,(u'How about that?', u'http://www.telegraph.co.uk/news/newstopics/howaboutthat/rss' )
]

53
recipes/todays_zaman.recipe Normal file
View File

@ -0,0 +1,53 @@
from calibre.web.feeds.news import BasicNewsRecipe
class TodaysZaman_en(BasicNewsRecipe):
title = u'Todays Zaman'
__author__ = u'thomass'
description = 'a Turkey based daily for national and international news in the fields of business, diplomacy, politics, culture, arts, sports and economics, in addition to commentaries, specials and features'
oldest_article = 2
max_articles_per_feed =100
no_stylesheets = True
#delay = 1
#use_embedded_content = False
encoding = 'utf-8'
#publisher = ' '
category = 'news, haberler,TR,gazete'
language = 'en_TR'
publication_type = 'newspaper'
#extra_css = ' body{ font-family: Verdana,Helvetica,Arial,sans-serif } .introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} '
#keep_only_tags = [dict(name='font', attrs={'class':['newsDetail','agenda2NewsSpot']}),dict(name='span', attrs={'class':['agenda2Title']}),dict(name='div', attrs={'id':['gallery']})]
keep_only_tags = [dict(name='h1', attrs={'class':['georgia_30']}),dict(name='span', attrs={'class':['left-date','detailDate','detailCName']}),dict(name='td', attrs={'id':['newsSpot','newsText']})] #resim ekleme: ,dict(name='div', attrs={'id':['gallery','detailDate',]})
remove_attributes = ['aria-describedby']
remove_tags = [dict(name='img', attrs={'src':['/images/icon_print.gif','http://gmodules.com/ig/images/plus_google.gif','/images/template/jazz/agenda/i1.jpg', 'http://medya.todayszaman.com/todayszaman/images/logo/logo.bmp']}),dict(name='hr', attrs={'class':[ 'interactive-hr']}),dict(name='div', attrs={'class':[ 'empty_height_18','empty_height_9']}) ,dict(name='td', attrs={'id':[ 'superTitle']}),dict(name='span', attrs={'class':[ 't-count enabled t-count-focus']}),dict(name='a', attrs={'id':[ 'count']}),dict(name='td', attrs={'class':[ 'left-date']}) ]
cover_img_url = 'http://medya.todayszaman.com/todayszaman/images/logo/logo.bmp'
masthead_url = 'http://medya.todayszaman.com/todayszaman/images/logo/logo.bmp'
remove_empty_feeds= True
# remove_attributes = ['width','height']
feeds = [
( u'Home', u'http://www.todayszaman.com/rss?sectionId=0'),
( u'News', u'http://www.todayszaman.com/rss?sectionId=100'),
( u'Business', u'http://www.todayszaman.com/rss?sectionId=105'),
( u'Interviews', u'http://www.todayszaman.com/rss?sectionId=8'),
( u'Columnists', u'http://www.todayszaman.com/rss?sectionId=6'),
( u'Op-Ed', u'http://www.todayszaman.com/rss?sectionId=109'),
( u'Arts & Culture', u'http://www.todayszaman.com/rss?sectionId=110'),
( u'Expat Zone', u'http://www.todayszaman.com/rss?sectionId=132'),
( u'Sports', u'http://www.todayszaman.com/rss?sectionId=5'),
( u'Features', u'http://www.todayszaman.com/rss?sectionId=116'),
( u'Travel', u'http://www.todayszaman.com/rss?sectionId=117'),
( u'Leisure', u'http://www.todayszaman.com/rss?sectionId=118'),
( u'Weird But True', u'http://www.todayszaman.com/rss?sectionId=134'),
( u'Life', u'http://www.todayszaman.com/rss?sectionId=133'),
( u'Health', u'http://www.todayszaman.com/rss?sectionId=126'),
( u'Press Review', u'http://www.todayszaman.com/rss?sectionId=130'),
( u'Todays think tanks', u'http://www.todayszaman.com/rss?sectionId=159'),
]
#def preprocess_html(self, soup):
# return self.adeify_images(soup)
#def print_version(self, url): #there is a probem caused by table format
#return url.replace('http://www.todayszaman.com/newsDetail_getNewsById.action?load=detay&', 'http://www.todayszaman.com/newsDetail_openPrintPage.action?')

136
recipes/wprost.recipe
View File

@ -2,90 +2,92 @@
__license__ = 'GPL v3'
__copyright__ = '2010, matek09, matek09@gmail.com'
__copyright__ = 'Modified 2011, Mariusz Wolek <mariusz_dot_wolek @ gmail dot com>'
from calibre.web.feeds.news import BasicNewsRecipe
import re
class Wprost(BasicNewsRecipe):
EDITION = 0
FIND_LAST_FULL_ISSUE = True
EXCLUDE_LOCKED = True
ICO_BLOCKED = 'http://www.wprost.pl/G/icons/ico_blocked.gif'
EDITION = 0
FIND_LAST_FULL_ISSUE = True
EXCLUDE_LOCKED = True
ICO_BLOCKED = 'http://www.wprost.pl/G/icons/ico_blocked.gif'
title = u'Wprost'
__author__ = 'matek09'
description = 'Weekly magazine'
encoding = 'ISO-8859-2'
no_stylesheets = True
language = 'pl'
remove_javascript = True
title = u'Wprost'
__author__ = 'matek09'
description = 'Weekly magazine'
encoding = 'ISO-8859-2'
no_stylesheets = True
language = 'pl'
remove_javascript = True
remove_tags_before = dict(dict(name = 'div', attrs = {'id' : 'print-layer'}))
remove_tags_after = dict(dict(name = 'div', attrs = {'id' : 'print-layer'}))
remove_tags_before = dict(dict(name = 'div', attrs = {'id' : 'print-layer'}))
remove_tags_after = dict(dict(name = 'div', attrs = {'id' : 'print-layer'}))
'''keep_only_tags =[]
keep_only_tags.append(dict(name = 'table', attrs = {'id' : 'title-table'}))
keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'div-header'}))
keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'div-content'}))
keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'def element-autor'}))'''
'''keep_only_tags =[]
keep_only_tags.append(dict(name = 'table', attrs = {'id' : 'title-table'}))
keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'div-header'}))
keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'div-content'}))
keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'def element-autor'}))'''
preprocess_regexps = [(re.compile(r'style="display: none;"'), lambda match: ''),
(re.compile(r'display: block;'), lambda match: '')]
preprocess_regexps = [(re.compile(r'style="display: none;"'), lambda match: ''),
(re.compile(r'display: block;'), lambda match: ''),
(re.compile(r'\<td\>\<tr\>\<\/table\>'), lambda match: ''),
(re.compile(r'\<table .*?\>'), lambda match: ''),
(re.compile(r'\<tr>'), lambda match: ''),
(re.compile(r'\<td .*?\>'), lambda match: '')]
remove_tags =[]
remove_tags.append(dict(name = 'div', attrs = {'class' : 'def element-date'}))
remove_tags.append(dict(name = 'div', attrs = {'class' : 'def silver'}))
remove_tags.append(dict(name = 'div', attrs = {'id' : 'content-main-column-right'}))
remove_tags =[]
remove_tags.append(dict(name = 'div', attrs = {'class' : 'def element-date'}))
remove_tags.append(dict(name = 'div', attrs = {'class' : 'def silver'}))
remove_tags.append(dict(name = 'div', attrs = {'id' : 'content-main-column-right'}))
extra_css = '''
.div-header {font-size: x-small; font-weight: bold}
'''
extra_css = '''
.div-header {font-size: x-small; font-weight: bold}
'''
#h2 {font-size: x-large; font-weight: bold}
def is_blocked(self, a):
if a.findNextSibling('img') is None:
return False
else:
return True
def is_blocked(self, a):
if a.findNextSibling('img') is None:
return False
else:
return True
def find_last_issue(self):
soup = self.index_to_soup('http://www.wprost.pl/archiwum/')
a = 0
if self.FIND_LAST_FULL_ISSUE:
ico_blocked = soup.findAll('img', attrs={'src' : self.ICO_BLOCKED})
a = ico_blocked[-1].findNext('a', attrs={'title' : re.compile('Zobacz spis tre.ci')})
else:
a = soup.find('a', attrs={'title' : re.compile('Zobacz spis tre.ci')})
self.EDITION = a['href'].replace('/tygodnik/?I=', '')
self.cover_url = a.img['src']
def find_last_issue(self):
soup = self.index_to_soup('http://www.wprost.pl/archiwum/')
a = 0
if self.FIND_LAST_FULL_ISSUE:
ico_blocked = soup.findAll('img', attrs={'src' : self.ICO_BLOCKED})
a = ico_blocked[-1].findNext('a', attrs={'title' : re.compile('Zobacz spis tre.ci')})
else:
a = soup.find('a', attrs={'title' : re.compile('Zobacz spis tre.ci')})
self.EDITION = a['href'].replace('/tygodnik/?I=', '')
self.cover_url = a.img['src']
def parse_index(self):
self.find_last_issue()
soup = self.index_to_soup('http://www.wprost.pl/tygodnik/?I=' + self.EDITION)
feeds = []
for main_block in soup.findAll(attrs={'class':'main-block-s3 s3-head head-red3'}):
articles = list(self.find_articles(main_block))
if len(articles) > 0:
section = self.tag_to_string(main_block)
feeds.append((section, articles))
return feeds
def find_articles(self, main_block):
for a in main_block.findAllNext( attrs={'style':['','padding-top: 15px;']}):
if a.name in "td":
break
if self.EXCLUDE_LOCKED & self.is_blocked(a):
continue
yield {
'title' : self.tag_to_string(a),
'url' : 'http://www.wprost.pl' + a['href'],
'date' : '',
'description' : ''
}
def parse_index(self):
self.find_last_issue()
soup = self.index_to_soup('http://www.wprost.pl/tygodnik/?I=' + self.EDITION)
feeds = []
for main_block in soup.findAll(attrs={'class':'main-block-s3 s3-head head-red3'}):
articles = list(self.find_articles(main_block))
if len(articles) > 0:
section = self.tag_to_string(main_block)
feeds.append((section, articles))
return feeds
def find_articles(self, main_block):
for a in main_block.findAllNext( attrs={'style':['','padding-top: 15px;']}):
if a.name in "td":
break
if self.EXCLUDE_LOCKED & self.is_blocked(a):
continue
yield {
'title' : self.tag_to_string(a),
'url' : 'http://www.wprost.pl' + a['href'],
'date' : '',
'description' : ''
}

67
recipes/zaman.recipe
View File

@ -1,20 +1,55 @@
# -*- coding: utf-8 -*-
from calibre.web.feeds.news import BasicNewsRecipe
class ZamanRecipe(BasicNewsRecipe):
title = u'Zaman'
__author__ = u'Deniz Og\xfcz'
language = 'tr'
oldest_article = 1
max_articles_per_feed = 10
class Zaman (BasicNewsRecipe):
cover_url = 'http://medya.zaman.com.tr/zamantryeni/pics/zamanonline.gif'
feeds = [(u'Gundem', u'http://www.zaman.com.tr/gundem.rss'),
(u'Son Dakika', u'http://www.zaman.com.tr/sondakika.rss'),
(u'Spor', u'http://www.zaman.com.tr/spor.rss'),
(u'Ekonomi', u'http://www.zaman.com.tr/ekonomi.rss'),
(u'Politika', u'http://www.zaman.com.tr/politika.rss'),
(u'D\u0131\u015f Haberler', u'http://www.zaman.com.tr/dishaberler.rss'),
(u'Yazarlar', u'http://www.zaman.com.tr/yazarlar.rss'),]
title = u'ZAMAN Gazetesi'
__author__ = u'thomass'
oldest_article = 2
max_articles_per_feed =100
# no_stylesheets = True
#delay = 1
#use_embedded_content = False
encoding = 'ISO 8859-9'
publisher = 'Zaman'
category = 'news, haberler,TR,gazete'
language = 'tr'
publication_type = 'newspaper '
extra_css = ' body{ font-family: Verdana,Helvetica,Arial,sans-serif } .introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} '
conversion_options = {
'tags' : category
,'language' : language
,'publisher' : publisher
,'linearize_tables': False
}
cover_img_url = 'https://fbcdn-profile-a.akamaihd.net/hprofile-ak-snc4/188140_81722291869_2111820_n.jpg'
masthead_url = 'http://medya.zaman.com.tr/extentions/zaman.com.tr/img/section/logo-section.png'
def print_version(self, url):
return url.replace('www.zaman.com.tr/haber.do?', 'www.zaman.com.tr/yazdir.do?')
keep_only_tags = [dict(name='div', attrs={'id':[ 'news-detail-content']}), dict(name='td', attrs={'class':['columnist-detail','columnist_head']}) ]
remove_tags = [ dict(name='div', attrs={'id':['news-detail-news-text-font-size','news-detail-gallery','news-detail-news-bottom-social']}),dict(name='div', attrs={'class':['radioEmbedBg','radyoProgramAdi']}),dict(name='a', attrs={'class':['webkit-html-attribute-value webkit-html-external-link']}),dict(name='table', attrs={'id':['yaziYorumTablosu']}),dict(name='img', attrs={'src':['http://medya.zaman.com.tr/pics/paylas.gif','http://medya.zaman.com.tr/extentions/zaman.com.tr/img/columnist/ma-16.png']})]
#remove_attributes = ['width','height']
remove_empty_feeds= True
feeds = [
( u'Anasayfa', u'http://www.zaman.com.tr/anasayfa.rss'),
( u'Son Dakika', u'http://www.zaman.com.tr/sondakika.rss'),
( u'En çok Okunanlar', u'http://www.zaman.com.tr/max_all.rss'),
( u'Gündem', u'http://www.zaman.com.tr/gundem.rss'),
( u'Yazarlar', u'http://www.zaman.com.tr/yazarlar.rss'),
( u'Politika', u'http://www.zaman.com.tr/politika.rss'),
( u'Ekonomi', u'http://www.zaman.com.tr/ekonomi.rss'),
( u'Dış Haberler', u'http://www.zaman.com.tr/dishaberler.rss'),
( u'Yorumlar', u'http://www.zaman.com.tr/yorumlar.rss'),
( u'Röportaj', u'http://www.zaman.com.tr/roportaj.rss'),
( u'Spor', u'http://www.zaman.com.tr/spor.rss'),
( u'Kürsü', u'http://www.zaman.com.tr/kursu.rss'),
( u'Kültür Sanat', u'http://www.zaman.com.tr/kultursanat.rss'),
( u'Televizyon', u'http://www.zaman.com.tr/televizyon.rss'),
( u'Manşet', u'http://www.zaman.com.tr/manset.rss'),
]

8
resources/default_tweaks.py
View File

@ -292,13 +292,17 @@ maximum_resort_levels = 5
generate_cover_title_font = None
generate_cover_foot_font = None
#: Control behavior of double clicks on the book list
# Behavior of doubleclick on the books list. Choices: open_viewer, do_nothing,
#: Control behavior of the book list
# You can control the behavior of doubleclicks on the books list.
# Choices: open_viewer, do_nothing,
# edit_cell, edit_metadata. Selecting edit_metadata has the side effect of
# disabling editing a field using a single click.
# Default: open_viewer.
# Example: doubleclick_on_library_view = 'do_nothing'
# You can also control whether the book list scrolls horizontal per column or
# per pixel. Default is per column.
doubleclick_on_library_view = 'open_viewer'
horizontal_scrolling_per_column = True
#: Language to use when sorting.
# Setting this tweak will force sorting to use the

3
resources/metadata_sqlite.sql
View File

@ -1,6 +1,7 @@
CREATE TABLE authors ( id INTEGER PRIMARY KEY,
name TEXT NOT NULL COLLATE NOCASE,
sort TEXT COLLATE NOCASE,
link TEXT NOT NULL DEFAULT "",
UNIQUE(name)
);
CREATE TABLE books ( id INTEGER PRIMARY KEY AUTOINCREMENT,
@ -545,4 +546,4 @@ CREATE TRIGGER series_update_trg
BEGIN
UPDATE series SET sort=NEW.name WHERE id=NEW.id;
END;
pragma user_version=20;
pragma user_version=21;

BIN
resources/quick_start.epub
View File

Binary file not shown.

7
setup/installer/windows/notes.rst
View File

@ -53,6 +53,13 @@ SQLite
Put sqlite3*.h from the sqlite windows amlgamation in ~/sw/include
APSW
-----
Download source from http://code.google.com/p/apsw/downloads/list and run in visual studio prompt
python setup.py fetch --all build --missing-checksum-ok --enable-all-extensions install test
OpenSSL
--------

8
src/calibre/__init__.py
View File

@ -106,10 +106,12 @@ def sanitize_file_name(name, substitute='_', as_unicode=False):
name = name.encode(filesystem_encoding, 'ignore')
one = _filename_sanitize.sub(substitute, name)
one = re.sub(r'\s', ' ', one).strip()
one = re.sub(r'^\.+$', '_', one)
bname, ext = os.path.splitext(one)
one = re.sub(r'^\.+$', '_', bname)
if as_unicode:
one = one.decode(filesystem_encoding)
one = one.replace('..', substitute)
one += ext
# Windows doesn't like path components that end with a period
if one and one[-1] in ('.', ' '):
one = one[:-1]+'_'
@ -132,8 +134,10 @@ def sanitize_file_name_unicode(name, substitute='_'):
name]
one = u''.join(chars)
one = re.sub(r'\s', ' ', one).strip()
one = re.sub(r'^\.+$', '_', one)
bname, ext = os.path.splitext(one)
one = re.sub(r'^\.+$', '_', bname)
one = one.replace('..', substitute)
one += ext
# Windows doesn't like path components that end with a period or space
if one and one[-1] in ('.', ' '):
one = one[:-1]+'_'

2
src/calibre/constants.py
View File

@ -4,7 +4,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
__appname__ = u'calibre'
numeric_version = (0, 8, 7)
numeric_version = (0, 8, 8)
__version__ = u'.'.join(map(unicode, numeric_version))
__author__ = u"Kovid Goyal <kovid@kovidgoyal.net>"

13
src/calibre/customize/builtins.py
View File

@ -611,7 +611,7 @@ from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS,
from calibre.devices.sne.driver import SNE
from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL,
GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR,
TREKSTOR, EEEREADER, NEXTBOOK, ADAM)
TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK)
from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
from calibre.devices.kobo.driver import KOBO
from calibre.devices.bambook.driver import BAMBOOK
@ -746,6 +746,7 @@ plugins += [
EEEREADER,
NEXTBOOK,
ADAM,
MOOVYBOOK,
ITUNES,
BOEYE_BEX,
BOEYE_BDX,
@ -1382,18 +1383,9 @@ class StoreOpenBooksStore(StoreBase):
name = 'Open Books'
description = u'Comprehensive listing of DRM free ebooks from a variety of sources provided by users of calibre.'
actual_plugin = 'calibre.gui2.store.stores.open_books_plugin:OpenBooksStore'
drm_free_only = True
headquarters = 'US'
class StoreOpenLibraryStore(StoreBase):
name = 'Open Library'
description = u'One web page for every book ever published. The goal is to be a true online library. Over 20 million records from a variety of large catalogs as well as single contributions, with more on the way.'
actual_plugin = 'calibre.gui2.store.stores.open_library_plugin:OpenLibraryStore'
drm_free_only = True
headquarters = 'US'
formats = ['DAISY', 'DJVU', 'EPUB', 'MOBI', 'PDF', 'TXT']
class StoreOReillyStore(StoreBase):
name = 'OReilly'
@ -1513,7 +1505,6 @@ plugins += [
StoreMobileReadStore,
StoreNextoStore,
StoreOpenBooksStore,
StoreOpenLibraryStore,
StoreOReillyStore,
StorePragmaticBookshelfStore,
StoreSmashwordsStore,

2
src/calibre/db/__init__.py
View File

@ -63,5 +63,5 @@ Various things that require other things before they can be migrated:
columns/categories/searches info into
self.field_metadata. Finally, implement metadata dirtied
functionality.
2. Test Schema upgrades
'''

261
src/calibre/db/backend.py
View File

@ -22,7 +22,7 @@ from calibre.library.field_metadata import FieldMetadata
from calibre.ebooks.metadata import title_sort, author_to_author_sort
from calibre.utils.icu import strcmp
from calibre.utils.config import to_json, from_json, prefs, tweaks
from calibre.utils.date import utcfromtimestamp
from calibre.utils.date import utcfromtimestamp, parse_date
from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable,
SizeTable, FormatsTable, AuthorsTable, IdentifiersTable)
# }}}
@ -248,7 +248,10 @@ class DB(object, SchemaUpgrade):
UPDATE authors SET sort=author_to_author_sort(name) WHERE sort IS NULL;
''')
def initialize_prefs(self, default_prefs):
self.initialize_custom_columns()
self.initialize_tables()
def initialize_prefs(self, default_prefs): # {{{
self.prefs = DBPrefs(self)
if default_prefs is not None and not self._exists:
@ -339,6 +342,222 @@ class DB(object, SchemaUpgrade):
cats_changed = True
if cats_changed:
self.prefs.set('user_categories', user_cats)
# }}}
def initialize_custom_columns(self): # {{{
with self.conn:
# Delete previously marked custom columns
for record in self.conn.get(
'SELECT id FROM custom_columns WHERE mark_for_delete=1'):
num = record[0]
table, lt = self.custom_table_names(num)
self.conn.execute('''\
DROP INDEX IF EXISTS {table}_idx;
DROP INDEX IF EXISTS {lt}_aidx;
DROP INDEX IF EXISTS {lt}_bidx;
DROP TRIGGER IF EXISTS fkc_update_{lt}_a;
DROP TRIGGER IF EXISTS fkc_update_{lt}_b;
DROP TRIGGER IF EXISTS fkc_insert_{lt};
DROP TRIGGER IF EXISTS fkc_delete_{lt};
DROP TRIGGER IF EXISTS fkc_insert_{table};
DROP TRIGGER IF EXISTS fkc_delete_{table};
DROP VIEW IF EXISTS tag_browser_{table};
DROP VIEW IF EXISTS tag_browser_filtered_{table};
DROP TABLE IF EXISTS {table};
DROP TABLE IF EXISTS {lt};
'''.format(table=table, lt=lt)
)
self.conn.execute('DELETE FROM custom_columns WHERE mark_for_delete=1')
# Load metadata for custom columns
self.custom_column_label_map, self.custom_column_num_map = {}, {}
triggers = []
remove = []
custom_tables = self.custom_tables
for record in self.conn.get(
'SELECT label,name,datatype,editable,display,normalized,id,is_multiple FROM custom_columns'):
data = {
'label':record[0],
'name':record[1],
'datatype':record[2],
'editable':bool(record[3]),
'display':json.loads(record[4]),
'normalized':bool(record[5]),
'num':record[6],
'is_multiple':bool(record[7]),
}
if data['display'] is None:
data['display'] = {}
# set up the is_multiple separator dict
if data['is_multiple']:
if data['display'].get('is_names', False):
seps = {'cache_to_list': '|', 'ui_to_list': '&', 'list_to_ui': ' & '}
elif data['datatype'] == 'composite':
seps = {'cache_to_list': ',', 'ui_to_list': ',', 'list_to_ui': ', '}
else:
seps = {'cache_to_list': '|', 'ui_to_list': ',', 'list_to_ui': ', '}
else:
seps = {}
data['multiple_seps'] = seps
table, lt = self.custom_table_names(data['num'])
if table not in custom_tables or (data['normalized'] and lt not in
custom_tables):
remove.append(data)
continue
self.custom_column_label_map[data['label']] = data['num']
self.custom_column_num_map[data['num']] = \
self.custom_column_label_map[data['label']] = data
# Create Foreign Key triggers
if data['normalized']:
trigger = 'DELETE FROM %s WHERE book=OLD.id;'%lt
else:
trigger = 'DELETE FROM %s WHERE book=OLD.id;'%table
triggers.append(trigger)
if remove:
with self.conn:
for data in remove:
prints('WARNING: Custom column %r not found, removing.' %
data['label'])
self.conn.execute('DELETE FROM custom_columns WHERE id=?',
(data['num'],))
if triggers:
with self.conn:
self.conn.execute('''\
CREATE TEMP TRIGGER custom_books_delete_trg
AFTER DELETE ON books
BEGIN
%s
END;
'''%(' \n'.join(triggers)))
# Setup data adapters
def adapt_text(x, d):
if d['is_multiple']:
if x is None:
return []
if isinstance(x, (str, unicode, bytes)):
x = x.split(d['multiple_seps']['ui_to_list'])
x = [y.strip() for y in x if y.strip()]
x = [y.decode(preferred_encoding, 'replace') if not isinstance(y,
unicode) else y for y in x]
return [u' '.join(y.split()) for y in x]
else:
return x if x is None or isinstance(x, unicode) else \
x.decode(preferred_encoding, 'replace')
def adapt_datetime(x, d):
if isinstance(x, (str, unicode, bytes)):
x = parse_date(x, assume_utc=False, as_utc=False)
return x
def adapt_bool(x, d):
if isinstance(x, (str, unicode, bytes)):
x = x.lower()
if x == 'true':
x = True
elif x == 'false':
x = False
elif x == 'none':
x = None
else:
x = bool(int(x))
return x
def adapt_enum(x, d):
v = adapt_text(x, d)
if not v:
v = None
return v
def adapt_number(x, d):
if x is None:
return None
if isinstance(x, (str, unicode, bytes)):
if x.lower() == 'none':
return None
if d['datatype'] == 'int':
return int(x)
return float(x)
self.custom_data_adapters = {
'float': adapt_number,
'int': adapt_number,
'rating':lambda x,d : x if x is None else min(10., max(0., float(x))),
'bool': adapt_bool,
'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),
'datetime' : adapt_datetime,
'text':adapt_text,
'series':adapt_text,
'enumeration': adapt_enum
}
# Create Tag Browser categories for custom columns
for k in sorted(self.custom_column_label_map.iterkeys()):
v = self.custom_column_label_map[k]
if v['normalized']:
is_category = True
else:
is_category = False
is_m = v['multiple_seps']
tn = 'custom_column_{0}'.format(v['num'])
self.field_metadata.add_custom_field(label=v['label'],
table=tn, column='value', datatype=v['datatype'],
colnum=v['num'], name=v['name'], display=v['display'],
is_multiple=is_m, is_category=is_category,
is_editable=v['editable'], is_csp=False)
# }}}
def initialize_tables(self): # {{{
tables = self.tables = {}
for col in ('title', 'sort', 'author_sort', 'series_index', 'comments',
'timestamp', 'published', 'uuid', 'path', 'cover',
'last_modified'):
metadata = self.field_metadata[col].copy()
if metadata['table'] is None:
metadata['table'], metadata['column'] == 'books', ('has_cover'
if col == 'cover' else col)
tables[col] = OneToOneTable(col, metadata)
for col in ('series', 'publisher', 'rating'):
tables[col] = ManyToOneTable(col, self.field_metadata[col].copy())
for col in ('authors', 'tags', 'formats', 'identifiers'):
cls = {
'authors':AuthorsTable,
'formats':FormatsTable,
'identifiers':IdentifiersTable,
}.get(col, ManyToManyTable)
tables[col] = cls(col, self.field_metadata[col].copy())
tables['size'] = SizeTable('size', self.field_metadata['size'].copy())
for label, data in self.custom_column_label_map.iteritems():
metadata = self.field_metadata[label].copy()
link_table = self.custom_table_names(data['num'])[1]
if data['normalized']:
if metadata['is_multiple']:
tables[label] = ManyToManyTable(label, metadata,
link_table=link_table)
else:
tables[label] = ManyToOneTable(label, metadata,
link_table=link_table)
if metadata['datatype'] == 'series':
# Create series index table
label += '_index'
metadata = self.field_metadata[label].copy()
metadata['column'] = 'extra'
metadata['table'] = link_table
tables[label] = OneToOneTable(label, metadata)
else:
tables[label] = OneToOneTable(label, metadata)
# }}}
@property
def conn(self):
@ -372,6 +591,15 @@ class DB(object, SchemaUpgrade):
# Database layer API {{{
def custom_table_names(self, num):
return 'custom_column_%d'%num, 'books_custom_column_%d_link'%num
@property
def custom_tables(self):
return set([x[0] for x in self.conn.get(
'SELECT name FROM sqlite_master WHERE type="table" AND '
'(name GLOB "custom_column_*" OR name GLOB "books_custom_column_*")')])
@classmethod
def exists_at(cls, path):
return path and os.path.exists(os.path.join(path, 'metadata.db'))
@ -405,39 +633,18 @@ class DB(object, SchemaUpgrade):
return utcfromtimestamp(os.stat(self.dbpath).st_mtime)
def read_tables(self):
tables = {}
for col in ('title', 'sort', 'author_sort', 'series_index', 'comments',
'timestamp', 'published', 'uuid', 'path', 'cover',
'last_modified'):
metadata = self.field_metadata[col].copy()
if metadata['table'] is None:
metadata['table'], metadata['column'] == 'books', ('has_cover'
if col == 'cover' else col)
tables[col] = OneToOneTable(col, metadata)
for col in ('series', 'publisher', 'rating'):
tables[col] = ManyToOneTable(col, self.field_metadata[col].copy())
for col in ('authors', 'tags', 'formats', 'identifiers'):
cls = {
'authors':AuthorsTable,
'formats':FormatsTable,
'identifiers':IdentifiersTable,
}.get(col, ManyToManyTable)
tables[col] = cls(col, self.field_metadata[col].copy())
tables['size'] = SizeTable('size', self.field_metadata['size'].copy())
'''
Read all data from the db into the python in-memory tables
'''
with self.conn: # Use a single transaction, to ensure nothing modifies
# the db while we are reading
for table in tables.itervalues():
for table in self.tables.itervalues():
try:
table.read()
except:
prints('Failed to read table:', table.name)
raise
return tables
# }}}

36
src/calibre/db/tables.py
View File

@ -32,7 +32,7 @@ def _c_convert_timestamp(val):
class Table(object):
def __init__(self, name, metadata):
def __init__(self, name, metadata, link_table=None):
self.name, self.metadata = name, metadata
# self.adapt() maps values from the db to python objects
@ -46,8 +46,17 @@ class Table(object):
# Legacy
self.adapt = lambda x: x.replace('|', ',') if x else None
self.link_table = (link_table if link_table else
'books_%s_link'%self.metadata['table'])
class OneToOneTable(Table):
'''
Represents data that is unique per book (it may not actually be unique) but
each item is assigned to a book in a one-to-one mapping. For example: uuid,
timestamp, size, etc.
'''
def read(self, db):
self.book_col_map = {}
idcol = 'id' if self.metadata['table'] == 'books' else 'book'
@ -66,6 +75,13 @@ class SizeTable(OneToOneTable):
class ManyToOneTable(Table):
'''
Represents data where one data item can map to many books, for example:
series or publisher.
Each book however has only one value for data of this type.
'''
def read(self, db):
self.id_map = {}
self.extra_map = {}
@ -82,8 +98,8 @@ class ManyToOneTable(Table):
def read_maps(self, db):
for row in db.conn.execute(
'SELECT book, {0} FROM books_{1}_link'.format(
self.metadata['link_column'], self.metadata['table'])):
'SELECT book, {0} FROM {1}'.format(
self.metadata['link_column'], self.link_table)):
if row[1] not in self.col_book_map:
self.col_book_map[row[1]] = []
self.col_book_map.append(row[0])
@ -91,10 +107,16 @@ class ManyToOneTable(Table):
class ManyToManyTable(ManyToOneTable):
'''
Represents data that has a many-to-many mapping with books. i.e. each book
can have more than one value and each value can be mapped to more than one
book. For example: tags or authors.
'''
def read_maps(self, db):
for row in db.conn.execute(
'SELECT book, {0} FROM books_{1}_link'.format(
self.metadata['link_column'], self.metadata['table'])):
'SELECT book, {0} FROM {1}'.format(
self.metadata['link_column'], self.link_table)):
if row[1] not in self.col_book_map:
self.col_book_map[row[1]] = []
self.col_book_map.append(row[0])
@ -105,11 +127,13 @@ class ManyToManyTable(ManyToOneTable):
class AuthorsTable(ManyToManyTable):
def read_id_maps(self, db):
self.alink_map = {}
for row in db.conn.execute(
'SELECT id, name, sort FROM authors'):
'SELECT id, name, sort, link FROM authors'):
self.id_map[row[0]] = row[1]
self.extra_map[row[0]] = (row[2] if row[2] else
author_to_author_sort(row[1]))
self.alink_map[row[0]] = row[3]
class FormatsTable(ManyToManyTable):

12
src/calibre/devices/android/driver.py
View File

@ -19,10 +19,11 @@ class ANDROID(USBMS):
VENDOR_ID = {
# HTC
0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226, 0x222],
0x0c01 : [0x100, 0x0227, 0x0226],
0x0ff9 : [0x0100, 0x0227, 0x0226],
0x0c87 : [0x0100, 0x0227, 0x0226],
0x0bb4 : { 0xc02 : [0x100, 0x0227, 0x0226, 0x222],
0xc01 : [0x100, 0x0227, 0x0226],
0xff9 : [0x0100, 0x0227, 0x0226],
0xc87 : [0x0100, 0x0227, 0x0226],
0xc91 : [0x0100, 0x0227, 0x0226],
0xc92 : [0x100],
0xc97 : [0x226],
0xc99 : [0x0100],
@ -100,6 +101,9 @@ class ANDROID(USBMS):
# ZTE
0x19d2 : { 0x1353 : [0x226] },
# Advent
0x0955 : { 0x7100 : [0x9999] }, # This is the same as the Notion Ink Adam
}
EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to '

103
src/calibre/devices/apple/driver.py
View File

@ -5,7 +5,7 @@ __copyright__ = '2010, Gregory Riker'
__docformat__ = 'restructuredtext en'
import cStringIO, ctypes, datetime, os, re, sys, tempfile, time
import cStringIO, ctypes, datetime, os, re, shutil, sys, tempfile, time
from calibre.constants import __appname__, __version__, DEBUG
from calibre import fit_image, confirm_config_name
from calibre.constants import isosx, iswindows
@ -119,11 +119,17 @@ class DriverBase(DeviceConfig, DevicePlugin):
'iBooks Category'),
_('Cache covers from iTunes/iBooks') +
':::' +
_('Enable to cache and display covers from iTunes/iBooks')
_('Enable to cache and display covers from iTunes/iBooks'),
_(u'"Copy files to iTunes Media folder %s" is enabled in iTunes Preferences|Advanced')%u'\u2026' +
':::' +
_("<p>This setting should match your iTunes <i>Preferences</i>|<i>Advanced</i> setting.</p>"
"<p>Disabling will store copies of books transferred to iTunes in your calibre configuration directory.</p>"
"<p>Enabling indicates that iTunes is configured to store copies in your iTunes Media folder.</p>")
]
EXTRA_CUSTOMIZATION_DEFAULT = [
True,
True,
False,
]
@ -193,6 +199,7 @@ class ITUNES(DriverBase):
# EXTRA_CUSTOMIZATION_MESSAGE indexes
USE_SERIES_AS_CATEGORY = 0
CACHE_COVERS = 1
USE_ITUNES_STORAGE = 2
OPEN_FEEDBACK_MESSAGE = _(
'Apple device detected, launching iTunes, please wait ...')
@ -281,6 +288,7 @@ class ITUNES(DriverBase):
description_prefix = "added by calibre"
ejected = False
iTunes= None
iTunes_local_storage = None
library_orphans = None
log = Log()
manual_sync_mode = False
@ -825,7 +833,7 @@ class ITUNES(DriverBase):
# Confirm/create thumbs archive
if not os.path.exists(self.cache_dir):
if DEBUG:
self.log.info(" creating thumb cache '%s'" % self.cache_dir)
self.log.info(" creating thumb cache at '%s'" % self.cache_dir)
os.makedirs(self.cache_dir)
if not os.path.exists(self.archive_path):
@ -837,6 +845,17 @@ class ITUNES(DriverBase):
if DEBUG:
self.log.info(" existing thumb cache at '%s'" % self.archive_path)
# If enabled in config options, create/confirm an iTunes storage folder
if not self.settings().extra_customization[self.USE_ITUNES_STORAGE]:
self.iTunes_local_storage = os.path.join(config_dir,'iTunes storage')
if not os.path.exists(self.iTunes_local_storage):
if DEBUG:
self.log(" creating iTunes_local_storage at '%s'" % self.iTunes_local_storage)
os.mkdir(self.iTunes_local_storage)
else:
if DEBUG:
self.log(" existing iTunes_local_storage at '%s'" % self.iTunes_local_storage)
def remove_books_from_metadata(self, paths, booklists):
'''
Remove books from the metadata list. This function must not communicate
@ -1281,50 +1300,27 @@ class ITUNES(DriverBase):
if DEBUG:
self.log.info(" ITUNES._add_new_copy()")
def _save_last_known_iTunes_storage(lb_added):
if isosx:
fp = lb_added.location().path
index = fp.rfind('/Books') + len('/Books')
last_known_iTunes_storage = fp[:index]
elif iswindows:
fp = lb_added.Location
index = fp.rfind('\Books') + len('\Books')
last_known_iTunes_storage = fp[:index]
dynamic['last_known_iTunes_storage'] = last_known_iTunes_storage
self.log.warning(" last_known_iTunes_storage: %s" % last_known_iTunes_storage)
db_added = None
lb_added = None
# If using iTunes_local_storage, copy the file, redirect iTunes to use local copy
if not self.settings().extra_customization[self.USE_ITUNES_STORAGE]:
local_copy = os.path.join(self.iTunes_local_storage, str(metadata.uuid) + os.path.splitext(fpath)[1])
shutil.copyfile(fpath,local_copy)
fpath = local_copy
if self.manual_sync_mode:
'''
This is the unsupported direct-connect mode.
In an attempt to avoid resetting the iTunes library Media folder, don't try to
add the book to iTunes if the last_known_iTunes_storage path is inaccessible.
This means that the path has to be set at least once, probably by using
'Connect to iTunes' and doing a transfer.
Unsupported direct-connect mode.
'''
self.log.warning(" unsupported direct connect mode")
db_added = self._add_device_book(fpath, metadata)
last_known_iTunes_storage = dynamic.get('last_known_iTunes_storage', None)
if last_known_iTunes_storage is not None:
if os.path.exists(last_known_iTunes_storage):
if DEBUG:
self.log.warning(" iTunes storage online, adding to library")
lb_added = self._add_library_book(fpath, metadata)
else:
if DEBUG:
self.log.warning(" iTunes storage not online, can't add to library")
if lb_added:
_save_last_known_iTunes_storage(lb_added)
lb_added = self._add_library_book(fpath, metadata)
if not lb_added and DEBUG:
self.log.warn(" failed to add '%s' to iTunes, iTunes Media folder inaccessible" % metadata.title)
else:
lb_added = self._add_library_book(fpath, metadata)
if lb_added:
_save_last_known_iTunes_storage(lb_added)
else:
if not lb_added:
raise UserFeedback("iTunes Media folder inaccessible",
details="Failed to add '%s' to iTunes" % metadata.title,
level=UserFeedback.WARN)
@ -1520,7 +1516,7 @@ class ITUNES(DriverBase):
else:
self.log.error(" book_playlist not found")
if len(dev_books):
if dev_books is not None and len(dev_books):
first_book = dev_books[0]
if False:
self.log.info(" determing manual mode by modifying '%s' by %s" % (first_book.name(), first_book.artist()))
@ -1551,7 +1547,7 @@ class ITUNES(DriverBase):
dev_books = pl.Tracks
break
if dev_books.Count:
if dev_books is not None and dev_books.Count:
first_book = dev_books.Item(1)
#if DEBUG:
#self.log.info(" determing manual mode by modifying '%s' by %s" % (first_book.Name, first_book.Artist))
@ -2526,7 +2522,15 @@ class ITUNES(DriverBase):
self.log.info(" processing %s" % fp)
if fp.startswith(prefs['library_path']):
self.log.info(" '%s' stored in calibre database, not removed" % cached_book['title'])
elif not self.settings().extra_customization[self.USE_ITUNES_STORAGE] and \
fp.startswith(self.iTunes_local_storage) and \
os.path.exists(fp):
# Delete the copy in iTunes_local_storage
os.remove(fp)
if DEBUG:
self.log(" removing from iTunes_local_storage")
else:
# Delete from iTunes Media folder
if os.path.exists(fp):
os.remove(fp)
if DEBUG:
@ -2544,12 +2548,6 @@ class ITUNES(DriverBase):
os.rmdir(author_storage_path)
if DEBUG:
self.log.info(" removing empty author directory")
'''
else:
if DEBUG:
self.log.info(" author_storage_path not empty:")
self.log.info(" %s" % '\n'.join(author_files))
'''
else:
self.log.info(" '%s' does not exist at storage location" % cached_book['title'])
@ -2586,7 +2584,15 @@ class ITUNES(DriverBase):
self.log.info(" processing %s" % fp)
if fp.startswith(prefs['library_path']):
self.log.info(" '%s' stored in calibre database, not removed" % cached_book['title'])
elif not self.settings().extra_customization[self.USE_ITUNES_STORAGE] and \
fp.startswith(self.iTunes_local_storage) and \
os.path.exists(fp):
# Delete the copy in iTunes_local_storage
os.remove(fp)
if DEBUG:
self.log(" removing from iTunes_local_storage")
else:
# Delete from iTunes Media folder
if os.path.exists(fp):
os.remove(fp)
if DEBUG:
@ -3234,6 +3240,17 @@ class ITUNES_ASYNC(ITUNES):
if DEBUG:
self.log.info(" existing thumb cache at '%s'" % self.archive_path)
# If enabled in config options, create/confirm an iTunes storage folder
if not self.settings().extra_customization[self.USE_ITUNES_STORAGE]:
self.iTunes_local_storage = os.path.join(config_dir,'iTunes storage')
if not os.path.exists(self.iTunes_local_storage):
if DEBUG:
self.log(" creating iTunes_local_storage at '%s'" % self.iTunes_local_storage)
os.mkdir(self.iTunes_local_storage)
else:
if DEBUG:
self.log(" existing iTunes_local_storage at '%s'" % self.iTunes_local_storage)
def sync_booklists(self, booklists, end_session=True):
'''
Update metadata on device.

6
src/calibre/devices/iriver/driver.py
View File

@ -20,11 +20,11 @@ class IRIVER_STORY(USBMS):
FORMATS = ['epub', 'fb2', 'pdf', 'djvu', 'txt']
VENDOR_ID = [0x1006]
PRODUCT_ID = [0x4023, 0x4024, 0x4025]
BCD = [0x0323]
PRODUCT_ID = [0x4023, 0x4024, 0x4025, 0x4034]
BCD = [0x0323, 0x0326]
VENDOR_NAME = 'IRIVER'
WINDOWS_MAIN_MEM = ['STORY', 'STORY_EB05', 'STORY_WI-FI']
WINDOWS_MAIN_MEM = ['STORY', 'STORY_EB05', 'STORY_WI-FI', 'STORY_EB07']
WINDOWS_CARD_A_MEM = ['STORY', 'STORY_SD']
#OSX_MAIN_MEM = 'Kindle Internal Storage Media'

18
src/calibre/devices/linux_mount_helper.c
View File

@ -64,14 +64,24 @@ int do_mount(const char *dev, const char *mp) {
snprintf(options, 1000, "rw,noexec,nosuid,sync,nodev");
snprintf(uids, 100, "%d", getuid());
snprintf(gids, 100, "%d", getgid());
#else
#ifdef __FreeBSD__
snprintf(options, 1000, "rw,noexec,nosuid,sync,-u=%d,-g=%d",getuid(),getgid());
#else
snprintf(options, 1000, "rw,noexec,nosuid,sync,nodev,quiet,shortname=mixed,uid=%d,gid=%d,umask=077,fmask=0177,dmask=0077,utf8,iocharset=iso8859-1", getuid(), getgid());
#endif
#endif
ensure_root();
#ifdef __NetBSD__
execlp("mount_msdos", "mount_msdos", "-u", uids, "-g", gids, "-o", options, dev, mp, NULL);
#else
#ifdef __FreeBSD__
execlp("mount", "mount", "-t", "msdosfs", "-o", options, dev, mp, NULL);
#else
execlp("mount", "mount", "-t", "auto", "-o", options, dev, mp, NULL);
#endif
#endif
errsv = errno;
fprintf(stderr, "Failed to mount with error: %s\n", strerror(errsv));
@ -91,8 +101,12 @@ int call_eject(const char *dev, const char *mp) {
ensure_root();
#ifdef __NetBSD__
execlp("eject", "eject", dev, NULL);
#else
#ifdef __FreeBSD__
execlp("umount", "umount", dev, NULL);
#else
execlp("eject", "eject", "-s", dev, NULL);
#endif
#endif
/* execlp failed */
errsv = errno;
@ -121,7 +135,11 @@ int call_umount(const char *dev, const char *mp) {
if (pid == 0) { /* Child process */
ensure_root();
#ifdef __FreeBSD__
execlp("umount", "umount", mp, NULL);
#else
execlp("umount", "umount", "-l", mp, NULL);
#endif
/* execlp failed */
errsv = errno;
fprintf(stderr, "Failed to umount with error: %s\n", strerror(errsv));

22
src/calibre/devices/misc.py
View File

@ -329,3 +329,25 @@ class NEXTBOOK(USBMS):
f.write(metadata.thumbnail[-1])
'''
class MOOVYBOOK(USBMS):
name = 'Moovybook device interface'
gui_name = 'Moovybook'
description = _('Communicate with the Moovybook Reader')
author = 'Kovid Goyal'
supported_platforms = ['windows', 'osx', 'linux']
# Ordered list of supported formats
FORMATS = ['epub', 'txt', 'pdf']
VENDOR_ID = [0x1cae]
PRODUCT_ID = [0x9b08]
BCD = [0x02]
EBOOK_DIR_MAIN = ''
SUPPORTS_SUB_DIRS = True
def get_main_ebook_dir(self, for_upload=False):
return 'Books' if for_upload else self.EBOOK_DIR_MAIN

4
src/calibre/devices/usbms/books.py
View File

@ -14,7 +14,7 @@ from calibre.constants import preferred_encoding
from calibre import isbytestring, force_unicode
from calibre.utils.config import prefs, tweaks
from calibre.utils.icu import strcmp
from calibre.utils.formatter import eval_formatter
from calibre.utils.formatter import EvalFormatter
class Book(Metadata):
def __init__(self, prefix, lpath, size=None, other=None):
@ -116,7 +116,7 @@ class CollectionsBookList(BookList):
field_name = field_meta['name']
else:
field_name = ''
cat_name = eval_formatter.safe_format(
cat_name = EvalFormatter().safe_format(
fmt=tweaks['sony_collection_name_template'],
kwargs={'category':field_name, 'value':field_value},
error_value='GET_CATEGORY', book=None)

160
src/calibre/devices/usbms/device.py
View File

@ -17,7 +17,7 @@ from itertools import repeat
from calibre.devices.interface import DevicePlugin
from calibre.devices.errors import DeviceError, FreeSpaceError
from calibre.devices.usbms.deviceconfig import DeviceConfig
from calibre.constants import iswindows, islinux, isosx, plugins
from calibre.constants import iswindows, islinux, isosx, isfreebsd, plugins
from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to
if isosx:
@ -701,7 +701,152 @@ class Device(DeviceConfig, DevicePlugin):
self._card_a_prefix = self._card_b_prefix
self._card_b_prefix = None
# ------------------------------------------------------
#
# open for FreeBSD
# find the device node or nodes that match the S/N we already have from the scanner
# and attempt to mount each one
# 1. get list of disk devices from sysctl
# 2. compare that list with the one from camcontrol
# 3. and see if it has a matching s/n
# 6. find any partitions/slices associated with each node
# 7. attempt to mount, using calibre-mount-helper, each one
# 8. when finished, we have a list of mount points and associated device nodes
#
def open_freebsd(self):
# this gives us access to the S/N, etc. of the reader that the scanner has found
# and the match routines for some of that data, like s/n, vendor ID, etc.
d=self.detected_device
if not d.serial:
raise DeviceError("Device has no S/N. Can't continue")
return False
devs={}
di=0
ndevs=4 # number of possible devices per reader (main, carda, cardb, launcher)
#get list of disk devices
p=subprocess.Popen(["sysctl", "kern.disks"], stdout=subprocess.PIPE)
kdsks=subprocess.Popen(["sed", "s/kern.disks: //"], stdin=p.stdout, stdout=subprocess.PIPE).communicate()[0]
p.stdout.close()
#print kdsks
for dvc in kdsks.split():
# for each one that's also in the list of cam devices ...
p=subprocess.Popen(["camcontrol", "devlist"], stdout=subprocess.PIPE)
devmatch=subprocess.Popen(["grep", dvc], stdin=p.stdout, stdout=subprocess.PIPE).communicate()[0]
p.stdout.close()
if devmatch:
#print "Checking ", devmatch
# ... see if we can get a S/N from the actual device node
sn=subprocess.Popen(["camcontrol", "inquiry", dvc, "-S"], stdout=subprocess.PIPE).communicate()[0]
sn=sn[0:-1] # drop the trailing newline
#print "S/N = ", sn
if sn and d.match_serial(sn):
# we have a matching s/n, record this device node
#print "match found: ", dvc
devs[di]=dvc
di += 1
# sort the list of devices
for i in range(1,ndevs+1):
for j in reversed(range(1,i)):
if devs[j-1] > devs[j]:
x=devs[j-1]
devs[j-1]=devs[j]
devs[j]=x
#print devs
# now we need to see if any of these have slices/partitions
mtd=0
label="READER" # could use something more unique, like S/N or productID...
cmd = '/usr/local/bin/calibre-mount-helper'
cmd = [cmd, 'mount']
for i in range(0,ndevs):
cmd2="ls /dev/"+devs[i]+"*"
p=subprocess.Popen(cmd2, shell=True, stdout=subprocess.PIPE)
devs[i]=subprocess.Popen(["cut", "-d", "/", "-f" "3"], stdin=p.stdout, stdout=subprocess.PIPE).communicate()[0]
p.stdout.close()
# try all the nodes to see what we can mount
for dev in devs[i].split():
mp='/media/'+label+'-'+dev
#print "trying ", dev, "on", mp
try:
p = subprocess.Popen(cmd + ["/dev/"+dev, mp])
except OSError:
raise DeviceError(_('Could not find mount helper: %s.')%cmd[0])
while p.poll() is None:
time.sleep(0.1)
if p.returncode == 0:
#print " mounted", dev
if i == 0:
self._main_prefix = mp
self._main_dev = "/dev/"+dev
#print "main = ", self._main_dev, self._main_prefix
if i == 1:
self._card_a_prefix = mp
self._card_a_dev = "/dev/"+dev
#print "card a = ", self._card_a_dev, self._card_a_prefix
if i == 2:
self._card_b_prefix = mp
self._card_b_dev = "/dev/"+dev
#print "card b = ", self._card_b_dev, self._card_b_prefix
mtd += 1
break
if mtd > 0:
return True
else :
return False
#
# ------------------------------------------------------
#
# this one is pretty simple:
# just umount each of the previously
# mounted filesystems, using the mount helper
#
def eject_freebsd(self):
cmd = '/usr/local/bin/calibre-mount-helper'
cmd = [cmd, 'eject']
if self._main_prefix:
#print "umount main:", cmd, self._main_dev, self._main_prefix
try:
p = subprocess.Popen(cmd + [self._main_dev, self._main_prefix])
except OSError:
raise DeviceError(
_('Could not find mount helper: %s.')%cmd[0])
while p.poll() is None:
time.sleep(0.1)
if self._card_a_prefix:
#print "umount card a:", cmd, self._card_a_dev, self._card_a_prefix
try:
p = subprocess.Popen(cmd + [self._card_a_dev, self._card_a_prefix])
except OSError:
raise DeviceError(
_('Could not find mount helper: %s.')%cmd[0])
while p.poll() is None:
time.sleep(0.1)
if self._card_b_prefix:
#print "umount card b:", cmd, self._card_b_dev, self._card_b_prefix
try:
p = subprocess.Popen(cmd + [self._card_b_dev, self._card_b_prefix])
except OSError:
raise DeviceError(
_('Could not find mount helper: %s.')%cmd[0])
while p.poll() is None:
time.sleep(0.1)
self._main_prefix = None
self._card_a_prefix = None
self._card_b_prefix = None
# ------------------------------------------------------
def open(self, library_uuid):
time.sleep(5)
@ -712,6 +857,14 @@ class Device(DeviceConfig, DevicePlugin):
except DeviceError:
time.sleep(7)
self.open_linux()
if isfreebsd:
self._main_dev = self._card_a_dev = self._card_b_dev = None
try:
self.open_freebsd()
except DeviceError:
subprocess.Popen(["camcontrol", "rescan", "all"])
time.sleep(2)
self.open_freebsd()
if iswindows:
try:
self.open_windows()
@ -800,6 +953,11 @@ class Device(DeviceConfig, DevicePlugin):
self.eject_linux()
except:
pass
if isfreebsd:
try:
self.eject_freebsd()
except:
pass
if iswindows:
try:
self.eject_windows()

2
src/calibre/ebooks/compression/palmdoc.c
View File

@ -54,7 +54,7 @@ cpalmdoc_decompress(PyObject *self, PyObject *args) {
// Map chars to bytes
for (j = 0; j < input_len; j++)
input[j] = (_input[j] < 0) ? _input[j]+256 : _input[j];
output = (char *)PyMem_Malloc(sizeof(char)*(MAX(BUFFER, 5*input_len)));
output = (char *)PyMem_Malloc(sizeof(char)*(MAX(BUFFER, 8*input_len)));
if (output == NULL) return PyErr_NoMemory();
while (i < input_len) {

2
src/calibre/ebooks/metadata/book/__init__.py
View File

@ -86,6 +86,8 @@ CALIBRE_METADATA_FIELDS = frozenset([
# a dict of user category names, where the value is a list of item names
# from the book that are in that category
'user_categories',
# a dict of author to an associated hyperlink
'author_link_map',
]
)

8
src/calibre/ebooks/metadata/book/base.py
View File

@ -34,6 +34,7 @@ NULL_VALUES = {
'authors' : [_('Unknown')],
'title' : _('Unknown'),
'user_categories' : {},
'author_link_map' : {},
'language' : 'und'
}
@ -70,6 +71,7 @@ class SafeFormat(TemplateFormatter):
return ''
return v
# DEPRECATED. This is not thread safe. Do not use.
composite_formatter = SafeFormat()
class Metadata(object):
@ -110,6 +112,7 @@ class Metadata(object):
# List of strings or []
self.author = list(authors) if authors else []# Needed for backward compatibility
self.authors = list(authors) if authors else []
self.formatter = SafeFormat()
def is_null(self, field):
'''
@ -146,7 +149,7 @@ class Metadata(object):
return val
if val is None:
d['#value#'] = 'RECURSIVE_COMPOSITE FIELD (Metadata) ' + field
val = d['#value#'] = composite_formatter.safe_format(
val = d['#value#'] = self.formatter.safe_format(
d['display']['composite_template'],
self,
_('TEMPLATE ERROR'),
@ -423,11 +426,12 @@ class Metadata(object):
'''
if not ops:
return
formatter = SafeFormat()
for op in ops:
try:
src = op[0]
dest = op[1]
val = composite_formatter.safe_format\
val = formatter.safe_format\
(src, other, 'PLUGBOARD TEMPLATE ERROR', other)
if dest == 'tags':
self.set(dest, [f.strip() for f in val.split(',') if f.strip()])

13
src/calibre/ebooks/metadata/opf2.py
View File

@ -474,7 +474,7 @@ def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8))
metadata_elem.append(meta)
def dump_user_categories(cats):
def dump_dict(cats):
if not cats:
cats = {}
from calibre.ebooks.metadata.book.json_codec import object_to_unicode
@ -537,8 +537,9 @@ class OPF(object): # {{{
formatter=parse_date, renderer=isoformat)
user_categories = MetadataField('user_categories', is_dc=False,
formatter=json.loads,
renderer=dump_user_categories)
renderer=dump_dict)
author_link_map = MetadataField('author_link_map', is_dc=False,
formatter=json.loads, renderer=dump_dict)
def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True,
populate_spine=True):
@ -1039,7 +1040,7 @@ class OPF(object): # {{{
for attr in ('title', 'authors', 'author_sort', 'title_sort',
'publisher', 'series', 'series_index', 'rating',
'isbn', 'tags', 'category', 'comments',
'pubdate', 'user_categories'):
'pubdate', 'user_categories', 'author_link_map'):
val = getattr(mi, attr, None)
if val is not None and val != [] and val != (None, None):
setattr(self, attr, val)
@ -1336,6 +1337,8 @@ def metadata_to_opf(mi, as_string=True):
for tag in mi.tags:
factory(DC('subject'), tag)
meta = lambda n, c: factory('meta', name='calibre:'+n, content=c)
if getattr(mi, 'author_link_map', None) is not None:
meta('author_link_map', dump_dict(mi.author_link_map))
if mi.series:
meta('series', mi.series)
if mi.series_index is not None:
@ -1349,7 +1352,7 @@ def metadata_to_opf(mi, as_string=True):
if mi.title_sort:
meta('title_sort', mi.title_sort)
if mi.user_categories:
meta('user_categories', dump_user_categories(mi.user_categories))
meta('user_categories', dump_dict(mi.user_categories))
serialize_user_metadata(metadata, mi.get_all_user_metadata(False))

5
src/calibre/ebooks/mobi/reader.py
View File

@ -957,7 +957,10 @@ def get_metadata(stream):
return get_metadata(stream)
from calibre.utils.logging import Log
log = Log()
mi = MetaInformation(os.path.basename(stream.name), [_('Unknown')])
try:
mi = MetaInformation(os.path.basename(stream.name), [_('Unknown')])
except:
mi = MetaInformation(_('Unknown'), [_('Unknown')])
mh = MetadataHeader(stream, log)
if mh.title and mh.title != _('Unknown'):
mi.title = mh.title

13
src/calibre/gui2/__init__.py
View File

@ -7,12 +7,13 @@ from urllib import unquote
from PyQt4.Qt import (QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt,
QByteArray, QTranslator, QCoreApplication, QThread,
QEvent, QTimer, pyqtSignal, QDate, QDesktopServices,
QFileDialog, QFileIconProvider,
QFileDialog, QFileIconProvider, QSettings,
QIcon, QApplication, QDialog, QUrl, QFont)
ORG_NAME = 'KovidsBrain'
APP_UID = 'libprs500'
from calibre.constants import islinux, iswindows, isbsd, isfrozen, isosx
from calibre.constants import (islinux, iswindows, isbsd, isfrozen, isosx,
config_dir)
from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig
from calibre.utils.localization import set_qt_translator
from calibre.ebooks.metadata import MetaInformation
@ -82,13 +83,14 @@ gprefs.defaults['tags_browser_partition_method'] = 'first letter'
gprefs.defaults['tags_browser_collapse_at'] = 100
gprefs.defaults['edit_metadata_single_layout'] = 'default'
gprefs.defaults['book_display_fields'] = [
('title', False), ('authors', False), ('formats', True),
('title', False), ('authors', True), ('formats', True),
('series', True), ('identifiers', True), ('tags', True),
('path', True), ('publisher', False), ('rating', False),
('author_sort', False), ('sort', False), ('timestamp', False),
('uuid', False), ('comments', True), ('id', False), ('pubdate', False),
('last_modified', False), ('size', False),
]
gprefs.defaults['default_author_link'] = 'http://en.wikipedia.org/w/index.php?search={author}'
# }}}
@ -192,6 +194,11 @@ def _config(): # {{{
config = _config()
# }}}
QSettings.setPath(QSettings.IniFormat, QSettings.UserScope, config_dir)
QSettings.setPath(QSettings.IniFormat, QSettings.SystemScope,
config_dir)
QSettings.setDefaultFormat(QSettings.IniFormat)
# Turn off DeprecationWarnings in windows GUI
if iswindows:
import warnings

3
src/calibre/gui2/actions/choose_library.py
View File

@ -260,7 +260,8 @@ class ChooseLibraryAction(InterfaceAction):
'The files remain on your computer, if you want '
'to delete them, you will have to do so manually.') % loc,
show=True)
open_local_file(loc)
if os.path.exists(loc):
open_local_file(loc)
def backup_status(self, location):
dirty_text = 'no'

3
src/calibre/gui2/actions/show_quickview.py
View File

@ -38,3 +38,6 @@ class ShowQuickviewAction(InterfaceAction):
Quickview(self.gui, self.gui.library_view, index)
self.current_instance.show()
def library_changed(self, db):
if self.current_instance and not self.current_instance.is_closed:
self.current_instance.set_database(db)

31
src/calibre/gui2/book_details.py
View File

@ -5,7 +5,6 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
from PyQt4.Qt import (QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl,
QPropertyAnimation, QEasingCurve, QApplication, QFontInfo,
QSizePolicy, QPainter, QRect, pyqtProperty, QLayout, QPalette, QMenu)
@ -23,6 +22,7 @@ from calibre.library.comments import comments_to_html
from calibre.gui2 import (config, open_local_file, open_url, pixmap_to_data,
gprefs)
from calibre.utils.icu import sort_key
from calibre.utils.formatter import EvalFormatter
def render_html(mi, css, vertical, widget, all_fields=False): # {{{
table = render_data(mi, all_fields=all_fields,
@ -98,6 +98,14 @@ def render_data(mi, use_roman_numbers=True, all_fields=False):
val = force_unicode(val)
ans.append((field,
u'<td class="comments" colspan="2">%s</td>'%comments_to_html(val)))
elif metadata['datatype'] == 'composite' and \
metadata['display'].get('contains_html', False):
val = getattr(mi, field)
if val:
val = force_unicode(val)
ans.append((field,
u'<td class="title">%s</td><td>%s</td>'%
(name, comments_to_html(val))))
elif field == 'path':
if mi.path:
path = force_unicode(mi.path, filesystem_encoding)
@ -121,6 +129,27 @@ def render_data(mi, use_roman_numbers=True, all_fields=False):
if links:
ans.append((field, u'<td class="title">%s</td><td>%s</td>'%(
_('Ids')+':', links)))
elif field == 'authors' and not isdevice:
authors = []
formatter = EvalFormatter()
for aut in mi.authors:
if mi.author_link_map[aut]:
link = mi.author_link_map[aut]
elif gprefs.get('default_author_link'):
vals = {'author': aut.replace(' ', '+')}
try:
vals['author_sort'] = mi.author_sort_map[aut].replace(' ', '+')
except:
vals['author_sort'] = aut.replace(' ', '+')
link = formatter.safe_format(
gprefs.get('default_author_link'), vals, '', vals)
if link:
link = prepare_string_for_xml(link)
authors.append(u'<a href="%s">%s</a>'%(link, aut))
else:
authors.append(aut)
ans.append((field, u'<td class="title">%s</td><td>%s</td>'%(name,
u' & '.join(authors))))
else:
val = mi.format_field(field)[-1]
if val is None:

62
src/calibre/gui2/dialogs/edit_authors_dialog.py
View File

@ -4,10 +4,11 @@ __docformat__ = 'restructuredtext en'
__license__ = 'GPL v3'
from PyQt4.Qt import (Qt, QDialog, QTableWidgetItem, QAbstractItemView, QIcon,
QDialogButtonBox, QFrame, QLabel, QTimer, QMenu, QApplication)
QDialogButtonBox, QFrame, QLabel, QTimer, QMenu, QApplication,
QByteArray)
from calibre.ebooks.metadata import author_to_author_sort
from calibre.gui2 import error_dialog
from calibre.gui2 import error_dialog, gprefs
from calibre.gui2.dialogs.edit_authors_dialog_ui import Ui_EditAuthorsDialog
from calibre.utils.icu import sort_key
@ -20,7 +21,7 @@ class tableItem(QTableWidgetItem):
class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
def __init__(self, parent, db, id_to_select, select_sort):
def __init__(self, parent, db, id_to_select, select_sort, select_link):
QDialog.__init__(self, parent)
Ui_EditAuthorsDialog.__init__(self)
self.setupUi(self)
@ -29,11 +30,19 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint))
self.setWindowIcon(icon)
try:
self.table_column_widths = \
gprefs.get('manage_authors_table_widths', None)
geom = gprefs.get('manage_authors_dialog_geometry', bytearray(''))
self.restoreGeometry(QByteArray(geom))
except:
pass
self.buttonBox.accepted.connect(self.accepted)
# Set up the column headings
self.table.setSelectionMode(QAbstractItemView.SingleSelection)
self.table.setColumnCount(2)
self.table.setColumnCount(3)
self.down_arrow_icon = QIcon(I('arrow-down.png'))
self.up_arrow_icon = QIcon(I('arrow-up.png'))
self.blank_icon = QIcon(I('blank.png'))
@ -43,26 +52,35 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
self.aus_col = QTableWidgetItem(_('Author sort'))
self.table.setHorizontalHeaderItem(1, self.aus_col)
self.aus_col.setIcon(self.up_arrow_icon)
self.aul_col = QTableWidgetItem(_('Link'))
self.table.setHorizontalHeaderItem(2, self.aul_col)
self.aus_col.setIcon(self.blank_icon)
# Add the data
self.authors = {}
auts = db.get_authors_with_ids()
self.table.setRowCount(len(auts))
select_item = None
for row, (id, author, sort) in enumerate(auts):
for row, (id, author, sort, link) in enumerate(auts):
author = author.replace('|', ',')
self.authors[id] = (author, sort)
self.authors[id] = (author, sort, link)
aut = tableItem(author)
aut.setData(Qt.UserRole, id)
sort = tableItem(sort)
link = tableItem(link)
self.table.setItem(row, 0, aut)
self.table.setItem(row, 1, sort)
self.table.setItem(row, 2, link)
if id == id_to_select:
if select_sort:
select_item = sort
elif select_link:
select_item = link
else:
select_item = aut
self.table.resizeColumnsToContents()
if self.table.columnWidth(2) < 200:
self.table.setColumnWidth(2, 200)
# set up the cellChanged signal only after the table is filled
self.table.cellChanged.connect(self.cell_changed)
@ -115,6 +133,28 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
self.table.setContextMenuPolicy(Qt.CustomContextMenu)
self.table.customContextMenuRequested .connect(self.show_context_menu)
def save_state(self):
self.table_column_widths = []
for c in range(0, self.table.columnCount()):
self.table_column_widths.append(self.table.columnWidth(c))
gprefs['manage_authors_table_widths'] = self.table_column_widths
gprefs['manage_authors_dialog_geometry'] = bytearray(self.saveGeometry())
def resizeEvent(self, *args):
QDialog.resizeEvent(self, *args)
if self.table_column_widths is not None:
for c,w in enumerate(self.table_column_widths):
self.table.setColumnWidth(c, w)
else:
# the vertical scroll bar might not be rendered, so might not yet
# have a width. Assume 25. Not a problem because user-changed column
# widths will be remembered
w = self.table.width() - 25 - self.table.verticalHeader().width()
w /= self.table.columnCount()
for c in range(0, self.table.columnCount()):
self.table.setColumnWidth(c, w)
self.save_state()
def show_context_menu(self, point):
self.context_item = self.table.itemAt(point)
case_menu = QMenu(_('Change Case'))
@ -231,14 +271,16 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
self.auth_col.setIcon(self.blank_icon)
def accepted(self):
self.save_state()
self.result = []
for row in range(0,self.table.rowCount()):
id = self.table.item(row, 0).data(Qt.UserRole).toInt()[0]
aut = unicode(self.table.item(row, 0).text()).strip()
sort = unicode(self.table.item(row, 1).text()).strip()
orig_aut,orig_sort = self.authors[id]
if orig_aut != aut or orig_sort != sort:
self.result.append((id, orig_aut, aut, sort))
link = unicode(self.table.item(row, 2).text()).strip()
orig_aut,orig_sort,orig_link = self.authors[id]
if orig_aut != aut or orig_sort != sort or orig_link != link:
self.result.append((id, orig_aut, aut, sort, link))
def do_recalc_author_sort(self):
self.table.cellChanged.disconnect()
@ -276,6 +318,6 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
c.setText(author_to_author_sort(aut))
item = c
else:
item = self.table.item(row, 1)
item = self.table.item(row, col)
self.table.setCurrentItem(item)
self.table.scrollToItem(item)

4
src/calibre/gui2/dialogs/metadata_bulk.py
View File

@ -12,7 +12,7 @@ from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
from calibre.gui2.dialogs.tag_editor import TagEditor
from calibre.ebooks.metadata import string_to_authors, authors_to_string, title_sort
from calibre.ebooks.metadata.book.base import composite_formatter
from calibre.ebooks.metadata.book.base import SafeFormat
from calibre.gui2.custom_column_widgets import populate_metadata_page
from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE, \
gprefs, question_dialog
@ -499,7 +499,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
def s_r_get_field(self, mi, field):
if field:
if field == '{template}':
v = composite_formatter.safe_format\
v = SafeFormat().safe_format\
(unicode(self.s_r_template.text()), mi, _('S/R TEMPLATE ERROR'), mi)
return [v]
fm = self.db.metadata_for_field(field)

53
src/calibre/gui2/dialogs/quickview.py
View File

@ -18,16 +18,29 @@ class TableItem(QTableWidgetItem):
A QTableWidgetItem that sorts on a separate string and uses ICU rules
'''
def __init__(self, val, sort):
def __init__(self, val, sort, idx=0):
self.sort = sort
self.sort_idx = idx
QTableWidgetItem.__init__(self, val)
self.setFlags(Qt.ItemIsEnabled|Qt.ItemIsSelectable)
def __ge__(self, other):
return sort_key(self.sort) >= sort_key(other.sort)
l = sort_key(self.sort)
r = sort_key(other.sort)
if l > r:
return 1
if l == r:
return self.sort_idx >= other.sort_idx
return 0
def __lt__(self, other):
return sort_key(self.sort) < sort_key(other.sort)
l = sort_key(self.sort)
r = sort_key(other.sort)
if l < r:
return 1
if l == r:
return self.sort_idx < other.sort_idx
return 0
class Quickview(QDialog, Ui_Quickview):
@ -60,6 +73,7 @@ class Quickview(QDialog, Ui_Quickview):
self.last_search = None
self.current_column = None
self.current_item = None
self.no_valid_items = False
self.items.setSelectionMode(QAbstractItemView.SingleSelection)
self.items.currentTextChanged.connect(self.item_selected)
@ -95,8 +109,19 @@ class Quickview(QDialog, Ui_Quickview):
self.search_button.clicked.connect(self.do_search)
view.model().new_bookdisplay_data.connect(self.book_was_changed)
def set_database(self, db):
self.db = db
self.items.blockSignals(True)
self.books_table.blockSignals(True)
self.items.clear()
self.books_table.setRowCount(0)
self.books_table.blockSignals(False)
self.items.blockSignals(False)
# search button
def do_search(self):
if self.no_valid_items:
return
if self.last_search is not None:
self.gui.search.set_search_string(self.last_search)
@ -110,6 +135,8 @@ class Quickview(QDialog, Ui_Quickview):
# clicks on the items listWidget
def item_selected(self, txt):
if self.no_valid_items:
return
self.fill_in_books_box(unicode(txt))
# Given a cell in the library view, display the information
@ -122,6 +149,7 @@ class Quickview(QDialog, Ui_Quickview):
# Only show items for categories
if not self.db.field_metadata[key]['is_category']:
if self.current_key is None:
self.indicate_no_items()
return
key = self.current_key
self.items_label.setText('{0} ({1})'.format(
@ -135,6 +163,7 @@ class Quickview(QDialog, Ui_Quickview):
vals = mi.get(key, None)
if vals:
self.no_valid_items = False
if not isinstance(vals, list):
vals = [vals]
vals.sort(key=sort_key)
@ -148,8 +177,19 @@ class Quickview(QDialog, Ui_Quickview):
self.current_key = key
self.fill_in_books_box(vals[0])
else:
self.indicate_no_items()
self.items.blockSignals(False)
def indicate_no_items(self):
print 'no items'
self.no_valid_items = True
self.items.clear()
self.items.addItem(QListWidgetItem(_('**No items found**')))
self.books_label.setText(_('Click in a column in the library view '
'to see the information for that book'))
def fill_in_books_box(self, selected_item):
self.current_item = selected_item
# Do a bit of fix-up on the items so that the search works.
@ -163,7 +203,8 @@ class Quickview(QDialog, Ui_Quickview):
self.db.data.search_restriction)
self.books_table.setRowCount(len(books))
self.books_label.setText(_('Books with selected item: {0}').format(len(books)))
self.books_label.setText(_('Books with selected item "{0}": {1}').
format(selected_item, len(books)))
select_item = None
self.books_table.setSortingEnabled(False)
@ -185,7 +226,7 @@ class Quickview(QDialog, Ui_Quickview):
series = mi.format_field('series')[1]
if series is None:
series = ''
a = TableItem(series, series)
a = TableItem(series, mi.series, mi.series_index)
a.setToolTip(tt)
self.books_table.setItem(row, 2, a)
self.books_table.setRowHeight(row, self.books_table_row_height)
@ -213,6 +254,8 @@ class Quickview(QDialog, Ui_Quickview):
self.save_state()
def book_doubleclicked(self, row, column):
if self.no_valid_items:
return
book_id = self.books_table.item(row, 0).data(Qt.UserRole).toInt()[0]
self.view.select_rows([book_id])
modifiers = int(QApplication.keyboardModifiers())

13
src/calibre/gui2/dialogs/quickview.ui
View File

@ -57,19 +57,6 @@
</property>
</widget>
</item>
<item row="2" column="1">
<spacer>
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>0</width>
<height>0</height>
</size>
</property>
</spacer>
</item>
<item row="3" column="0" colspan="2">
<layout class="QHBoxLayout">
<item>

2
src/calibre/gui2/dialogs/restore_library.py
View File

@ -54,7 +54,7 @@ class DBRestore(QDialog):
def reject(self):
self.rejected = True
self.restorer.progress_callback = lambda x, y: x
QDialog.rejecet(self)
QDialog.reject(self)
def update(self):
if self.restorer.is_alive():

4
src/calibre/gui2/dialogs/template_dialog.py
View File

@ -11,7 +11,7 @@ from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter, QFont,
from calibre.gui2 import error_dialog
from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog
from calibre.utils.formatter_functions import formatter_functions
from calibre.ebooks.metadata.book.base import composite_formatter, Metadata
from calibre.ebooks.metadata.book.base import SafeFormat, Metadata
from calibre.library.coloring import (displayable_columns)
@ -270,7 +270,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
self.highlighter.regenerate_paren_positions()
self.text_cursor_changed()
self.template_value.setText(
composite_formatter.safe_format(cur_text, self.mi,
SafeFormat().safe_format(cur_text, self.mi,
_('EXCEPTION: '), self.mi))
def text_cursor_changed(self):

5
src/calibre/gui2/library/models.py
View File

@ -14,7 +14,7 @@ from PyQt4.Qt import (QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage,
from calibre.gui2 import NONE, UNDEFINED_QDATE
from calibre.utils.pyparsing import ParseException
from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors
from calibre.ebooks.metadata.book.base import composite_formatter
from calibre.ebooks.metadata.book.base import SafeFormat
from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.config import tweaks, prefs
from calibre.utils.date import dt_factory, qt_to_dt
@ -91,6 +91,7 @@ class BooksModel(QAbstractTableModel): # {{{
self.current_highlighted_idx = None
self.highlight_only = False
self.colors = frozenset([unicode(c) for c in QColor.colorNames()])
self.formatter = SafeFormat()
self.read_config()
def change_alignment(self, colname, alignment):
@ -711,7 +712,7 @@ class BooksModel(QAbstractTableModel): # {{{
try:
if mi is None:
mi = self.db.get_metadata(id_, index_is_id=True)
color = composite_formatter.safe_format(fmt, mi, '', mi)
color = self.formatter.safe_format(fmt, mi, '', mi)
if color in self.colors:
color = QColor(color)
if color.isValid():

31
src/calibre/gui2/library/views.py
View File

@ -51,6 +51,9 @@ class BooksView(QTableView): # {{{
def __init__(self, parent, modelcls=BooksModel, use_edit_metadata_dialog=True):
QTableView.__init__(self, parent)
if not tweaks['horizontal_scrolling_per_column']:
self.setHorizontalScrollMode(self.ScrollPerPixel)
self.setEditTriggers(self.EditKeyPressed)
if tweaks['doubleclick_on_library_view'] == 'edit_cell':
self.setEditTriggers(self.DoubleClicked|self.editTriggers())
@ -110,6 +113,7 @@ class BooksView(QTableView): # {{{
self.column_header.sectionMoved.connect(self.save_state)
self.column_header.setContextMenuPolicy(Qt.CustomContextMenu)
self.column_header.customContextMenuRequested.connect(self.show_column_header_context_menu)
self.column_header.sectionResized.connect(self.column_resized, Qt.QueuedConnection)
# }}}
self._model.database_changed.connect(self.database_changed)
@ -214,6 +218,9 @@ class BooksView(QTableView): # {{{
self.column_header_context_menu.addSeparator()
self.column_header_context_menu.addAction(
_('Shrink column if it is too wide to fit'),
partial(self.resize_column_to_fit, column=self.column_map[idx]))
self.column_header_context_menu.addAction(
_('Restore default layout'),
partial(self.column_header_context_handler,
@ -235,13 +242,8 @@ class BooksView(QTableView): # {{{
self.selected_ids = [idc(r) for r in selected_rows]
def sorting_done(self, indexc):
if self.selected_ids:
indices = [self.model().index(indexc(i), 0) for i in
self.selected_ids]
sm = self.selectionModel()
for idx in indices:
sm.select(idx, sm.Select|sm.Rows)
self.scroll_to_row(indices[0].row())
self.select_rows(self.selected_ids, using_ids=True, change_current=True,
scroll=True)
self.selected_ids = []
def sort_by_named_field(self, field, order, reset=True):
@ -456,7 +458,9 @@ class BooksView(QTableView): # {{{
traceback.print_exc()
old_state['sort_history'] = sh
self.column_header.blockSignals(True)
self.apply_state(old_state)
self.column_header.blockSignals(False)
# Resize all rows to have the correct height
if self.model().rowCount(QModelIndex()) > 0:
@ -465,6 +469,19 @@ class BooksView(QTableView): # {{{
self.was_restored = True
def resize_column_to_fit(self, column):
col = self.column_map.index(column)
self.column_resized(col, self.columnWidth(col), self.columnWidth(col))
def column_resized(self, col, old_size, new_size):
# arbitrary: scroll bar + header + some
max_width = self.width() - (self.verticalScrollBar().width() +
self.verticalHeader().width() + 10)
if new_size > max_width:
self.column_header.blockSignals(True)
self.setColumnWidth(col, max_width)
self.column_header.blockSignals(False)
# }}}
# Initialization/Delegate Setup {{{

7
src/calibre/gui2/metadata/basic_widgets.py
View File

@ -1092,11 +1092,12 @@ class IdentifiersEdit(QLineEdit): # {{{
for x in parts:
c = x.split(':')
if len(c) > 1:
if c[0] == 'isbn':
itype = c[0].lower()
if itype == 'isbn':
v = check_isbn(c[1])
if v is not None:
c[1] = v
ans[c[0]] = c[1]
ans[itype] = c[1]
return ans
def fset(self, val):
if not val:
@ -1112,7 +1113,7 @@ class IdentifiersEdit(QLineEdit): # {{{
if v is not None:
val[k] = v
ids = sorted(val.iteritems(), key=keygen)
txt = ', '.join(['%s:%s'%(k, v) for k, v in ids])
txt = ', '.join(['%s:%s'%(k.lower(), v) for k, v in ids])
self.setText(txt.strip())
self.setCursorPosition(0)
return property(fget=fget, fset=fset)

20
src/calibre/gui2/preferences/create_custom_column.py
View File

@ -127,6 +127,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
self.composite_sort_by.setCurrentIndex(sb)
self.composite_make_category.setChecked(
c['display'].get('make_category', False))
self.composite_make_category.setChecked(
c['display'].get('contains_html', False))
elif ct == 'enumeration':
self.enum_box.setText(','.join(c['display'].get('enum_values', [])))
self.enum_colors.setText(','.join(c['display'].get('enum_colors', [])))
@ -141,6 +143,21 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
all_colors = [unicode(s) for s in list(QColor.colorNames())]
self.enum_colors_label.setToolTip('<p>' + ', '.join(all_colors) + '</p>')
self.composite_contains_html.setToolTip('<p>' +
_('If checked, this column will be displayed as HTML in '
'book details and the content server. This can be used to '
'construct links with the template language. For example, '
'the template '
'<pre>&lt;big&gt;&lt;b&gt;{title}&lt;/b&gt;&lt;/big&gt;'
'{series:| [|}{series_index:| [|]]}</pre>'
'will create a field displaying the title in bold large '
'characters, along with the series, for example <br>"<big><b>'
'An Oblique Approach</b></big> [Belisarius [1]]". The template '
'<pre>&lt;a href="http://www.beam-ebooks.de/ebook/{identifiers'
':select(beam)}"&gt;Beam book&lt;/a&gt;</pre> '
'will generate a link to the book on the Beam ebooks site.')
+ '</p>')
self.exec_()
def shortcut_activated(self, url):
@ -179,7 +196,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime')
getattr(self, 'number_format_'+x).setVisible(col_type in ['int', 'float'])
for x in ('box', 'default_label', 'label', 'sort_by', 'sort_by_label',
'make_category'):
'make_category', 'contains_html'):
getattr(self, 'composite_'+x).setVisible(col_type in ['composite', '*composite'])
for x in ('box', 'default_label', 'label', 'colors', 'colors_label'):
getattr(self, 'enum_'+x).setVisible(col_type == 'enumeration')
@ -257,6 +274,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
'composite_sort': ['text', 'number', 'date', 'bool']
[self.composite_sort_by.currentIndex()],
'make_category': self.composite_make_category.isChecked(),
'contains_html': self.composite_contains_html.isChecked(),
}
elif col_type == 'enumeration':
if not unicode(self.enum_box.text()).strip():

7
src/calibre/gui2/preferences/create_custom_column.ui
View File

@ -294,6 +294,13 @@ and end with &lt;code&gt;}&lt;/code&gt; You can have text before and after the f
</property>
</widget>
</item>
<item>
<widget class="QCheckBox" name="composite_contains_html">
<property name="text">
<string>Show as HTML in book details</string>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_24">
<property name="sizePolicy">

1
src/calibre/gui2/preferences/look_feel.py
View File

@ -138,6 +138,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
(_('Partitioned'), 'partition')]
r('tags_browser_partition_method', gprefs, choices=choices)
r('tags_browser_collapse_at', gprefs)
r('default_author_link', gprefs)
choices = set([k for k in db.field_metadata.all_field_keys()
if db.field_metadata[k]['is_category'] and

27
src/calibre/gui2/preferences/look_feel.ui
View File

@ -192,7 +192,7 @@
<string>Book Details</string>
</attribute>
<layout class="QGridLayout" name="gridLayout_12">
<item row="0" column="0" rowspan="2">
<item row="1" column="0" rowspan="2">
<widget class="QGroupBox" name="groupBox">
<property name="title">
<string>Select displayed metadata</string>
@ -243,6 +243,31 @@
</layout>
</widget>
</item>
<item row="0" column="0">
<layout class="QHBoxLayout">
<item>
<widget class="QLabel" name="label">
<property name="text">
<string>Default author link template:</string>
</property>
<property name="buddy">
<cstring>opt_default_author_link</cstring>
</property>
</widget>
</item>
<item>
<widget class="QLineEdit" name="opt_default_author_link">
<property name="toolTip">
<string>&lt;p&gt;Enter a template to be used to create a link for
an author in the books information dialog. This template will
be used when no link has been provided for the author using
Manage Authors. You can use the values {author} and
{author_sort}, and any template function.</string>
</property>
</widget>
</item>
</layout>
</item>
<item row="0" column="1">
<widget class="QCheckBox" name="opt_use_roman_numerals_for_series_number">
<property name="text">

1
src/calibre/gui2/preferences/main.py
View File

@ -357,7 +357,6 @@ class Preferences(QMainWindow):
bytearray(self.saveGeometry()))
if self.committed:
self.gui.must_restart_before_config = self.must_restart
self.gui.tags_view.set_new_model() # in case columns changed
self.gui.tags_view.recount()
self.gui.create_device_menu()
self.gui.set_device_menu_items_state(bool(self.gui.device_connected))

2
src/calibre/gui2/preferences/save_template.py
View File

@ -31,7 +31,7 @@ class SaveTemplate(QWidget, Ui_Form):
(var, FORMAT_ARG_DESCS[var]))
rows.append(u'<tr><td>%s&nbsp;</td><td>&nbsp;</td><td>%s</td></tr>'%(
_('Any custom field'),
_('The lookup name of any custom field. These names begin with "#")')))
_('The lookup name of any custom field (these names begin with "#").')))
table = u'<table>%s</table>'%(u'\n'.join(rows))
self.template_variables.setText(table)

2
src/calibre/gui2/preferences/search.py
View File

@ -173,7 +173,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
def refresh_gui(self, gui):
gui.set_highlight_only_button_icon()
if self.muc_changed:
gui.tags_view.set_new_model()
gui.tags_view.recount()
gui.search.search_as_you_type(config['search_as_you_type'])
gui.search.do_search()

0
src/calibre/gui2/store/mobileread/__init__.py
View File

119
src/calibre/gui2/store/mobileread/adv_search_builder.py
View File

@ -1,119 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import (unicode_literals, division, absolute_import, print_function)
__license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
import re
from PyQt4.Qt import (QDialog, QDialogButtonBox)
from calibre.gui2.store.mobileread.adv_search_builder_ui import Ui_Dialog
from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH
class AdvSearchBuilderDialog(QDialog, Ui_Dialog):
def __init__(self, parent):
QDialog.__init__(self, parent)
self.setupUi(self)
self.buttonBox.accepted.connect(self.advanced_search_button_pushed)
self.tab_2_button_box.accepted.connect(self.accept)
self.tab_2_button_box.rejected.connect(self.reject)
self.clear_button.clicked.connect(self.clear_button_pushed)
self.adv_search_used = False
self.mc = ''
self.tabWidget.setCurrentIndex(0)
self.tabWidget.currentChanged[int].connect(self.tab_changed)
self.tab_changed(0)
def tab_changed(self, idx):
if idx == 1:
self.tab_2_button_box.button(QDialogButtonBox.Ok).setDefault(True)
else:
self.buttonBox.button(QDialogButtonBox.Ok).setDefault(True)
def advanced_search_button_pushed(self):
self.adv_search_used = True
self.accept()
def clear_button_pushed(self):
self.title_box.setText('')
self.author_box.setText('')
self.format_box.setText('')
def tokens(self, raw):
phrases = re.findall(r'\s*".*?"\s*', raw)
for f in phrases:
raw = raw.replace(f, ' ')
phrases = [t.strip('" ') for t in phrases]
return ['"' + self.mc + t + '"' for t in phrases + [r.strip() for r in raw.split()]]
def search_string(self):
if self.adv_search_used:
return self.adv_search_string()
else:
return self.box_search_string()
def adv_search_string(self):
mk = self.matchkind.currentIndex()
if mk == CONTAINS_MATCH:
self.mc = ''
elif mk == EQUALS_MATCH:
self.mc = '='
else:
self.mc = '~'
all, any, phrase, none = map(lambda x: unicode(x.text()),
(self.all, self.any, self.phrase, self.none))
all, any, none = map(self.tokens, (all, any, none))
phrase = phrase.strip()
all = ' and '.join(all)
any = ' or '.join(any)
none = ' and not '.join(none)
ans = ''
if phrase:
ans += '"%s"'%phrase
if all:
ans += (' and ' if ans else '') + all
if none:
ans += (' and not ' if ans else 'not ') + none
if any:
ans += (' or ' if ans else '') + any
return ans
def token(self):
txt = unicode(self.text.text()).strip()
if txt:
if self.negate.isChecked():
txt = '!'+txt
tok = self.FIELDS[unicode(self.field.currentText())]+txt
if re.search(r'\s', tok):
tok = '"%s"'%tok
return tok
def box_search_string(self):
mk = self.matchkind.currentIndex()
if mk == CONTAINS_MATCH:
self.mc = ''
elif mk == EQUALS_MATCH:
self.mc = '='
else:
self.mc = '~'
ans = []
self.box_last_values = {}
title = unicode(self.title_box.text()).strip()
if title:
ans.append('title:"' + self.mc + title + '"')
author = unicode(self.author_box.text()).strip()
if author:
ans.append('author:"' + self.mc + author + '"')
format = unicode(self.format_box.text()).strip()
if format:
ans.append('format:"' + self.mc + format + '"')
if ans:
return ' and '.join(ans)
return ''

350
src/calibre/gui2/store/mobileread/adv_search_builder.ui
View File

@ -1,350 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>Dialog</class>
<widget class="QDialog" name="Dialog">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>752</width>
<height>472</height>
</rect>
</property>
<property name="windowTitle">
<string>Advanced Search</string>
</property>
<property name="windowIcon">
<iconset>
<normaloff>:/images/search.png</normaloff>:/images/search.png</iconset>
</property>
<layout class="QGridLayout" name="gridLayout_2">
<item row="0" column="0">
<widget class="QLabel" name="label_5">
<property name="text">
<string>&amp;What kind of match to use:</string>
</property>
<property name="buddy">
<cstring>matchkind</cstring>
</property>
</widget>
</item>
<item row="0" column="1">
<widget class="QComboBox" name="matchkind">
<item>
<property name="text">
<string>Contains: the word or phrase matches anywhere in the metadata field</string>
</property>
</item>
<item>
<property name="text">
<string>Equals: the word or phrase must match the entire metadata field</string>
</property>
</item>
<item>
<property name="text">
<string>Regular expression: the expression must match anywhere in the metadata field</string>
</property>
</item>
</widget>
</item>
<item row="2" column="0" colspan="2">
<widget class="QTabWidget" name="tabWidget">
<property name="currentIndex">
<number>0</number>
</property>
<widget class="QWidget" name="tab">
<attribute name="title">
<string>A&amp;dvanced Search</string>
</attribute>
<layout class="QGridLayout" name="gridLayout_3">
<item row="0" column="0">
<widget class="QGroupBox" name="groupBox">
<property name="title">
<string>Find entries that have...</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QLabel" name="label">
<property name="text">
<string>&amp;All these words:</string>
</property>
<property name="buddy">
<cstring>all</cstring>
</property>
</widget>
</item>
<item>
<widget class="QLineEdit" name="all"/>
</item>
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
<widget class="QLabel" name="label_2">
<property name="text">
<string>This exact &amp;phrase:</string>
</property>
<property name="buddy">
<cstring>all</cstring>
</property>
</widget>
</item>
<item>
<widget class="QLineEdit" name="phrase"/>
</item>
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_3">
<item>
<widget class="QLabel" name="label_3">
<property name="text">
<string>&amp;One or more of these words:</string>
</property>
<property name="buddy">
<cstring>all</cstring>
</property>
</widget>
</item>
<item>
<widget class="QLineEdit" name="any"/>
</item>
</layout>
</item>
</layout>
</widget>
</item>
<item row="1" column="0">
<widget class="QGroupBox" name="groupBox_2">
<property name="title">
<string>But dont show entries that have...</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout_2">
<item>
<layout class="QHBoxLayout" name="horizontalLayout_4">
<item>
<widget class="QLabel" name="label_4">
<property name="text">
<string>Any of these &amp;unwanted words:</string>
</property>
<property name="buddy">
<cstring>all</cstring>
</property>
</widget>
</item>
<item>
<widget class="QLineEdit" name="none"/>
</item>
</layout>
</item>
<item>
<widget class="QLabel" name="label_6">
<property name="maximumSize">
<size>
<width>16777215</width>
<height>30</height>
</size>
</property>
<property name="text">
<string>See the &lt;a href=&quot;http://calibre-ebook.com/user_manual/gui.html#the-search-interface&quot;&gt;User Manual&lt;/a&gt; for more help</string>
</property>
<property name="openExternalLinks">
<bool>true</bool>
</property>
</widget>
</item>
</layout>
</widget>
</item>
<item row="2" column="0">
<spacer name="verticalSpacer_2">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
<item row="3" column="0">
<widget class="QDialogButtonBox" name="buttonBox">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
</layout>
</widget>
<widget class="QWidget" name="tab_2">
<attribute name="title">
<string>Titl&amp;e/Author/Price ...</string>
</attribute>
<layout class="QGridLayout" name="gridLayout">
<item row="1" column="0">
<widget class="QLabel" name="label_7">
<property name="text">
<string>&amp;Title:</string>
</property>
<property name="buddy">
<cstring>title_box</cstring>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="EnLineEdit" name="title_box">
<property name="toolTip">
<string>Enter the title.</string>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="label_8">
<property name="text">
<string>&amp;Author:</string>
</property>
<property name="buddy">
<cstring>author_box</cstring>
</property>
</widget>
</item>
<item row="5" column="0" colspan="2">
<layout class="QHBoxLayout" name="horizontalLayout_6">
<item>
<widget class="QPushButton" name="clear_button">
<property name="text">
<string>&amp;Clear</string>
</property>
</widget>
</item>
<item>
<widget class="QDialogButtonBox" name="tab_2_button_box">
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
</layout>
</item>
<item row="4" column="1">
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
<item row="0" column="0" colspan="2">
<widget class="QLabel" name="label_11">
<property name="text">
<string>Search only in specific fields:</string>
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="EnLineEdit" name="author_box"/>
</item>
<item row="3" column="1">
<widget class="QLineEdit" name="format_box"/>
</item>
<item row="3" column="0">
<widget class="QLabel" name="label_10">
<property name="text">
<string>&amp;Format:</string>
</property>
<property name="buddy">
<cstring>format_box</cstring>
</property>
</widget>
</item>
</layout>
</widget>
</widget>
</item>
<item row="1" column="1">
<spacer name="verticalSpacer_3">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
</layout>
</widget>
<customwidgets>
<customwidget>
<class>EnLineEdit</class>
<extends>QLineEdit</extends>
<header>widgets.h</header>
</customwidget>
</customwidgets>
<tabstops>
<tabstop>all</tabstop>
<tabstop>phrase</tabstop>
<tabstop>any</tabstop>
<tabstop>none</tabstop>
<tabstop>buttonBox</tabstop>
<tabstop>title_box</tabstop>
<tabstop>author_box</tabstop>
<tabstop>format_box</tabstop>
<tabstop>clear_button</tabstop>
<tabstop>tab_2_button_box</tabstop>
<tabstop>tabWidget</tabstop>
<tabstop>matchkind</tabstop>
</tabstops>
<resources>
<include location="../../../../resources/images.qrc"/>
</resources>
<connections>
<connection>
<sender>buttonBox</sender>
<signal>accepted()</signal>
<receiver>Dialog</receiver>
<slot>accept()</slot>
<hints>
<hint type="sourcelabel">
<x>248</x>
<y>254</y>
</hint>
<hint type="destinationlabel">
<x>157</x>
<y>274</y>
</hint>
</hints>
</connection>
<connection>
<sender>buttonBox</sender>
<signal>rejected()</signal>
<receiver>Dialog</receiver>
<slot>reject()</slot>
<hints>
<hint type="sourcelabel">
<x>316</x>
<y>260</y>
</hint>
<hint type="destinationlabel">
<x>286</x>
<y>274</y>
</hint>
</hints>
</connection>
</connections>
</ui>

62
src/calibre/gui2/store/mobileread/cache_progress_dialog.py
View File

@ -1,62 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import (unicode_literals, division, absolute_import, print_function)
__license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from PyQt4.Qt import QDialog
from calibre.gui2.store.mobileread.cache_progress_dialog_ui import Ui_Dialog
class CacheProgressDialog(QDialog, Ui_Dialog):
def __init__(self, parent=None, total=None):
QDialog.__init__(self, parent)
self.setupUi(self)
self.completed = 0
self.canceled = False
self.progress.setValue(0)
self.progress.setMinimum(0)
self.progress.setMaximum(total if total else 0)
def exec_(self):
self.completed = 0
self.canceled = False
QDialog.exec_(self)
def open(self):
self.completed = 0
self.canceled = False
QDialog.open(self)
def reject(self):
self.canceled = True
QDialog.reject(self)
def update_progress(self):
'''
completed is an int from 0 to total representing the number
records that have bee completed.
'''
self.set_progress(self.completed + 1)
def set_message(self, msg):
self.message.setText(msg)
def set_details(self, msg):
self.details.setText(msg)
def set_progress(self, completed):
'''
completed is an int from 0 to total representing the number
records that have bee completed.
'''
self.completed = completed
self.progress.setValue(self.completed)
def set_total(self, total):
self.progress.setMaximum(total)

104
src/calibre/gui2/store/mobileread/cache_progress_dialog.ui
View File

@ -1,104 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>Dialog</class>
<widget class="QDialog" name="Dialog">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>402</width>
<height>138</height>
</rect>
</property>
<property name="windowTitle">
<string>Dialog</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<widget class="QLabel" name="message">
<property name="text">
<string>Updating book cache</string>
</property>
<property name="alignment">
<set>Qt::AlignCenter</set>
</property>
</widget>
</item>
<item>
<widget class="QProgressBar" name="progress">
<property name="value">
<number>24</number>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="details">
<property name="text">
<string/>
</property>
<property name="alignment">
<set>Qt::AlignCenter</set>
</property>
</widget>
</item>
<item>
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QDialogButtonBox" name="buttonBox">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="standardButtons">
<set>QDialogButtonBox::Cancel</set>
</property>
</widget>
</item>
</layout>
</widget>
<resources/>
<connections>
<connection>
<sender>buttonBox</sender>
<signal>accepted()</signal>
<receiver>Dialog</receiver>
<slot>accept()</slot>
<hints>
<hint type="sourcelabel">
<x>248</x>
<y>254</y>
</hint>
<hint type="destinationlabel">
<x>157</x>
<y>274</y>
</hint>
</hints>
</connection>
<connection>
<sender>buttonBox</sender>
<signal>rejected()</signal>
<receiver>Dialog</receiver>
<slot>reject()</slot>
<hints>
<hint type="sourcelabel">
<x>316</x>
<y>260</y>
</hint>
<hint type="destinationlabel">
<x>286</x>
<y>274</y>
</hint>
</hints>
</connection>
</connections>
</ui>

94
src/calibre/gui2/store/mobileread/cache_update_thread.py
View File

@ -1,94 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import (unicode_literals, division, absolute_import, print_function)
__license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
import time
from contextlib import closing
from threading import Thread
from lxml import html
from PyQt4.Qt import (pyqtSignal, QObject)
from calibre import browser
from calibre.gui2.store.search_result import SearchResult
class CacheUpdateThread(Thread, QObject):
total_changed = pyqtSignal(int)
update_progress = pyqtSignal(int)
update_details = pyqtSignal(unicode)
def __init__(self, config, seralize_books_function, timeout):
Thread.__init__(self)
QObject.__init__(self)
self.daemon = True
self.config = config
self.seralize_books = seralize_books_function
self.timeout = timeout
self._run = True
def abort(self):
self._run = False
def run(self):
url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html'
self.update_details.emit(_('Checking last download date.'))
last_download = self.config.get('last_download', None)
# Don't update the book list if our cache is less than one week old.
if last_download and (time.time() - last_download) < 604800:
return
self.update_details.emit(_('Downloading book list from MobileRead.'))
# Download the book list HTML file from MobileRead.
br = browser()
raw_data = None
try:
with closing(br.open(url, timeout=self.timeout)) as f:
raw_data = f.read()
except:
return
if not raw_data or not self._run:
return
self.update_details.emit(_('Processing books.'))
# Turn books listed in the HTML file into SearchResults's.
books = []
try:
data = html.fromstring(raw_data)
raw_books = data.xpath('//ul/li')
self.total_changed.emit(len(raw_books))
for i, book_data in enumerate(raw_books):
self.update_details.emit(_('%s of %s books processed.') % (i, len(raw_books)))
book = SearchResult()
book.detail_item = ''.join(book_data.xpath('.//a/@href'))
book.formats = ''.join(book_data.xpath('.//i/text()'))
book.formats = book.formats.strip()
text = ''.join(book_data.xpath('.//a/text()'))
if ':' in text:
book.author, q, text = text.partition(':')
book.author = book.author.strip()
book.title = text.strip()
books.append(book)
if not self._run:
books = []
break
else:
self.update_progress.emit(i)
except:
pass
# Save the book list and it's create time.
if books:
self.config['book_list'] = self.seralize_books(books)
self.config['last_download'] = time.time()

105
src/calibre/gui2/store/mobileread/mobileread_plugin.py
View File

@ -1,105 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import (unicode_literals, division, absolute_import, print_function)
__license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from threading import Lock
from PyQt4.Qt import (QUrl, QCoreApplication)
from calibre.gui2 import open_url
from calibre.gui2.store import StorePlugin
from calibre.gui2.store.basic_config import BasicStoreConfig
from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
from calibre.gui2.store.mobileread.models import SearchFilter
from calibre.gui2.store.mobileread.cache_progress_dialog import CacheProgressDialog
from calibre.gui2.store.mobileread.cache_update_thread import CacheUpdateThread
from calibre.gui2.store.mobileread.store_dialog import MobileReadStoreDialog
class MobileReadStore(BasicStoreConfig, StorePlugin):
def genesis(self):
self.lock = Lock()
def open(self, parent=None, detail_item=None, external=False):
url = 'http://www.mobileread.com/'
if external or self.config.get('open_external', False):
open_url(QUrl(detail_item if detail_item else url))
else:
if detail_item:
d = WebStoreDialog(self.gui, url, parent, detail_item)
d.setWindowTitle(self.name)
d.set_tags(self.config.get('tags', ''))
d.exec_()
else:
self.update_cache(parent, 30)
d = MobileReadStoreDialog(self, parent)
d.setWindowTitle(self.name)
d.exec_()
def search(self, query, max_results=10, timeout=60):
books = self.get_book_list()
sf = SearchFilter(books)
matches = sf.parse(query)
for book in matches:
book.price = '$0.00'
book.drm = SearchResult.DRM_UNLOCKED
yield book
def update_cache(self, parent=None, timeout=10, force=False, suppress_progress=False):
if self.lock.acquire(False):
try:
update_thread = CacheUpdateThread(self.config, self.seralize_books, timeout)
if not suppress_progress:
progress = CacheProgressDialog(parent)
progress.set_message(_('Updating MobileRead book cache...'))
update_thread.total_changed.connect(progress.set_total)
update_thread.update_progress.connect(progress.set_progress)
update_thread.update_details.connect(progress.set_details)
progress.rejected.connect(update_thread.abort)
progress.open()
update_thread.start()
while update_thread.is_alive() and not progress.canceled:
QCoreApplication.processEvents()
if progress.isVisible():
progress.accept()
return not progress.canceled
else:
update_thread.start()
finally:
self.lock.release()
def get_book_list(self):
return self.deseralize_books(self.config.get('book_list', []))
def seralize_books(self, books):
sbooks = []
for b in books:
data = {}
data['author'] = b.author
data['title'] = b.title
data['detail_item'] = b.detail_item
data['formats'] = b.formats
sbooks.append(data)
return sbooks
def deseralize_books(self, sbooks):
books = []
for s in sbooks:
b = SearchResult()
b.author = s.get('author', '')
b.title = s.get('title', '')
b.detail_item = s.get('detail_item', '')
b.formats = s.get('formats', '')
books.append(b)
return books

191
src/calibre/gui2/store/mobileread/models.py
View File

@ -1,191 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import (unicode_literals, division, absolute_import, print_function)
__license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from operator import attrgetter
from PyQt4.Qt import (Qt, QAbstractItemModel, QModelIndex, QVariant, pyqtSignal)
from calibre.gui2 import NONE
from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
REGEXP_MATCH
from calibre.utils.icu import sort_key
from calibre.utils.search_query_parser import SearchQueryParser
class BooksModel(QAbstractItemModel):
total_changed = pyqtSignal(int)
HEADERS = [_('Title'), _('Author(s)'), _('Format')]
def __init__(self, all_books):
QAbstractItemModel.__init__(self)
self.books = all_books
self.all_books = all_books
self.filter = ''
self.search_filter = SearchFilter(all_books)
self.sort_col = 0
self.sort_order = Qt.AscendingOrder
def get_book(self, index):
row = index.row()
if row < len(self.books):
return self.books[row]
else:
return None
def search(self, filter):
self.filter = filter.strip()
if not self.filter:
self.books = self.all_books
else:
try:
self.books = list(self.search_filter.parse(self.filter))
except:
self.books = self.all_books
self.layoutChanged.emit()
self.sort(self.sort_col, self.sort_order)
self.total_changed.emit(self.rowCount())
def index(self, row, column, parent=QModelIndex()):
return self.createIndex(row, column)
def parent(self, index):
if not index.isValid() or index.internalId() == 0:
return QModelIndex()
return self.createIndex(0, 0)
def rowCount(self, *args):
return len(self.books)
def columnCount(self, *args):
return len(self.HEADERS)
def headerData(self, section, orientation, role):
if role != Qt.DisplayRole:
return NONE
text = ''
if orientation == Qt.Horizontal:
if section < len(self.HEADERS):
text = self.HEADERS[section]
return QVariant(text)
else:
return QVariant(section+1)
def data(self, index, role):
row, col = index.row(), index.column()
result = self.books[row]
if role == Qt.DisplayRole:
if col == 0:
return QVariant(result.title)
elif col == 1:
return QVariant(result.author)
elif col == 2:
return QVariant(result.formats)
return NONE
def data_as_text(self, result, col):
text = ''
if col == 0:
text = result.title
elif col == 1:
text = result.author
elif col == 2:
text = result.formats
return text
def sort(self, col, order, reset=True):
self.sort_col = col
self.sort_order = order
if not self.books:
return
descending = order == Qt.DescendingOrder
self.books.sort(None,
lambda x: sort_key(unicode(self.data_as_text(x, col))),
descending)
if reset:
self.reset()
class SearchFilter(SearchQueryParser):
USABLE_LOCATIONS = [
'all',
'author',
'authors',
'format',
'formats',
'title',
]
def __init__(self, all_books=[]):
SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS)
self.srs = set(all_books)
def universal_set(self):
return self.srs
def get_matches(self, location, query):
location = location.lower().strip()
if location == 'authors':
location = 'author'
elif location == 'formats':
location = 'format'
matchkind = CONTAINS_MATCH
if len(query) > 1:
if query.startswith('\\'):
query = query[1:]
elif query.startswith('='):
matchkind = EQUALS_MATCH
query = query[1:]
elif query.startswith('~'):
matchkind = REGEXP_MATCH
query = query[1:]
if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D
query = query.lower()
if location not in self.USABLE_LOCATIONS:
return set([])
matches = set([])
all_locs = set(self.USABLE_LOCATIONS) - set(['all'])
locations = all_locs if location == 'all' else [location]
q = {
'author': lambda x: x.author.lower(),
'format': attrgetter('formats'),
'title': lambda x: x.title.lower(),
}
for x in ('author', 'format'):
q[x+'s'] = q[x]
for sr in self.srs:
for locvalue in locations:
accessor = q[locvalue]
if query == 'true':
if accessor(sr) is not None:
matches.add(sr)
continue
if query == 'false':
if accessor(sr) is None:
matches.add(sr)
continue
try:
### Can't separate authors because comma is used for name sep and author sep
### Exact match might not get what you want. For that reason, turn author
### exactmatch searches into contains searches.
if locvalue == 'author' and matchkind == EQUALS_MATCH:
m = CONTAINS_MATCH
else:
m = matchkind
vals = [accessor(sr)]
if _match(query, vals, m):
matches.add(sr)
break
except ValueError: # Unicode errors
import traceback
traceback.print_exc()
return matches

84
src/calibre/gui2/store/mobileread/store_dialog.py
View File

@ -1,84 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import (unicode_literals, division, absolute_import, print_function)
__license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from PyQt4.Qt import (Qt, QDialog, QIcon, QComboBox)
from calibre.gui2.store.mobileread.adv_search_builder import AdvSearchBuilderDialog
from calibre.gui2.store.mobileread.models import BooksModel
from calibre.gui2.store.mobileread.store_dialog_ui import Ui_Dialog
class MobileReadStoreDialog(QDialog, Ui_Dialog):
def __init__(self, plugin, *args):
QDialog.__init__(self, *args)
self.setupUi(self)
self.plugin = plugin
self.search_query.initialize('store_mobileread_search')
self.search_query.setSizeAdjustPolicy(QComboBox.AdjustToMinimumContentsLengthWithIcon)
self.search_query.setMinimumContentsLength(25)
self.adv_search_button.setIcon(QIcon(I('search.png')))
self._model = BooksModel(self.plugin.get_book_list())
self.results_view.setModel(self._model)
self.total.setText('%s' % self.results_view.model().rowCount())
self.search_button.clicked.connect(self.do_search)
self.adv_search_button.clicked.connect(self.build_adv_search)
self.results_view.activated.connect(self.open_store)
self.results_view.model().total_changed.connect(self.update_book_total)
self.finished.connect(self.dialog_closed)
self.restore_state()
def do_search(self):
self.results_view.model().search(unicode(self.search_query.text()))
def open_store(self, index):
result = self.results_view.model().get_book(index)
if result:
self.plugin.open(self, result.detail_item)
def update_book_total(self, total):
self.total.setText('%s' % total)
def build_adv_search(self):
adv = AdvSearchBuilderDialog(self)
if adv.exec_() == QDialog.Accepted:
self.search_query.setText(adv.search_string())
def restore_state(self):
geometry = self.plugin.config.get('dialog_geometry', None)
if geometry:
self.restoreGeometry(geometry)
results_cwidth = self.plugin.config.get('dialog_results_view_column_width')
if results_cwidth:
for i, x in enumerate(results_cwidth):
if i >= self.results_view.model().columnCount():
break
self.results_view.setColumnWidth(i, x)
else:
for i in xrange(self.results_view.model().columnCount()):
self.results_view.resizeColumnToContents(i)
self.results_view.model().sort_col = self.plugin.config.get('dialog_sort_col', 0)
self.results_view.model().sort_order = self.plugin.config.get('dialog_sort_order', Qt.AscendingOrder)
self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order)
self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order)
def save_state(self):
self.plugin.config['dialog_geometry'] = bytearray(self.saveGeometry())
self.plugin.config['dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.results_view.model().columnCount())]
self.plugin.config['dialog_sort_col'] = self.results_view.model().sort_col
self.plugin.config['dialog_sort_order'] = self.results_view.model().sort_order
def dialog_closed(self, result):
self.save_state()

143
src/calibre/gui2/store/mobileread/store_dialog.ui
View File

@ -1,143 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>Dialog</class>
<widget class="QDialog" name="Dialog">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>691</width>
<height>614</height>
</rect>
</property>
<property name="windowTitle">
<string>Dialog</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
<widget class="QLabel" name="label">
<property name="text">
<string>&amp;Query:</string>
</property>
<property name="buddy">
<cstring>search_query</cstring>
</property>
</widget>
</item>
<item>
<widget class="QToolButton" name="adv_search_button">
<property name="text">
<string>...</string>
</property>
</widget>
</item>
<item>
<widget class="HistoryLineEdit" name="search_query">
<property name="sizePolicy">
<sizepolicy hsizetype="Expanding" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="search_button">
<property name="text">
<string>Search</string>
</property>
</widget>
</item>
</layout>
</item>
<item>
<widget class="QTreeView" name="results_view">
<property name="alternatingRowColors">
<bool>true</bool>
</property>
<property name="rootIsDecorated">
<bool>false</bool>
</property>
<property name="itemsExpandable">
<bool>false</bool>
</property>
<property name="sortingEnabled">
<bool>true</bool>
</property>
<property name="expandsOnDoubleClick">
<bool>false</bool>
</property>
<attribute name="headerCascadingSectionResizes">
<bool>false</bool>
</attribute>
</widget>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QLabel" name="label_2">
<property name="text">
<string>Books:</string>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="total">
<property name="text">
<string>0</string>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>308</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QPushButton" name="close_button">
<property name="text">
<string>Close</string>
</property>
</widget>
</item>
</layout>
</item>
</layout>
</widget>
<customwidgets>
<customwidget>
<class>HistoryLineEdit</class>
<extends>QLineEdit</extends>
<header>widgets.h</header>
</customwidget>
</customwidgets>
<resources/>
<connections>
<connection>
<sender>close_button</sender>
<signal>clicked()</signal>
<receiver>Dialog</receiver>
<slot>accept()</slot>
<hints>
<hint type="sourcelabel">
<x>440</x>
<y>432</y>
</hint>
<hint type="destinationlabel">
<x>245</x>
<y>230</y>
</hint>
</hints>
</connection>
</connections>
</ui>

5
src/calibre/gui2/store/opensearch_store.py
View File

@ -7,7 +7,6 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
import mimetypes
import urllib
from contextlib import closing
from lxml import etree
@ -22,7 +21,7 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
from calibre.utils.opensearch.description import Description
from calibre.utils.opensearch.query import Query
class OpenSearchStore(StorePlugin):
class OpenSearchOPDSStore(StorePlugin):
open_search_url = ''
web_url = ''
@ -50,7 +49,7 @@ class OpenSearchStore(StorePlugin):
oquery = Query(url_template)
# set up initial values
oquery.searchTerms = urllib.quote_plus(query)
oquery.searchTerms = query
oquery.count = max_results
url = oquery.url()

2
src/calibre/gui2/store/search/models.py
View File

@ -22,6 +22,7 @@ from calibre.utils.icu import sort_key
from calibre.utils.search_query_parser import SearchQueryParser
def comparable_price(text):
text = re.sub(r'[^0-9.,]', '', text)
if len(text) < 3 or text[-3] not in ('.', ','):
text += '00'
text = re.sub(r'\D', '', text)
@ -293,6 +294,7 @@ class SearchFilter(SearchQueryParser):
return self.srs
def get_matches(self, location, query):
query = query.strip()
location = location.lower().strip()
if location == 'authors':
location = 'author'

5
src/calibre/gui2/store/search/search.py
View File

@ -22,6 +22,7 @@ from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog
from calibre.gui2.store.search.download_thread import SearchThreadPool, \
CacheUpdateThreadPool
from calibre.gui2.store.search.search_ui import Ui_Dialog
from calibre.utils.filenames import ascii_filename
class SearchDialog(QDialog, Ui_Dialog):
@ -349,7 +350,9 @@ class SearchDialog(QDialog, Ui_Dialog):
d = ChooseFormatDialog(self, _('Choose format to download to your library.'), result.downloads.keys())
if d.exec_() == d.Accepted:
ext = d.format()
self.gui.download_ebook(result.downloads[ext])
fname = result.title + '.' + ext.lower()
fname = ascii_filename(fname)
self.gui.download_ebook(result.downloads[ext], filename=fname)
def open_store(self, result):
self.gui.istores[result.store_name].open(self, result.detail_item, self.open_external.isChecked())

21
src/calibre/gui2/store/stores/archive_org_plugin.py
View File

@ -6,24 +6,19 @@ __license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from contextlib import closing
from lxml import html
from calibre import browser
from calibre.gui2.store.basic_config import BasicStoreConfig
from calibre.gui2.store.opensearch_store import OpenSearchStore
from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore
from calibre.gui2.store.search_result import SearchResult
class ArchiveOrgStore(BasicStoreConfig, OpenSearchStore):
class ArchiveOrgStore(BasicStoreConfig, OpenSearchOPDSStore):
open_search_url = 'http://bookserver.archive.org/catalog/opensearch.xml'
web_url = 'http://www.archive.org/details/texts'
# http://bookserver.archive.org/catalog/
def search(self, query, max_results=10, timeout=60):
for s in OpenSearchStore.search(self, query, max_results, timeout):
for s in OpenSearchOPDSStore.search(self, query, max_results, timeout):
s.detail_item = 'http://www.archive.org/details/' + s.detail_item.split(':')[-1]
s.price = '$0.00'
s.drm = SearchResult.DRM_UNLOCKED
@ -34,10 +29,14 @@ class ArchiveOrgStore(BasicStoreConfig, OpenSearchStore):
The opensearch feed only returns a subset of formats that are available.
We want to get a list of all formats that the user can get.
'''
from calibre import browser
from contextlib import closing
from lxml import html
br = browser()
with closing(br.open(search_result.detail_item, timeout=timeout)) as nf:
idata = html.fromstring(nf.read())
formats = ', '.join(idata.xpath('//p[@id="dl" and @class="content"]//a/text()'))
search_result.formats = formats.upper()
return True

29
src/calibre/gui2/store/stores/bn_plugin.py
View File

@ -7,7 +7,6 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
import random
import re
from contextlib import closing
from lxml import html
@ -22,20 +21,18 @@ from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
class BNStore(BasicStoreConfig, StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
pub_id = '21000000000352219'
pub_id = 'sHa5EXvYOwA'
# Use Kovid's affiliate id 30% of the time.
if random.randint(1, 10) in (1, 2, 3):
pub_id = '21000000000352583'
url = 'http://gan.doubleclick.net/gan_click?lid=41000000028437369&pubid=' + pub_id
pub_id = '0dsO3kDu/AU'
base_url = 'http://click.linksynergy.com/fs-bin/click?id=%s&subid=&offerid=229293.1&type=10&tmpid=8433&RD_PARM1=' % pub_id
url = base_url + 'http%253A%252F%252Fwww.barnesandnoble.com%252F'
if detail_item:
mo = re.search(r'(?<=/)(?P<isbn>\d+)(?=/|$)', detail_item)
if mo:
isbn = mo.group('isbn')
detail_item = 'http://gan.doubleclick.net/gan_click?lid=41000000012871747&pid=' + isbn + '&adurl=' + detail_item + '&pubid=' + pub_id
detail_item = base_url + detail_item
if external or self.config.get('open_external', False):
open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
@ -48,27 +45,27 @@ class BNStore(BasicStoreConfig, StorePlugin):
def search(self, query, max_results=10, timeout=60):
query = query.replace(' ', '-')
url = 'http://www.barnesandnoble.com/s/%s?store=ebook&sze=%s' % (query, max_results)
br = browser()
counter = max_results
with closing(br.open(url, timeout=timeout)) as f:
doc = html.fromstring(f.read())
for data in doc.xpath('//ul[contains(@class, "result-set")]/li[contains(@class, "result")]'):
if counter <= 0:
break
id = ''.join(data.xpath('.//div[contains(@class, "image")]/a/@href'))
if not id:
continue
cover_url = ''.join(data.xpath('.//div[contains(@class, "image")]//img/@src'))
title = ''.join(data.xpath('.//p[@class="title"]//span[@class="name"]/text()'))
author = ', '.join(data.xpath('.//ul[@class="contributors"]//li[position()>1]//a/text()'))
price = ''.join(data.xpath('.//table[@class="displayed-formats"]//a[@class="subtle"]/text()'))
counter -= 1
s = SearchResult()
s.cover_url = cover_url
s.title = title.strip()

6
src/calibre/gui2/store/stores/epubbud_plugin.py
View File

@ -7,10 +7,10 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from calibre.gui2.store.basic_config import BasicStoreConfig
from calibre.gui2.store.opensearch_store import OpenSearchStore
from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore
from calibre.gui2.store.search_result import SearchResult
class EpubBudStore(BasicStoreConfig, OpenSearchStore):
class EpubBudStore(BasicStoreConfig, OpenSearchOPDSStore):
open_search_url = 'http://www.epubbud.com/feeds/opensearch.xml'
web_url = 'http://www.epubbud.com/'
@ -18,7 +18,7 @@ class EpubBudStore(BasicStoreConfig, OpenSearchStore):
# http://www.epubbud.com/feeds/catalog.atom
def search(self, query, max_results=10, timeout=60):
for s in OpenSearchStore.search(self, query, max_results, timeout):
for s in OpenSearchOPDSStore.search(self, query, max_results, timeout):
s.price = '$0.00'
s.drm = SearchResult.DRM_UNLOCKED
s.formats = 'EPUB'

6
src/calibre/gui2/store/stores/feedbooks_plugin.py
View File

@ -7,10 +7,10 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from calibre.gui2.store.basic_config import BasicStoreConfig
from calibre.gui2.store.opensearch_store import OpenSearchStore
from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore
from calibre.gui2.store.search_result import SearchResult
class FeedbooksStore(BasicStoreConfig, OpenSearchStore):
class FeedbooksStore(BasicStoreConfig, OpenSearchOPDSStore):
open_search_url = 'http://assets0.feedbooks.net/opensearch.xml?t=1253087147'
web_url = 'http://feedbooks.com/'
@ -18,7 +18,7 @@ class FeedbooksStore(BasicStoreConfig, OpenSearchStore):
# http://www.feedbooks.com/catalog
def search(self, query, max_results=10, timeout=60):
for s in OpenSearchStore.search(self, query, max_results, timeout):
for s in OpenSearchOPDSStore.search(self, query, max_results, timeout):
if s.downloads:
s.drm = SearchResult.DRM_UNLOCKED
s.price = '$0.00'

70
src/calibre/gui2/store/stores/gutenberg_plugin.py
View File

@ -6,6 +6,7 @@ __license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
import mimetypes
import urllib
from contextlib import closing
@ -23,70 +24,67 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
class GutenbergStore(BasicStoreConfig, StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
url = 'http://m.gutenberg.org/'
ext_url = 'http://gutenberg.org/'
url = 'http://gutenberg.org/'
if detail_item:
detail_item = url_slash_cleaner(url + detail_item)
if external or self.config.get('open_external', False):
if detail_item:
ext_url = ext_url + detail_item
open_url(QUrl(url_slash_cleaner(ext_url)))
open_url(QUrl(detail_item if detail_item else url))
else:
detail_url = None
if detail_item:
detail_url = url + detail_item
d = WebStoreDialog(self.gui, url, parent, detail_url)
d = WebStoreDialog(self.gui, url, parent, detail_item)
d.setWindowTitle(self.name)
d.set_tags(self.config.get('tags', ''))
d.exec_()
def search(self, query, max_results=10, timeout=60):
# Gutenberg's website does not allow searching both author and title.
# Using a google search so we can search on both fields at once.
url = 'http://www.google.com/xhtml?q=site:gutenberg.org+' + urllib.quote_plus(query)
url = 'http://m.gutenberg.org/ebooks/search.mobile/?default_prefix=all&sort_order=title&query=' + urllib.quote_plus(query)
br = browser()
counter = max_results
with closing(br.open(url, timeout=timeout)) as f:
doc = html.fromstring(f.read())
for data in doc.xpath('//div[@class="edewpi"]//div[@class="r ld"]'):
for data in doc.xpath('//ol[@class="results"]//li[contains(@class, "icon_title")]'):
if counter <= 0:
break
id = ''.join(data.xpath('./a/@href'))
id = id.split('.mobile')[0]
url = ''
url_a = data.xpath('div[@class="jd"]/a')
if url_a:
url_a = url_a[0]
url = url_a.get('href', None)
if url:
url = url.split('u=')[-1].split('&')[0]
if '/ebooks/' not in url:
continue
id = url.split('/')[-1]
url_a = html.fromstring(html.tostring(url_a))
heading = ''.join(url_a.xpath('//text()'))
title, _, author = heading.rpartition('by ')
author = author.split('-')[0]
price = '$0.00'
title = ''.join(data.xpath('.//span[@class="title"]/text()'))
author = ''.join(data.xpath('.//span[@class="subtitle"]/text()'))
counter -= 1
s = SearchResult()
s.cover_url = ''
s.detail_item = id.strip()
s.title = title.strip()
s.author = author.strip()
s.price = price.strip()
s.detail_item = '/ebooks/' + id.strip()
s.price = '$0.00'
s.drm = SearchResult.DRM_UNLOCKED
yield s
def get_details(self, search_result, timeout):
url = 'http://m.gutenberg.org/'
url = url_slash_cleaner('http://m.gutenberg.org/' + search_result.detail_item + '.mobile')
br = browser()
with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf:
idata = html.fromstring(nf.read())
search_result.formats = ', '.join(idata.xpath('//a[@type!="application/atom+xml"]//span[@class="title"]/text()'))
return True
with closing(br.open(url, timeout=timeout)) as nf:
doc = html.fromstring(nf.read())
for save_item in doc.xpath('//li[contains(@class, "icon_save")]/a'):
type = save_item.get('type')
href = save_item.get('href')
if type:
ext = mimetypes.guess_extension(type)
if ext:
ext = ext[1:].upper().strip()
search_result.downloads[ext] = href
search_result.formats = ', '.join(search_result.downloads.keys())
return True

149
src/calibre/gui2/store/stores/manybooks_plugin.py
View File

@ -6,90 +6,101 @@ __license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
import re
import urllib
import mimetypes
from contextlib import closing
from lxml import html
from lxml import etree
from PyQt4.Qt import QUrl
from calibre import browser, url_slash_cleaner
from calibre.gui2 import open_url
from calibre.gui2.store import StorePlugin
from calibre import browser
from calibre.gui2.store.basic_config import BasicStoreConfig
from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore
from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
class ManyBooksStore(BasicStoreConfig, StorePlugin):
from calibre.utils.opensearch.description import Description
from calibre.utils.opensearch.query import Query
def open(self, parent=None, detail_item=None, external=False):
url = 'http://manybooks.net/'
class ManyBooksStore(BasicStoreConfig, OpenSearchOPDSStore):
detail_url = None
if detail_item:
detail_url = url + detail_item
if external or self.config.get('open_external', False):
open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
else:
d = WebStoreDialog(self.gui, url, parent, detail_url)
d.setWindowTitle(self.name)
d.set_tags(self.config.get('tags', ''))
d.exec_()
open_search_url = 'http://www.manybooks.net/opds/'
web_url = 'http://manybooks.net'
def search(self, query, max_results=10, timeout=60):
# ManyBooks website separates results for title and author.
# It also doesn't do a clear job of references authors and
# secondary titles. Google is also faster.
# Using a google search so we can search on both fields at once.
url = 'http://www.google.com/xhtml?q=site:manybooks.net+' + urllib.quote_plus(query)
br = browser()
'''
Manybooks uses a very strange opds feed. The opds
main feed is structured like a stanza feed. The
search result entries give very little information
and requires you to go to a detail link. The detail
link has the wrong type specified (text/html instead
of application/atom+xml).
'''
if not hasattr(self, 'open_search_url'):
return
description = Description(self.open_search_url)
url_template = description.get_best_template()
if not url_template:
return
oquery = Query(url_template)
# set up initial values
oquery.searchTerms = query
oquery.count = max_results
url = oquery.url()
counter = max_results
br = browser()
with closing(br.open(url, timeout=timeout)) as f:
doc = html.fromstring(f.read())
for data in doc.xpath('//div[@class="edewpi"]//div[@class="r ld"]'):
doc = etree.fromstring(f.read())
for data in doc.xpath('//*[local-name() = "entry"]'):
if counter <= 0:
break
url = ''
url_a = data.xpath('div[@class="jd"]/a')
if url_a:
url_a = url_a[0]
url = url_a.get('href', None)
if url:
url = url.split('u=')[-1][:-2]
if '/titles/' not in url:
continue
id = url.split('/')[-1]
id = id.strip()
url_a = html.fromstring(html.tostring(url_a))
heading = ''.join(url_a.xpath('//text()'))
title, _, author = heading.rpartition('by ')
author = author.split('-')[0]
price = '$0.00'
cover_url = ''
mo = re.match('^\D+', id)
if mo:
cover_name = mo.group()
cover_name = cover_name.replace('etext', '')
cover_id = id.split('.')[0]
cover_url = 'http://www.manybooks.net/images/' + id[0] + '/' + cover_name + '/' + cover_id + '-thumb.jpg'
print(cover_url)
counter -= 1
s = SearchResult()
s.cover_url = cover_url
s.title = title.strip()
s.author = author.strip()
s.price = price.strip()
s.detail_item = '/titles/' + id
s.drm = SearchResult.DRM_UNLOCKED
s.formts = 'EPUB, PDB (eReader, PalmDoc, zTXT, Plucker, iSilo), FB2, ZIP, AZW, MOBI, PRC, LIT, PKG, PDF, TXT, RB, RTF, LRF, TCR, JAR'
detail_links = data.xpath('./*[local-name() = "link" and @type = "text/html"]')
if not detail_links:
continue
detail_link = detail_links[0]
detail_href = detail_link.get('href')
if not detail_href:
continue
s.detail_item = 'http://manybooks.net/titles/' + detail_href.split('tid=')[-1] + '.html'
# These can have HTML inside of them. We are going to get them again later
# just in case.
s.title = ''.join(data.xpath('./*[local-name() = "title"]//text()')).strip()
s.author = ', '.join(data.xpath('./*[local-name() = "author"]//text()')).strip()
# Follow the detail link to get the rest of the info.
with closing(br.open(detail_href, timeout=timeout/4)) as df:
ddoc = etree.fromstring(df.read())
ddata = ddoc.xpath('//*[local-name() = "entry"][1]')
if ddata:
ddata = ddata[0]
# This is the real title and author info we want. We got
# it previously just in case it's not specified here for some reason.
s.title = ''.join(ddata.xpath('./*[local-name() = "title"]//text()')).strip()
s.author = ', '.join(ddata.xpath('./*[local-name() = "author"]//text()')).strip()
if s.author.startswith(','):
s.author = s.author[1:]
if s.author.endswith(','):
s.author = s.author[:-1]
s.cover_url = ''.join(ddata.xpath('./*[local-name() = "link" and @rel = "http://opds-spec.org/thumbnail"][1]/@href')).strip()
for link in ddata.xpath('./*[local-name() = "link" and @rel = "http://opds-spec.org/acquisition"]'):
type = link.get('type')
href = link.get('href')
if type:
ext = mimetypes.guess_extension(type)
if ext:
ext = ext[1:].upper().strip()
s.downloads[ext] = href
s.price = '$0.00'
s.drm = SearchResult.DRM_UNLOCKED
s.formats = 'EPUB, PDB (eReader, PalmDoc, zTXT, Plucker, iSilo), FB2, ZIP, AZW, MOBI, PRC, LIT, PKG, PDF, TXT, RB, RTF, LRF, TCR, JAR'
yield s

84
src/calibre/gui2/store/stores/open_library_plugin.py
View File

@ -1,84 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import (unicode_literals, division, absolute_import, print_function)
__license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
import urllib2
from contextlib import closing
from lxml import html
from PyQt4.Qt import QUrl
from calibre import browser, url_slash_cleaner
from calibre.gui2 import open_url
from calibre.gui2.store import StorePlugin
from calibre.gui2.store.basic_config import BasicStoreConfig
from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
class OpenLibraryStore(BasicStoreConfig, StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
url = 'http://openlibrary.org/'
if external or self.config.get('open_external', False):
if detail_item:
url = url + detail_item
open_url(QUrl(url_slash_cleaner(url)))
else:
detail_url = None
if detail_item:
detail_url = url + detail_item
d = WebStoreDialog(self.gui, url, parent, detail_url)
d.setWindowTitle(self.name)
d.set_tags(self.config.get('tags', ''))
d.exec_()
def search(self, query, max_results=10, timeout=60):
url = 'http://openlibrary.org/search?q=' + urllib2.quote(query) + '&has_fulltext=true'
br = browser()
counter = max_results
with closing(br.open(url, timeout=timeout)) as f:
doc = html.fromstring(f.read())
for data in doc.xpath('//div[@id="searchResults"]/ul[@id="siteSearch"]/li'):
if counter <= 0:
break
# Don't include books that don't have downloadable files.
if not data.xpath('boolean(./span[@class="actions"]//span[@class="label" and contains(text(), "Read")])'):
continue
id = ''.join(data.xpath('./span[@class="bookcover"]/a/@href'))
if not id:
continue
cover_url = ''.join(data.xpath('./span[@class="bookcover"]/a/img/@src'))
title = ''.join(data.xpath('.//h3[@class="booktitle"]/a[@class="results"]/text()'))
author = ''.join(data.xpath('.//span[@class="bookauthor"]/a/text()'))
price = '$0.00'
counter -= 1
s = SearchResult()
s.cover_url = cover_url
s.title = title.strip()
s.author = author.strip()
s.price = price
s.detail_item = id.strip()
s.drm = SearchResult.DRM_UNLOCKED
yield s
def get_details(self, search_result, timeout):
url = 'http://openlibrary.org/'
br = browser()
with closing(br.open(url_slash_cleaner(url + search_result.detail_item), timeout=timeout)) as nf:
idata = html.fromstring(nf.read())
search_result.formats = ', '.join(list(set(idata.xpath('//a[contains(@title, "Download")]/text()'))))
return True

6
src/calibre/gui2/store/stores/pragmatic_bookshelf_plugin.py
View File

@ -7,10 +7,10 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from calibre.gui2.store.basic_config import BasicStoreConfig
from calibre.gui2.store.opensearch_store import OpenSearchStore
from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore
from calibre.gui2.store.search_result import SearchResult
class PragmaticBookshelfStore(BasicStoreConfig, OpenSearchStore):
class PragmaticBookshelfStore(BasicStoreConfig, OpenSearchOPDSStore):
open_search_url = 'http://pragprog.com/catalog/search-description'
web_url = 'http://pragprog.com/'
@ -18,7 +18,7 @@ class PragmaticBookshelfStore(BasicStoreConfig, OpenSearchStore):
# http://pragprog.com/catalog.opds
def search(self, query, max_results=10, timeout=60):
for s in OpenSearchStore.search(self, query, max_results, timeout):
for s in OpenSearchOPDSStore.search(self, query, max_results, timeout):
s.drm = SearchResult.DRM_UNLOCKED
s.formats = 'EPUB, PDF, MOBI'
yield s

9
src/calibre/gui2/store/stores/smashwords_plugin.py
View File

@ -77,9 +77,12 @@ class SmashwordsStore(BasicStoreConfig, StorePlugin):
title = ''.join(data.xpath('//a[@class="bookTitle"]/text()'))
subnote = ''.join(data.xpath('//span[@class="subnote"]/text()'))
author = ''.join(data.xpath('//span[@class="subnote"]/a/text()'))
price = subnote.partition('$')[2]
price = price.split(u'\xa0')[0]
price = '$' + price
if '$' in subnote:
price = subnote.partition('$')[2]
price = price.split(u'\xa0')[0]
price = '$' + price
else:
price = '$0.00'
counter -= 1

916
src/calibre/gui2/tag_browser/model.py
View File

File diff suppressed because it is too large Load Diff

28
src/calibre/gui2/tag_browser/ui.py
View File

@ -91,10 +91,10 @@ class TagBrowserMixin(object): # {{{
# Add the new category
user_cats[new_cat] = []
db.prefs.set('user_categories', user_cats)
self.tags_view.set_new_model()
self.tags_view.recount()
m = self.tags_view.model()
idx = m.index_for_path(m.find_category_node('@' + new_cat))
m.show_item_at_index(idx)
self.tags_view.show_item_at_index(idx)
# Open the editor on the new item to rename it
if new_category_name is None:
self.tags_view.edit(idx)
@ -111,7 +111,7 @@ class TagBrowserMixin(object): # {{{
for k in d.categories:
db.field_metadata.add_user_category('@' + k, k)
db.data.change_search_locations(db.field_metadata.get_search_terms())
self.tags_view.set_new_model()
self.tags_view.recount()
def do_delete_user_category(self, category_name):
'''
@ -144,7 +144,7 @@ class TagBrowserMixin(object): # {{{
elif k.startswith(category_name + '.'):
del user_cats[k]
db.prefs.set('user_categories', user_cats)
self.tags_view.set_new_model()
self.tags_view.recount()
def do_del_item_from_user_cat(self, user_cat, item_name, item_category):
'''
@ -262,20 +262,22 @@ class TagBrowserMixin(object): # {{{
self.library_view.select_rows(ids)
# refreshing the tags view happens at the emit()/call() site
def do_author_sort_edit(self, parent, id, select_sort=True):
def do_author_sort_edit(self, parent, id, select_sort=True, select_link=False):
'''
Open the manage authors dialog
'''
db = self.library_view.model().db
editor = EditAuthorsDialog(parent, db, id, select_sort)
editor = EditAuthorsDialog(parent, db, id, select_sort, select_link)
d = editor.exec_()
if d:
for (id, old_author, new_author, new_sort) in editor.result:
for (id, old_author, new_author, new_sort, new_link) in editor.result:
if old_author != new_author:
# The id might change if the new author already exists
id = db.rename_author(id, new_author)
db.set_sort_field_for_author(id, unicode(new_sort),
commit=False, notify=False)
db.set_link_field_for_author(id, unicode(new_link),
commit=False, notify=False)
db.commit()
self.library_view.model().refresh()
self.tags_view.recount()
@ -413,13 +415,14 @@ class TagBrowserWidget(QWidget): # {{{
txt = unicode(self.item_search.currentText()).strip()
if txt.startswith('*'):
self.tags_view.set_new_model(filter_categories_by=txt[1:])
model.filter_categories_by = txt[1:]
self.tags_view.recount()
self.current_find_position = None
return
if model.get_filter_categories_by():
self.tags_view.set_new_model(filter_categories_by=None)
if model.filter_categories_by:
model.filter_categories_by = None
self.tags_view.recount()
self.current_find_position = None
model = self.tags_view.model()
if not txt:
return
@ -437,8 +440,9 @@ class TagBrowserWidget(QWidget): # {{{
self.current_find_position = \
model.find_item_node(key, txt, self.current_find_position)
if self.current_find_position:
model.show_item_at_path(self.current_find_position, box=True)
self.tags_view.show_item_at_path(self.current_find_position, box=True)
elif self.item_search.text():
self.not_found_label.setVisible(True)
if self.tags_view.verticalScrollBar().isVisible():

179
src/calibre/gui2/tag_browser/view.py
View File

@ -7,11 +7,12 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import cPickle, traceback
import cPickle
from functools import partial
from itertools import izip
from PyQt4.Qt import (QItemDelegate, Qt, QTreeView, pyqtSignal, QSize, QIcon,
QApplication, QMenu, QPoint)
QApplication, QMenu, QPoint, QModelIndex, QToolTip, QCursor)
from calibre.gui2.tag_browser.model import (TagTreeItem, TAG_SEARCH_STATES,
TagsModel)
@ -65,7 +66,7 @@ class TagsView(QTreeView): # {{{
tag_list_edit = pyqtSignal(object, object)
saved_search_edit = pyqtSignal(object)
rebuild_saved_searches = pyqtSignal()
author_sort_edit = pyqtSignal(object, object)
author_sort_edit = pyqtSignal(object, object, object, object)
tag_item_renamed = pyqtSignal()
search_item_renamed = pyqtSignal()
drag_drop_finished = pyqtSignal(object)
@ -90,55 +91,59 @@ class TagsView(QTreeView): # {{{
self.setDropIndicatorShown(True)
self.setAutoExpandDelay(500)
self.pane_is_visible = False
if gprefs['tags_browser_collapse_at'] == 0:
self.collapse_model = 'disable'
else:
self.collapse_model = gprefs['tags_browser_partition_method']
self.search_icon = QIcon(I('search.png'))
self.user_category_icon = QIcon(I('tb_folder.png'))
self.delete_icon = QIcon(I('list_remove.png'))
self.rename_icon = QIcon(I('edit-undo.png'))
self._model = TagsModel(self)
self._model.search_item_renamed.connect(self.search_item_renamed)
self._model.refresh_required.connect(self.refresh_required,
type=Qt.QueuedConnection)
self._model.tag_item_renamed.connect(self.tag_item_renamed)
self._model.restriction_error.connect(self.restriction_error)
self._model.user_categories_edited.connect(self.user_categories_edited,
type=Qt.QueuedConnection)
self._model.drag_drop_finished.connect(self.drag_drop_finished)
@property
def hidden_categories(self):
return self._model.hidden_categories
@property
def db(self):
return self._model.db
@property
def collapse_model(self):
return self._model.collapse_model
def set_pane_is_visible(self, to_what):
pv = self.pane_is_visible
self.pane_is_visible = to_what
if to_what and not pv:
self.recount()
def get_state(self):
state_map = {}
expanded_categories = []
for row, category in enumerate(self._model.category_nodes):
if self.isExpanded(self._model.index(row, 0, QModelIndex())):
expanded_categories.append(category.category_key)
states = [c.tag.state for c in category.child_tags()]
names = [(c.tag.name, c.tag.category) for c in category.child_tags()]
state_map[category.category_key] = dict(izip(names, states))
return expanded_categories, state_map
def reread_collapse_parameters(self):
if gprefs['tags_browser_collapse_at'] == 0:
self.collapse_model = 'disable'
else:
self.collapse_model = gprefs['tags_browser_partition_method']
self.set_new_model(self._model.get_filter_categories_by())
self._model.reread_collapse_model(self.get_state()[1])
def set_database(self, db, tag_match, sort_by):
hidden_cats = db.prefs.get('tag_browser_hidden_categories', None)
self.hidden_categories = []
# migrate from config to db prefs
if hidden_cats is None:
hidden_cats = config['tag_browser_hidden_categories']
# strip out any non-existence field keys
for cat in hidden_cats:
if cat in db.field_metadata:
self.hidden_categories.append(cat)
db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories))
self.hidden_categories = set(self.hidden_categories)
self._model.set_database(db)
old = getattr(self, '_model', None)
if old is not None:
old.break_cycles()
self._model = TagsModel(db, parent=self,
hidden_categories=self.hidden_categories,
search_restriction=None,
drag_drop_finished=self.drag_drop_finished,
collapse_model=self.collapse_model,
state_map={})
self.pane_is_visible = True # because TagsModel.init did a recount
self.pane_is_visible = True # because TagsModel.set_database did a recount
self.sort_by = sort_by
self.tag_match = tag_match
self.db = db
self.search_restriction = None
self.setModel(self._model)
self.setContextMenuPolicy(Qt.CustomContextMenu)
pop = config['sort_tags_by']
@ -164,6 +169,13 @@ class TagsView(QTreeView): # {{{
self.refresh_signal_processed = False
self.refresh_required.emit()
def user_categories_edited(self, user_cats, nkey):
state_map = self.get_state()[1]
self.db.prefs.set('user_categories', user_cats)
self._model.rebuild_node_tree(state_map=state_map)
p = self._model.find_category_node('@'+nkey)
self.show_item_at_path(p)
@property
def match_all(self):
return self.tag_match and self.tag_match.currentIndex() > 0
@ -179,11 +191,8 @@ class TagsView(QTreeView): # {{{
pass
def set_search_restriction(self, s):
if s:
self.search_restriction = s
else:
self.search_restriction = None
self.set_new_model()
s = s if s else None
self._model.set_search_restriction(s)
def mouseReleaseEvent(self, event):
# Swallow everything except leftButton so context menus work correctly
@ -268,23 +277,29 @@ class TagsView(QTreeView): # {{{
self.saved_search_edit.emit(category)
return
if action == 'edit_author_sort':
self.author_sort_edit.emit(self, index)
self.author_sort_edit.emit(self, index, True, False)
return
if action == 'edit_author_link':
self.author_sort_edit.emit(self, index, False, True)
return
reset_filter_categories = True
if action == 'hide':
self.hidden_categories.add(category)
elif action == 'show':
self.hidden_categories.discard(category)
elif action == 'categorization':
changed = self.collapse_model != category
self.collapse_model = category
self._model.collapse_model = category
if changed:
self.set_new_model(self._model.get_filter_categories_by())
reset_filter_categories = False
gprefs['tags_browser_partition_method'] = category
elif action == 'defaults':
self.hidden_categories.clear()
self.db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories))
self.set_new_model()
if reset_filter_categories:
self._model.filter_categories_by = None
self._model.rebuild_node_tree()
except:
return
@ -334,6 +349,9 @@ class TagsView(QTreeView): # {{{
self.context_menu.addAction(_('Edit sort for %s')%display_name(tag),
partial(self.context_menu_handler,
action='edit_author_sort', index=tag.id))
self.context_menu.addAction(_('Edit link for %s')%display_name(tag),
partial(self.context_menu_handler,
action='edit_author_link', index=tag.id))
# is_editable is also overloaded to mean 'can be added
# to a user category'
@ -475,10 +493,25 @@ class TagsView(QTreeView): # {{{
pa.setCheckable(True)
pa.setChecked(True)
if config['sort_tags_by'] != "name":
fla.setEnabled(False)
m.hovered.connect(self.collapse_menu_hovered)
fla.setToolTip(_('First letter is usable only when sorting by name'))
# Apparently one cannot set a tooltip to empty, so use a star and
# deal with it in the hover method
da.setToolTip('*')
pa.setToolTip('*')
if not self.context_menu.isEmpty():
self.context_menu.popup(self.mapToGlobal(point))
return True
def collapse_menu_hovered(self, action):
tip = action.toolTip()
if tip == '*':
tip = ''
QToolTip.showText(QCursor.pos(), tip)
def dragMoveEvent(self, event):
QTreeView.dragMoveEvent(self, event)
self.setDropIndicatorShown(False)
@ -487,6 +520,8 @@ class TagsView(QTreeView): # {{{
return
src_is_tb = event.mimeData().hasFormat('application/calibre+from_tag_browser')
item = index.data(Qt.UserRole).toPyObject()
if item.type == TagTreeItem.ROOT:
return
flags = self._model.flags(index)
if item.type == TagTreeItem.TAG and flags & Qt.ItemIsDropEnabled:
self.setDropIndicatorShown(not src_is_tb)
@ -537,11 +572,35 @@ class TagsView(QTreeView): # {{{
if not ci.isValid():
ci = self.indexAt(QPoint(10, 10))
path = self.model().path_for_index(ci) if self.is_visible(ci) else None
expanded_categories, state_map = self.model().get_state()
self.set_new_model(state_map=state_map)
expanded_categories, state_map = self.get_state()
self._model.rebuild_node_tree(state_map=state_map)
for category in expanded_categories:
self.expand(self.model().index_for_category(category))
self._model.show_item_at_path(path)
idx = self._model.index_for_category(category)
if idx is not None and idx.isValid():
self.expand(idx)
self.show_item_at_path(path)
def show_item_at_path(self, path, box=False,
position=QTreeView.PositionAtCenter):
'''
Scroll the browser and open categories to show the item referenced by
path. If possible, the item is placed in the center. If box=True, a
box is drawn around the item.
'''
if path:
self.show_item_at_index(self._model.index_for_path(path), box=box,
position=position)
def show_item_at_index(self, idx, box=False,
position=QTreeView.PositionAtCenter):
if idx.isValid() and idx.data(Qt.UserRole).toPyObject() is not self._model.root_item:
self.expand(self._model.parent(idx)) # Needed otherwise Qt sometimes segfaults if the
# node is buried in a collapsed, off
# screen hierarchy
self.setCurrentIndex(idx)
self.scrollTo(idx, position)
if box:
self._model.set_boxed(idx)
def item_expanded(self, idx):
'''
@ -549,30 +608,6 @@ class TagsView(QTreeView): # {{{
'''
self.setCurrentIndex(idx)
def set_new_model(self, filter_categories_by=None, state_map={}):
'''
There are cases where we need to rebuild the category tree without
attempting to reposition the current node.
'''
try:
old = getattr(self, '_model', None)
if old is not None:
old.break_cycles()
self._model = TagsModel(self.db, parent=self,
hidden_categories=self.hidden_categories,
search_restriction=self.search_restriction,
drag_drop_finished=self.drag_drop_finished,
filter_categories_by=filter_categories_by,
collapse_model=self.collapse_model,
state_map=state_map)
self.setModel(self._model)
except:
# The DB must be gone. Set the model to None and hope that someone
# will call set_database later. I don't know if this in fact works.
# But perhaps a Bad Thing Happened, so print the exception
traceback.print_exc()
self._model = None
self.setModel(None)
# }}}

10
src/calibre/library/caches.py
View File

@ -1024,7 +1024,15 @@ class SortKeyGenerator(object):
dt = 'datetime'
elif sb == 'number':
try:
val = float(val)
val = val.replace(',', '').strip()
p = 1
for i, candidate in enumerate(
(' B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
if val.endswith(candidate):
p = 1024**(i)
val = val[:-len(candidate)].strip()
break
val = float(val) * p
except:
val = 0.0
dt = 'float'

59
src/calibre/library/database2.py
View File

@ -8,6 +8,7 @@ The database used to store ebook metadata
'''
import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
json, uuid, tempfile, hashlib
from collections import defaultdict
import threading, random
from itertools import repeat
from math import ceil
@ -367,7 +368,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
'uuid',
'has_cover',
('au_map', 'authors', 'author',
'aum_sortconcat(link.id, authors.name, authors.sort)'),
'aum_sortconcat(link.id, authors.name, authors.sort, authors.link)'),
'last_modified',
'(SELECT identifiers_concat(type, val) FROM identifiers WHERE identifiers.book=books.id) identifiers',
]
@ -487,6 +488,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
self.refresh()
self.last_update_check = self.last_modified()
self.format_metadata_cache = defaultdict(dict)
def break_cycles(self):
self.data.break_cycles()
@ -894,13 +896,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
aut_list = []
aum = []
aus = {}
for (author, author_sort) in aut_list:
aum.append(author.replace('|', ','))
aus[author] = author_sort.replace('|', ',')
aul = {}
for (author, author_sort, link) in aut_list:
aut = author.replace('|', ',')
aum.append(aut)
aus[aut] = author_sort.replace('|', ',')
aul[aut] = link
mi.title = row[fm['title']]
mi.authors = aum
mi.author_sort = row[fm['author_sort']]
mi.author_sort_map = aus
mi.author_link_map = aul
mi.comments = row[fm['comments']]
mi.publisher = row[fm['publisher']]
mi.timestamp = row[fm['timestamp']]
@ -910,11 +916,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.book_size = row[fm['size']]
mi.ondevice_col= row[fm['ondevice']]
mi.last_modified = row[fm['last_modified']]
id = idx if index_is_id else self.id(idx)
formats = row[fm['formats']]
mi.format_metadata = {}
if not formats:
formats = None
else:
formats = formats.split(',')
for f in formats:
mi.format_metadata[f] = self.format_metadata(id, f)
mi.formats = formats
tags = row[fm['tags']]
if tags:
@ -923,7 +933,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if mi.series:
mi.series_index = row[fm['series_index']]
mi.rating = row[fm['rating']]
id = idx if index_is_id else self.id(idx)
mi.set_identifiers(self.get_identifiers(id, index_is_id=True))
mi.application_id = id
mi.id = id
@ -959,6 +968,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
mi.cover_data = ('jpeg', cdata)
else:
mi.cover = self.cover(id, index_is_id=True, as_path=True)
mi.has_cover = _('Yes') if self.has_cover(id) else ''
return mi
def has_book(self, mi):
@ -1122,13 +1132,21 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if m:
return m['mtime']
def format_metadata(self, id_, fmt):
def format_metadata(self, id_, fmt, allow_cache=True):
if not fmt:
return {}
fmt = fmt.upper()
if allow_cache:
x = self.format_metadata_cache[id_].get(fmt, None)
if x is not None:
return x
path = self.format_abspath(id_, fmt, index_is_id=True)
ans = {}
if path is not None:
stat = os.stat(path)
ans['size'] = stat.st_size
ans['mtime'] = utcfromtimestamp(stat.st_mtime)
self.format_metadata_cache[id_][fmt] = ans
return ans
def format_hash(self, id_, fmt):
@ -1245,6 +1263,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE)
shutil.copyfileobj(f, ret)
ret.seek(0)
# Various bits of code try to use the name as the default
# title when reading metadata, so set it
ret.name = f.name
else:
ret = f.read()
return ret
@ -1261,6 +1282,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def add_format(self, index, format, stream, index_is_id=False, path=None,
notify=True, replace=True):
id = index if index_is_id else self.id(index)
if format:
self.format_metadata_cache[id].pop(format.upper(), None)
if path is None:
path = os.path.join(self.library_path, self.path(id, index_is_id=True))
name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False)
@ -1313,6 +1336,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def remove_format(self, index, format, index_is_id=False, notify=True,
commit=True, db_only=False):
id = index if index_is_id else self.id(index)
if format:
self.format_metadata_cache[id].pop(format.upper(), None)
name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False)
if name:
if not db_only:
@ -1442,7 +1467,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
raise ValueError('sort ' + sort + ' not a valid value')
self.books_list_filter.change([] if not ids else ids)
id_filter = None if not ids else frozenset(ids)
id_filter = None if ids is None else frozenset(ids)
tb_cats = self.field_metadata
tcategories = {}
@ -1520,7 +1545,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
rating_dex = self.FIELD_MAP['rating']
tag_class = LibraryDatabase2.TCat_Tag
for book in self.data.iterall():
if id_filter and book[id_dex] not in id_filter:
if id_filter is not None and book[id_dex] not in id_filter:
continue
rating = book[rating_dex]
# We kept track of all possible category field_map positions above
@ -2038,13 +2063,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def authors_with_sort_strings(self, id, index_is_id=False):
id = id if index_is_id else self.id(id)
aut_strings = self.conn.get('''
SELECT authors.id, authors.name, authors.sort
SELECT authors.id, authors.name, authors.sort, authors.link
FROM authors, books_authors_link as bl
WHERE bl.book=? and authors.id=bl.author
ORDER BY bl.id''', (id,))
result = []
for (id_, author, sort,) in aut_strings:
result.append((id_, author.replace('|', ','), sort))
for (id_, author, sort, link) in aut_strings:
result.append((id_, author.replace('|', ','), sort, link))
return result
# Given a book, return the author_sort string for authors of the book
@ -2084,7 +2109,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
aum = self.authors_with_sort_strings(id_, index_is_id=True)
self.data.set(id_, self.FIELD_MAP['au_map'],
':#:'.join([':::'.join((au.replace(',', '|'), aus)) for (_, au, aus) in aum]),
':#:'.join([':::'.join((au.replace(',', '|'), aus, aul))
for (_, au, aus, aul) in aum]),
row_is_id=True)
def _set_authors(self, id, authors, allow_case_change=False):
@ -2435,7 +2461,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.conn.commit()
def get_authors_with_ids(self):
result = self.conn.get('SELECT id,name,sort FROM authors')
result = self.conn.get('SELECT id,name,sort,link FROM authors')
if not result:
return []
return result
@ -2446,6 +2472,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
(author,), all=False)
return result
def set_link_field_for_author(self, aid, link, commit=True, notify=False):
if not link:
link = ''
self.conn.execute('UPDATE authors SET link=? WHERE id=?', (link.strip(), aid))
if commit:
self.conn.commit()
def set_sort_field_for_author(self, old_id, new_sort, commit=True, notify=False):
self.conn.execute('UPDATE authors SET sort=? WHERE id=?', \
(new_sort.strip(), old_id))

12
src/calibre/library/restore.py
View File

@ -53,6 +53,7 @@ class Restore(Thread):
self.mismatched_dirs = []
self.successes = 0
self.tb = None
self.authors_links = {}
@property
def errors_occurred(self):
@ -160,6 +161,12 @@ class Restore(Thread):
else:
self.mismatched_dirs.append(dirpath)
alm = mi.get('author_link_map', {})
for author, link in alm.iteritems():
existing_link, timestamp = self.authors_links.get(author, (None, None))
if existing_link is None or existing_link != link and timestamp < mi.timestamp:
self.authors_links[author] = (link, mi.timestamp)
def create_cc_metadata(self):
self.books.sort(key=itemgetter('timestamp'))
self.custom_columns = {}
@ -206,6 +213,11 @@ class Restore(Thread):
self.failed_restores.append((book, traceback.format_exc()))
self.progress_callback(book['mi'].title, i+1)
for author in self.authors_links.iterkeys():
link, ign = self.authors_links[author]
db.conn.execute('UPDATE authors SET link=? WHERE name=?',
(link, author.replace(',', '|')))
db.conn.commit()
db.conn.close()
def restore_book(self, book, db):

10
src/calibre/library/schema_upgrades.py
View File

@ -600,4 +600,14 @@ class SchemaUpgrade(object):
with open(os.path.join(bdir, fname), 'wb') as f:
f.write(script)
def upgrade_version_20(self):
'''
Add a link column to the authors table.
'''
script = '''
ALTER TABLE authors ADD COLUMN link TEXT NOT NULL DEFAULT "";
'''
self.conn.executescript(script)

4
src/calibre/library/server/browse.py
View File

@ -795,7 +795,9 @@ class BrowseServer(object):
list(mi.get_all_user_metadata(False).items()):
if m['is_custom'] and field not in displayed_custom_fields:
continue
if m['datatype'] == 'comments' or field == 'comments':
if m['datatype'] == 'comments' or field == 'comments' or (
m['datatype'] == 'composite' and \
m['display'].get('contains_html', False)):
val = mi.get(field, '')
if val and val.strip():
comments.append((m['name'], comments_to_html(val)))

Some files were not shown because too many files have changed in this diff Show More