Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

KG updates

Browse Source
This commit is contained in:
GRiker 2010-10-27 03:50:57 -07:00
commit a116f8930c
74 changed files with 1573 additions and 977 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
resources/content_server/browse/browse.js
View File

@ -109,7 +109,7 @@ function toplevel_layout() {
var last = $(".toplevel li").last(); var last = $(".toplevel li").last();
var title = $('.toplevel h3').first(); var title = $('.toplevel h3').first();
var bottom = last.position().top + last.height() - title.position().top; var bottom = last.position().top + last.height() - title.position().top;
$("#main").height(Math.max(200, bottom)); $("#main").height(Math.max(200, bottom+75));
} }
function toplevel() { function toplevel() {
@ -156,6 +156,7 @@ function category() {
if (href) { if (href) {
$.ajax({ $.ajax({
url:href, url:href,
cache: false,
data:{'sort':cookie(sort_cookie_name)}, data:{'sort':cookie(sort_cookie_name)},
success: function(data) { success: function(data) {
this.children(".loaded").html(data); this.children(".loaded").html(data);
@ -212,6 +213,7 @@ function load_page(elem) {
url: href, url: href,
context: elem, context: elem,
dataType: "json", dataType: "json",
cache : false,
type: 'POST', type: 'POST',
timeout: 600000, //milliseconds (10 minutes) timeout: 600000, //milliseconds (10 minutes)
data: {'ids': ids}, data: {'ids': ids},
@ -263,6 +265,7 @@ function show_details(a_dom) {
$.ajax({ $.ajax({
url: book.find('.details-href').attr('title'), url: book.find('.details-href').attr('title'),
context: bd, context: bd,
cache: false,
dataType: "json", dataType: "json",
timeout: 600000, //milliseconds (10 minutes) timeout: 600000, //milliseconds (10 minutes)
error: function(xhr, stat, err) { error: function(xhr, stat, err) {

19
resources/content_server/mobile.css
View File

@ -1,5 +1,9 @@
/* CSS for the mobile version of the content server webpage */ /* CSS for the mobile version of the content server webpage */
.body {
font-family: sans-serif;
}
.navigation table.buttons { .navigation table.buttons {
width: 100%; width: 100%;
} }
@ -53,6 +57,7 @@ div.navigation {
} }
#listing td { #listing td {
padding: 0.25em; padding: 0.25em;
vertical-align: middle;
} }
#listing td.thumbnail { #listing td.thumbnail {
@ -73,6 +78,7 @@ div.navigation {
overflow: hidden; overflow: hidden;
text-align: center; text-align: center;
text-decoration: none; text-decoration: none;
vertical-align: middle;
} }
#logo { #logo {
@ -83,4 +89,17 @@ div.navigation {
clear: both; clear: both;
} }
.data-container {
display: inline-block;
vertical-align: middle;
}
.first-line {
font-size: larger;
font-weight: bold;
}
.second-line {
margin-top: 0.75ex;
display: block;
}

29
resources/default_tweaks.py
View File

@ -106,7 +106,8 @@ title_sort_articles=r'^(A|The|An)\s+'
auto_connect_to_folder = '' auto_connect_to_folder = ''
# Specify renaming rules for sony collections. Collections on Sonys are named # Specify renaming rules for sony collections. This tweak is only applicable if
# metadata management is set to automatic. Collections on Sonys are named
# depending upon whether the field is standard or custom. A collection derived # depending upon whether the field is standard or custom. A collection derived
# from a standard field is named for the value in that field. For example, if # from a standard field is named for the value in that field. For example, if
# the standard 'series' column contains the name 'Darkover', then the series # the standard 'series' column contains the name 'Darkover', then the series
@ -137,6 +138,24 @@ auto_connect_to_folder = ''
sony_collection_renaming_rules={} sony_collection_renaming_rules={}
# Specify how sony collections are sorted. This tweak is only applicable if
# metadata management is set to automatic. You can indicate which metadata is to
# be used to sort on a collection-by-collection basis. The format of the tweak
# is a list of metadata fields from which collections are made, followed by the
# name of the metadata field containing the sort value.
# Example: The following indicates that collections built from pubdate and tags
# are to be sorted by the value in the custom column '#mydate', that collections
# built from 'series' are to be sorted by 'series_index', and that all other
# collections are to be sorted by title. If a collection metadata field is not
# named, then if it is a series- based collection it is sorted by series order,
# otherwise it is sorted by title order.
# [(['pubdate', 'tags'],'#mydate'), (['series'],'series_index'), (['*'], 'title')]
# Note that the bracketing and parentheses are required. The syntax is
# [ ( [list of fields], sort field ) , ( [ list of fields ] , sort field ) ]
# Default: empty (no rules), so no collection attributes are named.
sony_collection_sorting_rules = []
# Create search terms to apply a query across several built-in search terms. # Create search terms to apply a query across several built-in search terms.
# Syntax: {'new term':['existing term 1', 'term 2', ...], 'new':['old'...] ...} # Syntax: {'new term':['existing term 1', 'term 2', ...], 'new':['old'...] ...}
# Example: create the term 'myseries' that when used as myseries:foo would # Example: create the term 'myseries' that when used as myseries:foo would
@ -184,3 +203,11 @@ content_server_wont_display = ['']
# level sorts, and if you are seeing a slowdown, reduce the value of this tweak. # level sorts, and if you are seeing a slowdown, reduce the value of this tweak.
maximum_resort_levels = 5 maximum_resort_levels = 5
# Absolute path to a TTF font file to use as the font for the title and author
# when generating a default cover. Useful if the default font (Liberation
# Serif) does not contain glyphs for the language of the books in your library.
generate_cover_title_font = None
# Absolute path to a TTF font file to use as the font for the footer in the
# default cover
generate_cover_foot_font = None

BIN
resources/images/news/theecocolapse.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB

8
resources/recipes/atlantic.recipe
View File

@ -71,7 +71,9 @@ class TheAtlantic(BasicNewsRecipe):
for poem in soup.findAll('div', attrs={'class':'poem'}): for poem in soup.findAll('div', attrs={'class':'poem'}):
title = self.tag_to_string(poem.find('h4')) title = self.tag_to_string(poem.find('h4'))
desc = self.tag_to_string(poem.find(attrs={'class':'author'})) desc = self.tag_to_string(poem.find(attrs={'class':'author'}))
url = 'http://www.theatlantic.com'+poem.find('a')['href'] url = poem.find('a')['href']
if url.startswith('/'):
url = 'http://www.theatlantic.com' + url
self.log('\tFound article:', title, 'at', url) self.log('\tFound article:', title, 'at', url)
self.log('\t\t', desc) self.log('\t\t', desc)
poems.append({'title':title, 'url':url, 'description':desc, poems.append({'title':title, 'url':url, 'description':desc,
@ -83,7 +85,9 @@ class TheAtlantic(BasicNewsRecipe):
if div is not None: if div is not None:
self.log('Found section: Advice') self.log('Found section: Advice')
title = self.tag_to_string(div.find('h4')) title = self.tag_to_string(div.find('h4'))
url = 'http://www.theatlantic.com'+div.find('a')['href'] url = div.find('a')['href']
if url.startswith('/'):
url = 'http://www.theatlantic.com' + url
desc = self.tag_to_string(div.find('p')) desc = self.tag_to_string(div.find('p'))
self.log('\tFound article:', title, 'at', url) self.log('\tFound article:', title, 'at', url)
self.log('\t\t', desc) self.log('\t\t', desc)

32
resources/recipes/cubadebate.recipe
View File

@ -1,9 +1,7 @@
#!/usr/bin/env python
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>' __copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
''' '''
newyorker.com cubadebate.cu
''' '''
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
@ -13,32 +11,44 @@ class CubaDebate(BasicNewsRecipe):
__author__ = 'Darko Miletic' __author__ = 'Darko Miletic'
description = 'Contra el Terorismo Mediatico' description = 'Contra el Terorismo Mediatico'
oldest_article = 15 oldest_article = 15
language = 'es' language = 'es'
max_articles_per_feed = 100 max_articles_per_feed = 100
no_stylesheets = True no_stylesheets = True
use_embedded_content = False use_embedded_content = False
publisher = 'Cubadebate' publisher = 'Cubadebate'
category = 'news, politics, Cuba' category = 'news, politics, Cuba'
encoding = 'utf-8' encoding = 'utf-8'
extra_css = ' #BlogTitle{font-size: x-large; font-weight: bold} ' masthead_url = 'http://www.cubadebate.cu/wp-content/themes/cubadebate/images/logo.gif'
publication_type = 'newsportal'
extra_css = """
#BlogTitle{font-size: xx-large; font-weight: bold}
body{font-family: Verdana, Arial, Tahoma, sans-serif}
"""
conversion_options = { conversion_options = {
'comments' : description 'comments' : description
,'tags' : category ,'tags' : category
,'language' : 'es' ,'language' : language
,'publisher' : publisher ,'publisher' : publisher
,'pretty_print': True
} }
keep_only_tags = [dict(name='div', attrs={'id':'Outline'})] keep_only_tags = [dict(name='div', attrs={'id':'Outline'})]
remove_tags_after = dict(name='div',attrs={'id':'BlogContent'}) remove_tags_after = dict(name='div',attrs={'id':'BlogContent'})
remove_tags = [dict(name='link')] remove_tags = [
dict(name=['link','base','embed','object','meta','iframe'])
,dict(attrs={'id':'addthis_container'})
]
feeds = [(u'Articulos', u'http://www.cubadebate.cu/feed/')] feeds = [(u'Articulos', u'http://www.cubadebate.cu/feed/')]
remove_attributes=['width','height','lang']
def print_version(self, url): def print_version(self, url):
return url + 'print/' return url + 'print/'
def preprocess_html(self, soup): def preprocess_html(self, soup):
return self.adeify_images(soup) for item in soup.findAll(style=True):
del item['style']
for item in soup.findAll('img'):
if not item.has_key('alt'):
item['alt'] = 'image'
return soup

13
resources/recipes/el_pais.recipe
View File

@ -2,7 +2,7 @@
__license__ = 'GPL v3' __license__ = 'GPL v3'
__author__ = 'Jordi Balcells, based on an earlier version by Lorenzo Vigentini & Kovid Goyal' __author__ = 'Jordi Balcells, based on an earlier version by Lorenzo Vigentini & Kovid Goyal'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
description = 'Main daily newspaper from Spain - v1.03 (03, September 2010)' description = 'Main daily newspaper from Spain - v1.04 (19, October 2010)'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
''' '''
@ -32,19 +32,16 @@ class ElPais(BasicNewsRecipe):
remove_javascript = True remove_javascript = True
no_stylesheets = True no_stylesheets = True
keep_only_tags = [ dict(name='div', attrs={'class':['cabecera_noticia','cabecera_noticia_reportaje','cabecera_noticia_opinion','contenido_noticia','caja_despiece','presentacion']})] keep_only_tags = [ dict(name='div', attrs={'class':['cabecera_noticia_reportaje estirar','cabecera_noticia_opinion estirar','cabecera_noticia estirar','contenido_noticia','caja_despiece']})]
extra_css = ''' extra_css = ' p{text-align: justify; font-size: 100%} body{ text-align: left; font-family: serif; font-size: 100% } h1{ font-family: sans-serif; font-size:200%; font-weight: bolder; text-align: justify; } h2{ font-family: sans-serif; font-size:150%; font-weight: 500; text-align: justify } h3{ font-family: sans-serif; font-size:125%; font-weight: 500; text-align: justify } img{margin-bottom: 0.4em} '
p{style:normal size:12 serif}
'''
remove_tags = [ remove_tags = [
dict(name='div', attrs={'class':['zona_superior','pie_enlaces_inferiores','contorno_f','ampliar']}), dict(name='div', attrs={'class':['zona_superior','pie_enlaces_inferiores','contorno_f','ampliar']}),
dict(name='div', attrs={'class':['limpiar','mod_apoyo','borde_sup','votos','info_complementa','info_relacionada','buscador_m','nav_ant_sig']}), dict(name='div', attrs={'class':['limpiar','mod_apoyo','borde_sup','votos estirar','info_complementa','info_relacionada','buscador_m','nav_ant_sig']}),
dict(name='div', attrs={'id':['suscribirse suscrito','google_noticia','utilidades','coment','foros_not','pie','lomas','calendar']}), dict(name='div', attrs={'id':['suscribirse suscrito','google_noticia','utilidades','coment','foros_not','pie','lomas','calendar']}),
dict(name='p', attrs={'class':'nav_meses'}), dict(name='p', attrs={'class':'nav_meses'}),
dict(attrs={'class':['enlaces_m','miniaturas_m']}) dict(attrs={'class':['enlaces_m','miniaturas_m','nav_miniaturas_m']})
] ]
feeds = [ feeds = [

39
resources/recipes/foxnews.recipe
View File

@ -4,7 +4,6 @@ __copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
foxnews.com foxnews.com
''' '''
import re
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
class FoxNews(BasicNewsRecipe): class FoxNews(BasicNewsRecipe):
@ -21,11 +20,10 @@ class FoxNews(BasicNewsRecipe):
language = 'en' language = 'en'
publication_type = 'newsportal' publication_type = 'newsportal'
remove_empty_feeds = True remove_empty_feeds = True
extra_css = ' body{font-family: Arial,sans-serif } img{margin-bottom: 0.4em} .caption{font-size: x-small} ' extra_css = """
body{font-family: Arial,sans-serif }
preprocess_regexps = [ .caption{font-size: x-small}
(re.compile(r'</title>.*?</head>', re.DOTALL|re.IGNORECASE),lambda match: '</title></head>') """
]
conversion_options = { conversion_options = {
'comment' : description 'comment' : description
@ -34,27 +32,15 @@ class FoxNews(BasicNewsRecipe):
, 'language' : language , 'language' : language
} }
remove_attributes = ['xmlns'] remove_attributes = ['xmlns','lang']
keep_only_tags = [
dict(name='div', attrs={'id' :['story','browse-story-content']})
,dict(name='div', attrs={'class':['posts articles','slideshow']})
,dict(name='h4' , attrs={'class':'storyDate'})
,dict(name='h1' , attrs={'xmlns:functx':'http://www.functx.com'})
,dict(name='div', attrs={'class':'authInfo'})
,dict(name='div', attrs={'id':'articleCont'})
]
remove_tags = [ remove_tags = [
dict(name='div', attrs={'class':['share-links','quigo quigo2','share-text','storyControls','socShare','btm-links']}) dict(name=['object','embed','link','script','iframe','meta','base'])
,dict(name='div', attrs={'id' :['otherMedia','loomia_display','img-all-path','story-vcmId','story-url','pane-browse-story-comments','story_related']}) ,dict(attrs={'class':['user-control','url-description','ad-context']})
,dict(name='ul' , attrs={'class':['tools','tools alt','tools alt2','tabs']})
,dict(name='a' , attrs={'class':'join-discussion'})
,dict(name='ul' , attrs={'class':['tools','tools alt','tools alt2']})
,dict(name='p' , attrs={'class':'see_fullarchive'})
,dict(name=['object','embed','link','script'])
] ]
remove_tags_before=dict(name='h1')
remove_tags_after =dict(attrs={'class':'url-description'})
feeds = [ feeds = [
(u'Latest Headlines', u'http://feeds.foxnews.com/foxnews/latest' ) (u'Latest Headlines', u'http://feeds.foxnews.com/foxnews/latest' )
@ -67,8 +53,5 @@ class FoxNews(BasicNewsRecipe):
,(u'Entertainment' , u'http://feeds.foxnews.com/foxnews/entertainment' ) ,(u'Entertainment' , u'http://feeds.foxnews.com/foxnews/entertainment' )
] ]
def preprocess_html(self, soup): def print_version(self, url):
for item in soup.findAll(style=True): return url + 'print'
del item['style']
return self.adeify_images(soup)

64
resources/recipes/ming_pao.recipe Normal file
View File

@ -0,0 +1,64 @@
cense__ = 'GPL v3'
__copyright__ = '2010, Eddie Lau'
'''
modified from Singtao Toronto calibre recipe by rty
'''
import datetime
from calibre.web.feeds.recipes import BasicNewsRecipe
class AdvancedUserRecipe1278063072(BasicNewsRecipe):
title = 'Ming Pao - Hong Kong'
oldest_article = 1
max_articles_per_feed = 100
__author__ = 'Eddie Lau'
description = 'Hong Kong Chinese Newspaper'
publisher = 'news.mingpao.com'
category = 'Chinese, News, Hong Kong'
remove_javascript = True
use_embedded_content = False
no_stylesheets = True
language = 'zh'
encoding = 'Big5-HKSCS'
recursions = 0
conversion_options = {'linearize_tables':True}
masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif'
keep_only_tags = [dict(name='h1'),
dict(attrs={'id':['newscontent01','newscontent02']})]
def get_fetchdate(self):
dt_utc = datetime.datetime.utcnow()
# convert UTC to local hk time
dt_local = dt_utc - datetime.timedelta(-8.0/24)
return dt_local.strftime("%Y%m%d")
def parse_index(self):
feeds = []
dateStr = self.get_fetchdate()
for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm'), (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), (u'\u793e\u8a55\u2027\u7b46\u9663 Editorial', 'http://news.mingpao.com/' + dateStr + '/mrindex.htm'), (u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'), (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm'), ('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm'), (u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),]:
articles = self.parse_section(url)
if articles:
feeds.append((title, articles))
return feeds
def parse_section(self, url):
dateStr = self.get_fetchdate()
soup = self.index_to_soup(url)
divs = soup.findAll(attrs={'class': ['bullet']})
current_articles = []
for i in divs:
a = i.find('a', href = True)
title = self.tag_to_string(a)
url = a.get('href', False)
url = 'http://news.mingpao.com/' + dateStr + '/' +url
current_articles.append({'title': title, 'url': url, 'description':''})
return current_articles
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
for item in soup.findAll(width=True):
del item['width']
return soup

29
resources/recipes/new_scientist.recipe
View File

@ -8,11 +8,11 @@ import re
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
class NewScientist(BasicNewsRecipe): class NewScientist(BasicNewsRecipe):
title = 'New Scientist - Online News' title = 'New Scientist - Online News w. subscription'
__author__ = 'Darko Miletic' __author__ = 'Darko Miletic'
description = 'Science news and science articles from New Scientist.' description = 'Science news and science articles from New Scientist.'
language = 'en' language = 'en'
publisher = 'New Scientist' publisher = 'Reed Business Information Ltd.'
category = 'science news, science articles, science jobs, drugs, cancer, depression, computer software' category = 'science news, science articles, science jobs, drugs, cancer, depression, computer software'
oldest_article = 7 oldest_article = 7
max_articles_per_feed = 100 max_articles_per_feed = 100
@ -21,7 +21,12 @@ class NewScientist(BasicNewsRecipe):
cover_url = 'http://www.newscientist.com/currentcover.jpg' cover_url = 'http://www.newscientist.com/currentcover.jpg'
masthead_url = 'http://www.newscientist.com/img/misc/ns_logo.jpg' masthead_url = 'http://www.newscientist.com/img/misc/ns_logo.jpg'
encoding = 'utf-8' encoding = 'utf-8'
extra_css = ' body{font-family: Arial,sans-serif} img{margin-bottom: 0.8em} ' needs_subscription = 'optional'
extra_css = """
body{font-family: Arial,sans-serif}
img{margin-bottom: 0.8em}
.quotebx{font-size: x-large; font-weight: bold; margin-right: 2em; margin-left: 2em}
"""
conversion_options = { conversion_options = {
'comment' : description 'comment' : description
@ -33,15 +38,27 @@ class NewScientist(BasicNewsRecipe):
keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})] keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})]
def get_browser(self):
br = BasicNewsRecipe.get_browser()
br.open('http://www.newscientist.com/')
if self.username is not None and self.password is not None:
br.open('https://www.newscientist.com/user/login?redirectURL=')
br.select_form(nr=2)
br['loginId' ] = self.username
br['password'] = self.password
br.submit()
return br
remove_tags = [ remove_tags = [
dict(name='div' , attrs={'class':['hldBd','adline','pnl','infotext' ]}) dict(name='div' , attrs={'class':['hldBd','adline','pnl','infotext' ]})
,dict(name='div' , attrs={'id' :['compnl','artIssueInfo','artTools','comments','blgsocial','sharebtns']}) ,dict(name='div' , attrs={'id' :['compnl','artIssueInfo','artTools','comments','blgsocial','sharebtns']})
,dict(name='p' , attrs={'class':['marker','infotext' ]}) ,dict(name='p' , attrs={'class':['marker','infotext' ]})
,dict(name='meta' , attrs={'name' :'description' }) ,dict(name='meta' , attrs={'name' :'description' })
,dict(name='a' , attrs={'rel' :'tag' }) ,dict(name='a' , attrs={'rel' :'tag' })
,dict(name=['link','base','meta','iframe','object','embed'])
] ]
remove_tags_after = dict(attrs={'class':['nbpcopy','comments']}) remove_tags_after = dict(attrs={'class':['nbpcopy','comments']})
remove_attributes = ['height','width'] remove_attributes = ['height','width','lang']
feeds = [ feeds = [
(u'Latest Headlines' , u'http://feeds.newscientist.com/science-news' ) (u'Latest Headlines' , u'http://feeds.newscientist.com/science-news' )
@ -62,6 +79,8 @@ class NewScientist(BasicNewsRecipe):
return url + '?full=true&print=true' return url + '?full=true&print=true'
def preprocess_html(self, soup): def preprocess_html(self, soup):
for item in soup.findAll(['quote','quotetext']):
item.name='p'
for tg in soup.findAll('a'): for tg in soup.findAll('a'):
if tg.string == 'Home': if tg.string == 'Home':
tg.parent.extract() tg.parent.extract()

551
resources/recipes/nytimes_sub.recipe
View File

@ -4,149 +4,79 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
''' '''
nytimes.com nytimes.com
V5 - One picture per article, moved to top:
Headline
Image
Byline
Story
''' '''
import re, string, time import string, re, time
from calibre import strftime from calibre import strftime
from calibre.web.feeds.recipes import BasicNewsRecipe from calibre.web.feeds.recipes import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup, NavigableString, Tag from calibre.ebooks.BeautifulSoup import BeautifulSoup
def decode(self, src):
enc = 'utf-8'
if 'iso-8859-1' in src:
enc = 'cp1252'
return src.decode(enc, 'ignore')
class NYTimes(BasicNewsRecipe): class NYTimes(BasicNewsRecipe):
title = 'The New York Times' title = u'New York Times'
__author__ = 'GRiker' __author__ = 'Kovid Goyal/Nick Redding'
language = 'en' language = 'en'
requires_version = (0, 7, 5) requires_version = (0, 6, 36)
description = 'Daily news from the New York Times (subscription version)' description = 'Daily news from the New York Times (subscription version)'
allSectionKeywords = ['The Front Page', 'International','National','Obituaries','Editorials', timefmt = ' [%b %d]'
'New York','Business Day','Science Times','Sports','Dining','Arts',
'Home','Styles','Sunday Business','Week In Review','Travel','Magazine',
'Book Review','Weddings','Real Estate','Automobiles',"T Men's Fashion",
"T Women's Fashion"]
# List of sections to exclude
# To add a section, copy the section name from the allSectionKeywords list above
# For example, to exclude 'Dining' and 'Weddings':
#excludeSectionKeywords = ['Dining','Weddings']
excludeSectionKeywords = []
# List of sections to include (test and debug only)
# By default, any sections in today's paper that are not listed in excludeSectionKeywords
# are downloaded. fetch_only specifies that only certain sections are to be downloaded.
# This should only be used for testing and debugging.
# For example, to download only 'The Front Page' section:
# fetch_only = set(['The Front Page'])
fetch_only = set([])
if fetch_only:
excludeSectionKeywords = list(set(allSectionKeywords) ^ fetch_only)
# one_picture_per_article specifies that calibre should only use the first image
# from an article (if one exists). If one_picture_per_article = True, the image
# will be moved to a location between the headline and the byline.
# If one_picture_per_article = False, all images from the article will be included
# and shown in their original location.
one_picture_per_article = True
timefmt = ''
needs_subscription = True needs_subscription = True
remove_tags_before = dict(id='article') remove_tags_before = dict(id='article')
remove_tags_after = dict(id='article') remove_tags_after = dict(id='article')
remove_tags = [dict(attrs={'class':[ remove_tags = [dict(attrs={'class':['articleTools', 'post-tools', 'side_tool','nextArticleLink',
'articleFooter', 'nextArticleLink clearfix','columnGroup doubleRule','doubleRule','entry-meta',
'articleTools', 'icon enlargeThis','columnGroup last','relatedSearchesModule']}),
'columnGroup doubleRule', dict({'class':re.compile('^subNavigation')}),
'columnGroup singleRule', dict({'class':re.compile('^leaderboard')}),
'columnGroup last', dict({'class':re.compile('^module')}),
'columnGroup last', dict({'class':'metaFootnote'}),
'doubleRule', dict(id=['inlineBox','footer', 'toolsRight', 'articleInline','login','masthead',
'dottedLine', 'navigation', 'archive', 'side_search', 'blog_sidebar','cCol','portfolioInline',
'entry-meta', 'side_tool', 'side_index','header','readerReviewsCount','readerReviews',
'entry-response module', 'relatedArticles', 'relatedTopics', 'adxSponLink']),
'icon enlargeThis', dict(name=['script', 'noscript', 'style','form','hr'])]
'leftNavTabs', encoding = decode
'module box nav',
'nextArticleLink',
'nextArticleLink clearfix',
'post-tools',
'relatedSearchesModule',
'side_tool',
'singleAd',
'subNavigation clearfix',
'subNavigation tabContent active',
'subNavigation tabContent active clearfix',
]}),
dict(id=[
'adxLeaderboard',
'archive',
'articleExtras',
'articleInline',
'blog_sidebar',
'businessSearchBar',
'cCol',
'entertainmentSearchBar',
'footer',
'header',
'header_search',
'login',
'masthead',
'masthead-nav',
'memberTools',
'navigation',
'portfolioInline',
'relatedArticles',
'respond',
'side_search',
'side_index',
'side_tool',
'toolsRight',
]),
dict(name=['script', 'noscript', 'style'])]
masthead_url = 'http://graphics8.nytimes.com/images/misc/nytlogo379x64.gif'
cover_margins = (18,18,'grey99')
no_stylesheets = True no_stylesheets = True
extra_css = '.headline {text-align: left;}\n \ extra_css = '''
.byline {font-family: monospace; \ .articleHeadline { margin-top:0.5em; margin-bottom:0.25em; }
text-align: left; \ .credit { font-size: small; font-style:italic; line-height:1em; margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; }
margin-top: 0px; \ .byline { font-size: small; font-style:italic; line-height:1em; margin-top:10px; margin-left:0; margin-right:0; margin-bottom: 0; }
margin-bottom: 0px;}\n \ .dateline { font-size: small; line-height:1em;margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; }
.dateline {font-size: small; \ .kicker { font-size: small; line-height:1em;margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; }
margin-top: 0px; \ .timestamp { font-size: small; }
margin-bottom: 0px;}\n \ .caption { font-size: small; line-height:1em; margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; }
.timestamp {font-size: small; \ a:link {text-decoration: none; }'''
margin-top: 0px; \
margin-bottom: 0px;}\n \
.source {text-align: left;}\n \
.image {text-align: center;}\n \
.credit {text-align: right; \
font-size: small; \
margin-top: 0px; \
margin-bottom: 0px;}\n \
.articleBody {text-align: left;}\n \
.authorId {text-align: left; \
font-style: italic;}\n '
def get_browser(self): def get_browser(self):
br = BasicNewsRecipe.get_browser() br = BasicNewsRecipe.get_browser()
if self.username is not None and self.password is not None: if self.username is not None and self.password is not None:
try: br.open('http://www.nytimes.com/auth/login')
br.open('http://www.nytimes.com/auth/login') br.select_form(name='login')
br.select_form(name='login') br['USERID'] = self.username
br['USERID'] = self.username br['PASSWORD'] = self.password
br['PASSWORD'] = self.password raw = br.submit().read()
raw = br.submit().read() if 'Sorry, we could not find the combination you entered. Please try again.' in raw:
if 'Sorry, we could not find the combination you entered. Please try again.' in raw: raise Exception('Your username and password are incorrect')
raise Exception('Your username and password are incorrect') #open('/t/log.html', 'wb').write(raw)
#open('/t/log.html', 'wb').write(raw)
except:
self.log("\nFailed to login")
return br return br
def get_masthead_url(self):
masthead = 'http://graphics8.nytimes.com/images/misc/nytlogo379x64.gif'
#masthead = 'http://members.cox.net/nickredding/nytlogo.gif'
br = BasicNewsRecipe.get_browser()
try:
br.open(masthead)
except:
self.log("\nMasthead unavailable")
masthead = None
return masthead
def get_cover_url(self): def get_cover_url(self):
cover = None cover = None
st = time.localtime() st = time.localtime()
@ -162,316 +92,101 @@ class NYTimes(BasicNewsRecipe):
cover = None cover = None
return cover return cover
def get_masthead_title(self): def short_title(self):
return self.title return 'New York Times'
def dump_ans(self, ans):
total_article_count = 0
for section in ans :
if self.verbose:
self.log("section %s: %d articles" % (section[0], len(section[1])) )
for article in section[1]:
total_article_count += 1
if self.verbose:
self.log("\t%-40.40s... \t%-60.60s..." % (article['title'].encode('mac-roman','replace'),
article['url'].encode('mac-roman','replace')))
self.log( "Queued %d articles" % total_article_count )
def dump_hex(self, src, length=16):
''' Diagnostic '''
FILTER=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)])
N=0; result=''
while src:
s,src = src[:length],src[length:]
hexa = ' '.join(["%02X"%ord(x) for x in s])
s = s.translate(FILTER)
result += "%04X %-*s %s\n" % (N, length*3, hexa, s)
N+=length
print result
def fixChars(self,string):
# Replace lsquo (\x91)
fixed = re.sub("\x91","&#8216;",string)
# Replace rsquo (\x92)
fixed = re.sub("\x92","&#8217;",fixed)
# Replace ldquo (\x93)
fixed = re.sub("\x93","&#8220;",fixed)
# Replace rdquo (\x94)
fixed = re.sub("\x94","&#8221;",fixed)
# Replace ndash (\x96)
fixed = re.sub("\x96","&#8211;",fixed)
# Replace mdash (\x97)
fixed = re.sub("\x97","&#8212;",fixed)
return fixed
def massageNCXText(self, description):
# Kindle TOC descriptions won't render certain characters
if description:
massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES))
# Replace '&' with '&#38;'
massaged = re.sub("&","&#38;", massaged)
return self.fixChars(massaged)
else:
return description
def parse_index(self): def parse_index(self):
self.encoding = 'cp1252'
soup = self.index_to_soup('http://www.nytimes.com/pages/todayspaper/index.html') soup = self.index_to_soup('http://www.nytimes.com/pages/todayspaper/index.html')
self.encoding = decode
def feed_title(div): def feed_title(div):
return ''.join(div.findAll(text=True, recursive=False)).strip() return ''.join(div.findAll(text=True, recursive=True)).strip()
articles = {} articles = {}
key = None key = None
ans = [] ans = []
# Find each instance of class="section-headline", class="story", class="story headline" url_list = []
for div in soup.findAll(True,
attrs={'class':['section-headline', 'story', 'story headline']}):
if div['class'] == 'section-headline': def handle_article(div):
key = string.capwords(feed_title(div)) a = div.find('a', href=True)
if self.excludeSectionKeywords: if not a:
excluded = re.compile('|'.join(self.excludeSectionKeywords)) return
if excluded.search(key): url = re.sub(r'\?.*', '', a['href'])
self.log("Skipping section %s" % key) if not url.startswith("http"):
continue return
articles[key] = [] if not url.endswith(".html"):
ans.append(key) return
if 'podcast' in url:
elif div['class'] in ['story', 'story headline'] : return
a = div.find('a', href=True) url += '?pagewanted=all'
if not a: if url in url_list:
continue return
url = re.sub(r'\?.*', '', a['href']) url_list.append(url)
url += '?pagewanted=all' title = self.tag_to_string(a, use_alt=True).strip()
#self.log("Title: %s" % title)
title = self.massageNCXText(self.tag_to_string(a, use_alt=True).strip()) description = ''
pubdate = strftime('%a, %d %b')
description = '' summary = div.find(True, attrs={'class':'summary'})
pubdate = strftime('%a, %d %b') if summary:
summary = div.find(True, attrs={'class':'summary'}) description = self.tag_to_string(summary, use_alt=False)
if summary: author = ''
description = self.massageNCXText(self.tag_to_string(summary, use_alt=False)) authorAttribution = div.find(True, attrs={'class':'byline'})
if authorAttribution:
author = '' author = self.tag_to_string(authorAttribution, use_alt=False)
authorAttribution = div.find(True, attrs={'class':'storyheadline-author'}) else:
authorAttribution = div.find(True, attrs={'class':'byline'})
if authorAttribution: if authorAttribution:
author = self.tag_to_string(authorAttribution, use_alt=False) author = self.tag_to_string(authorAttribution, use_alt=False)
else: feed = key if key is not None else 'Uncategorized'
authorAttribution = div.find(True, attrs={'class':'byline'}) if not articles.has_key(feed):
if authorAttribution: articles[feed] = []
author = self.tag_to_string(authorAttribution, use_alt=False) articles[feed].append(
# Kill commas - Kindle switches to '&' dict(title=title, url=url, date=pubdate,
author = re.sub(',','',author) description=description, author=author,
content=''))
feed = key if key is not None else 'Uncategorized'
if not articles.has_key(feed):
articles[feed] = [] # Find each instance of class="section-headline", class="story", class="story headline"
if not 'podcasts' in url: for div in soup.findAll(True,
articles[feed].append( attrs={'class':['section-headline', 'story', 'story headline','sectionHeader','headlinesOnly multiline flush']}):
dict(title=title, url=url, date=pubdate,
description=description, author=author, if div['class'] in ['section-headline','sectionHeader']:
content='')) key = string.capwords(feed_title(div))
ans = self.sort_index_by(ans, {'The Front Page':-1, articles[key] = []
'Dining In, Dining Out':1, ans.append(key)
'Obituaries':2}) #self.log('Section: %s' % key)
elif div['class'] in ['story', 'story headline'] :
handle_article(div)
elif div['class'] == 'headlinesOnly multiline flush':
for lidiv in div.findAll('li'):
handle_article(lidiv)
# ans = self.sort_index_by(ans, {'The Front Page':-1,
# 'Dining In, Dining Out':1,
# 'Obituaries':2})
ans = [(key, articles[key]) for key in ans if articles.has_key(key)] ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
self.dump_ans(ans)
return ans return ans
def skip_ad_pages(self, soup):
# Skip ad pages served before actual article
skip_tag = soup.find(True, {'name':'skip'})
if skip_tag is not None:
self.log.warn("Found forwarding link: %s" % skip_tag.parent['href'])
url = 'http://www.nytimes.com' + re.sub(r'\?.*', '', skip_tag.parent['href'])
url += '?pagewanted=all'
self.log.warn("Skipping ad to article at '%s'" % url)
return self.index_to_soup(url, raw=True)
def preprocess_html(self, soup): def preprocess_html(self, soup):
return self.strip_anchors(soup) kicker_tag = soup.find(attrs={'class':'kicker'})
if kicker_tag:
tagline = self.tag_to_string(kicker_tag)
#self.log("FOUND KICKER %s" % tagline)
if tagline=='Op-Ed Columnist':
img_div = soup.find('div','inlineImage module')
#self.log("Searching for photo")
if img_div:
img_div.extract()
#self.log("Photo deleted")
refresh = soup.find('meta', {'http-equiv':'refresh'})
if refresh is None:
return soup
content = refresh.get('content').partition('=')[2]
raw = self.browser.open_novisit('http://www.nytimes.com'+content).read()
return BeautifulSoup(raw.decode('cp1252', 'replace'))
def postprocess_html(self,soup, True):
print "\npostprocess_html()\n"
if self.one_picture_per_article:
# Remove all images after first
largeImg = soup.find(True, {'class':'articleSpanImage'})
inlineImgs = soup.findAll(True, {'class':'inlineImage module'})
if largeImg:
for inlineImg in inlineImgs:
inlineImg.extract()
else:
if inlineImgs:
firstImg = inlineImgs[0]
for inlineImg in inlineImgs[1:]:
inlineImg.extract()
# Move firstImg after headline
cgFirst = soup.find(True, {'class':'columnGroup first'})
if cgFirst:
# Strip all sibling NavigableStrings: noise
navstrings = cgFirst.findAll(text=True, recursive=False)
[ns.extract() for ns in navstrings]
headline_found = False
tag = cgFirst.find(True)
insertLoc = 0
while True:
insertLoc += 1
if hasattr(tag,'class') and tag['class'] == 'articleHeadline':
headline_found = True
break
tag = tag.nextSibling
if not tag:
headline_found = False
break
if headline_found:
cgFirst.insert(insertLoc,firstImg)
else:
self.log(">>> No class:'columnGroup first' found <<<")
# Change class="kicker" to <h3>
kicker = soup.find(True, {'class':'kicker'})
if kicker and kicker.contents and kicker.contents[0]:
h3Tag = Tag(soup, "h3")
h3Tag.insert(0, self.fixChars(self.tag_to_string(kicker,
use_alt=False)))
kicker.replaceWith(h3Tag)
# Change captions to italic -1
for caption in soup.findAll(True, {'class':'caption'}) :
if caption and caption.contents[0]:
emTag = Tag(soup, "em")
c = self.fixChars(self.tag_to_string(caption,use_alt=False)).strip()
mp_off = c.find("More Photos")
if mp_off >= 0:
c = c[:mp_off]
emTag.insert(0, c)
#hrTag = Tag(soup, 'hr')
#hrTag['class'] = 'caption_divider'
hrTag = Tag(soup, 'div')
hrTag['class'] = 'divider'
emTag.insert(1, hrTag)
caption.replaceWith(emTag)
# Change <nyt_headline> to <h2>
h1 = soup.find('h1')
if h1:
headline = h1.find("nyt_headline")
if headline:
tag = Tag(soup, "h2")
tag['class'] = "headline"
tag.insert(0, self.fixChars(headline.contents[0]))
h1.replaceWith(tag)
else:
# Blog entry - replace headline, remove <hr> tags
headline = soup.find('title')
if headline:
tag = Tag(soup, "h2")
tag['class'] = "headline"
tag.insert(0, self.fixChars(headline.contents[0]))
soup.insert(0, tag)
hrs = soup.findAll('hr')
for hr in hrs:
hr.extract()
# Change <h1> to <h3> - used in editorial blogs
masthead = soup.find("h1")
if masthead:
# Nuke the href
if masthead.a:
del(masthead.a['href'])
tag = Tag(soup, "h3")
tag.insert(0, self.fixChars(masthead.contents[0]))
masthead.replaceWith(tag)
# Change <span class="bold"> to <b>
for subhead in soup.findAll(True, {'class':'bold'}) :
if subhead.contents:
bTag = Tag(soup, "b")
bTag.insert(0, subhead.contents[0])
subhead.replaceWith(bTag)
# Synthesize a section header
dsk = soup.find('meta', attrs={'name':'dsk'})
if dsk and dsk.has_key('content'):
hTag = Tag(soup,'h3')
hTag['class'] = 'section'
hTag.insert(0,NavigableString(dsk['content']))
articleTag = soup.find(True, attrs={'id':'article'})
if articleTag:
articleTag.insert(0,hTag)
# Add class="articleBody" to <div> so we can format with CSS
divTag = soup.find('div',attrs={'id':'articleBody'})
if divTag:
divTag['class'] = divTag['id']
# Add class="authorId" to <div> so we can format with CSS
divTag = soup.find('div',attrs={'id':'authorId'})
if divTag and divTag.contents[0]:
tag = Tag(soup, "p")
tag['class'] = "authorId"
tag.insert(0, self.fixChars(self.tag_to_string(divTag.contents[0],
use_alt=False)))
divTag.replaceWith(tag)
return soup
def populate_article_metadata(self,article,soup,first):
'''
Extract author and description from article, add to article metadata
'''
def extract_author(soup):
byline = soup.find('meta',attrs={'name':['byl','CLMST']})
if byline :
author = byline['content']
else :
# Try for <div class="byline">
byline = soup.find('div', attrs={'class':'byline'})
if byline:
author = byline.renderContents()
else:
print soup.prettify()
return None
return author
def extract_description(soup):
description = soup.find('meta',attrs={'name':['description','description ']})
if description :
return self.massageNCXText(description['content'])
else:
# Take first paragraph of article
articlebody = soup.find('div',attrs={'id':'articlebody'})
if not articlebody:
# Try again with class instead of id
articlebody = soup.find('div',attrs={'class':'articlebody'})
if not articlebody:
print 'postprocess_book.extract_description(): Did not find <div id="articlebody">:'
print soup.prettify()
return None
paras = articlebody.findAll('p')
for p in paras:
if p.renderContents() > '' :
return self.massageNCXText(self.tag_to_string(p,use_alt=False))
return None
if not article.author:
article.author = extract_author(soup)
if not article.summary:
article.summary = article.text_summary = extract_description(soup)
def strip_anchors(self,soup):
paras = soup.findAll(True)
for para in paras:
aTags = para.findAll('a')
for a in aTags:
if a.img is None:
a.replaceWith(a.renderContents().decode('utf-8','replace'))
#a.replaceWith(a.renderContents().decode('cp1252','replace'))
return soup

60
resources/recipes/stnn.recipe Normal file
View File

@ -0,0 +1,60 @@
__license__ = 'GPL v3'
__copyright__ = '2010, Larry Chan <larry1chan at gmail.com>'
'''
Singtao STNN
'''
from calibre.web.feeds.recipes import BasicNewsRecipe
class SingtaoSTNN(BasicNewsRecipe):
title = 'Singtao STNN'
__author__ = 'Larry Chan, larry1chan'
description = 'Chinese News'
oldest_article = 2
max_articles_per_feed = 100
simultaneous_downloads = 5
no_stylesheets = True
#delay = 1
use_embedded_content = False
encoding = 'gb2312'
publisher = 'Singtao STNN'
category = 'news, China, world'
language = 'zh'
publication_type = 'newsportal'
extra_css = ' body{ font-family: Verdana,Helvetica,Arial,sans-serif } .introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} '
masthead_url = 'http://www.stnn.cc/images/0806/logo_080728.gif'
conversion_options = {
'comments' : description
,'tags' : category
,'language' : language
,'publisher' : publisher
,'linearize_tables': True
}
remove_tags_before = dict(name='div', attrs={'class':['page_box']})
remove_tags_after = dict(name='div', attrs={'class':['pagelist']})
keep_only_tags = [
dict(name='div', attrs={'class':['font_title clearfix']}),
dict(name='div', attrs={'id':['content_zoom']})
]
remove_attributes = ['width','height','href']
# for a full list of rss check out [url]http://www.stnn.cc/rss/[/url]
feeds = [ (u'Headline News', u'http://www.stnn.cc/rss/news/index.xml'),
(u'Breaking News', u'http://www.stnn.cc/rss/tufa/index.xml'),
(u'Finance', u'http://www.stnn.cc/rss/fin/index.xml'),
(u'Entertainment', u'http://www.stnn.cc/rss/ent/index.xml'),
(u'International', u'http://www.stnn.cc/rss/guoji/index.xml'),
(u'China', u'http://www.stnn.cc/rss/china/index.xml'),
(u'Opnion', u'http://www.stnn.cc/rss/fin_op/index.xml'),
(u'Blog', u'http://blog.stnn.cc/uploadfile/rssblogtypehotlog.xml'),
(u'Hong Kong', u'http://www.stnn.cc/rss/hongkong/index.xml')
]

46
resources/recipes/theecocolapse.recipe Normal file
View File

@ -0,0 +1,46 @@
__license__ = 'GPL v3'
__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
'''
theeconomiccollapseblog.com
'''
from calibre.web.feeds.news import BasicNewsRecipe
class TheEconomicCollapse(BasicNewsRecipe):
title = 'The Economic Collapse'
__author__ = 'Darko Miletic'
description = 'Are You Prepared For The Coming Economic Collapse And The Next Great Depression?'
publisher = 'The Economic Collapse'
category = 'news, politics, USA, economy'
oldest_article = 2
max_articles_per_feed = 200
no_stylesheets = True
encoding = 'utf8'
use_embedded_content = False
language = 'en'
remove_empty_feeds = True
extra_css = """
body{font-family: Tahoma,Arial,sans-serif }
img{margin-bottom: 0.4em}
"""
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
}
remove_tags = [
dict(attrs={'class':'sociable'})
,dict(name=['iframe','object','embed','meta','link','base'])
]
remove_attributes=['lang','onclick','width','height']
keep_only_tags=[dict(attrs={'class':['post-headline','post-bodycopy clearfix','']})]
feeds = [(u'Posts', u'http://theeconomiccollapseblog.com/feed')]
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
return self.adeify_images(soup)

8
resources/recipes/theeconomictimes_india.recipe
View File

@ -21,8 +21,9 @@ class TheEconomicTimes(BasicNewsRecipe):
language = 'en_IN' language = 'en_IN'
publication_type = 'newspaper' publication_type = 'newspaper'
masthead_url = 'http://economictimes.indiatimes.com/photo/2676871.cms' masthead_url = 'http://economictimes.indiatimes.com/photo/2676871.cms'
extra_css = """ body{font-family: Arial,Helvetica,sans-serif} extra_css = """
.heading1{font-size: xx-large; font-weight: bold} """ body{font-family: Arial,Helvetica,sans-serif}
"""
conversion_options = { conversion_options = {
'comment' : description 'comment' : description
@ -31,8 +32,9 @@ class TheEconomicTimes(BasicNewsRecipe):
, 'language' : language , 'language' : language
} }
keep_only_tags = [dict(attrs={'class':['heading1','headingnext','Normal']})] keep_only_tags = [dict(attrs={'class':'printdiv'})]
remove_tags = [dict(name=['object','link','embed','iframe','base','table','meta'])] remove_tags = [dict(name=['object','link','embed','iframe','base','table','meta'])]
remove_attributes = ['name']
feeds = [(u'All articles', u'http://economictimes.indiatimes.com/rssfeedsdefault.cms')] feeds = [(u'All articles', u'http://economictimes.indiatimes.com/rssfeedsdefault.cms')]

2
setup/extensions.py
View File

@ -54,7 +54,7 @@ reflow_error = poppler_error if poppler_error else magick_error
pdfreflow_libs = [] pdfreflow_libs = []
if iswindows: if iswindows:
pdfreflow_libs = ['advapi32', 'User32', 'Gdi32'] pdfreflow_libs = ['advapi32', 'User32', 'Gdi32', 'zlib']
extensions = [ extensions = [

2
setup/installer/windows/notes.rst
View File

@ -213,7 +213,7 @@ It contains correct fonts.conf etc.
poppler poppler
------------- -------------
In Cmake: disable GTK, Qt, OPenjpeg, zlib, lcms, gtk_tests, qt_tests. Enable qt4, jpeg, png and zlib In Cmake: disable GTK, Qt, OPenjpeg, cpp, lcms, gtk_tests, qt_tests. Enable qt4, jpeg, png and zlib
NOTE: poppler must be built as a static library, unless you build the qt4 bindings NOTE: poppler must be built as a static library, unless you build the qt4 bindings

5
src/calibre/customize/conversion.py
View File

@ -294,3 +294,8 @@ class OutputFormatPlugin(Plugin):
''' '''
raise NotImplementedError raise NotImplementedError
@property
def is_periodical(self):
return self.oeb.metadata.publication_type and \
unicode(self.oeb.metadata.publication_type[0]).startswith('periodical:')

13
src/calibre/customize/profiles.py
View File

@ -4,6 +4,7 @@ __license__ = 'GPL 3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import sys
from itertools import izip from itertools import izip
from xml.sax.saxutils import escape from xml.sax.saxutils import escape
@ -417,6 +418,13 @@ class iPadOutput(OutputProfile):
''' '''
# }}} # }}}
class TabletOutput(iPadOutput):
name = 'Tablet'
short_name = 'tablet'
description = _('Intended for generic tablet devices, does no resizing of images')
screen_size = (sys.maxint, sys.maxint)
comic_screen_size = (sys.maxint, sys.maxint)
class SonyReaderOutput(OutputProfile): class SonyReaderOutput(OutputProfile):
@ -583,7 +591,8 @@ class KindleDXOutput(OutputProfile):
# Screen size is a best guess # Screen size is a best guess
screen_size = (744, 1022) screen_size = (744, 1022)
dpi = 150.0 dpi = 150.0
comic_screen_size = (741, 1022) comic_screen_size = (771, 1116)
#comic_screen_size = (741, 1022)
supports_mobi_indexing = True supports_mobi_indexing = True
periodical_date_in_title = False periodical_date_in_title = False
mobi_ems_per_blockquote = 2.0 mobi_ems_per_blockquote = 2.0
@ -663,7 +672,7 @@ class BambookOutput(OutputProfile):
output_profiles = [OutputProfile, SonyReaderOutput, SonyReader300Output, output_profiles = [OutputProfile, SonyReaderOutput, SonyReader300Output,
SonyReader900Output, MSReaderOutput, MobipocketOutput, HanlinV3Output, SonyReader900Output, MSReaderOutput, MobipocketOutput, HanlinV3Output,
HanlinV5Output, CybookG3Output, CybookOpusOutput, KindleOutput, HanlinV5Output, CybookG3Output, CybookOpusOutput, KindleOutput,
iPadOutput, KoboReaderOutput, iPadOutput, KoboReaderOutput, TabletOutput,
SonyReaderLandscapeOutput, KindleDXOutput, IlliadOutput, SonyReaderLandscapeOutput, KindleDXOutput, IlliadOutput,
IRexDR1000Output, IRexDR800Output, JetBook5Output, NookOutput, IRexDR1000Output, IRexDR800Output, JetBook5Output, NookOutput,
BambookOutput, ] BambookOutput, ]

3
src/calibre/devices/android/driver.py
View File

@ -20,7 +20,8 @@ class ANDROID(USBMS):
VENDOR_ID = { VENDOR_ID = {
# HTC # HTC
0x0bb4 : { 0x0c02 : [0x100, 0x0227], 0x0c01 : [0x100, 0x0227], 0x0ff9 0x0bb4 : { 0x0c02 : [0x100, 0x0227], 0x0c01 : [0x100, 0x0227], 0x0ff9
: [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226]}, : [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226],
0xc92 : [0x100]},
# Motorola # Motorola
0x22b8 : { 0x41d9 : [0x216], 0x2d67 : [0x100], 0x41db : [0x216], 0x22b8 : { 0x41d9 : [0x216], 0x2d67 : [0x100], 0x41db : [0x216],

2
src/calibre/devices/cybook/driver.py
View File

@ -42,7 +42,7 @@ class CYBOOK(USBMS):
DELETE_EXTS = ['.mbp', '.dat', '.bin', '_6090.t2b', '.thn'] DELETE_EXTS = ['.mbp', '.dat', '.bin', '_6090.t2b', '.thn']
SUPPORTS_SUB_DIRS = True SUPPORTS_SUB_DIRS = True
def upload_cover(self, path, filename, metadata): def upload_cover(self, path, filename, metadata, filepath):
coverdata = getattr(metadata, 'thumbnail', None) coverdata = getattr(metadata, 'thumbnail', None)
if coverdata and coverdata[2]: if coverdata and coverdata[2]:
coverdata = coverdata[2] coverdata = coverdata[2]

4
src/calibre/devices/hanvon/driver.py
View File

@ -77,7 +77,7 @@ class ALEX(N516):
name = os.path.splitext(os.path.basename(file_abspath))[0] + '.png' name = os.path.splitext(os.path.basename(file_abspath))[0] + '.png'
return os.path.join(base, 'covers', name) return os.path.join(base, 'covers', name)
def upload_cover(self, path, filename, metadata): def upload_cover(self, path, filename, metadata, filepath):
from calibre.ebooks import calibre_cover from calibre.ebooks import calibre_cover
from calibre.utils.magick.draw import thumbnail from calibre.utils.magick.draw import thumbnail
coverdata = getattr(metadata, 'thumbnail', None) coverdata = getattr(metadata, 'thumbnail', None)
@ -129,7 +129,7 @@ class AZBOOKA(ALEX):
def can_handle(self, device_info, debug=False): def can_handle(self, device_info, debug=False):
return not is_alex(device_info) return not is_alex(device_info)
def upload_cover(self, path, filename, metadata): def upload_cover(self, path, filename, metadata, filepath):
pass pass
class EB511(USBMS): class EB511(USBMS):

63
src/calibre/devices/kobo/driver.py
View File

@ -22,7 +22,9 @@ class KOBO(USBMS):
gui_name = 'Kobo Reader' gui_name = 'Kobo Reader'
description = _('Communicate with the Kobo Reader') description = _('Communicate with the Kobo Reader')
author = 'Timothy Legge and Kovid Goyal' author = 'Timothy Legge and Kovid Goyal'
version = (1, 0, 6) version = (1, 0, 7)
dbversion = 0
supported_platforms = ['windows', 'osx', 'linux'] supported_platforms = ['windows', 'osx', 'linux']
@ -92,7 +94,7 @@ class KOBO(USBMS):
if lpath.startswith(os.sep): if lpath.startswith(os.sep):
lpath = lpath[len(os.sep):] lpath = lpath[len(os.sep):]
lpath = lpath.replace('\\', '/') lpath = lpath.replace('\\', '/')
# print "LPATH: " + lpath # debug_print("LPATH: ", lpath, " - Title: " , title)
playlist_map = {} playlist_map = {}
@ -112,7 +114,7 @@ class KOBO(USBMS):
#print "Image name Normalized: " + imagename #print "Image name Normalized: " + imagename
if imagename is not None: if imagename is not None:
bl[idx].thumbnail = ImageWrapper(imagename) bl[idx].thumbnail = ImageWrapper(imagename)
if ContentType != '6': if (ContentType != '6'and self.dbversion < 8) or (self.dbversion >= 8):
if self.update_metadata_item(bl[idx]): if self.update_metadata_item(bl[idx]):
# print 'update_metadata_item returned true' # print 'update_metadata_item returned true'
changed = True changed = True
@ -120,10 +122,16 @@ class KOBO(USBMS):
playlist_map[lpath] not in bl[idx].device_collections: playlist_map[lpath] not in bl[idx].device_collections:
bl[idx].device_collections.append(playlist_map[lpath]) bl[idx].device_collections.append(playlist_map[lpath])
else: else:
if ContentType == '6': if ContentType == '6' and self.dbversion < 8:
book = Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=1048576) book = Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=1048576)
else: else:
book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID) try:
book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID)
except:
debug_print("prefix: ", prefix, "lpath: ", lpath, "title: ", title, "authors: ", authors, \
"mime: ", mime, "date: ", date, "ContentType: ", ContentType, "ImageID: ", ImageID)
raise
# print 'Update booklist' # print 'Update booklist'
book.device_collections = [playlist_map[lpath]] if lpath in playlist_map else [] book.device_collections = [playlist_map[lpath]] if lpath in playlist_map else []
@ -143,6 +151,13 @@ class KOBO(USBMS):
# numrows = row[0] # numrows = row[0]
#cursor.close() #cursor.close()
# Determine the database version
# 4 - Bluetooth Kobo Rev 2 (1.4)
# 8 - WIFI KOBO Rev 1
cursor.execute('select version from dbversion')
result = cursor.fetchone()
self.dbversion = result[0]
query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
'ImageID, ReadStatus from content where BookID is Null' 'ImageID, ReadStatus from content where BookID is Null'
@ -153,7 +168,8 @@ class KOBO(USBMS):
# self.report_progress((i+1) / float(numrows), _('Getting list of books on device...')) # self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
path = self.path_from_contentid(row[3], row[5], oncard) path = self.path_from_contentid(row[3], row[5], oncard)
mime = mime_type_ext(path_to_ext(row[3])) mime = mime_type_ext(path_to_ext(path)) if path.find('kepub') == -1 else 'application/epub+zip'
# debug_print("mime:", mime)
if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"): if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"):
changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7]) changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7])
@ -206,7 +222,7 @@ class KOBO(USBMS):
cursor.close() cursor.close()
cursor = connection.cursor() cursor = connection.cursor()
if ContentType == 6: if ContentType == 6 and self.dbversion < 8:
# Delete the shortcover_pages first # Delete the shortcover_pages first
cursor.execute('delete from shortcover_page where shortcoverid in (select ContentID from content where BookID = ?)', t) cursor.execute('delete from shortcover_page where shortcoverid in (select ContentID from content where BookID = ?)', t)
@ -249,7 +265,7 @@ class KOBO(USBMS):
path = self.normalize_path(path) path = self.normalize_path(path)
# print "Delete file normalized path: " + path # print "Delete file normalized path: " + path
extension = os.path.splitext(path)[1] extension = os.path.splitext(path)[1]
ContentType = self.get_content_type_from_extension(extension) ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(path)
ContentID = self.contentid_from_path(path, ContentType) ContentID = self.contentid_from_path(path, ContentType)
@ -332,9 +348,14 @@ class KOBO(USBMS):
def contentid_from_path(self, path, ContentType): def contentid_from_path(self, path, ContentType):
if ContentType == 6: if ContentType == 6:
ContentID = os.path.splitext(path)[0] if self.dbversion < 8:
# Remove the prefix on the file. it could be either ContentID = os.path.splitext(path)[0]
ContentID = ContentID.replace(self._main_prefix, '') # Remove the prefix on the file. it could be either
ContentID = ContentID.replace(self._main_prefix, '')
else:
ContentID = path
ContentID = ContentID.replace(self._main_prefix + '.kobo/kepub/', '')
if self._card_a_prefix is not None: if self._card_a_prefix is not None:
ContentID = ContentID.replace(self._card_a_prefix, '') ContentID = ContentID.replace(self._card_a_prefix, '')
elif ContentType == 999: # HTML Files elif ContentType == 999: # HTML Files
@ -350,6 +371,13 @@ class KOBO(USBMS):
ContentID = ContentID.replace("\\", '/') ContentID = ContentID.replace("\\", '/')
return ContentID return ContentID
def get_content_type_from_path(self, path):
# Strictly speaking the ContentType could be 6 or 10
# however newspapers have the same storage format
if path.find('kepub') >= 0:
ContentType = 6
return ContentType
def get_content_type_from_extension(self, extension): def get_content_type_from_extension(self, extension):
if extension == '.kobo': if extension == '.kobo':
# Kobo books do not have book files. They do have some images though # Kobo books do not have book files. They do have some images though
@ -369,19 +397,22 @@ class KOBO(USBMS):
print 'path from_contentid cardb' print 'path from_contentid cardb'
elif oncard == 'carda': elif oncard == 'carda':
path = path.replace("file:///mnt/sd/", self._card_a_prefix) path = path.replace("file:///mnt/sd/", self._card_a_prefix)
# print "SD Card: " + filename # print "SD Card: " + path
else: else:
if ContentType == "6": if ContentType == "6" and self.dbversion < 8:
# This is a hack as the kobo files do not exist # This is a hack as the kobo files do not exist
# but the path is required to make a unique id # but the path is required to make a unique id
# for calibre's reference # for calibre's reference
path = self._main_prefix + path + '.kobo' path = self._main_prefix + path + '.kobo'
# print "Path: " + path # print "Path: " + path
elif (ContentType == "6" or ContentType == "10") and self.dbversion >= 8:
path = self._main_prefix + '.kobo/kepub/' + path
# print "Internal: " + path
else: else:
# if path.startswith("file:///mnt/onboard/"): # if path.startswith("file:///mnt/onboard/"):
path = path.replace("file:///mnt/onboard/", self._main_prefix) path = path.replace("file:///mnt/onboard/", self._main_prefix)
path = path.replace("/mnt/onboard/", self._main_prefix) path = path.replace("/mnt/onboard/", self._main_prefix)
# print "Internal: " + filename # print "Internal: " + path
return path return path
@ -469,7 +500,7 @@ class KOBO(USBMS):
book.device_collections = ['Im_Reading'] book.device_collections = ['Im_Reading']
extension = os.path.splitext(book.path)[1] extension = os.path.splitext(book.path)[1]
ContentType = self.get_content_type_from_extension(extension) ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
ContentID = self.contentid_from_path(book.path, ContentType) ContentID = self.contentid_from_path(book.path, ContentType)
datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())
@ -505,7 +536,7 @@ class KOBO(USBMS):
book.device_collections = ['Read'] book.device_collections = ['Read']
extension = os.path.splitext(book.path)[1] extension = os.path.splitext(book.path)[1]
ContentType = self.get_content_type_from_extension(extension) ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
ContentID = self.contentid_from_path(book.path, ContentType) ContentID = self.contentid_from_path(book.path, ContentType)
# datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) # datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())

2
src/calibre/devices/misc.py
View File

@ -102,7 +102,7 @@ class PDNOVEL(USBMS):
DELETE_EXTS = ['.jpg', '.jpeg', '.png'] DELETE_EXTS = ['.jpg', '.jpeg', '.png']
def upload_cover(self, path, filename, metadata): def upload_cover(self, path, filename, metadata, filepath):
coverdata = getattr(metadata, 'thumbnail', None) coverdata = getattr(metadata, 'thumbnail', None)
if coverdata and coverdata[2]: if coverdata and coverdata[2]:
with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile: with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile:

10
src/calibre/devices/nokia/driver.py
View File

@ -36,15 +36,15 @@ class N770(USBMS):
class N810(N770): class N810(N770):
name = 'Nokia 810 Device Interface' name = 'Nokia 810 Device Interface'
gui_name = 'Nokia 810' gui_name = 'Nokia 810/900'
description = _('Communicate with the Nokia 810 internet tablet.') description = _('Communicate with the Nokia 810/900 internet tablet.')
PRODUCT_ID = [0x96] PRODUCT_ID = [0x96, 0x1c7]
BCD = [0x316] BCD = [0x316]
WINDOWS_MAIN_MEM = 'N810' WINDOWS_MAIN_MEM = ['N810', 'N900']
MAIN_MEMORY_VOLUME_LABEL = 'N810 Main Memory' MAIN_MEMORY_VOLUME_LABEL = 'Nokia Tablet Main Memory'
class E71X(USBMS): class E71X(USBMS):

2
src/calibre/devices/nook/driver.py
View File

@ -45,7 +45,7 @@ class NOOK(USBMS):
DELETE_EXTS = ['.jpg'] DELETE_EXTS = ['.jpg']
SUPPORTS_SUB_DIRS = True SUPPORTS_SUB_DIRS = True
def upload_cover(self, path, filename, metadata): def upload_cover(self, path, filename, metadata, filepath):
try: try:
from PIL import Image, ImageDraw from PIL import Image, ImageDraw
Image, ImageDraw Image, ImageDraw

6
src/calibre/devices/prs505/__init__.py
View File

@ -2,5 +2,11 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
MEDIA_XML = 'database/cache/media.xml' MEDIA_XML = 'database/cache/media.xml'
MEDIA_EXT = 'database/cache/cacheExt.xml'
CACHE_XML = 'Sony Reader/database/cache.xml' CACHE_XML = 'Sony Reader/database/cache.xml'
CACHE_EXT = 'Sony Reader/database/cacheExt.xml'
MEDIA_THUMBNAIL = 'database/thumbnail'
CACHE_THUMBNAIL = 'Sony Reader/database/thumbnail'

49
src/calibre/devices/prs505/driver.py
View File

@ -9,10 +9,10 @@ Device driver for the SONY devices
import os, time, re import os, time, re
from calibre.devices.usbms.driver import USBMS, debug_print from calibre.devices.usbms.driver import USBMS, debug_print
from calibre.devices.prs505 import MEDIA_XML from calibre.devices.prs505 import MEDIA_XML, MEDIA_EXT, CACHE_XML, CACHE_EXT, \
from calibre.devices.prs505 import CACHE_XML MEDIA_THUMBNAIL, CACHE_THUMBNAIL
from calibre.devices.prs505.sony_cache import XMLCache from calibre.devices.prs505.sony_cache import XMLCache
from calibre import __appname__ from calibre import __appname__, prints
from calibre.devices.usbms.books import CollectionsBookList from calibre.devices.usbms.books import CollectionsBookList
class PRS505(USBMS): class PRS505(USBMS):
@ -66,6 +66,8 @@ class PRS505(USBMS):
plugboard = None plugboard = None
plugboard_func = None plugboard_func = None
THUMBNAIL_HEIGHT = 200
def windows_filter_pnp_id(self, pnp_id): def windows_filter_pnp_id(self, pnp_id):
return '_LAUNCHER' in pnp_id return '_LAUNCHER' in pnp_id
@ -116,20 +118,21 @@ class PRS505(USBMS):
return fname return fname
def initialize_XML_cache(self): def initialize_XML_cache(self):
paths, prefixes = {}, {} paths, prefixes, ext_paths = {}, {}, {}
for prefix, path, source_id in [ for prefix, path, ext_path, source_id in [
('main', MEDIA_XML, 0), ('main', MEDIA_XML, MEDIA_EXT, 0),
('card_a', CACHE_XML, 1), ('card_a', CACHE_XML, CACHE_EXT, 1),
('card_b', CACHE_XML, 2) ('card_b', CACHE_XML, CACHE_EXT, 2)
]: ]:
prefix = getattr(self, '_%s_prefix'%prefix) prefix = getattr(self, '_%s_prefix'%prefix)
if prefix is not None and os.path.exists(prefix): if prefix is not None and os.path.exists(prefix):
paths[source_id] = os.path.join(prefix, *(path.split('/'))) paths[source_id] = os.path.join(prefix, *(path.split('/')))
ext_paths[source_id] = os.path.join(prefix, *(ext_path.split('/')))
prefixes[source_id] = prefix prefixes[source_id] = prefix
d = os.path.dirname(paths[source_id]) d = os.path.dirname(paths[source_id])
if not os.path.exists(d): if not os.path.exists(d):
os.makedirs(d) os.makedirs(d)
return XMLCache(paths, prefixes, self.settings().use_author_sort) return XMLCache(paths, ext_paths, prefixes, self.settings().use_author_sort)
def books(self, oncard=None, end_session=True): def books(self, oncard=None, end_session=True):
debug_print('PRS505: starting fetching books for card', oncard) debug_print('PRS505: starting fetching books for card', oncard)
@ -174,3 +177,31 @@ class PRS505(USBMS):
def set_plugboards(self, plugboards, pb_func): def set_plugboards(self, plugboards, pb_func):
self.plugboards = plugboards self.plugboards = plugboards
self.plugboard_func = pb_func self.plugboard_func = pb_func
def upload_cover(self, path, filename, metadata, filepath):
if metadata.thumbnail and metadata.thumbnail[-1]:
path = path.replace('/', os.sep)
is_main = path.startswith(self._main_prefix)
thumbnail_dir = MEDIA_THUMBNAIL if is_main else CACHE_THUMBNAIL
prefix = None
if is_main:
prefix = self._main_prefix
else:
if self._card_a_prefix and \
path.startswith(self._card_a_prefix):
prefix = self._card_a_prefix
elif self._card_b_prefix and \
path.startswith(self._card_b_prefix):
prefix = self._card_b_prefix
if prefix is None:
prints('WARNING: Failed to find prefix for:', filepath)
return
thumbnail_dir = os.path.join(prefix, *thumbnail_dir.split('/'))
relpath = os.path.relpath(filepath, prefix)
thumbnail_dir = os.path.join(thumbnail_dir, relpath)
if not os.path.exists(thumbnail_dir):
os.makedirs(thumbnail_dir)
with open(os.path.join(thumbnail_dir, 'main_thumbnail.jpg'), 'wb') as f:
f.write(metadata.thumbnail[-1])

137
src/calibre/devices/prs505/sony_cache.py
View File

@ -9,6 +9,7 @@ import os, time
from base64 import b64decode from base64 import b64decode
from uuid import uuid4 from uuid import uuid4
from lxml import etree from lxml import etree
from datetime import date
from calibre import prints, guess_type, isbytestring from calibre import prints, guess_type, isbytestring
from calibre.devices.errors import DeviceError from calibre.devices.errors import DeviceError
@ -18,6 +19,20 @@ from calibre.ebooks.chardet import xml_to_unicode
from calibre.ebooks.metadata import authors_to_string, title_sort, \ from calibre.ebooks.metadata import authors_to_string, title_sort, \
authors_to_sort_string authors_to_sort_string
'''
cahceExt.xml
Periodical identifier sample from a PRS-650:
<?xml version="1.0" encoding="UTF-8"?>
<cacheExt xmlns="http://www.sony.com/xmlns/product/prs/device/1">
<text conformsTo="http://xmlns.sony.net/e-book/prs/periodicals/1.0/newspaper/1.0" periodicalName="The Atlantic" description="Current affairs and politics focussed on the US" publicationDate="Tue, 19 Oct 2010 00:00:00 GMT" path="database/media/books/calibre/Atlantic [Mon, 18 Oct 2010], The - calibre_1701.epub">
<thumbnail width="167" height="217">main_thumbnail.jpg</thumbnail>
</text>
</cacheExt>
'''
# Utility functions {{{ # Utility functions {{{
EMPTY_CARD_CACHE = '''\ EMPTY_CARD_CACHE = '''\
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
@ -25,6 +40,12 @@ EMPTY_CARD_CACHE = '''\
</cache> </cache>
''' '''
EMPTY_EXT_CACHE = '''\
<?xml version="1.0" encoding="UTF-8"?>
<cacheExt xmlns="http://www.sony.com/xmlns/product/prs/device/1">
</cacheExt>
'''
MIME_MAP = { MIME_MAP = {
"lrf" : "application/x-sony-bbeb", "lrf" : "application/x-sony-bbeb",
'lrx' : 'application/x-sony-bbeb', 'lrx' : 'application/x-sony-bbeb',
@ -63,7 +84,7 @@ def uuid():
class XMLCache(object): class XMLCache(object):
def __init__(self, paths, prefixes, use_author_sort): def __init__(self, paths, ext_paths, prefixes, use_author_sort):
if DEBUG: if DEBUG:
debug_print('Building XMLCache...', paths) debug_print('Building XMLCache...', paths)
self.paths = paths self.paths = paths
@ -76,8 +97,8 @@ class XMLCache(object):
for source_id, path in paths.items(): for source_id, path in paths.items():
if source_id == 0: if source_id == 0:
if not os.path.exists(path): if not os.path.exists(path):
raise DeviceError('The SONY XML cache media.xml does not exist. Try' raise DeviceError(('The SONY XML cache %r does not exist. Try'
' disconnecting and reconnecting your reader.') ' disconnecting and reconnecting your reader.')%repr(path))
with open(path, 'rb') as f: with open(path, 'rb') as f:
raw = f.read() raw = f.read()
else: else:
@ -85,14 +106,34 @@ class XMLCache(object):
if os.access(path, os.R_OK): if os.access(path, os.R_OK):
with open(path, 'rb') as f: with open(path, 'rb') as f:
raw = f.read() raw = f.read()
self.roots[source_id] = etree.fromstring(xml_to_unicode( self.roots[source_id] = etree.fromstring(xml_to_unicode(
raw, strip_encoding_pats=True, assume_utf8=True, raw, strip_encoding_pats=True, assume_utf8=True,
verbose=DEBUG)[0], verbose=DEBUG)[0],
parser=parser) parser=parser)
if self.roots[source_id] is None: if self.roots[source_id] is None:
raise Exception(('The SONY database at %s is corrupted. Try ' raise Exception(('The SONY database at %r is corrupted. Try '
' disconnecting and reconnecting your reader.')%path) ' disconnecting and reconnecting your reader.')%path)
self.ext_paths, self.ext_roots = {}, {}
for source_id, path in ext_paths.items():
if not os.path.exists(path):
try:
with open(path, 'wb') as f:
f.write(EMPTY_EXT_CACHE)
except:
pass
if os.access(path, os.W_OK):
try:
with open(path, 'rb') as f:
self.ext_roots[source_id] = etree.fromstring(
xml_to_unicode(f.read(),
strip_encoding_pats=True, assume_utf8=True,
verbose=DEBUG)[0], parser=parser)
self.ext_paths[source_id] = path
except:
pass
# }}} # }}}
recs = self.roots[0].xpath('//*[local-name()="records"]') recs = self.roots[0].xpath('//*[local-name()="records"]')
@ -352,12 +393,18 @@ class XMLCache(object):
debug_print('Updating XML Cache:', i) debug_print('Updating XML Cache:', i)
root = self.record_roots[i] root = self.record_roots[i]
lpath_map = self.build_lpath_map(root) lpath_map = self.build_lpath_map(root)
ext_root = self.ext_roots[i] if i in self.ext_roots else None
ext_lpath_map = None
if ext_root is not None:
ext_lpath_map = self.build_lpath_map(ext_root)
gtz_count = ltz_count = 0 gtz_count = ltz_count = 0
use_tz_var = False use_tz_var = False
for book in booklist: for book in booklist:
path = os.path.join(self.prefixes[i], *(book.lpath.split('/'))) path = os.path.join(self.prefixes[i], *(book.lpath.split('/')))
record = lpath_map.get(book.lpath, None) record = lpath_map.get(book.lpath, None)
created = False
if record is None: if record is None:
created = True
record = self.create_text_record(root, i, book.lpath) record = self.create_text_record(root, i, book.lpath)
if plugboard is not None: if plugboard is not None:
newmi = book.deepcopy_metadata() newmi = book.deepcopy_metadata()
@ -373,6 +420,13 @@ class XMLCache(object):
if book.device_collections is None: if book.device_collections is None:
book.device_collections = [] book.device_collections = []
book.device_collections = playlist_map.get(book.lpath, []) book.device_collections = playlist_map.get(book.lpath, [])
if created and ext_root is not None and \
ext_lpath_map.get(book.lpath, None) is None:
ext_record = self.create_ext_text_record(ext_root, i,
book.lpath, book.thumbnail)
self.periodicalize_book(book, ext_record)
debug_print('Timezone votes: %d GMT, %d LTZ, use_tz_var=%s'% debug_print('Timezone votes: %d GMT, %d LTZ, use_tz_var=%s'%
(gtz_count, ltz_count, use_tz_var)) (gtz_count, ltz_count, use_tz_var))
self.update_playlists(i, root, booklist, collections_attributes) self.update_playlists(i, root, booklist, collections_attributes)
@ -386,6 +440,47 @@ class XMLCache(object):
self.fix_ids() self.fix_ids()
debug_print('Finished update') debug_print('Finished update')
def is_sony_periodical(self, book):
if _('News') not in book.tags:
return False
if not book.lpath.lower().endswith('.epub'):
return False
if book.pubdate.date() < date(2010, 10, 17):
return False
return True
def periodicalize_book(self, book, record):
if not self.is_sony_periodical(book):
return
record.set('conformsTo',
"http://xmlns.sony.net/e-book/prs/periodicals/1.0/newspaper/1.0")
record.set('description', '')
name = None
if '[' in book.title:
name = book.title.split('[')[0].strip()
if len(name) < 4:
name = None
if not name:
try:
name = [t for t in book.tags if t != _('News')][0]
except:
name = None
if not name:
name = book.title
record.set('periodicalName', name)
try:
pubdate = strftime(book.pubdate.utctimetuple(),
zone=lambda x : x)
record.set('publicationDate', pubdate)
except:
pass
def rebuild_collections(self, booklist, bl_index): def rebuild_collections(self, booklist, bl_index):
if bl_index not in self.record_roots: if bl_index not in self.record_roots:
return return
@ -472,6 +567,28 @@ class XMLCache(object):
root.append(ans) root.append(ans)
return ans return ans
def create_ext_text_record(self, root, bl_id, lpath, thumbnail):
namespace = root.nsmap[None]
attrib = { 'path': lpath }
ans = root.makeelement('{%s}text'%namespace, attrib=attrib,
nsmap=root.nsmap)
ans.tail = '\n'
if len(root) > 0:
root[-1].tail = '\n\t'
else:
root.text = '\n\t'
root.append(ans)
if thumbnail and thumbnail[-1]:
ans.text = '\n' + '\t\t'
t = root.makeelement('{%s}thumbnail'%namespace,
attrib={'width':str(thumbnail[0]), 'height':str(thumbnail[1])},
nsmap=root.nsmap)
t.text = 'main_thumbnail.jpg'
ans.append(t)
t.tail = '\n\t'
return ans
def update_text_record(self, record, book, path, bl_index, def update_text_record(self, record, book, path, bl_index,
gtz_count, ltz_count, use_tz_var): gtz_count, ltz_count, use_tz_var):
''' '''
@ -589,6 +706,18 @@ class XMLCache(object):
'<?xml version="1.0" encoding="UTF-8"?>') '<?xml version="1.0" encoding="UTF-8"?>')
with open(path, 'wb') as f: with open(path, 'wb') as f:
f.write(raw) f.write(raw)
for i, path in self.ext_paths.items():
try:
raw = etree.tostring(self.ext_roots[i], encoding='UTF-8',
xml_declaration=True)
except:
continue
raw = raw.replace("<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>')
with open(path, 'wb') as f:
f.write(raw)
# }}} # }}}
# Utility methods {{{ # Utility methods {{{

20
src/calibre/devices/udisks.py
View File

@ -5,8 +5,7 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import dbus import dbus, os
import os
def node_mountpoint(node): def node_mountpoint(node):
@ -56,15 +55,6 @@ class UDisks(object):
parent = device_node_path parent = device_node_path
while parent[-1] in '0123456789': while parent[-1] in '0123456789':
parent = parent[:-1] parent = parent[:-1]
devices = [str(x) for x in self.main.EnumerateDeviceFiles()]
for d in devices:
if d.startswith(parent) and d != parent:
try:
self.unmount(d)
except:
import traceback
print 'Failed to unmount:', d
traceback.print_exc()
d = self.device(parent) d = self.device(parent)
d.DriveEject([]) d.DriveEject([])
@ -76,13 +66,19 @@ def eject(node_path):
u = UDisks() u = UDisks()
u.eject(node_path) u.eject(node_path)
def umount(node_path):
u = UDisks()
u.unmount(node_path)
if __name__ == '__main__': if __name__ == '__main__':
import sys import sys
dev = sys.argv[1] dev = sys.argv[1]
print 'Testing with node', dev print 'Testing with node', dev
u = UDisks() u = UDisks()
print 'Mounted at:', u.mount(dev) print 'Mounted at:', u.mount(dev)
print 'Ejecting' print 'Unmounting'
u.unmount(dev)
print 'Ejecting:'
u.eject(dev) u.eject(dev)

38
src/calibre/devices/usbms/books.py
View File

@ -99,6 +99,13 @@ class CollectionsBookList(BookList):
def supports_collections(self): def supports_collections(self):
return True return True
def in_category_sort_rules(self, attr):
sorts = tweaks['sony_collection_sorting_rules']
for attrs,sortattr in sorts:
if attr in attrs or '*' in attrs:
return sortattr
return None
def compute_category_name(self, attr, category, field_meta): def compute_category_name(self, attr, category, field_meta):
renames = tweaks['sony_collection_renaming_rules'] renames = tweaks['sony_collection_renaming_rules']
attr_name = renames.get(attr, None) attr_name = renames.get(attr, None)
@ -116,6 +123,7 @@ class CollectionsBookList(BookList):
from calibre.devices.usbms.driver import debug_print from calibre.devices.usbms.driver import debug_print
debug_print('Starting get_collections:', prefs['manage_device_metadata']) debug_print('Starting get_collections:', prefs['manage_device_metadata'])
debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules']) debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules'])
debug_print('Sorting rules:', tweaks['sony_collection_sorting_rules'])
# Complexity: we can use renaming rules only when using automatic # Complexity: we can use renaming rules only when using automatic
# management. Otherwise we don't always have the metadata to make the # management. Otherwise we don't always have the metadata to make the
@ -171,6 +179,7 @@ class CollectionsBookList(BookList):
else: else:
val = [val] val = [val]
sort_attr = self.in_category_sort_rules(attr)
for category in val: for category in val:
is_series = False is_series = False
if doing_dc: if doing_dc:
@ -199,22 +208,41 @@ class CollectionsBookList(BookList):
if cat_name not in collections: if cat_name not in collections:
collections[cat_name] = {} collections[cat_name] = {}
if is_series: if use_renaming_rules and sort_attr:
sort_val = book.get(sort_attr, None)
collections[cat_name][lpath] = \
(book, sort_val, book.get('title_sort', 'zzzz'))
elif is_series:
if doing_dc: if doing_dc:
collections[cat_name][lpath] = \ collections[cat_name][lpath] = \
(book, book.get('series_index', sys.maxint)) (book, book.get('series_index', sys.maxint), '')
else: else:
collections[cat_name][lpath] = \ collections[cat_name][lpath] = \
(book, book.get(attr+'_index', sys.maxint)) (book, book.get(attr+'_index', sys.maxint), '')
else: else:
if lpath not in collections[cat_name]: if lpath not in collections[cat_name]:
collections[cat_name][lpath] = \ collections[cat_name][lpath] = \
(book, book.get('title_sort', 'zzzz')) (book, book.get('title_sort', 'zzzz'), '')
# Sort collections # Sort collections
result = {} result = {}
def none_cmp(xx, yy):
x = xx[1]
y = yy[1]
if x is None and y is None:
return cmp(xx[2], yy[2])
if x is None:
return 1
if y is None:
return -1
c = cmp(x, y)
if c != 0:
return c
return cmp(xx[2], yy[2])
for category, lpaths in collections.items(): for category, lpaths in collections.items():
books = lpaths.values() books = lpaths.values()
books.sort(cmp=lambda x,y:cmp(x[1], y[1])) books.sort(cmp=none_cmp)
result[category] = [x[0] for x in books] result[category] = [x[0] for x in books]
return result return result

49
src/calibre/devices/usbms/device.py
View File

@ -523,7 +523,8 @@ class Device(DeviceConfig, DevicePlugin):
devnodes.append(node) devnodes.append(node)
devnodes += list(repeat(None, 3)) devnodes += list(repeat(None, 3))
ans = tuple(['/dev/'+x if ok.get(x, False) else None for x in devnodes[:3]]) ans = ['/dev/'+x if ok.get(x, False) else None for x in devnodes[:3]]
ans.sort(key=lambda x: x[5:] if x else 'zzzzz')
return self.linux_swap_drives(ans) return self.linux_swap_drives(ans)
def linux_swap_drives(self, drives): def linux_swap_drives(self, drives):
@ -732,24 +733,36 @@ class Device(DeviceConfig, DevicePlugin):
pass pass
def eject_linux(self): def eject_linux(self):
try: from calibre.devices.udisks import eject, umount
from calibre.devices.udisks import eject drives = [d for d in self.find_device_nodes() if d]
return eject(self._linux_main_device_node) for d in drives:
except: try:
pass umount(d)
drives = self.find_device_nodes() except:
pass
failures = False
for d in drives:
try:
eject(d)
except Exception, e:
print 'Udisks eject call for:', d, 'failed:'
print '\t', e
failures = True
if not failures:
return
for drive in drives: for drive in drives:
if drive: cmd = 'calibre-mount-helper'
cmd = 'calibre-mount-helper' if getattr(sys, 'frozen_path', False):
if getattr(sys, 'frozen_path', False): cmd = os.path.join(sys.frozen_path, cmd)
cmd = os.path.join(sys.frozen_path, cmd) cmd = [cmd, 'eject']
cmd = [cmd, 'eject'] mp = getattr(self, "_linux_mount_map", {}).get(drive,
mp = getattr(self, "_linux_mount_map", {}).get(drive, 'dummy/')[:-1]
'dummy/')[:-1] try:
try: subprocess.Popen(cmd + [drive, mp]).wait()
subprocess.Popen(cmd + [drive, mp]).wait() except:
except: pass
pass
def eject(self): def eject(self):
if islinux: if islinux:

10
src/calibre/devices/usbms/driver.py
View File

@ -186,7 +186,8 @@ class USBMS(CLI, Device):
self.put_file(infile, filepath, replace_file=True) self.put_file(infile, filepath, replace_file=True)
try: try:
self.upload_cover(os.path.dirname(filepath), self.upload_cover(os.path.dirname(filepath),
os.path.splitext(os.path.basename(filepath))[0], mdata) os.path.splitext(os.path.basename(filepath))[0],
mdata, filepath)
except: # Failure to upload cover is not catastrophic except: # Failure to upload cover is not catastrophic
import traceback import traceback
traceback.print_exc() traceback.print_exc()
@ -197,14 +198,15 @@ class USBMS(CLI, Device):
debug_print('USBMS: finished uploading %d books'%(len(files))) debug_print('USBMS: finished uploading %d books'%(len(files)))
return zip(paths, cycle([on_card])) return zip(paths, cycle([on_card]))
def upload_cover(self, path, filename, metadata): def upload_cover(self, path, filename, metadata, filepath):
''' '''
Upload book cover to the device. Default implementation does nothing. Upload book cover to the device. Default implementation does nothing.
:param path: the full path were the associated book is located. :param path: The full path to the directory where the associated book is located.
:param filename: the name of the book file without the extension. :param filename: The name of the book file without the extension.
:param metadata: metadata belonging to the book. Use metadata.thumbnail :param metadata: metadata belonging to the book. Use metadata.thumbnail
for cover for cover
:param filepath: The full path to the ebook file
''' '''
pass pass

6
src/calibre/ebooks/conversion/plumber.py
View File

@ -14,7 +14,7 @@ from calibre.ebooks.conversion.preprocess import HTMLPreProcessor
from calibre.ptempfile import PersistentTemporaryDirectory from calibre.ptempfile import PersistentTemporaryDirectory
from calibre.utils.date import parse_date from calibre.utils.date import parse_date
from calibre.utils.zipfile import ZipFile from calibre.utils.zipfile import ZipFile
from calibre import extract, walk from calibre import extract, walk, isbytestring, filesystem_encoding
from calibre.constants import __version__ from calibre.constants import __version__
DEBUG_README=u''' DEBUG_README=u'''
@ -77,6 +77,10 @@ class Plumber(object):
:param input: Path to input file. :param input: Path to input file.
:param output: Path to output file/directory :param output: Path to output file/directory
''' '''
if isbytestring(input):
input = input.decode(filesystem_encoding)
if isbytestring(output):
output = output.decode(filesystem_encoding)
self.original_input_arg = input self.original_input_arg = input
self.input = os.path.abspath(input) self.input = os.path.abspath(input)
self.output = os.path.abspath(output) self.output = os.path.abspath(output)

16
src/calibre/ebooks/epub/__init__.py
View File

@ -15,22 +15,30 @@ def rules(stylesheets):
if r.type == r.STYLE_RULE: if r.type == r.STYLE_RULE:
yield r yield r
def initialize_container(path_to_container, opf_name='metadata.opf'): def initialize_container(path_to_container, opf_name='metadata.opf',
extra_entries=[]):
''' '''
Create an empty EPUB document, with a default skeleton. Create an empty EPUB document, with a default skeleton.
''' '''
CONTAINER='''\ rootfiles = ''
for path, mimetype, _ in extra_entries:
rootfiles += u'<rootfile full-path="{0}" media-type="{1}"/>'.format(
path, mimetype)
CONTAINER = u'''\
<?xml version="1.0"?> <?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container"> <container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles> <rootfiles>
<rootfile full-path="%s" media-type="application/oebps-package+xml"/> <rootfile full-path="{0}" media-type="application/oebps-package+xml"/>
{extra_entries}
</rootfiles> </rootfiles>
</container> </container>
'''%opf_name '''.format(opf_name, extra_entries=rootfiles).encode('utf-8')
zf = ZipFile(path_to_container, 'w') zf = ZipFile(path_to_container, 'w')
zf.writestr('mimetype', 'application/epub+zip', compression=ZIP_STORED) zf.writestr('mimetype', 'application/epub+zip', compression=ZIP_STORED)
zf.writestr('META-INF/', '', 0700) zf.writestr('META-INF/', '', 0700)
zf.writestr('META-INF/container.xml', CONTAINER) zf.writestr('META-INF/container.xml', CONTAINER)
for path, _, data in extra_entries:
zf.writestr(path, data)
return zf return zf

34
src/calibre/ebooks/epub/input.py
View File

@ -108,6 +108,27 @@ class EPUBInput(InputFormatPlugin):
open('calibre_raster_cover.jpg', 'wb').write( open('calibre_raster_cover.jpg', 'wb').write(
renderer) renderer)
def find_opf(self):
def attr(n, attr):
for k, v in n.attrib.items():
if k.endswith(attr):
return v
try:
with open('META-INF/container.xml') as f:
root = etree.fromstring(f.read())
for r in root.xpath('//*[local-name()="rootfile"]'):
if attr(r, 'media-type') != "application/oebps-package+xml":
continue
path = attr(r, 'full-path')
if not path:
continue
path = os.path.join(os.getcwdu(), *path.split('/'))
if os.path.exists(path):
return path
except:
import traceback
traceback.print_exc()
def convert(self, stream, options, file_ext, log, accelerators): def convert(self, stream, options, file_ext, log, accelerators):
from calibre.utils.zipfile import ZipFile from calibre.utils.zipfile import ZipFile
from calibre import walk from calibre import walk
@ -116,12 +137,13 @@ class EPUBInput(InputFormatPlugin):
zf = ZipFile(stream) zf = ZipFile(stream)
zf.extractall(os.getcwd()) zf.extractall(os.getcwd())
encfile = os.path.abspath(os.path.join('META-INF', 'encryption.xml')) encfile = os.path.abspath(os.path.join('META-INF', 'encryption.xml'))
opf = None opf = self.find_opf()
for f in walk(u'.'): if opf is None:
if f.lower().endswith('.opf') and '__MACOSX' not in f and \ for f in walk(u'.'):
not os.path.basename(f).startswith('.'): if f.lower().endswith('.opf') and '__MACOSX' not in f and \
opf = os.path.abspath(f) not os.path.basename(f).startswith('.'):
break opf = os.path.abspath(f)
break
path = getattr(stream, 'name', 'stream') path = getattr(stream, 'name', 'stream')
if opf is None: if opf is None:

13
src/calibre/ebooks/epub/output.py
View File

@ -106,6 +106,7 @@ class EPUBOutput(OutputFormatPlugin):
recommendations = set([('pretty_print', True, OptionRecommendation.HIGH)]) recommendations = set([('pretty_print', True, OptionRecommendation.HIGH)])
def workaround_webkit_quirks(self): # {{{ def workaround_webkit_quirks(self): # {{{
from calibre.ebooks.oeb.base import XPath from calibre.ebooks.oeb.base import XPath
for x in self.oeb.spine: for x in self.oeb.spine:
@ -183,6 +184,12 @@ class EPUBOutput(OutputFormatPlugin):
with TemporaryDirectory('_epub_output') as tdir: with TemporaryDirectory('_epub_output') as tdir:
from calibre.customize.ui import plugin_for_output_format from calibre.customize.ui import plugin_for_output_format
metadata_xml = None
extra_entries = []
if self.is_periodical:
from calibre.ebooks.epub.periodical import sony_metadata
metadata_xml, atom_xml = sony_metadata(oeb)
extra_entries = [('atom.xml', 'application/atom+xml', atom_xml)]
oeb_output = plugin_for_output_format('oeb') oeb_output = plugin_for_output_format('oeb')
oeb_output.convert(oeb, tdir, input_plugin, opts, log) oeb_output.convert(oeb, tdir, input_plugin, opts, log)
opf = [x for x in os.listdir(tdir) if x.endswith('.opf')][0] opf = [x for x in os.listdir(tdir) if x.endswith('.opf')][0]
@ -194,10 +201,14 @@ class EPUBOutput(OutputFormatPlugin):
encryption = self.encrypt_fonts(encrypted_fonts, tdir, uuid) encryption = self.encrypt_fonts(encrypted_fonts, tdir, uuid)
from calibre.ebooks.epub import initialize_container from calibre.ebooks.epub import initialize_container
epub = initialize_container(output_path, os.path.basename(opf)) epub = initialize_container(output_path, os.path.basename(opf),
extra_entries=extra_entries)
epub.add_dir(tdir) epub.add_dir(tdir)
if encryption is not None: if encryption is not None:
epub.writestr('META-INF/encryption.xml', encryption) epub.writestr('META-INF/encryption.xml', encryption)
if metadata_xml is not None:
epub.writestr('META-INF/metadata.xml',
metadata_xml.encode('utf-8'))
if opts.extract_to is not None: if opts.extract_to is not None:
if os.path.exists(opts.extract_to): if os.path.exists(opts.extract_to):
shutil.rmtree(opts.extract_to) shutil.rmtree(opts.extract_to)

173
src/calibre/ebooks/epub/periodical.py Normal file
View File

@ -0,0 +1,173 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
__license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
from uuid import uuid4
from calibre.constants import __appname__, __version__
from calibre import strftime, prepare_string_for_xml as xml
SONY_METADATA = u'''\
<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:dcterms="http://purl.org/dc/terms/"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:prs="http://xmlns.sony.net/e-book/prs/">
<rdf:Description rdf:about="">
<dc:title>{title}</dc:title>
<dc:publisher>{publisher}</dc:publisher>
<dcterms:alternative>{short_title}</dcterms:alternative>
<dcterms:issued>{issue_date}</dcterms:issued>
<dc:language>{language}</dc:language>
<dcterms:conformsTo rdf:resource="http://xmlns.sony.net/e-book/prs/periodicals/1.0/newspaper/1.0"/>
<dcterms:type rdf:resource="http://xmlns.sony.net/e-book/prs/datatype/newspaper"/>
<dcterms:type rdf:resource="http://xmlns.sony.net/e-book/prs/datatype/periodical"/>
</rdf:Description>
</rdf:RDF>
'''
SONY_ATOM = u'''\
<?xml version="1.0" encoding="utf-8" ?>
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:dcterms="http://purl.org/dc/terms/"
xmlns:prs="http://xmlns.sony.net/e-book/prs/"
xmlns:media="http://video.search.yahoo.com/mrss"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<title>{short_title}</title>
<updated>{updated}</updated>
<id>{id}</id>
{entries}
</feed>
'''
SONY_ATOM_SECTION = u'''\
<entry rdf:ID="{title}">
<title>{title}</title>
<link href="{href}"/>
<id>{id}</id>
<updated>{updated}</updated>
<summary>{desc}</summary>
<category term="{short_title}/{title}"
scheme="http://xmlns.sony.net/e-book/terms/" label="{title}"/>
<dc:type xsi:type="prs:datatype">newspaper/section</dc:type>
<dcterms:isReferencedBy rdf:resource=""/>
</entry>
'''
SONY_ATOM_ENTRY = u'''\
<entry>
<title>{title}</title>
<author><name>{author}</name></author>
<link href="{href}"/>
<id>{id}</id>
<updated>{updated}</updated>
<summary>{desc}</summary>
<category term="{short_title}/{section_title}"
scheme="http://xmlns.sony.net/e-book/terms/" label="{section_title}"/>
<dcterms:extent xsi:type="prs:word-count">{word_count}</dcterms:extent>
<dc:type xsi:type="prs:datatype">newspaper/article</dc:type>
<dcterms:isReferencedBy rdf:resource="#{section_title}"/>
</entry>
'''
def sony_metadata(oeb):
m = oeb.metadata
title = short_title = unicode(m.title[0])
publisher = __appname__ + ' ' + __version__
try:
pt = unicode(oeb.metadata.publication_type[0])
short_title = u':'.join(pt.split(':')[2:])
except:
pass
try:
date = unicode(m.date[0]).split('T')[0]
except:
date = strftime('%Y-%m-%d')
try:
language = unicode(m.language[0]).replace('_', '-')
except:
language = 'en'
short_title = xml(short_title, True)
metadata = SONY_METADATA.format(title=xml(title),
short_title=short_title,
publisher=xml(publisher), issue_date=xml(date),
language=xml(language))
updated = strftime('%Y-%m-%dT%H:%M:%SZ')
def cal_id(x):
for k, v in x.attrib.items():
if k.endswith('scheme') and v == 'uuid':
return True
try:
base_id = unicode(list(filter(cal_id, m.identifier))[0])
except:
base_id = str(uuid4())
entries = []
seen_titles = set([])
for i, section in enumerate(oeb.toc):
if not section.href:
continue
secid = 'section%d'%i
sectitle = section.title
if not sectitle:
sectitle = _('Unknown')
d = 1
bsectitle = sectitle
while sectitle in seen_titles:
sectitle = bsectitle + ' ' + str(d)
d += 1
seen_titles.add(sectitle)
sectitle = xml(sectitle, True)
secdesc = section.description
if not secdesc:
secdesc = ''
secdesc = xml(secdesc)
entries.append(SONY_ATOM_SECTION.format(title=sectitle,
href=section.href, id=xml(base_id)+'/'+secid,
short_title=short_title, desc=secdesc, updated=updated))
for j, article in enumerate(section):
if not article.href:
continue
atitle = article.title
btitle = atitle
d = 1
while atitle in seen_titles:
atitle = btitle + ' ' + str(d)
d += 1
auth = article.author if article.author else ''
desc = section.description
if not desc:
desc = ''
aid = 'article%d'%j
entries.append(SONY_ATOM_ENTRY.format(
title=xml(atitle),
author=xml(auth),
updated=updated,
desc=desc,
short_title=short_title,
section_title=sectitle,
href=article.href,
word_count=str(1),
id=xml(base_id)+'/'+secid+'/'+aid
))
atom = SONY_ATOM.format(short_title=short_title,
entries='\n\n'.join(entries), updated=updated,
id=xml(base_id)).encode('utf-8')
return metadata, atom

4
src/calibre/ebooks/metadata/book/base.py
View File

@ -43,7 +43,7 @@ class SafeFormat(TemplateFormatter):
b = self.book.get_user_metadata(key, False) b = self.book.get_user_metadata(key, False)
if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0: if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
v = '' v = ''
elif b and b['datatype'] == 'float' and b.get(key, 0.0) == 0.0: elif b and b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0:
v = '' v = ''
else: else:
ign, v = self.book.format_field(key.lower(), series_with_index=False) ign, v = self.book.format_field(key.lower(), series_with_index=False)
@ -501,7 +501,7 @@ class Metadata(object):
if key.startswith('#') and key.endswith('_index'): if key.startswith('#') and key.endswith('_index'):
tkey = key[:-6] # strip the _index tkey = key[:-6] # strip the _index
cmeta = self.get_user_metadata(tkey, make_copy=False) cmeta = self.get_user_metadata(tkey, make_copy=False)
if cmeta['datatype'] == 'series': if cmeta and cmeta['datatype'] == 'series':
if self.get(tkey): if self.get(tkey):
res = self.get_extra(tkey) res = self.get_extra(tkey)
return (unicode(cmeta['name']+'_index'), return (unicode(cmeta['name']+'_index'),

25
src/calibre/ebooks/metadata/opf2.py
View File

@ -382,11 +382,13 @@ class Guide(ResourceCollection): # {{{
class MetadataField(object): class MetadataField(object):
def __init__(self, name, is_dc=True, formatter=None, none_is=None): def __init__(self, name, is_dc=True, formatter=None, none_is=None,
renderer=lambda x: unicode(x)):
self.name = name self.name = name
self.is_dc = is_dc self.is_dc = is_dc
self.formatter = formatter self.formatter = formatter
self.none_is = none_is self.none_is = none_is
self.renderer = renderer
def __real_get__(self, obj, type=None): def __real_get__(self, obj, type=None):
ans = obj.get_metadata_element(self.name) ans = obj.get_metadata_element(self.name)
@ -418,7 +420,7 @@ class MetadataField(object):
return return
if elem is None: if elem is None:
elem = obj.create_metadata_element(self.name, is_dc=self.is_dc) elem = obj.create_metadata_element(self.name, is_dc=self.is_dc)
obj.set_text(elem, unicode(val)) obj.set_text(elem, self.renderer(val))
def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)): def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)):
@ -489,10 +491,11 @@ class OPF(object): # {{{
series = MetadataField('series', is_dc=False) series = MetadataField('series', is_dc=False)
series_index = MetadataField('series_index', is_dc=False, formatter=float, none_is=1) series_index = MetadataField('series_index', is_dc=False, formatter=float, none_is=1)
rating = MetadataField('rating', is_dc=False, formatter=int) rating = MetadataField('rating', is_dc=False, formatter=int)
pubdate = MetadataField('date', formatter=parse_date) pubdate = MetadataField('date', formatter=parse_date,
renderer=isoformat)
publication_type = MetadataField('publication_type', is_dc=False) publication_type = MetadataField('publication_type', is_dc=False)
timestamp = MetadataField('timestamp', is_dc=False, timestamp = MetadataField('timestamp', is_dc=False,
formatter=parse_date) formatter=parse_date, renderer=isoformat)
def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True, def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True,
@ -826,11 +829,10 @@ class OPF(object): # {{{
def fset(self, val): def fset(self, val):
matches = self.isbn_path(self.metadata) matches = self.isbn_path(self.metadata)
if val is None: if not val:
if matches: for x in matches:
for x in matches: x.getparent().remove(x)
x.getparent().remove(x) return
return
if not matches: if not matches:
attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'ISBN'} attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'ISBN'}
matches = [self.create_metadata_element('identifier', matches = [self.create_metadata_element('identifier',
@ -987,11 +989,14 @@ class OPF(object): # {{{
def smart_update(self, mi, replace_metadata=False): def smart_update(self, mi, replace_metadata=False):
for attr in ('title', 'authors', 'author_sort', 'title_sort', for attr in ('title', 'authors', 'author_sort', 'title_sort',
'publisher', 'series', 'series_index', 'rating', 'publisher', 'series', 'series_index', 'rating',
'isbn', 'language', 'tags', 'category', 'comments', 'isbn', 'tags', 'category', 'comments',
'pubdate'): 'pubdate'):
val = getattr(mi, attr, None) val = getattr(mi, attr, None)
if val is not None and val != [] and val != (None, None): if val is not None and val != [] and val != (None, None):
setattr(self, attr, val) setattr(self, attr, val)
lang = getattr(mi, 'language', None)
if lang and lang != 'und':
self.language = lang
temp = self.to_book_metadata() temp = self.to_book_metadata()
temp.smart_update(mi, replace_metadata=replace_metadata) temp.smart_update(mi, replace_metadata=replace_metadata)
self._user_metadata_ = temp.get_all_user_metadata(True) self._user_metadata_ = temp.get_all_user_metadata(True)

9
src/calibre/ebooks/mobi/output.py
View File

@ -42,11 +42,10 @@ class MOBIOutput(OutputFormatPlugin):
]) ])
def check_for_periodical(self): def check_for_periodical(self):
if self.oeb.metadata.publication_type and \ if self.is_periodical:
unicode(self.oeb.metadata.publication_type[0]).startswith('periodical:'): self.periodicalize_toc()
self.periodicalize_toc() self.check_for_masthead()
self.check_for_masthead() self.opts.mobi_periodical = True
self.opts.mobi_periodical = True
else: else:
self.opts.mobi_periodical = False self.opts.mobi_periodical = False

50
src/calibre/ebooks/oeb/transforms/rescale.py
View File

@ -6,8 +6,6 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import cStringIO
from calibre import fit_image from calibre import fit_image
class RescaleImages(object): class RescaleImages(object):
@ -19,13 +17,7 @@ class RescaleImages(object):
self.rescale(qt=is_ok_to_use_qt()) self.rescale(qt=is_ok_to_use_qt())
def rescale(self, qt=True): def rescale(self, qt=True):
from PyQt4.Qt import QImage, Qt from calibre.utils.magick.draw import Image
from calibre.gui2 import pixmap_to_data
try:
from PIL import Image as PILImage
PILImage
except ImportError:
import Image as PILImage
is_image_collection = getattr(self.opts, 'is_image_collection', False) is_image_collection = getattr(self.opts, 'is_image_collection', False)
@ -35,6 +27,7 @@ class RescaleImages(object):
page_width, page_height = self.opts.dest.width, self.opts.dest.height page_width, page_height = self.opts.dest.width, self.opts.dest.height
page_width -= (self.opts.margin_left + self.opts.margin_right) * self.opts.dest.dpi/72. page_width -= (self.opts.margin_left + self.opts.margin_right) * self.opts.dest.dpi/72.
page_height -= (self.opts.margin_top + self.opts.margin_bottom) * self.opts.dest.dpi/72. page_height -= (self.opts.margin_top + self.opts.margin_bottom) * self.opts.dest.dpi/72.
for item in self.oeb.manifest: for item in self.oeb.manifest:
if item.media_type.startswith('image'): if item.media_type.startswith('image'):
ext = item.media_type.split('/')[-1].upper() ext = item.media_type.split('/')[-1].upper()
@ -44,42 +37,25 @@ class RescaleImages(object):
raw = item.data raw = item.data
if not raw: continue if not raw: continue
if qt: try:
img = QImage(10, 10, QImage.Format_ARGB32_Premultiplied) img = Image()
try: img.load(raw)
if not img.loadFromData(raw): continue except:
except: continue
continue width, height = img.size
width, height = img.width(), img.height()
else:
f = cStringIO.StringIO(raw)
try:
im = PILImage.open(f)
except IOError:
continue
width, height = im.size
scaled, new_width, new_height = fit_image(width, height, scaled, new_width, new_height = fit_image(width, height,
page_width, page_height) page_width, page_height)
if scaled: if scaled:
data = None
self.log('Rescaling image from %dx%d to %dx%d'%( self.log('Rescaling image from %dx%d to %dx%d'%(
width, height, new_width, new_height), item.href) width, height, new_width, new_height), item.href)
if qt: try:
img = img.scaled(new_width, new_height, img.size = (new_width, new_height)
Qt.IgnoreAspectRatio, Qt.SmoothTransformation) data = img.export(ext.lower())
data = pixmap_to_data(img, format=ext) except:
self.log.exception('Failed to rescale image')
else: else:
try:
im = im.resize((int(new_width), int(new_height)), PILImage.ANTIALIAS)
of = cStringIO.StringIO()
im.convert('RGB').save(of, ext)
data = of.getvalue()
except:
self.log.exception('Failed to rescale image')
if data is not None:
item.data = data item.data = data
item.unload_data_from_memory() item.unload_data_from_memory()

4
src/calibre/gui2/__init__.py
View File

@ -514,7 +514,7 @@ class FileDialog(QObject):
if f and os.path.exists(f): if f and os.path.exists(f):
self.selected_files.append(f) self.selected_files.append(f)
else: else:
opts = QFileDialog.ShowDirsOnly if mode == QFileDialog.DirectoryOnly else QFileDialog.Option() opts = QFileDialog.ShowDirsOnly if mode == QFileDialog.Directory else QFileDialog.Option()
f = unicode(QFileDialog.getExistingDirectory(parent, title, initial_dir, opts)) f = unicode(QFileDialog.getExistingDirectory(parent, title, initial_dir, opts))
if os.path.exists(f): if os.path.exists(f):
self.selected_files.append(f) self.selected_files.append(f)
@ -534,7 +534,7 @@ class FileDialog(QObject):
def choose_dir(window, name, title, default_dir='~'): def choose_dir(window, name, title, default_dir='~'):
fd = FileDialog(title=title, filters=[], add_all_files_filter=False, fd = FileDialog(title=title, filters=[], add_all_files_filter=False,
parent=window, name=name, mode=QFileDialog.DirectoryOnly, parent=window, name=name, mode=QFileDialog.Directory,
default_dir=default_dir) default_dir=default_dir)
dir = fd.get_files() dir = fd.get_files()
if dir: if dir:

5
src/calibre/gui2/actions/fetch_news.py
View File

@ -9,7 +9,6 @@ from PyQt4.Qt import Qt
from calibre.gui2 import Dispatcher from calibre.gui2 import Dispatcher
from calibre.gui2.tools import fetch_scheduled_recipe from calibre.gui2.tools import fetch_scheduled_recipe
from calibre.utils.config import dynamic
from calibre.gui2.actions import InterfaceAction from calibre.gui2.actions import InterfaceAction
class FetchNewsAction(InterfaceAction): class FetchNewsAction(InterfaceAction):
@ -60,9 +59,9 @@ class FetchNewsAction(InterfaceAction):
return self.gui.job_exception(job) return self.gui.job_exception(job)
id = self.gui.library_view.model().add_news(pt.name, arg) id = self.gui.library_view.model().add_news(pt.name, arg)
self.gui.library_view.model().reset() self.gui.library_view.model().reset()
sync = dynamic.get('news_to_be_synced', set([])) sync = self.gui.news_to_be_synced
sync.add(id) sync.add(id)
dynamic.set('news_to_be_synced', sync) self.gui.news_to_be_synced = sync
self.scheduler.recipe_downloaded(arg) self.scheduler.recipe_downloaded(arg)
self.gui.status_bar.show_message(arg['title'] + _(' fetched.'), 3000) self.gui.status_bar.show_message(arg['title'] + _(' fetched.'), 3000)
self.gui.email_news(id) self.gui.email_news(id)

33
src/calibre/gui2/custom_column_widgets.py
View File

@ -429,7 +429,38 @@ class BulkBase(Base):
self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify) self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify)
class BulkBool(BulkBase, Bool): class BulkBool(BulkBase, Bool):
pass
def get_initial_value(self, book_ids):
value = None
for book_id in book_ids:
val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None:
val = False
if value is not None and value != val:
return None
value = val
return value
def setup_ui(self, parent):
self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent),
QComboBox(parent)]
w = self.widgets[1]
items = [_('Yes'), _('No'), _('Undefined')]
icons = [I('ok.png'), I('list_remove.png'), I('blank.png')]
for icon, text in zip(icons, items):
w.addItem(QIcon(icon), text)
def setter(self, val):
val = {None: 2, False: 1, True: 0}[val]
self.widgets[1].setCurrentIndex(val)
def commit(self, book_ids, notify=False):
val = self.gui_val
val = self.normalize_ui_val(val)
if val != self.initial_val:
if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None:
val = False
self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify)
class BulkInt(BulkBase, Int): class BulkInt(BulkBase, Int):
pass pass

34
src/calibre/gui2/device.py
View File

@ -1102,12 +1102,35 @@ class DeviceMixin(object): # {{{
self.status_bar.show_message(_('Sending catalogs to device.'), 5000) self.status_bar.show_message(_('Sending catalogs to device.'), 5000)
@dynamic_property
def news_to_be_synced(self):
doc = 'Set of ids to be sent to device'
def fget(self):
ans = []
try:
ans = self.library_view.model().db.prefs.get('news_to_be_synced',
[])
except:
import traceback
traceback.print_exc()
return set(ans)
def fset(self, ids):
try:
self.library_view.model().db.prefs.set('news_to_be_synced',
list(ids))
except:
import traceback
traceback.print_exc()
return property(fget=fget, fset=fset, doc=doc)
def sync_news(self, send_ids=None, do_auto_convert=True): def sync_news(self, send_ids=None, do_auto_convert=True):
if self.device_connected: if self.device_connected:
del_on_upload = config['delete_news_from_library_on_upload'] del_on_upload = config['delete_news_from_library_on_upload']
settings = self.device_manager.device.settings() settings = self.device_manager.device.settings()
ids = list(dynamic.get('news_to_be_synced', set([]))) if send_ids is None else send_ids ids = list(self.news_to_be_synced) if send_ids is None else send_ids
ids = [id for id in ids if self.library_view.model().db.has_id(id)] ids = [id for id in ids if self.library_view.model().db.has_id(id)]
files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids( files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(
ids, settings.format_map, ids, settings.format_map,
@ -1139,7 +1162,7 @@ class DeviceMixin(object): # {{{
for f in files: for f in files:
f.deleted_after_upload = del_on_upload f.deleted_after_upload = del_on_upload
if not files: if not files:
dynamic.set('news_to_be_synced', set([])) self.news_to_be_synced = set([])
return return
metadata = self.library_view.model().metadata_for(ids) metadata = self.library_view.model().metadata_for(ids)
names = [] names = []
@ -1153,7 +1176,7 @@ class DeviceMixin(object): # {{{
if mi.cover and os.access(mi.cover, os.R_OK): if mi.cover and os.access(mi.cover, os.R_OK):
mi.thumbnail = self.cover_to_thumbnail(open(mi.cover, mi.thumbnail = self.cover_to_thumbnail(open(mi.cover,
'rb').read()) 'rb').read())
dynamic.set('news_to_be_synced', set([])) self.news_to_be_synced = set([])
if config['upload_news_to_device'] and files: if config['upload_news_to_device'] and files:
remove = ids if del_on_upload else [] remove = ids if del_on_upload else []
space = { self.location_manager.free[0] : None, space = { self.location_manager.free[0] : None,
@ -1347,8 +1370,9 @@ class DeviceMixin(object): # {{{
# If it does not, then do it here. # If it does not, then do it here.
if not self.set_books_in_library(self.booklists(), reset=True): if not self.set_books_in_library(self.booklists(), reset=True):
self.upload_booklists() self.upload_booklists()
self.book_on_device(None, reset=True) with self.library_view.preserve_selected_books:
self.refresh_ondevice() self.book_on_device(None, reset=True)
self.refresh_ondevice()
view = self.card_a_view if on_card == 'carda' else \ view = self.card_a_view if on_card == 'carda' else \
self.card_b_view if on_card == 'cardb' else self.memory_view self.card_b_view if on_card == 'cardb' else self.memory_view

7
src/calibre/gui2/dialogs/book_info.py
View File

@ -90,10 +90,15 @@ class BookInfo(QDialog, Ui_BookInfo):
row = row.row() row = row.row()
if row == self.current_row: if row == self.current_row:
return return
info = self.view.model().get_book_info(row)
if info is None:
# Indicates books was deleted from library, or row numbers have
# changed
return
self.previous_button.setEnabled(False if row == 0 else True) self.previous_button.setEnabled(False if row == 0 else True)
self.next_button.setEnabled(False if row == self.view.model().rowCount(QModelIndex())-1 else True) self.next_button.setEnabled(False if row == self.view.model().rowCount(QModelIndex())-1 else True)
self.current_row = row self.current_row = row
info = self.view.model().get_book_info(row)
self.setWindowTitle(info[_('Title')]) self.setWindowTitle(info[_('Title')])
self.title.setText('<b>'+info.pop(_('Title'))) self.title.setText('<b>'+info.pop(_('Title')))
comments = info.pop(_('Comments'), '') comments = info.pop(_('Comments'), '')

73
src/calibre/gui2/dialogs/check_library.py
View File

@ -3,11 +3,16 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
__license__ = 'GPL v3' __license__ = 'GPL v3'
import os
from PyQt4.Qt import QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel, \ from PyQt4.Qt import QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel, \
QPushButton, QDialogButtonBox, QApplication, QTreeWidgetItem, \ QPushButton, QDialogButtonBox, QApplication, QTreeWidgetItem, \
QLineEdit QLineEdit, Qt
from calibre.gui2.dialogs.confirm_delete import confirm
from calibre.library.check_library import CheckLibrary, CHECKS from calibre.library.check_library import CheckLibrary, CHECKS
from calibre.library.database2 import delete_file, delete_tree
from calibre import prints
class Item(QTreeWidgetItem): class Item(QTreeWidgetItem):
pass pass
@ -24,24 +29,25 @@ class CheckLibraryDialog(QDialog):
self.setLayout(self._layout) self.setLayout(self._layout)
self.log = QTreeWidget(self) self.log = QTreeWidget(self)
self.log.itemChanged.connect(self.item_changed)
self._layout.addWidget(self.log) self._layout.addWidget(self.log)
self.check = QPushButton(_('Run the check')) self.check = QPushButton(_('&Run the check'))
self.check.setDefault(False) self.check.setDefault(False)
self.check.clicked.connect(self.run_the_check) self.check.clicked.connect(self.run_the_check)
self.copy = QPushButton(_('Copy to clipboard')) self.copy = QPushButton(_('Copy &to clipboard'))
self.copy.setDefault(False) self.copy.setDefault(False)
self.copy.clicked.connect(self.copy_to_clipboard) self.copy.clicked.connect(self.copy_to_clipboard)
self.ok = QPushButton('&Done') self.ok = QPushButton('&Done')
self.ok.setDefault(True) self.ok.setDefault(True)
self.ok.clicked.connect(self.accept) self.ok.clicked.connect(self.accept)
self.cancel = QPushButton('&Cancel') self.delete = QPushButton('Delete &marked')
self.cancel.setDefault(False) self.delete.setDefault(False)
self.cancel.clicked.connect(self.reject) self.delete.clicked.connect(self.delete_marked)
self.bbox = QDialogButtonBox(self) self.bbox = QDialogButtonBox(self)
self.bbox.addButton(self.copy, QDialogButtonBox.ActionRole)
self.bbox.addButton(self.check, QDialogButtonBox.ActionRole) self.bbox.addButton(self.check, QDialogButtonBox.ActionRole)
self.bbox.addButton(self.cancel, QDialogButtonBox.RejectRole) self.bbox.addButton(self.delete, QDialogButtonBox.ActionRole)
self.bbox.addButton(self.copy, QDialogButtonBox.ActionRole)
self.bbox.addButton(self.ok, QDialogButtonBox.AcceptRole) self.bbox.addButton(self.ok, QDialogButtonBox.AcceptRole)
h = QHBoxLayout() h = QHBoxLayout()
@ -83,35 +89,70 @@ class CheckLibraryDialog(QDialog):
plaintext = [] plaintext = []
def builder(tree, checker, check): def builder(tree, checker, check):
attr = check[0] attr, h, checkable = check
list = getattr(checker, attr, None) list = getattr(checker, attr, None)
if list is None: if list is None:
return return
h = check[1]
tl = Item([h]) tl = Item([h])
for problem in list: for problem in list:
it = Item() it = Item()
if checkable:
it.setFlags(Qt.ItemIsEnabled | Qt.ItemIsUserCheckable)
it.setCheckState(1, False)
else:
it.setFlags(Qt.ItemIsEnabled)
it.setText(0, problem[0]) it.setText(0, problem[0])
it.setText(1, problem[1]) it.setText(1, problem[1])
p = ', '.join(problem[2])
it.setText(2, p)
tl.addChild(it) tl.addChild(it)
plaintext.append(','.join([h, problem[0], problem[1], p])) self.all_items.append(it)
plaintext.append(','.join([h, problem[0], problem[1]]))
tree.addTopLevelItem(tl) tree.addTopLevelItem(tl)
t = self.log t = self.log
t.clear() t.clear()
t.setColumnCount(3); t.setColumnCount(2);
t.setHeaderLabels([_('Name'), _('Path from library'), _('Additional Information')]) t.setHeaderLabels([_('Name'), _('Path from library')])
self.all_items = []
for check in CHECKS: for check in CHECKS:
builder(t, checker, check) builder(t, checker, check)
t.setColumnWidth(0, 200) t.setColumnWidth(0, 200)
t.setColumnWidth(1, 400) t.setColumnWidth(1, 400)
self.delete.setEnabled(False)
self.text_results = '\n'.join(plaintext) self.text_results = '\n'.join(plaintext)
def item_changed(self, item, column):
for it in self.all_items:
if it.checkState(1):
self.delete.setEnabled(True)
return
def delete_marked(self):
if not confirm('<p>'+_('The marked files and folders will be '
'<b>permanently deleted</b>. Are you sure?')
+'</p>', 'check_library_editor_delete', self):
return
# Sort the paths in reverse length order so that we can be sure that
# if an item is in another item, the sub-item will be deleted first.
items = sorted(self.all_items,
key=lambda x: len(x.text(1)),
reverse=True)
for it in items:
if it.checkState(1):
try:
p = os.path.join(self.db.library_path ,unicode(it.text(1)))
if os.path.isdir(p):
delete_tree(p)
else:
delete_file(p)
except:
prints('failed to delete',
os.path.join(self.db.library_path,
unicode(it.text(1))))
self.run_the_check()
def copy_to_clipboard(self): def copy_to_clipboard(self):
QApplication.clipboard().setText(self.text_results) QApplication.clipboard().setText(self.text_results)

3
src/calibre/gui2/dialogs/fetch_metadata.py
View File

@ -190,7 +190,8 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
if self.model.rowCount() < 1: if self.model.rowCount() < 1:
info_dialog(self, _('No metadata found'), info_dialog(self, _('No metadata found'),
_('No metadata found, try adjusting the title and author ' _('No metadata found, try adjusting the title and author '
'or the ISBN key.')).exec_() 'and/or removing the ISBN.')).exec_()
self.reject()
return return
self.matches.setModel(self.model) self.matches.setModel(self.model)

14
src/calibre/gui2/dialogs/metadata_bulk.py
View File

@ -16,6 +16,7 @@ from calibre.gui2.custom_column_widgets import populate_metadata_page
from calibre.gui2 import error_dialog from calibre.gui2 import error_dialog
from calibre.gui2.progress_indicator import ProgressIndicator from calibre.gui2.progress_indicator import ProgressIndicator
from calibre.utils.config import dynamic from calibre.utils.config import dynamic
from calibre.utils.titlecase import titlecase
class MyBlockingBusy(QDialog): class MyBlockingBusy(QDialog):
@ -50,6 +51,7 @@ class MyBlockingBusy(QDialog):
self.start() self.start()
self.args = args self.args = args
self.series_start_value = None
self.db = db self.db = db
self.ids = ids self.ids = ids
self.error = None self.error = None
@ -115,7 +117,7 @@ class MyBlockingBusy(QDialog):
aum = [a.strip().replace('|', ',') for a in aum.split(',')] aum = [a.strip().replace('|', ',') for a in aum.split(',')]
new_title = authors_to_string(aum) new_title = authors_to_string(aum)
if do_title_case: if do_title_case:
new_title = new_title.title() new_title = titlecase(new_title)
self.db.set_title(id, new_title, notify=False) self.db.set_title(id, new_title, notify=False)
title_set = True title_set = True
if title: if title:
@ -123,7 +125,7 @@ class MyBlockingBusy(QDialog):
self.db.set_authors(id, new_authors, notify=False) self.db.set_authors(id, new_authors, notify=False)
if do_title_case and not title_set: if do_title_case and not title_set:
title = self.db.title(id, index_is_id=True) title = self.db.title(id, index_is_id=True)
self.db.set_title(id, title.title(), notify=False) self.db.set_title(id, titlecase(title), notify=False)
if au: if au:
self.db.set_authors(id, string_to_authors(au), notify=False) self.db.set_authors(id, string_to_authors(au), notify=False)
elif self.current_phase == 2: elif self.current_phase == 2:
@ -147,8 +149,10 @@ class MyBlockingBusy(QDialog):
if do_series: if do_series:
if do_series_restart: if do_series_restart:
next = series_start_value if self.series_start_value is None:
series_start_value += 1 self.series_start_value = series_start_value
next = self.series_start_value
self.series_start_value += 1
else: else:
next = self.db.get_next_series_num_for(series) next = self.db.get_next_series_num_for(series)
self.db.set_series(id, series, notify=False, commit=False) self.db.set_series(id, series, notify=False, commit=False)
@ -179,7 +183,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
s_r_functions = { '' : lambda x: x, s_r_functions = { '' : lambda x: x,
_('Lower Case') : lambda x: x.lower(), _('Lower Case') : lambda x: x.lower(),
_('Upper Case') : lambda x: x.upper(), _('Upper Case') : lambda x: x.upper(),
_('Title Case') : lambda x: x.title(), _('Title Case') : lambda x: titlecase(x),
} }
s_r_match_modes = [ _('Character match'), s_r_match_modes = [ _('Character match'),

28
src/calibre/gui2/library/models.py
View File

@ -374,6 +374,8 @@ class BooksModel(QAbstractTableModel): # {{{
if isinstance(index, int): if isinstance(index, int):
index = self.index(index, 0) index = self.index(index, 0)
data = self.current_changed(index, None, False) data = self.current_changed(index, None, False)
if data is None:
return data
row = index.row() row = index.row()
data[_('Title')] = self.db.title(row) data[_('Title')] = self.db.title(row)
au = self.db.authors(row) au = self.db.authors(row)
@ -783,18 +785,22 @@ class BooksModel(QAbstractTableModel): # {{{
self.db.set_rating(id, val) self.db.set_rating(id, val)
elif column == 'series': elif column == 'series':
val = val.strip() val = val.strip()
pat = re.compile(r'\[([.0-9]+)\]') if not val:
match = pat.search(val)
if match is not None:
self.db.set_series_index(id, float(match.group(1)))
val = pat.sub('', val).strip()
elif val:
if tweaks['series_index_auto_increment'] == 'next':
ni = self.db.get_next_series_num_for(val)
if ni != 1:
self.db.set_series_index(id, ni)
if val:
self.db.set_series(id, val) self.db.set_series(id, val)
self.db.set_series_index(id, 1.0)
else:
pat = re.compile(r'\[([.0-9]+)\]')
match = pat.search(val)
if match is not None:
self.db.set_series_index(id, float(match.group(1)))
val = pat.sub('', val).strip()
elif val:
if tweaks['series_index_auto_increment'] == 'next':
ni = self.db.get_next_series_num_for(val)
if ni != 1:
self.db.set_series_index(id, ni)
if val:
self.db.set_series(id, val)
elif column == 'timestamp': elif column == 'timestamp':
if val.isNull() or not val.isValid(): if val.isNull() or not val.isValid():
return False return False

31
src/calibre/gui2/library/views.py
View File

@ -22,6 +22,26 @@ from calibre.gui2.library import DEFAULT_SORT
from calibre.constants import filesystem_encoding from calibre.constants import filesystem_encoding
from calibre import force_unicode from calibre import force_unicode
class PreserveSelection(object): # {{{
'''
Save the set of selected books at enter time. If at exit time there are no
selected books, restore the previous selection.
'''
def __init__(self, view):
self.view = view
self.selected_ids = []
def __enter__(self):
self.selected_ids = self.view.get_selected_ids()
def __exit__(self, *args):
current = self.view.get_selected_ids()
if not current:
self.view.select_rows(self.selected_ids, using_ids=True)
# }}}
class BooksView(QTableView): # {{{ class BooksView(QTableView): # {{{
files_dropped = pyqtSignal(object) files_dropped = pyqtSignal(object)
@ -58,6 +78,7 @@ class BooksView(QTableView): # {{{
self.setSelectionBehavior(QAbstractItemView.SelectRows) self.setSelectionBehavior(QAbstractItemView.SelectRows)
self.setSortingEnabled(True) self.setSortingEnabled(True)
self.selectionModel().currentRowChanged.connect(self._model.current_changed) self.selectionModel().currentRowChanged.connect(self._model.current_changed)
self.preserve_selected_books = PreserveSelection(self)
# {{{ Column Header setup # {{{ Column Header setup
self.can_add_columns = True self.can_add_columns = True
@ -613,6 +634,16 @@ class BooksView(QTableView): # {{{
sel.select(m.index(row, 0), m.index(row, max_col)) sel.select(m.index(row, 0), m.index(row, max_col))
sm.select(sel, sm.ClearAndSelect) sm.select(sel, sm.ClearAndSelect)
def get_selected_ids(self):
ans = []
m = self.model()
for idx in self.selectedIndexes():
r = idx.row()
i = m.id(r)
if i not in ans:
ans.append(i)
return ans
def close(self): def close(self):
self._model.close() self._model.close()

189
src/calibre/gui2/lrf_renderer/main.ui
View File

@ -1,7 +1,8 @@
<ui version="4.0" > <?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>MainWindow</class> <class>MainWindow</class>
<widget class="QMainWindow" name="MainWindow" > <widget class="QMainWindow" name="MainWindow">
<property name="geometry" > <property name="geometry">
<rect> <rect>
<x>0</x> <x>0</x>
<y>0</y> <y>0</y>
@ -9,75 +10,51 @@
<height>701</height> <height>701</height>
</rect> </rect>
</property> </property>
<property name="sizePolicy" > <property name="sizePolicy">
<sizepolicy vsizetype="Minimum" hsizetype="Minimum" > <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
<horstretch>0</horstretch> <horstretch>0</horstretch>
<verstretch>0</verstretch> <verstretch>0</verstretch>
</sizepolicy> </sizepolicy>
</property> </property>
<property name="windowTitle" > <property name="windowTitle">
<string>LRF Viewer</string> <string>LRF Viewer</string>
</property> </property>
<property name="windowIcon" > <property name="windowIcon">
<iconset resource="../../../../resources/images.qrc" > <iconset>
<normaloff>:/images/viewer.png</normaloff>:/images/viewer.png</iconset> <normaloff>:/images/viewer.png</normaloff>:/images/viewer.png</iconset>
</property> </property>
<widget class="QWidget" name="central_widget" > <widget class="QWidget" name="central_widget">
<property name="geometry" > <layout class="QVBoxLayout">
<rect> <property name="margin">
<x>0</x>
<y>39</y>
<width>601</width>
<height>662</height>
</rect>
</property>
<layout class="QVBoxLayout" >
<property name="margin" >
<number>0</number> <number>0</number>
</property> </property>
<item> <item>
<widget class="QStackedWidget" name="stack" > <widget class="QStackedWidget" name="stack">
<property name="currentIndex" > <property name="currentIndex">
<number>0</number> <number>0</number>
</property> </property>
<widget class="QWidget" name="viewer_page" > <widget class="QWidget" name="viewer_page">
<property name="geometry" > <layout class="QGridLayout">
<rect> <property name="margin">
<x>0</x>
<y>0</y>
<width>601</width>
<height>662</height>
</rect>
</property>
<layout class="QGridLayout" >
<property name="margin" >
<number>0</number> <number>0</number>
</property> </property>
<item row="0" column="0" > <item row="0" column="0">
<widget class="BookView" name="graphics_view" > <widget class="BookView" name="graphics_view">
<property name="mouseTracking" > <property name="mouseTracking">
<bool>true</bool> <bool>true</bool>
</property> </property>
</widget> </widget>
</item> </item>
</layout> </layout>
</widget> </widget>
<widget class="QWidget" name="bar_page" > <widget class="QWidget" name="bar_page">
<property name="geometry" > <layout class="QVBoxLayout">
<rect>
<x>0</x>
<y>0</y>
<width>601</width>
<height>701</height>
</rect>
</property>
<layout class="QVBoxLayout" >
<item> <item>
<spacer> <spacer>
<property name="orientation" > <property name="orientation">
<enum>Qt::Vertical</enum> <enum>Qt::Vertical</enum>
</property> </property>
<property name="sizeHint" stdset="0" > <property name="sizeHint" stdset="0">
<size> <size>
<width>20</width> <width>20</width>
<height>40</height> <height>40</height>
@ -86,34 +63,34 @@
</spacer> </spacer>
</item> </item>
<item> <item>
<widget class="QFrame" name="frame_2" > <widget class="QFrame" name="frame_2">
<property name="frameShape" > <property name="frameShape">
<enum>QFrame::StyledPanel</enum> <enum>QFrame::StyledPanel</enum>
</property> </property>
<property name="frameShadow" > <property name="frameShadow">
<enum>QFrame::Raised</enum> <enum>QFrame::Raised</enum>
</property> </property>
<layout class="QVBoxLayout" > <layout class="QVBoxLayout">
<item> <item>
<widget class="QProgressBar" name="progress_bar" > <widget class="QProgressBar" name="progress_bar">
<property name="maximum" > <property name="maximum">
<number>0</number> <number>0</number>
</property> </property>
<property name="value" > <property name="value">
<number>-1</number> <number>-1</number>
</property> </property>
</widget> </widget>
</item> </item>
<item> <item>
<widget class="QLabel" name="progress_label" > <widget class="QLabel" name="progress_label">
<property name="font" > <property name="font">
<font> <font>
<pointsize>11</pointsize> <pointsize>11</pointsize>
<weight>75</weight> <weight>75</weight>
<bold>true</bold> <bold>true</bold>
</font> </font>
</property> </property>
<property name="text" > <property name="text">
<string>Parsing LRF file</string> <string>Parsing LRF file</string>
</property> </property>
</widget> </widget>
@ -123,10 +100,10 @@
</item> </item>
<item> <item>
<spacer> <spacer>
<property name="orientation" > <property name="orientation">
<enum>Qt::Vertical</enum> <enum>Qt::Vertical</enum>
</property> </property>
<property name="sizeHint" stdset="0" > <property name="sizeHint" stdset="0">
<size> <size>
<width>20</width> <width>20</width>
<height>40</height> <height>40</height>
@ -140,93 +117,85 @@
</item> </item>
</layout> </layout>
</widget> </widget>
<widget class="QToolBar" name="tool_bar" > <widget class="QToolBar" name="tool_bar">
<property name="geometry" > <property name="windowTitle">
<rect>
<x>0</x>
<y>0</y>
<width>601</width>
<height>39</height>
</rect>
</property>
<property name="windowTitle" >
<string>LRF Viewer toolbar</string> <string>LRF Viewer toolbar</string>
</property> </property>
<property name="allowedAreas" > <property name="allowedAreas">
<set>Qt::AllToolBarAreas</set> <set>Qt::AllToolBarAreas</set>
</property> </property>
<attribute name="toolBarArea" > <attribute name="toolBarArea">
<enum>TopToolBarArea</enum> <enum>Qt::TopToolBarArea</enum>
</attribute> </attribute>
<attribute name="toolBarBreak" > <attribute name="toolBarBreak">
<bool>true</bool> <bool>false</bool>
</attribute> </attribute>
<addaction name="action_back" /> <addaction name="action_back"/>
<addaction name="action_forward" /> <addaction name="action_forward"/>
<addaction name="separator" /> <addaction name="separator"/>
<addaction name="action_open_ebook" /> <addaction name="action_open_ebook"/>
<addaction name="action_configure" /> <addaction name="action_configure"/>
<addaction name="separator" /> <addaction name="separator"/>
<addaction name="action_previous_page" /> <addaction name="action_previous_page"/>
<addaction name="action_next_page" /> <addaction name="action_next_page"/>
<addaction name="separator" /> <addaction name="separator"/>
</widget> </widget>
<action name="action_next_page" > <action name="action_next_page">
<property name="icon" > <property name="icon">
<iconset resource="../../../../resources/images.qrc" > <iconset>
<normaloff>:/images/next.png</normaloff>:/images/next.png</iconset> <normaloff>:/images/next.png</normaloff>:/images/next.png</iconset>
</property> </property>
<property name="text" > <property name="text">
<string>Next Page</string> <string>Next Page</string>
</property> </property>
</action> </action>
<action name="action_previous_page" > <action name="action_previous_page">
<property name="icon" > <property name="icon">
<iconset resource="../../../../resources/images.qrc" > <iconset>
<normaloff>:/images/previous.png</normaloff>:/images/previous.png</iconset> <normaloff>:/images/previous.png</normaloff>:/images/previous.png</iconset>
</property> </property>
<property name="text" > <property name="text">
<string>Previous Page</string> <string>Previous Page</string>
</property> </property>
</action> </action>
<action name="action_back" > <action name="action_back">
<property name="icon" > <property name="icon">
<iconset resource="../../../../resources/images.qrc" > <iconset>
<normaloff>:/images/back.png</normaloff>:/images/back.png</iconset> <normaloff>:/images/back.png</normaloff>:/images/back.png</iconset>
</property> </property>
<property name="text" > <property name="text">
<string>Back</string> <string>Back</string>
</property> </property>
</action> </action>
<action name="action_forward" > <action name="action_forward">
<property name="icon" > <property name="icon">
<iconset resource="../../../../resources/images.qrc" > <iconset>
<normaloff>:/images/forward.png</normaloff>:/images/forward.png</iconset> <normaloff>:/images/forward.png</normaloff>:/images/forward.png</iconset>
</property> </property>
<property name="text" > <property name="text">
<string>Forward</string> <string>Forward</string>
</property> </property>
</action> </action>
<action name="action_next_match" > <action name="action_next_match">
<property name="text" > <property name="text">
<string>Next match</string> <string>Next match</string>
</property> </property>
</action> </action>
<action name="action_open_ebook" > <action name="action_open_ebook">
<property name="icon" > <property name="icon">
<iconset resource="../../../../resources/images.qrc" > <iconset>
<normaloff>:/images/document_open.png</normaloff>:/images/document_open.png</iconset> <normaloff>:/images/document_open.png</normaloff>:/images/document_open.png</iconset>
</property> </property>
<property name="text" > <property name="text">
<string>Open ebook</string> <string>Open ebook</string>
</property> </property>
</action> </action>
<action name="action_configure" > <action name="action_configure">
<property name="icon" > <property name="icon">
<iconset resource="../../../../resources/images.qrc" > <iconset>
<normaloff>:/images/config.png</normaloff>:/images/config.png</iconset> <normaloff>:/images/config.png</normaloff>:/images/config.png</iconset>
</property> </property>
<property name="text" > <property name="text">
<string>Configure</string> <string>Configure</string>
</property> </property>
</action> </action>
@ -239,7 +208,7 @@
</customwidget> </customwidget>
</customwidgets> </customwidgets>
<resources> <resources>
<include location="../../../../resources/images.qrc" /> <include location="../../../../resources/images.qrc"/>
</resources> </resources>
<connections/> <connections/>
</ui> </ui>

2
src/calibre/gui2/ui.py
View File

@ -365,6 +365,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
except: except:
olddb = None olddb = None
db = LibraryDatabase2(newloc) db = LibraryDatabase2(newloc)
if self.content_server is not None:
self.content_server.set_database(db)
self.library_path = newloc self.library_path = newloc
self.book_on_device(None, reset=True) self.book_on_device(None, reset=True)
db.set_book_on_device_func(self.book_on_device) db.set_book_on_device_func(self.book_on_device)

3
src/calibre/gui2/viewer/main.py
View File

@ -716,6 +716,9 @@ View an ebook.
def main(args=sys.argv): def main(args=sys.argv):
# Ensure viewer can continue to function if GUI is closed
os.environ.pop('CALIBRE_WORKER_TEMP_DIR', None)
parser = option_parser() parser = option_parser()
opts, args = parser.parse_args(args) opts, args = parser.parse_args(args)
pid = os.fork() if False and (islinux or isfreebsd) else -1 pid = os.fork() if False and (islinux or isfreebsd) else -1

4
src/calibre/gui2/viewer/main.ui
View File

@ -108,7 +108,7 @@
</size> </size>
</property> </property>
<attribute name="toolBarArea"> <attribute name="toolBarArea">
<enum>LeftToolBarArea</enum> <enum>Qt::LeftToolBarArea</enum>
</attribute> </attribute>
<attribute name="toolBarBreak"> <attribute name="toolBarBreak">
<bool>false</bool> <bool>false</bool>
@ -136,7 +136,7 @@
</widget> </widget>
<widget class="QToolBar" name="tool_bar2"> <widget class="QToolBar" name="tool_bar2">
<attribute name="toolBarArea"> <attribute name="toolBarArea">
<enum>TopToolBarArea</enum> <enum>Qt::TopToolBarArea</enum>
</attribute> </attribute>
<attribute name="toolBarBreak"> <attribute name="toolBarBreak">
<bool>false</bool> <bool>false</bool>

6
src/calibre/library/caches.py
View File

@ -380,7 +380,7 @@ class ResultCache(SearchQueryParser): # {{{
field_count = 3 field_count = 3
else: else:
try: try:
qd = parse_date(query) qd = parse_date(query, as_utc=False)
except: except:
raise ParseException(query, len(query), 'Date conversion error', self) raise ParseException(query, len(query), 'Date conversion error', self)
if '-' in query: if '-' in query:
@ -816,6 +816,10 @@ class SortKeyGenerator(object):
if val is None: if val is None:
val = '' val = ''
val = val.lower() val = val.lower()
elif dt == 'bool':
val = {True: 1, False: 2, None: 3}.get(val, 3)
yield val yield val
# }}} # }}}

58
src/calibre/library/check_library.py
View File

@ -14,14 +14,14 @@ from calibre.ebooks import BOOK_EXTENSIONS
EBOOK_EXTENSIONS = frozenset(BOOK_EXTENSIONS) EBOOK_EXTENSIONS = frozenset(BOOK_EXTENSIONS)
NORMALS = frozenset(['metadata.opf', 'cover.jpg']) NORMALS = frozenset(['metadata.opf', 'cover.jpg'])
CHECKS = [('invalid_titles', _('Invalid titles')), CHECKS = [('invalid_titles', _('Invalid titles'), True),
('extra_titles', _('Extra titles')), ('extra_titles', _('Extra titles'), True),
('invalid_authors', _('Invalid authors')), ('invalid_authors', _('Invalid authors'), True),
('extra_authors', _('Extra authors')), ('extra_authors', _('Extra authors'), True),
('missing_formats', _('Missing book formats')), ('missing_formats', _('Missing book formats'), False),
('extra_formats', _('Extra book formats')), ('extra_formats', _('Extra book formats'), True),
('extra_files', _('Unknown files in books')), ('extra_files', _('Unknown files in books'), True),
('failed_folders', _('Folders raising exception')) ('failed_folders', _('Folders raising exception'), False)
] ]
@ -41,7 +41,6 @@ class CheckLibrary(object):
self.all_lc_dbpaths = frozenset([f.lower() for f in self.all_dbpaths]) self.all_lc_dbpaths = frozenset([f.lower() for f in self.all_dbpaths])
self.db_id_regexp = re.compile(r'^.* \((\d+)\)$') self.db_id_regexp = re.compile(r'^.* \((\d+)\)$')
self.bad_ext_pat = re.compile(r'[^a-z0-9]+')
self.dirs = [] self.dirs = []
self.book_dirs = [] self.book_dirs = []
@ -78,7 +77,7 @@ class CheckLibrary(object):
auth_path = os.path.join(lib, auth_dir) auth_path = os.path.join(lib, auth_dir)
# First check: author must be a directory # First check: author must be a directory
if not os.path.isdir(auth_path): if not os.path.isdir(auth_path):
self.invalid_authors.append((auth_dir, auth_dir, [])) self.invalid_authors.append((auth_dir, auth_dir))
continue continue
self.potential_authors[auth_dir] = {} self.potential_authors[auth_dir] = {}
@ -93,7 +92,7 @@ class CheckLibrary(object):
m = self.db_id_regexp.search(title_dir) m = self.db_id_regexp.search(title_dir)
# Second check: title must have an ID and must be a directory # Second check: title must have an ID and must be a directory
if m is None or not os.path.isdir(title_path): if m is None or not os.path.isdir(title_path):
self.invalid_titles.append((auth_dir, db_path, [title_dir])) self.invalid_titles.append((auth_dir, db_path))
continue continue
id = m.group(1) id = m.group(1)
@ -101,12 +100,12 @@ class CheckLibrary(object):
if self.is_case_sensitive: if self.is_case_sensitive:
if int(id) not in self.all_ids or \ if int(id) not in self.all_ids or \
db_path not in self.all_dbpaths: db_path not in self.all_dbpaths:
self.extra_titles.append((title_dir, db_path, [])) self.extra_titles.append((title_dir, db_path))
continue continue
else: else:
if int(id) not in self.all_ids or \ if int(id) not in self.all_ids or \
db_path.lower() not in self.all_lc_dbpaths: db_path.lower() not in self.all_lc_dbpaths:
self.extra_titles.append((title_dir, db_path, [])) self.extra_titles.append((title_dir, db_path))
continue continue
# Record the book to check its formats # Record the book to check its formats
@ -115,7 +114,7 @@ class CheckLibrary(object):
# Fourth check: author directories that contain no titles # Fourth check: author directories that contain no titles
if not found_titles: if not found_titles:
self.extra_authors.append((auth_dir, auth_dir, [])) self.extra_authors.append((auth_dir, auth_dir))
for x in self.book_dirs: for x in self.book_dirs:
try: try:
@ -132,9 +131,7 @@ class CheckLibrary(object):
ext = ext[1:].lower() ext = ext[1:].lower()
if ext in EBOOK_EXTENSIONS: if ext in EBOOK_EXTENSIONS:
return True return True
if self.bad_ext_pat.search(ext) is not None: return False
return False
return True
def process_book(self, lib, book_info): def process_book(self, lib, book_info):
(db_path, title_dir, book_id) = book_info (db_path, title_dir, book_id) = book_info
@ -148,18 +145,18 @@ class CheckLibrary(object):
if self.is_case_sensitive: if self.is_case_sensitive:
unknowns = frozenset(filenames-formats-NORMALS) unknowns = frozenset(filenames-formats-NORMALS)
# Check: any books that aren't formats or normally there? # Check: any books that aren't formats or normally there?
if unknowns: for u in unknowns:
self.extra_files.append((title_dir, db_path, unknowns)) self.extra_files.append((title_dir, os.path.join(db_path, u)))
# Check: any book formats that should be there? # Check: any book formats that should be there?
missing = book_formats - formats missing = book_formats - formats
if missing: for m in missing:
self.missing_formats.append((title_dir, db_path, missing)) self.missing_formats.append((title_dir, os.path.join(db_path, m)))
# Check: any book formats that shouldn't be there? # Check: any book formats that shouldn't be there?
extra = formats - book_formats - NORMALS extra = formats - book_formats - NORMALS
if extra: for e in extra:
self.extra_formats.append((title_dir, db_path, extra)) self.extra_formats.append((title_dir, os.path.join(db_path, e)))
else: else:
def lc_map(fnames, fset): def lc_map(fnames, fset):
m = {} m = {}
@ -171,19 +168,16 @@ class CheckLibrary(object):
formats_lc = frozenset([f.lower() for f in formats]) formats_lc = frozenset([f.lower() for f in formats])
unknowns = frozenset(filenames_lc-formats_lc-NORMALS) unknowns = frozenset(filenames_lc-formats_lc-NORMALS)
# Check: any books that aren't formats or normally there? # Check: any books that aren't formats or normally there?
if unknowns: for f in lc_map(filenames, unknowns):
self.extra_files.append((title_dir, db_path, self.extra_files.append((title_dir, os.path.join(db_path, f)))
lc_map(filenames, unknowns)))
book_formats_lc = frozenset([f.lower() for f in book_formats]) book_formats_lc = frozenset([f.lower() for f in book_formats])
# Check: any book formats that should be there? # Check: any book formats that should be there?
missing = book_formats_lc - formats_lc missing = book_formats_lc - formats_lc
if missing: for m in lc_map(book_formats, missing):
self.missing_formats.append((title_dir, db_path, self.missing_formats.append((title_dir, os.path.join(db_path, m)))
lc_map(book_formats, missing)))
# Check: any book formats that shouldn't be there? # Check: any book formats that shouldn't be there?
extra = formats_lc - book_formats_lc - NORMALS extra = formats_lc - book_formats_lc - NORMALS
if extra: for e in lc_map(formats, extra):
self.extra_formats.append((title_dir, db_path, self.extra_formats.append((title_dir, os.path.join(db_path, e)))
lc_map(formats, extra)))

4
src/calibre/library/cli.py
View File

@ -943,11 +943,11 @@ def command_check_library(args, dbpath):
return return
if opts.csv: if opts.csv:
for i in list: for i in list:
print check[1] + ',' + i[0] + ',' + i[1] + ',' + '|'.join(i[2]) print check[1] + ',' + i[0] + ',' + i[1]
else: else:
print check[1] print check[1]
for i in list: for i in list:
print ' %-30.30s - %-30.30s - %s'%(i[0], i[1], ', '.join(i[2])) print ' %-40.40s - %-40.40s'%(i[0], i[1])
db = LibraryDatabase2(dbpath) db = LibraryDatabase2(dbpath)
checker = CheckLibrary(dbpath, db) checker = CheckLibrary(dbpath, db)

22
src/calibre/library/database2.py
View File

@ -653,17 +653,21 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.dirtied(book_ids) self.dirtied(book_ids)
def get_metadata_for_dump(self, idx, remove_from_dirtied=True): def get_metadata_for_dump(self, idx, remove_from_dirtied=True):
path, mi = (None, None)
try: try:
path = os.path.join(self.abspath(idx, index_is_id=True), 'metadata.opf') # While a book is being created, the path is empty. Don't bother to
mi = self.get_metadata(idx, index_is_id=True) # try to write the opf, because it will go to the wrong folder.
# Always set cover to cover.jpg. Even if cover doesn't exist, if self.path(idx, index_is_id=True):
# no harm done. This way no need to call dirtied when path = os.path.join(self.abspath(idx, index_is_id=True), 'metadata.opf')
# cover is set/removed mi = self.get_metadata(idx, index_is_id=True)
mi.cover = 'cover.jpg' # Always set cover to cover.jpg. Even if cover doesn't exist,
# no harm done. This way no need to call dirtied when
# cover is set/removed
mi.cover = 'cover.jpg'
except: except:
# This almost certainly means that the book has been deleted while # This almost certainly means that the book has been deleted while
# the backup operation sat in the queue. # the backup operation sat in the queue.
path, mi = (None, None) pass
try: try:
# clear the dirtied indicator. The user must put it back if # clear the dirtied indicator. The user must put it back if
@ -748,10 +752,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
return False return False
def find_identical_books(self, mi): def find_identical_books(self, mi):
fuzzy_title_patterns = [(re.compile(pat), repl) for pat, repl in fuzzy_title_patterns = [(re.compile(pat, re.IGNORECASE), repl) for pat, repl in
[ [
(r'[\[\](){}<>\'";,:#]', ''), (r'[\[\](){}<>\'";,:#]', ''),
(r'^(the|a|an) ', ''), (tweaks.get('title_sort_articles', r'^(a|the|an)\s+'), ''),
(r'[-._]', ' '), (r'[-._]', ' '),
(r'\s+', ' ') (r'\s+', ' ')
] ]

27
src/calibre/library/restore.py
View File

@ -71,9 +71,17 @@ class Restore(Thread):
if self.conflicting_custom_cols: if self.conflicting_custom_cols:
ans += '\n\n' ans += '\n\n'
ans += 'The following custom columns were not fully restored:\n' ans += 'The following custom columns have conflicting definitions ' \
'and were not fully restored:\n'
for x in self.conflicting_custom_cols: for x in self.conflicting_custom_cols:
ans += '\t#'+x+'\n' ans += '\t#'+x+'\n'
ans += '\tused:\t%s, %s, %s, %s\n'%(self.custom_columns[x][1],
self.custom_columns[x][2],
self.custom_columns[x][3],
self.custom_columns[x][5])
for coldef in self.conflicting_custom_cols[x]:
ans += '\tother:\t%s, %s, %s, %s\n'%(coldef[1], coldef[2],
coldef[3], coldef[5])
if self.mismatched_dirs: if self.mismatched_dirs:
ans += '\n\n' ans += '\n\n'
@ -152,7 +160,7 @@ class Restore(Thread):
def create_cc_metadata(self): def create_cc_metadata(self):
self.books.sort(key=itemgetter('timestamp')) self.books.sort(key=itemgetter('timestamp'))
m = {} self.custom_columns = {}
fields = ('label', 'name', 'datatype', 'is_multiple', 'is_editable', fields = ('label', 'name', 'datatype', 'is_multiple', 'is_editable',
'display') 'display')
for b in self.books: for b in self.books:
@ -168,16 +176,17 @@ class Restore(Thread):
if len(args) == len(fields): if len(args) == len(fields):
# TODO: Do series type columns need special handling? # TODO: Do series type columns need special handling?
label = cfm['label'] label = cfm['label']
if label in m and args != m[label]: if label in self.custom_columns and args != self.custom_columns[label]:
if label not in self.conflicting_custom_cols: if label not in self.conflicting_custom_cols:
self.conflicting_custom_cols[label] = set([m[label]]) self.conflicting_custom_cols[label] = []
self.conflicting_custom_cols[label].add(args) if self.custom_columns[label] not in self.conflicting_custom_cols[label]:
m[cfm['label']] = args self.conflicting_custom_cols[label].append(self.custom_columns[label])
self.custom_columns[label] = args
db = RestoreDatabase(self.library_path) db = RestoreDatabase(self.library_path)
self.progress_callback(None, len(m)) self.progress_callback(None, len(self.custom_columns))
if len(m): if len(self.custom_columns):
for i,args in enumerate(m.values()): for i,args in enumerate(self.custom_columns.values()):
db.create_custom_column(*args) db.create_custom_column(*args)
self.progress_callback(_('creating custom column ')+args[0], i+1) self.progress_callback(_('creating custom column ')+args[0], i+1)
db.conn.close() db.conn.close()

20
src/calibre/library/save_to_disk.py
View File

@ -131,15 +131,14 @@ class SafeFormat(TemplateFormatter):
self.vformat(b['display']['composite_template'], [], kwargs) self.vformat(b['display']['composite_template'], [], kwargs)
return self.composite_values[key] return self.composite_values[key]
if key in kwargs: if key in kwargs:
return kwargs[key].replace('/', '_').replace('\\', '_') val = kwargs[key]
return val.replace('/', '_').replace('\\', '_')
return '' return ''
except: except:
if DEBUG: if DEBUG:
traceback.print_exc() traceback.print_exc()
return key return key
safe_formatter = SafeFormat()
def get_components(template, mi, id, timefmt='%b %Y', length=250, def get_components(template, mi, id, timefmt='%b %Y', length=250,
sanitize_func=ascii_filename, replace_whitespace=False, sanitize_func=ascii_filename, replace_whitespace=False,
to_lowercase=False): to_lowercase=False):
@ -173,17 +172,22 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
custom_metadata = mi.get_all_user_metadata(make_copy=False) custom_metadata = mi.get_all_user_metadata(make_copy=False)
for key in custom_metadata: for key in custom_metadata:
if key in format_args: if key in format_args:
cm = custom_metadata[key]
## TODO: NEWMETA: should ratings be divided by 2? The standard rating isn't... ## TODO: NEWMETA: should ratings be divided by 2? The standard rating isn't...
if custom_metadata[key]['datatype'] == 'series': if cm['datatype'] == 'series':
format_args[key] = tsfmt(format_args[key]) format_args[key] = tsfmt(format_args[key])
if key+'_index' in format_args: if key+'_index' in format_args:
format_args[key+'_index'] = fmt_sidx(format_args[key+'_index']) format_args[key+'_index'] = fmt_sidx(format_args[key+'_index'])
elif custom_metadata[key]['datatype'] == 'datetime': elif cm['datatype'] == 'datetime':
format_args[key] = strftime(timefmt, format_args[key].timetuple()) format_args[key] = strftime(timefmt, format_args[key].timetuple())
elif custom_metadata[key]['datatype'] == 'bool': elif cm['datatype'] == 'bool':
format_args[key] = _('yes') if format_args[key] else _('no') format_args[key] = _('yes') if format_args[key] else _('no')
elif cm['datatype'] in ['int', 'float']:
components = safe_formatter.safe_format(template, format_args, if format_args[key] != 0:
format_args[key] = unicode(format_args[key])
else:
format_args[key] = ''
components = SafeFormat().safe_format(template, format_args,
'G_C-EXCEPTION!', mi) 'G_C-EXCEPTION!', mi)
components = [x.strip() for x in components.split('/') if x.strip()] components = [x.strip() for x in components.split('/') if x.strip()]
components = [sanitize_func(x) for x in components if x] components = [sanitize_func(x) for x in components if x]

78
src/calibre/library/server/base.py
View File

@ -10,6 +10,7 @@ import logging
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
import cherrypy import cherrypy
from cherrypy.process.plugins import SimplePlugin
from calibre.constants import __appname__, __version__ from calibre.constants import __appname__, __version__
from calibre.utils.date import fromtimestamp from calibre.utils.date import fromtimestamp
@ -54,16 +55,43 @@ class DispatchController(object): # {{{
# }}} # }}}
class BonJour(SimplePlugin):
def __init__(self, engine, port=8080):
SimplePlugin.__init__(self, engine)
self.port = port
def start(self):
try:
publish_zeroconf('Books in calibre', '_stanza._tcp',
self.port, {'path':'/stanza'})
except:
import traceback
cherrypy.log.error('Failed to start BonJour:')
cherrypy.log.error(traceback.format_exc())
start.priority = 90
def stop(self):
try:
stop_zeroconf()
except:
import traceback
cherrypy.log.error('Failed to stop BonJour:')
cherrypy.log.error(traceback.format_exc())
stop.priority = 10
cherrypy.engine.bonjour = BonJour(cherrypy.engine)
class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache, class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache,
BrowseServer): BrowseServer):
server_name = __appname__ + '/' + __version__ server_name = __appname__ + '/' + __version__
def __init__(self, db, opts, embedded=False, show_tracebacks=True): def __init__(self, db, opts, embedded=False, show_tracebacks=True):
self.db = db
for item in self.db:
item
break
self.opts = opts self.opts = opts
self.embedded = embedded self.embedded = embedded
self.state_callback = None self.state_callback = None
@ -71,7 +99,14 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache,
map(int, self.opts.max_cover.split('x')) map(int, self.opts.max_cover.split('x'))
path = P('content_server') path = P('content_server')
self.build_time = fromtimestamp(os.stat(path).st_mtime) self.build_time = fromtimestamp(os.stat(path).st_mtime)
self.default_cover = open(P('content_server/default_cover.jpg'), 'rb').read() self.default_cover = open(P('content_server/default_cover.jpg'), 'rb').read()
cherrypy.engine.bonjour.port = opts.port
Cache.__init__(self)
self.set_database(db)
cherrypy.config.update({ cherrypy.config.update({
'log.screen' : opts.develop, 'log.screen' : opts.develop,
'engine.autoreload_on' : opts.develop, 'engine.autoreload_on' : opts.develop,
@ -97,18 +132,28 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache,
'tools.digest_auth.users' : {opts.username.strip():opts.password.strip()}, 'tools.digest_auth.users' : {opts.username.strip():opts.password.strip()},
} }
sr = getattr(opts, 'restriction', None)
sr = db.prefs.get('cs_restriction', '') if sr is None else sr
self.set_search_restriction(sr)
self.is_running = False self.is_running = False
self.exception = None self.exception = None
self.setup_loggers()
cherrypy.engine.bonjour.subscribe()
def set_database(self, db):
self.db = db
sr = getattr(self.opts, 'restriction', None)
sr = db.prefs.get('cs_restriction', '') if sr is None else sr
self.set_search_restriction(sr)
def graceful(self):
cherrypy.engine.graceful()
def set_search_restriction(self, restriction): def set_search_restriction(self, restriction):
self.search_restriction_name = restriction
if restriction: if restriction:
self.search_restriction = 'search:"%s"'%restriction self.search_restriction = 'search:"%s"'%restriction
else: else:
self.search_restriction = '' self.search_restriction = ''
self.reset_caches()
def setup_loggers(self): def setup_loggers(self):
access_file = log_access_file access_file = log_access_file
@ -140,7 +185,6 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache,
root_conf['request.dispatch'] = d.dispatcher root_conf['request.dispatch'] = d.dispatcher
self.config['/'] = root_conf self.config['/'] = root_conf
self.setup_loggers()
cherrypy.tree.mount(root=None, config=self.config) cherrypy.tree.mount(root=None, config=self.config)
try: try:
try: try:
@ -154,24 +198,14 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache,
cherrypy.engine.start() cherrypy.engine.start()
self.is_running = True self.is_running = True
try: #if hasattr(cherrypy.engine, 'signal_handler'):
publish_zeroconf('Books in calibre', '_stanza._tcp', # cherrypy.engine.signal_handler.subscribe()
self.opts.port, {'path':'/stanza'})
except:
import traceback
cherrypy.log.error('Failed to start BonJour:')
cherrypy.log.error(traceback.format_exc())
cherrypy.engine.block() cherrypy.engine.block()
except Exception, e: except Exception, e:
self.exception = e self.exception = e
finally: finally:
self.is_running = False self.is_running = False
try:
stop_zeroconf()
except:
import traceback
cherrypy.log.error('Failed to stop BonJour:')
cherrypy.log.error(traceback.format_exc())
try: try:
if callable(self.state_callback): if callable(self.state_callback):
self.state_callback(self.is_running) self.state_callback(self.is_running)

40
src/calibre/library/server/browse.py
View File

@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en'
import operator, os, json import operator, os, json
from binascii import hexlify, unhexlify from binascii import hexlify, unhexlify
from urllib import quote, unquote
import cherrypy import cherrypy
@ -115,7 +116,10 @@ def render_rating(rating, container='span', prefix=None): # {{{
# }}} # }}}
def get_category_items(category, items, db, datatype): # {{{ def get_category_items(category, items, restriction, datatype): # {{{
if category == 'search':
items = [x for x in items if x.name != restriction]
def item(i): def item(i):
templ = (u'<div title="{4}" class="category-item">' templ = (u'<div title="{4}" class="category-item">'
@ -136,7 +140,7 @@ def get_category_items(category, items, db, datatype): # {{{
q = i.category q = i.category
if not q: if not q:
q = category q = category
href = '/browse/matches/%s/%s'%(q, id_) href = '/browse/matches/%s/%s'%(quote(q), quote(id_))
return templ.format(xml(name), rating, return templ.format(xml(name), rating,
xml(desc), xml(href), rstring) xml(desc), xml(href), rstring)
@ -164,6 +168,9 @@ class Endpoint(object): # {{{
sort_val = cookie[eself.sort_cookie_name].value sort_val = cookie[eself.sort_cookie_name].value
kwargs[eself.sort_kwarg] = sort_val kwargs[eself.sort_kwarg] = sort_val
# Remove AJAX caching disabling jquery workaround arg
kwargs.pop('_', None)
ans = func(self, *args, **kwargs) ans = func(self, *args, **kwargs)
cherrypy.response.headers['Content-Type'] = eself.mimetype cherrypy.response.headers['Content-Type'] = eself.mimetype
updated = self.db.last_modified() updated = self.db.last_modified()
@ -298,6 +305,7 @@ class BrowseServer(object):
category_meta = self.db.field_metadata category_meta = self.db.field_metadata
cats = [ cats = [
(_('Newest'), 'newest', 'forward.png'), (_('Newest'), 'newest', 'forward.png'),
(_('All books'), 'allbooks', 'book.png'),
] ]
def getter(x): def getter(x):
@ -329,7 +337,7 @@ class BrowseServer(object):
cats = [('<li title="{2} {0}"><img src="{src}" alt="{0}" />' cats = [('<li title="{2} {0}"><img src="{src}" alt="{0}" />'
'<span class="label">{0}</span>' '<span class="label">{0}</span>'
'<span class="url">/browse/category/{1}</span></li>') '<span class="url">/browse/category/{1}</span></li>')
.format(xml(x, True), xml(y), xml(_('Browse books by')), .format(xml(x, True), xml(quote(y)), xml(_('Browse books by')),
src='/browse/icon/'+z) src='/browse/icon/'+z)
for x, y, z in cats] for x, y, z in cats]
@ -369,7 +377,8 @@ class BrowseServer(object):
if len(items) <= self.opts.max_opds_ungrouped_items: if len(items) <= self.opts.max_opds_ungrouped_items:
script = 'false' script = 'false'
items = get_category_items(category, items, self.db, datatype) items = get_category_items(category, items,
self.search_restriction_name, datatype)
else: else:
getter = lambda x: unicode(getattr(x, 'sort', x.name)) getter = lambda x: unicode(getattr(x, 'sort', x.name))
starts = set([]) starts = set([])
@ -439,7 +448,8 @@ class BrowseServer(object):
entries.append(x) entries.append(x)
sort = self.browse_sort_categories(entries, sort) sort = self.browse_sort_categories(entries, sort)
entries = get_category_items(category, entries, self.db, datatype) entries = get_category_items(category, entries,
self.search_restriction_name, datatype)
return json.dumps(entries, ensure_ascii=False) return json.dumps(entries, ensure_ascii=False)
@ -450,6 +460,8 @@ class BrowseServer(object):
ans = self.browse_toplevel() ans = self.browse_toplevel()
elif category == 'newest': elif category == 'newest':
raise cherrypy.InternalRedirect('/browse/matches/newest/dummy') raise cherrypy.InternalRedirect('/browse/matches/newest/dummy')
elif category == 'allbooks':
raise cherrypy.InternalRedirect('/browse/matches/allbooks/dummy')
else: else:
ans = self.browse_category(category, category_sort) ans = self.browse_category(category, category_sort)
@ -473,32 +485,40 @@ class BrowseServer(object):
@Endpoint(sort_type='list') @Endpoint(sort_type='list')
def browse_matches(self, category=None, cid=None, list_sort=None): def browse_matches(self, category=None, cid=None, list_sort=None):
if list_sort:
list_sort = unquote(list_sort)
if not cid: if not cid:
raise cherrypy.HTTPError(404, 'invalid category id: %r'%cid) raise cherrypy.HTTPError(404, 'invalid category id: %r'%cid)
categories = self.categories_cache() categories = self.categories_cache()
if category not in categories and category != 'newest': if category not in categories and \
category not in ('newest', 'allbooks'):
raise cherrypy.HTTPError(404, 'category not found') raise cherrypy.HTTPError(404, 'category not found')
fm = self.db.field_metadata fm = self.db.field_metadata
try: try:
category_name = fm[category]['name'] category_name = fm[category]['name']
dt = fm[category]['datatype'] dt = fm[category]['datatype']
except: except:
if category != 'newest': if category not in ('newest', 'allbooks'):
raise raise
category_name = _('Newest') category_name = {
'newest' : _('Newest'),
'allbooks' : _('All books'),
}[category]
dt = None dt = None
hide_sort = 'true' if dt == 'series' else 'false' hide_sort = 'true' if dt == 'series' else 'false'
if category == 'search': if category == 'search':
which = unhexlify(cid) which = unhexlify(cid).decode('utf-8')
try: try:
ids = self.search_cache('search:"%s"'%which) ids = self.search_cache('search:"%s"'%which)
except: except:
raise cherrypy.HTTPError(404, 'Search: %r not understood'%which) raise cherrypy.HTTPError(404, 'Search: %r not understood'%which)
elif category == 'newest': elif category == 'newest':
ids = list(self.db.data.iterallids()) ids = self.search_cache('')
hide_sort = 'true' hide_sort = 'true'
elif category == 'allbooks':
ids = self.search_cache('')
else: else:
q = category q = category
if q == 'news': if q == 'news':

5
src/calibre/library/server/cache.py
View File

@ -10,7 +10,10 @@ from calibre.utils.ordered_dict import OrderedDict
class Cache(object): class Cache(object):
def add_routes(self, c): def __init__(self):
self.reset_caches()
def reset_caches(self):
self._category_cache = OrderedDict() self._category_cache = OrderedDict()
self._search_cache = OrderedDict() self._search_cache = OrderedDict()

2
src/calibre/library/server/content.py
View File

@ -124,7 +124,7 @@ class ContentServer(object):
if want_mobile: if want_mobile:
return self.mobile() return self.mobile()
return self.static('index.html') return self.browse_toplevel()
def old(self, **kwargs): def old(self, **kwargs):
return self.static('index.html') return self.static('index.html')

44
src/calibre/library/server/main.py
View File

@ -5,7 +5,7 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import os, sys import sys
from threading import Thread from threading import Thread
from calibre.library.server import server_config as config from calibre.library.server import server_config as config
@ -38,50 +38,18 @@ def option_parser():
' in the GUI')) ' in the GUI'))
return parser return parser
def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
try:
pid = os.fork()
if pid > 0:
# exit first parent
sys.exit(0)
except OSError, e:
print >>sys.stderr, "fork #1 failed: %d (%s)" % (e.errno, e.strerror)
sys.exit(1)
# decouple from parent environment
os.chdir("/")
os.setsid()
os.umask(0)
# do second fork
try:
pid = os.fork()
if pid > 0:
# exit from second parent
sys.exit(0)
except OSError, e:
print >>sys.stderr, "fork #2 failed: %d (%s)" % (e.errno, e.strerror)
sys.exit(1)
# Redirect standard file descriptors.
si = file(stdin, 'r')
so = file(stdout, 'a+')
se = file(stderr, 'a+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
def main(args=sys.argv): def main(args=sys.argv):
from calibre.library.database2 import LibraryDatabase2 from calibre.library.database2 import LibraryDatabase2
parser = option_parser() parser = option_parser()
opts, args = parser.parse_args(args) opts, args = parser.parse_args(args)
if opts.daemonize and not iswindows: if opts.daemonize and not iswindows:
daemonize() from cherrypy.process.plugins import Daemonizer
d = Daemonizer(cherrypy.engine)
d.subscribe()
if opts.pidfile is not None: if opts.pidfile is not None:
with open(opts.pidfile, 'wb') as f: from cherrypy.process.plugins import PIDFile
f.write(str(os.getpid())) PIDFile(cherrypy.engine, opts.pidfile).subscribe()
cherrypy.log.screen = True cherrypy.log.screen = True
from calibre.utils.config import prefs from calibre.utils.config import prefs
if opts.with_library is None: if opts.with_library is None:

21
src/calibre/library/server/mobile.py
View File

@ -112,7 +112,6 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS):
CLASS('thumbnail')) CLASS('thumbnail'))
data = TD() data = TD()
last = None
for fmt in book['formats'].split(','): for fmt in book['formats'].split(','):
a = ascii_filename(book['authors']) a = ascii_filename(book['authors'])
t = ascii_filename(book['title']) t = ascii_filename(book['title'])
@ -124,9 +123,11 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS):
), ),
CLASS('button')) CLASS('button'))
s.tail = u'' s.tail = u''
last = s
data.append(s) data.append(s)
div = DIV(CLASS('data-container'))
data.append(div)
series = u'[%s - %s]'%(book['series'], book['series_index']) \ series = u'[%s - %s]'%(book['series'], book['series_index']) \
if book['series'] else '' if book['series'] else ''
tags = u'Tags=[%s]'%book['tags'] if book['tags'] else '' tags = u'Tags=[%s]'%book['tags'] if book['tags'] else ''
@ -137,13 +138,13 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS):
if val: if val:
ctext += '%s=[%s] '%tuple(val.split(':#:')) ctext += '%s=[%s] '%tuple(val.split(':#:'))
text = u'\u202f%s %s by %s - %s - %s %s %s' % (book['title'], series, first = SPAN(u'\u202f%s %s by %s' % (book['title'], series,
book['authors'], book['size'], book['timestamp'], tags, ctext) book['authors']), CLASS('first-line'))
div.append(first)
if last is None: second = SPAN(u'%s - %s %s %s' % ( book['size'],
data.text = text book['timestamp'],
else: tags, ctext), CLASS('second-line'))
last.tail += text div.append(second)
bookt.append(TR(thumbnail, data)) bookt.append(TR(thumbnail, data))
# }}} # }}}
@ -229,7 +230,7 @@ class MobileServer(object):
no_tag_count=True) no_tag_count=True)
book['title'] = record[FM['title']] book['title'] = record[FM['title']]
for x in ('timestamp', 'pubdate'): for x in ('timestamp', 'pubdate'):
book[x] = strftime('%Y/%m/%d %H:%M:%S', record[FM[x]]) book[x] = strftime('%b, %Y', record[FM[x]])
book['id'] = record[FM['id']] book['id'] = record[FM['id']]
books.append(book) books.append(book)
for key in CKEYS: for key in CKEYS:

30
src/calibre/manual/faq.rst
View File

@ -435,3 +435,33 @@ And since I'm sure someone will ask: The reason adding/saving books are in separ
Finally, the reason calibre keep workers alive and idle instead of launching on demand is to workaround the slow startup time of python processes. Finally, the reason calibre keep workers alive and idle instead of launching on demand is to workaround the slow startup time of python processes.
How do I run parts of |app| like news download and the content server on my own linux server?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
First, you must install |app| onto your linux server. If your server is using a modern linux distro, you should have no problems installing |app| onto it.
.. note::
If you bought into the notion that a real server must run a decade old version of Debian, then you will have to jump through a few hoops. First, compile a newer version of glibc (>= 2.10) on your server from source. Then get the |app| linux binary tarball from the |app| google code page for your server architecture. Extract it into :file:`/opt/calibre`. Put your previously compiled glibc into :file:`/opt/calibre` as :file:`libc.so.6`. You can now run the calibre binaries from :file:`/opt/calibre`.
You can run the |app| server via the command::
/opt/calibre/calibre-server --with-library /path/to/the/library/you/want/to/share
You can download news and convert it into an ebook with the command::
/opt/calibre/ebook-convert "Title of news source.recipe" outputfile.epub
If you want to generate MOBI, use outputfile.mobi instead.
You can email downloaded news with the command::
/opt/calibre/calibre-smtp
I leave figuring out the exact command line as an exercise for the reader.
Finally, you can add downloaded news to the |app| library with::
/opt/calibre/calibredb add --with-library /path/to/library outfile.epub
Remember to read the command line documentatation section of the |app| User Manual to learn more about these, and other commands.

3
src/calibre/utils/formatter.py
View File

@ -7,6 +7,7 @@ Created on 23 Sep 2010
import re, string, traceback import re, string, traceback
from calibre.constants import DEBUG from calibre.constants import DEBUG
from calibre.utils.titlecase import titlecase
class TemplateFormatter(string.Formatter): class TemplateFormatter(string.Formatter):
''' '''
@ -81,7 +82,7 @@ class TemplateFormatter(string.Formatter):
functions = { functions = {
'uppercase' : (0, lambda s,x: x.upper()), 'uppercase' : (0, lambda s,x: x.upper()),
'lowercase' : (0, lambda s,x: x.lower()), 'lowercase' : (0, lambda s,x: x.lower()),
'titlecase' : (0, lambda s,x: x.title()), 'titlecase' : (0, lambda s,x: titlecase(x)),
'capitalize' : (0, lambda s,x: x.capitalize()), 'capitalize' : (0, lambda s,x: x.capitalize()),
'contains' : (3, _contains), 'contains' : (3, _contains),
'ifempty' : (1, _ifempty), 'ifempty' : (1, _ifempty),

2
src/calibre/utils/ipc/worker.py
View File

@ -105,7 +105,7 @@ def main():
notifier.start() notifier.start()
result = func(*args, **kwargs) result = func(*args, **kwargs)
if result is not None: if result is not None and os.path.exists(os.path.dirname(resultf)):
cPickle.dump(result, open(resultf, 'wb'), -1) cPickle.dump(result, open(resultf, 'wb'), -1)
notifier.queue.put(None) notifier.queue.put(None)

10
src/calibre/utils/magick/draw.py
View File

@ -9,6 +9,7 @@ import os
from calibre.utils.magick import Image, DrawingWand, create_canvas from calibre.utils.magick import Image, DrawingWand, create_canvas
from calibre.constants import __appname__, __version__ from calibre.constants import __appname__, __version__
from calibre.utils.config import tweaks
from calibre import fit_image from calibre import fit_image
def normalize_format_name(fmt): def normalize_format_name(fmt):
@ -113,7 +114,9 @@ def add_borders_to_image(img_data, left=0, top=0, right=0, bottom=0,
def create_text_wand(font_size, font_path=None): def create_text_wand(font_size, font_path=None):
if font_path is None: if font_path is None:
font_path = P('fonts/liberation/LiberationSerif-Bold.ttf') font_path = tweaks['generate_cover_title_font']
if font_path is None:
font_path = P('fonts/liberation/LiberationSerif-Bold.ttf')
ans = DrawingWand() ans = DrawingWand()
ans.font = font_path ans.font = font_path
ans.font_size = font_size ans.font_size = font_size
@ -203,8 +206,11 @@ def create_cover_page(top_lines, logo_path, width=590, height=750,
bottom += line.bottom_margin bottom += line.bottom_margin
bottom -= top_lines[-1].bottom_margin bottom -= top_lines[-1].bottom_margin
foot_font = tweaks['generate_cover_foot_font']
if not foot_font:
foot_font = P('fonts/liberation/LiberationMono-Regular.ttf')
vanity = create_text_arc(__appname__ + ' ' + __version__, 24, vanity = create_text_arc(__appname__ + ' ' + __version__, 24,
font=P('fonts/liberation/LiberationMono-Regular.ttf')) font=foot_font)
lwidth, lheight = vanity.size lwidth, lheight = vanity.size
left = int(max(0, (width - lwidth)/2.)) left = int(max(0, (width - lwidth)/2.))
top = height - lheight - 10 top = height - lheight - 10

9
src/calibre/utils/mdns.py
View File

@ -58,11 +58,12 @@ def publish(desc, type, port, properties=None, add_hostname=True):
''' '''
port = int(port) port = int(port)
server = start_server() server = start_server()
try:
hostname = socket.gethostname().partition('.')[0]
except:
hostname = 'Unknown'
if add_hostname: if add_hostname:
try:
hostname = socket.gethostname().partition('.')[0]
except:
hostname = 'Unknown'
desc += ' (on %s)'%hostname desc += ' (on %s)'%hostname
local_ip = get_external_ip() local_ip = get_external_ip()
type = type+'.local.' type = type+'.local.'

5
src/calibre/web/feeds/news.py
View File

@ -842,6 +842,9 @@ class BasicNewsRecipe(Recipe):
except NotImplementedError: except NotImplementedError:
feeds = self.parse_feeds() feeds = self.parse_feeds()
if not feeds:
raise ValueError('No articles found, aborting')
#feeds = FeedCollection(feeds) #feeds = FeedCollection(feeds)
self.report_progress(0, _('Trying to download cover...')) self.report_progress(0, _('Trying to download cover...'))
@ -1104,7 +1107,7 @@ class BasicNewsRecipe(Recipe):
mi = MetaInformation(title, [__appname__]) mi = MetaInformation(title, [__appname__])
mi.publisher = __appname__ mi.publisher = __appname__
mi.author_sort = __appname__ mi.author_sort = __appname__
mi.publication_type = 'periodical:'+self.publication_type mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title()
mi.timestamp = nowf() mi.timestamp = nowf()
mi.comments = self.description mi.comments = self.description
if not isinstance(mi.comments, unicode): if not isinstance(mi.comments, unicode):