Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge from trunk

Browse Source
This commit is contained in:
Sengian 2010-12-07 20:44:42 +01:00
commit cf323f3c42
36 changed files with 656 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
resources/recipes/mainichi.recipe
View File

@ -4,6 +4,7 @@ __copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
www.mainichi.jp www.mainichi.jp
''' '''
import re
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
class MainichiDailyNews(BasicNewsRecipe): class MainichiDailyNews(BasicNewsRecipe):
@ -22,3 +23,18 @@ class MainichiDailyNews(BasicNewsRecipe):
remove_tags = [{'class':"RelatedArticle"}] remove_tags = [{'class':"RelatedArticle"}]
remove_tags_after = {'class':"Credit"} remove_tags_after = {'class':"Credit"}
def parse_feeds(self):
feeds = BasicNewsRecipe.parse_feeds(self)
for curfeed in feeds:
delList = []
for a,curarticle in enumerate(curfeed.articles):
if re.search(r'pheedo.jp', curarticle.url):
delList.append(curarticle)
if len(delList)>0:
for d in delList:
index = curfeed.articles.index(d)
curfeed.articles[index:index+1] = []
return feeds

16
resources/recipes/mainichi_it_news.recipe
View File

@ -14,5 +14,19 @@ class MainichiDailyITNews(BasicNewsRecipe):
remove_tags_before = {'class':"NewsTitle"} remove_tags_before = {'class':"NewsTitle"}
remove_tags = [{'class':"RelatedArticle"}] remove_tags = [{'class':"RelatedArticle"}]
remove_tags_after = {'class':"Credit"}
def parse_feeds(self):
feeds = BasicNewsRecipe.parse_feeds(self)
for curfeed in feeds:
delList = []
for a,curarticle in enumerate(curfeed.articles):
if re.search(r'pheedo.jp', curarticle.url):
delList.append(curarticle)
if len(delList)>0:
for d in delList:
index = curfeed.articles.index(d)
curfeed.articles[index:index+1] = []
return feeds remove_tags_after = {'class':"Credit"}

5
resources/recipes/nikkei_sub_life.recipe
View File

@ -32,12 +32,9 @@ class NikkeiNet_sub_life(BasicNewsRecipe):
remove_tags_after = {'class':"cmn-pr_list"} remove_tags_after = {'class':"cmn-pr_list"}
feeds = [ (u'\u304f\u3089\u3057', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kurashi'), feeds = [ (u'\u304f\u3089\u3057', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kurashi'),
(u'\u30b9\u30dd\u30fc\u30c4', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=sports'),
(u'\u793e\u4f1a', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=shakai'),
(u'\u30a8\u30b3', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=eco'), (u'\u30a8\u30b3', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=eco'),
(u'\u5065\u5eb7', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kenkou'), (u'\u5065\u5eb7', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kenkou'),
(u'\u7279\u96c6', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=special'), (u'\u7279\u96c6', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=special')
(u'\u30e9\u30f3\u30ad\u30f3\u30b0', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=ranking')
] ]
def get_browser(self): def get_browser(self):

102
resources/recipes/nikkei_sub_shakai.recipe Normal file
View File

@ -0,0 +1,102 @@
__license__ = 'GPL v3'
__copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
'''
www.nikkei.com
'''
import re
from calibre.web.feeds.recipes import BasicNewsRecipe
import mechanize
from calibre.ptempfile import PersistentTemporaryFile
class NikkeiNet_sub_life(BasicNewsRecipe):
title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(\u751f\u6d3b)'
__author__ = 'Hiroshi Miura'
description = 'News and current market affairs from Japan'
cover_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg'
masthead_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg'
needs_subscription = True
oldest_article = 2
max_articles_per_feed = 20
language = 'ja'
remove_javascript = False
temp_files = []
remove_tags_before = {'class':"cmn-section cmn-indent"}
remove_tags = [
{'class':"JSID_basePageMove JSID_baseAsyncSubmit cmn-form_area JSID_optForm_utoken"},
{'class':"cmn-article_keyword cmn-clearfix"},
{'class':"cmn-print_headline cmn-clearfix"},
]
remove_tags_after = {'class':"cmn-pr_list"}
feeds = [
(u'\u793e\u4f1a', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=shakai')
]
def get_browser(self):
br = BasicNewsRecipe.get_browser()
cj = mechanize.LWPCookieJar()
br.set_cookiejar(cj)
#br.set_debug_http(True)
#br.set_debug_redirects(True)
#br.set_debug_responses(True)
if self.username is not None and self.password is not None:
#print "----------------------------get login form--------------------------------------------"
# open login form
br.open('https://id.nikkei.com/lounge/nl/base/LA0010.seam')
response = br.response()
#print "----------------------------get login form---------------------------------------------"
#print "----------------------------set login form---------------------------------------------"
# remove disabled input which brings error on mechanize
response.set_data(response.get_data().replace("<input id=\"j_id48\"", "<!-- "))
response.set_data(response.get_data().replace("gm_home_on.gif\" />", " -->"))
br.set_response(response)
br.select_form(name='LA0010Form01')
br['LA0010Form01:LA0010Email'] = self.username
br['LA0010Form01:LA0010Password'] = self.password
br.form.find_control(id='LA0010Form01:LA0010AutoLoginOn',type="checkbox").get(nr=0).selected = True
br.submit()
br.response()
#print "----------------------------send login form---------------------------------------------"
#print "----------------------------open news main page-----------------------------------------"
# open news site
br.open('http://www.nikkei.com/')
br.response()
#print "----------------------------www.nikkei.com BODY --------------------------------------"
#print response2.get_data()
#print "-------------------------^^-got auto redirect form----^^--------------------------------"
# forced redirect in default
br.select_form(nr=0)
br.submit()
response3 = br.response()
# return some cookie which should be set by Javascript
#print response3.geturl()
raw = response3.get_data()
#print "---------------------------response to form --------------------------------------------"
# grab cookie from JS and set it
redirectflag = re.search(r"var checkValue = '(\d+)';", raw, re.M).group(1)
br.select_form(nr=0)
self.temp_files.append(PersistentTemporaryFile('_fa.html'))
self.temp_files[-1].write("#LWP-Cookies-2.0\n")
self.temp_files[-1].write("Set-Cookie3: Cookie-dummy=Cookie-value; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n")
self.temp_files[-1].write("Set-Cookie3: redirectFlag="+redirectflag+"; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n")
self.temp_files[-1].close()
cj.load(self.temp_files[-1].name)
br.submit()
#br.set_debug_http(False)
#br.set_debug_redirects(False)
#br.set_debug_responses(False)
return br

70
resources/recipes/st_louis_post_dispatch.recipe Normal file
View File

@ -0,0 +1,70 @@
from calibre.web.feeds.news import BasicNewsRecipe
class AdvancedUserRecipe1282093204(BasicNewsRecipe):
title = u'St Louis Post-Dispatch'
__author__ = 'cisaak'
language = 'en'
oldest_article = 1
max_articles_per_feed = 15
masthead_url = 'http://farm5.static.flickr.com/4118/4929686950_0e22e2c88a.jpg'
feeds = [
(u'News-Bill McClellan', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fcolumns%2Fbill-mclellan&f=rss&t=article'),
(u'News-Columns', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcolumns*&l=50&f=rss&t=article'),
(u'News-Crime & Courtshttp://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcrime-and-courts&l=50&f=rss&t=article'),
(u'News-Deb Peterson', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fcolumns%2Fdeb-peterson&f=rss&t=article'),
(u'News-Education', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2feducation&f=rss&t=article'),
(u'News-Government & Politics', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fgovt-and-politics&f=rss&t=article'),
(u'News-Local', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal&f=rss&t=article'),
(u'News-Metro', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fmetro&f=rss&t=article'),
(u'News-Metro East', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fillinois&f=rss&t=article'),
(u'News-Missouri Out State', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Fstate-and-regional%2FMissouri&l=50&f=rss&t=article'),
(u'Opinion-Colleen Carroll Campbell', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Fopinion%2fcolumns%2Fcolleen-carroll-campbell&f=rss&t=article'),
(u'Opinion-Editorial', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Fopinion%2feditorial&f=rss&t=article'),
(u'Opinion-Kevin Horrigan', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Fopinion%2fcolumns%2Fkevin-horrigan&f=rss&t=article'),
(u'Opinion-Mailbag', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Fopinion%2fmailbag&f=rss&t=article'),
(u'Business Columns-Savvy Consumer', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fsavvy-consumer&l=100&f=rss&t=article'),
(u'Business Columns-Lager Heads', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Flager-heads&l=100&f=rss&t=article'),
(u'Business Columns-Job Watch', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fjob-watch&l=100&f=rss&t=article'),
(u'Business Columns-Steve Geigerich', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fsteve-giegerich&l=100&f=rss&t=article'),
(u'Business Columns-David Nicklaus', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fdavid-nicklaus&l=100&f=rss&t=article'),
(u'Business Columns-Jim Gallagher', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fjim-gallagher&l=100&f=rss&t=article'),
(u'Business Columns-Building Blocks', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fbuilding-blocks&l=100&f=rss&t=article'),
(u'Business', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business*l&l=100&f=rss&t=article'),
(u'Business-Technology', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Ftechnology&l=50&f=rss&t=article'),
(u'Business-National', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fnational-and-international&l=50&f=rss&t=article'),
(u'Travel', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=travel*&l=100&f=rss&t=article'),
(u'Sports', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports*&f=rss&t=article'),
(u'Sports-Baseball', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fbaseball%2Fprofessional&l=100&f=rss&t=article'),
(u'Sports-Bernie Miklasz', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcolumns%2Fbernie-miklasz&l=50&f=rss&t=article'),
(u'Sports-Bryan Burwell', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcolumns%2Fbryan-burwell&l=50&f=rss&t=article'),
(u'Sports-College', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcollege*&l=100&f=rss&t=article'),
(u'Sports-Dan Caesar', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcolumns%2Fdan-caesar&l=50&f=rss&t=article'),
(u'Sports-Football', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Ffootball%2Fprofessional&l=100&f=rss&t=article'),
(u'Sports-Hockey', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fhockey%2Fprofessional&l=100&f=rss&t=article'),
(u'Sports-Illini', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcollege%2Fillini&l=100&f=rss&t=article'),
(u'Sports-Jeff Gordon', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcolumns%2Fjeff-gordon&l=100&f=rss&t=article'),
(u'Life & Style', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles&l=100&f=rss&t=article'),
(u'Life & Style-Debra Bass', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Ffashion-and-style%2Fdebra-bass&l=100&f=rss&t=article'),
(u'Life & Style-Food and Cooking', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Ffood-and-cooking&l=100&f=rss&t=article'),
(u'Life & Style-Health/Medicine/Fitness', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Fhealth-med-fit&l=100&f=rss&t=article'),
(u'Life & Style-Joe Holleman', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Fcolumns%2Fjoe-holleman&l=100&f=rss&t=article'),
(u'Life & Style-Steals-and-Deals', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Fcolumns%2Fsteals-and-deals&l=100&f=rss&t=article'),
(u'Life & Style-Tim Townsend', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Ffaith-and-values%2Ftim-townsend&l=100&f=rss&t=article'),
(u'Entertainment', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment&l=100&f=rss&t=article'),
(u'Entertainment-Arts & Theatre', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Farts-and-theatre&l=100&f=rss&t=article'),
(u'Entertainment-Books & Literature', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fbooks-and-literature&l=100&f=rss&t=article'),
(u'Entertainment-Dining', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fhockey%2Fprofessional&l=100&f=rss&t=article'),
(u'Entertainment-Events Calendar', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fevents-calendar&l=100&f=rss&t=article'),
(u'Entertainment-Gail Pennington', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Ftelevision%2Fgail-pennington&l=100&f=rss&t=article'),
(u'Entertainment-Hip Hops', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fdining%2Fbars-and-clubs-other%2Fhip-hops&l=100&f=rss&t=article'),
(u'Entertainment-House-O-Fun', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fhouse-o-fun&l=100&f=rss&t=article'),
(u'Entertainment-Kevin C. Johnson', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fmusic%2Fkevin-johnson&l=100&f=rss&t=article')
]
remove_empty_feeds = True
remove_tags = [dict(name='div', attrs={'id':'blox-logo'}),dict(name='a')]
keep_only_tags = [dict(name='h1'), dict(name='p', attrs={'class':'byline'}), dict(name="div", attrs={'id':'blox-story-text'})]
extra_css = 'p {text-align: left;}'

2
resources/recipes/yomiuri.recipe
View File

@ -21,7 +21,7 @@ class YOLNews(BasicNewsRecipe):
remove_javascript = True remove_javascript = True
masthead_title = u'YOMIURI ONLINE' masthead_title = u'YOMIURI ONLINE'
remove_tags_before = {'class':"article-def"} keep_only_tags = [{'class':"article-def"}]
remove_tags = [{'class':"RelatedArticle"}, remove_tags = [{'class':"RelatedArticle"},
{'class':"sbtns"} {'class':"sbtns"}
] ]

2
resources/recipes/yomiuri_world.recipe
View File

@ -21,7 +21,7 @@ class YOLNews(BasicNewsRecipe):
remove_javascript = True remove_javascript = True
masthead_title = u"YOMIURI ONLINE" masthead_title = u"YOMIURI ONLINE"
remove_tags_before = {'class':"article-def"} keep_only_tags = [{'class':"article-def"}]
remove_tags = [{'class':"RelatedArticle"}, remove_tags = [{'class':"RelatedArticle"},
{'class':"sbtns"} {'class':"sbtns"}
] ]

4
src/calibre/devices/android/driver.py
View File

@ -21,7 +21,7 @@ class ANDROID(USBMS):
# HTC # HTC
0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100, 0x0227], 0x0ff9 0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100, 0x0227], 0x0ff9
: [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226], : [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226],
0xc92 : [0x100]}, 0xc92 : [0x100], 0xc97: [0x226]},
# Eken # Eken
0x040d : { 0x8510 : [0x0001] }, 0x040d : { 0x8510 : [0x0001] },
@ -63,7 +63,7 @@ class ANDROID(USBMS):
WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
'__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
'SCH-I500_CARD'] 'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810']
WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID'] 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID']

6
src/calibre/devices/usbms/books.py
View File

@ -11,9 +11,9 @@ from calibre.ebooks.metadata.book.base import Metadata
from calibre.devices.mime import mime_type_ext from calibre.devices.mime import mime_type_ext
from calibre.devices.interface import BookList as _BookList from calibre.devices.interface import BookList as _BookList
from calibre.constants import preferred_encoding from calibre.constants import preferred_encoding
from calibre import isbytestring from calibre import isbytestring, force_unicode
from calibre.utils.config import prefs, tweaks from calibre.utils.config import prefs, tweaks
from calibre.utils.icu import sort_key, strcmp as icu_strcmp from calibre.utils.icu import strcmp
class Book(Metadata): class Book(Metadata):
def __init__(self, prefix, lpath, size=None, other=None): def __init__(self, prefix, lpath, size=None, other=None):
@ -241,7 +241,7 @@ class CollectionsBookList(BookList):
if y is None: if y is None:
return -1 return -1
if isinstance(x, (unicode, str)): if isinstance(x, (unicode, str)):
c = strcmp(x, y) c = strcmp(force_unicode(x), force_unicode(y))
else: else:
c = cmp(x, y) c = cmp(x, y)
if c != 0: if c != 0:

26
src/calibre/ebooks/metadata/amazonfr.py
View File

@ -10,7 +10,7 @@ from lxml import html
from lxml.html import soupparser from lxml.html import soupparser
from calibre.utils.date import parse_date, utcnow, replace_months from calibre.utils.date import parse_date, utcnow, replace_months
from calibre.utils.cleantext import clean_ascii_char from calibre.utils.cleantext import clean_ascii_chars
from calibre import browser, preferred_encoding from calibre import browser, preferred_encoding
from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.chardet import xml_to_unicode
from calibre.ebooks.metadata import MetaInformation, check_isbn, \ from calibre.ebooks.metadata import MetaInformation, check_isbn, \
@ -22,8 +22,8 @@ from calibre.library.comments import sanitize_comments_html
class AmazonFr(MetadataSource): class AmazonFr(MetadataSource):
name = 'Amazon french' name = 'Amazon French'
description = _('Downloads social metadata from amazon.fr') description = _('Downloads metadata from amazon.fr')
supported_platforms = ['windows', 'osx', 'linux'] supported_platforms = ['windows', 'osx', 'linux']
author = 'Sengian' author = 'Sengian'
version = (1, 0, 0) version = (1, 0, 0)
@ -39,8 +39,8 @@ class AmazonFr(MetadataSource):
class AmazonEs(MetadataSource): class AmazonEs(MetadataSource):
name = 'Amazon spanish' name = 'Amazon Spanish'
description = _('Downloads social metadata from amazon.com in spanish') description = _('Downloads metadata from amazon.com in spanish')
supported_platforms = ['windows', 'osx', 'linux'] supported_platforms = ['windows', 'osx', 'linux']
author = 'Sengian' author = 'Sengian'
version = (1, 0, 0) version = (1, 0, 0)
@ -56,8 +56,8 @@ class AmazonEs(MetadataSource):
class AmazonEn(MetadataSource): class AmazonEn(MetadataSource):
name = 'Amazon english' name = 'Amazon English'
description = _('Downloads social metadata from amazon.com in english') description = _('Downloads metadata from amazon.com in english')
supported_platforms = ['windows', 'osx', 'linux'] supported_platforms = ['windows', 'osx', 'linux']
author = 'Sengian' author = 'Sengian'
version = (1, 0, 0) version = (1, 0, 0)
@ -73,8 +73,8 @@ class AmazonEn(MetadataSource):
class AmazonDe(MetadataSource): class AmazonDe(MetadataSource):
name = 'Amazon german' name = 'Amazon German'
description = _('Downloads social metadata from amazon.de') description = _('Downloads metadata from amazon.de')
supported_platforms = ['windows', 'osx', 'linux'] supported_platforms = ['windows', 'osx', 'linux']
author = 'Sengian' author = 'Sengian'
version = (1, 0, 0) version = (1, 0, 0)
@ -91,7 +91,7 @@ class AmazonDe(MetadataSource):
class Amazon(MetadataSource): class Amazon(MetadataSource):
name = 'Amazon' name = 'Amazon'
description = _('Downloads social metadata from amazon.com') description = _('Downloads metadata from amazon.com')
supported_platforms = ['windows', 'osx', 'linux'] supported_platforms = ['windows', 'osx', 'linux']
author = 'Kovid Goyal & Sengian' author = 'Kovid Goyal & Sengian'
version = (1, 1, 0) version = (1, 1, 0)
@ -209,7 +209,7 @@ class Query(object):
except: except:
try: try:
#remove ASCII invalid chars #remove ASCII invalid chars
return soupparser.fromstring(clean_ascii_char(raw)) return soupparser.fromstring(clean_ascii_chars(raw))
except: except:
return None, self.urldata return None, self.urldata
@ -237,7 +237,7 @@ class Query(object):
except: except:
try: try:
#remove ASCII invalid chars #remove ASCII invalid chars
return soupparser.fromstring(clean_ascii_char(raw)) return soupparser.fromstring(clean_ascii_chars(raw))
except: except:
continue continue
pages.append(feed) pages.append(feed)
@ -429,7 +429,7 @@ class ResultList(list):
except: except:
try: try:
#remove ASCII invalid chars #remove ASCII invalid chars
return soupparser.fromstring(clean_ascii_char(raw)) return soupparser.fromstring(clean_ascii_chars(raw))
except: except:
report(verbose) report(verbose)
return return

7
src/calibre/ebooks/metadata/fictionwise.py
View File

@ -6,7 +6,6 @@ __docformat__ = 'restructuredtext en'
import sys, textwrap, re, traceback, socket import sys, textwrap, re, traceback, socket
from urllib import urlencode from urllib import urlencode
from lxml import html
from lxml.html import soupparser, tostring from lxml.html import soupparser, tostring
from calibre import browser, preferred_encoding from calibre import browser, preferred_encoding
@ -17,7 +16,7 @@ from calibre.library.comments import sanitize_comments_html
from calibre.ebooks.metadata.fetch import MetadataSource from calibre.ebooks.metadata.fetch import MetadataSource
from calibre.utils.config import OptionParser from calibre.utils.config import OptionParser
from calibre.utils.date import parse_date, utcnow from calibre.utils.date import parse_date, utcnow
from calibre.utils.cleantext import clean_ascii_char from calibre.utils.cleantext import clean_ascii_chars
class Fictionwise(MetadataSource): # {{{ class Fictionwise(MetadataSource): # {{{
@ -109,7 +108,7 @@ class Query(object):
except: except:
try: try:
#remove ASCII invalid chars #remove ASCII invalid chars
feed = soupparser.fromstring(clean_ascii_char(raw)) feed = soupparser.fromstring(clean_ascii_chars(raw))
except: except:
return None return None
@ -295,7 +294,7 @@ class ResultList(list):
except: except:
try: try:
#remove ASCII invalid chars #remove ASCII invalid chars
return soupparser.fromstring(clean_ascii_char(raw)) return soupparser.fromstring(clean_ascii_chars(raw))
except: except:
return None return None

8
src/calibre/ebooks/metadata/nicebooks.py
View File

@ -11,7 +11,7 @@ from copy import deepcopy
from lxml.html import soupparser from lxml.html import soupparser
from calibre.utils.date import parse_date, utcnow, replace_months from calibre.utils.date import parse_date, utcnow, replace_months
from calibre.utils.cleantext import clean_ascii_char from calibre.utils.cleantext import clean_ascii_chars
from calibre import browser, preferred_encoding from calibre import browser, preferred_encoding
from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.chardet import xml_to_unicode
from calibre.ebooks.metadata import MetaInformation, check_isbn, \ from calibre.ebooks.metadata import MetaInformation, check_isbn, \
@ -126,7 +126,7 @@ class Query(object):
except: except:
try: try:
#remove ASCII invalid chars #remove ASCII invalid chars
feed = soupparser.fromstring(clean_ascii_char(raw)) feed = soupparser.fromstring(clean_ascii_chars(raw))
except: except:
return None return None
@ -155,7 +155,7 @@ class Query(object):
except: except:
try: try:
#remove ASCII invalid chars #remove ASCII invalid chars
feed = soupparser.fromstring(clean_ascii_char(raw)) feed = soupparser.fromstring(clean_ascii_chars(raw))
except: except:
continue continue
pages.append(feed) pages.append(feed)
@ -251,7 +251,7 @@ class ResultList(list):
except: except:
try: try:
#remove ASCII invalid chars #remove ASCII invalid chars
feed = soupparser.fromstring(clean_ascii_char(raw)) feed = soupparser.fromstring(clean_ascii_chars(raw))
except: except:
return None return None

91
src/calibre/ebooks/metadata/worker.py
View File

@ -8,12 +8,12 @@ __docformat__ = 'restructuredtext en'
from threading import Thread from threading import Thread
from Queue import Empty from Queue import Empty
import os, time, sys, shutil import os, time, sys, shutil, json
from calibre.utils.ipc.job import ParallelJob from calibre.utils.ipc.job import ParallelJob
from calibre.utils.ipc.server import Server from calibre.utils.ipc.server import Server
from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory
from calibre import prints from calibre import prints, isbytestring
from calibre.constants import filesystem_encoding from calibre.constants import filesystem_encoding
@ -194,14 +194,42 @@ class SaveWorker(Thread):
self.daemon = True self.daemon = True
self.path, self.opts = path, opts self.path, self.opts = path, opts
self.ids = ids self.ids = ids
self.library_path = db.library_path self.db = db
self.canceled = False self.canceled = False
self.result_queue = result_queue self.result_queue = result_queue
self.error = None self.error = None
self.spare_server = spare_server self.spare_server = spare_server
self.start() self.start()
def collect_data(self, ids):
from calibre.ebooks.metadata.opf2 import metadata_to_opf
data = {}
for i in set(ids):
mi = self.db.get_metadata(i, index_is_id=True, get_cover=True)
opf = metadata_to_opf(mi)
if isbytestring(opf):
opf = opf.decode('utf-8')
cpath = None
if mi.cover:
cpath = mi.cover
if isbytestring(cpath):
cpath = cpath.decode(filesystem_encoding)
formats = {}
if mi.formats:
for fmt in mi.formats:
fpath = self.db.format_abspath(i, fmt, index_is_id=True)
if fpath is not None:
if isbytestring(fpath):
fpath = fpath.decode(filesystem_encoding)
formats[fmt.lower()] = fpath
data[i] = [opf, cpath, formats]
return data
def run(self): def run(self):
with TemporaryDirectory('save_to_disk_data') as tdir:
self._run(tdir)
def _run(self, tdir):
from calibre.library.save_to_disk import config from calibre.library.save_to_disk import config
server = Server() if self.spare_server is None else self.spare_server server = Server() if self.spare_server is None else self.spare_server
ids = set(self.ids) ids = set(self.ids)
@ -212,12 +240,19 @@ class SaveWorker(Thread):
for pref in c.preferences: for pref in c.preferences:
recs[pref.name] = getattr(self.opts, pref.name) recs[pref.name] = getattr(self.opts, pref.name)
plugboards = self.db.prefs.get('plugboards', {})
for i, task in enumerate(tasks): for i, task in enumerate(tasks):
tids = [x[-1] for x in task] tids = [x[-1] for x in task]
data = self.collect_data(tids)
dpath = os.path.join(tdir, '%d.json'%i)
with open(dpath, 'wb') as f:
f.write(json.dumps(data, ensure_ascii=False).encode('utf-8'))
job = ParallelJob('save_book', job = ParallelJob('save_book',
'Save books (%d of %d)'%(i, len(tasks)), 'Save books (%d of %d)'%(i, len(tasks)),
lambda x,y:x, lambda x,y:x,
args=[tids, self.library_path, self.path, recs]) args=[tids, dpath, plugboards, self.path, recs])
jobs.add(job) jobs.add(job)
server.add_job(job) server.add_job(job)
@ -226,21 +261,21 @@ class SaveWorker(Thread):
time.sleep(0.2) time.sleep(0.2)
running = False running = False
for job in jobs: for job in jobs:
job.update(consume_notifications=False) self.get_notifications(job, ids)
while True:
try:
id, title, ok, tb = job.notifications.get_nowait()[0]
if id in ids:
self.result_queue.put((id, title, ok, tb))
ids.remove(id)
except Empty:
break
if not job.is_finished: if not job.is_finished:
running = True running = True
if not running: if not running:
break break
for job in jobs:
if not job.result:
continue
for id_, title, ok, tb in job.result:
if id_ in ids:
self.result_queue.put((id_, title, ok, tb))
ids.remove(id_)
server.close() server.close()
time.sleep(1) time.sleep(1)
@ -257,21 +292,39 @@ class SaveWorker(Thread):
except: except:
pass pass
def get_notifications(self, job, ids):
job.update(consume_notifications=False)
while True:
try:
id, title, ok, tb = job.notifications.get_nowait()[0]
if id in ids:
self.result_queue.put((id, title, ok, tb))
ids.remove(id)
except Empty:
break
def save_book(task, library_path, path, recs, notification=lambda x,y:x):
from calibre.library.database2 import LibraryDatabase2 def save_book(ids, dpath, plugboards, path, recs, notification=lambda x,y:x):
db = LibraryDatabase2(library_path) from calibre.library.save_to_disk import config, save_serialized_to_disk
from calibre.library.save_to_disk import config, save_to_disk
from calibre.customize.ui import apply_null_metadata from calibre.customize.ui import apply_null_metadata
opts = config().parse() opts = config().parse()
for name in recs: for name in recs:
setattr(opts, name, recs[name]) setattr(opts, name, recs[name])
results = []
def callback(id, title, failed, tb): def callback(id, title, failed, tb):
results.append((id, title, not failed, tb))
notification((id, title, not failed, tb)) notification((id, title, not failed, tb))
return True return True
with apply_null_metadata: data_ = json.loads(open(dpath, 'rb').read().decode('utf-8'))
save_to_disk(db, task, path, opts, callback) data = {}
for k, v in data_.iteritems():
data[int(k)] = v
with apply_null_metadata:
save_serialized_to_disk(ids, data, plugboards, path, opts, callback)
return results

2
src/calibre/ebooks/oeb/reader.py
View File

@ -544,7 +544,7 @@ class OEBReader(object):
data = render_html_svg_workaround(path, self.logger) data = render_html_svg_workaround(path, self.logger)
if not data: if not data:
data = '' data = ''
id, href = self.oeb.manifest.generate('cover', 'cover.jpeg') id, href = self.oeb.manifest.generate('cover', 'cover.jpg')
item = self.oeb.manifest.add(id, href, JPEG_MIME, data=data) item = self.oeb.manifest.add(id, href, JPEG_MIME, data=data)
return item return item

4
src/calibre/gui2/__init__.py
View File

@ -123,8 +123,8 @@ def _config():
help=_('Download social metadata (tags/rating/etc.)')) help=_('Download social metadata (tags/rating/etc.)'))
c.add_opt('overwrite_author_title_metadata', default=True, c.add_opt('overwrite_author_title_metadata', default=True,
help=_('Overwrite author and title with new metadata')) help=_('Overwrite author and title with new metadata'))
c.add_opt('overwrite_cover_image', default=False, c.add_opt('auto_download_cover', default=False,
help=_('Overwrite cover with new new cover if existing')) help=_('Automatically download the cover, if available'))
c.add_opt('enforce_cpu_limit', default=True, c.add_opt('enforce_cpu_limit', default=True,
help=_('Limit max simultaneous jobs to number of CPUs')) help=_('Limit max simultaneous jobs to number of CPUs'))
c.add_opt('tag_browser_hidden_categories', default=set(), c.add_opt('tag_browser_hidden_categories', default=set(),

4
src/calibre/gui2/actions/add.py
View File

@ -61,6 +61,7 @@ class AddAction(InterfaceAction):
self._adder = Adder(self.gui, self._adder = Adder(self.gui,
self.gui.library_view.model().db, self.gui.library_view.model().db,
self.Dispatcher(self._files_added), spare_server=self.gui.spare_server) self.Dispatcher(self._files_added), spare_server=self.gui.spare_server)
self.gui.tags_view.disable_recounting = True
self._adder.add_recursive(root, single) self._adder.add_recursive(root, single)
def add_recursive_single(self, *args): def add_recursive_single(self, *args):
@ -201,9 +202,11 @@ class AddAction(InterfaceAction):
self._adder = Adder(self.gui, self._adder = Adder(self.gui,
None if to_device else self.gui.library_view.model().db, None if to_device else self.gui.library_view.model().db,
self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server) self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server)
self.gui.tags_view.disable_recounting = True
self._adder.add(paths) self._adder.add(paths)
def _files_added(self, paths=[], names=[], infos=[], on_card=None): def _files_added(self, paths=[], names=[], infos=[], on_card=None):
self.gui.tags_view.disable_recounting = False
if paths: if paths:
self.gui.upload_books(paths, self.gui.upload_books(paths,
list(map(ascii_filename, names)), list(map(ascii_filename, names)),
@ -214,6 +217,7 @@ class AddAction(InterfaceAction):
self.gui.library_view.model().books_added(self._adder.number_of_books_added) self.gui.library_view.model().books_added(self._adder.number_of_books_added)
if hasattr(self.gui, 'db_images'): if hasattr(self.gui, 'db_images'):
self.gui.db_images.reset() self.gui.db_images.reset()
self.gui.tags_view.recount()
if getattr(self._adder, 'merged_books', False): if getattr(self._adder, 'merged_books', False):
books = u'\n'.join([x if isinstance(x, unicode) else books = u'\n'.join([x if isinstance(x, unicode) else
x.decode(preferred_encoding, 'replace') for x in x.decode(preferred_encoding, 'replace') for x in

181
src/calibre/gui2/add.py
View File

@ -3,41 +3,55 @@ UI for adding books to the database and saving books to disk
''' '''
import os, shutil, time import os, shutil, time
from Queue import Queue, Empty from Queue import Queue, Empty
from threading import Thread from functools import partial
from PyQt4.Qt import QThread, SIGNAL, QObject, QTimer, Qt, \ from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer
QProgressDialog
from calibre.gui2.dialogs.progress import ProgressDialog from calibre.gui2.dialogs.progress import ProgressDialog
from calibre.gui2 import question_dialog, error_dialog, info_dialog from calibre.gui2 import question_dialog, error_dialog, info_dialog
from calibre.ebooks.metadata.opf2 import OPF from calibre.ebooks.metadata.opf2 import OPF
from calibre.ebooks.metadata import MetaInformation from calibre.ebooks.metadata import MetaInformation
from calibre.constants import preferred_encoding, filesystem_encoding from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG
from calibre.utils.config import prefs from calibre.utils.config import prefs
from calibre import prints
single_shot = partial(QTimer.singleShot, 75)
class DuplicatesAdder(QObject): # {{{
added = pyqtSignal(object)
adding_done = pyqtSignal()
class DuplicatesAdder(QThread): # {{{
# Add duplicate books
def __init__(self, parent, db, duplicates, db_adder): def __init__(self, parent, db, duplicates, db_adder):
QThread.__init__(self, parent) QObject.__init__(self, parent)
self.db, self.db_adder = db, db_adder self.db, self.db_adder = db, db_adder
self.duplicates = duplicates self.duplicates = list(duplicates)
self.count = 0
single_shot(self.add_one)
def add_one(self):
if not self.duplicates:
self.adding_done.emit()
return
mi, cover, formats = self.duplicates.pop()
formats = [f for f in formats if not f.lower().endswith('.opf')]
id = self.db.create_book_entry(mi, cover=cover,
add_duplicates=True)
# here we add all the formats for dupe book record created above
self.db_adder.add_formats(id, formats)
self.db_adder.number_of_books_added += 1
self.count += 1
self.added.emit(self.count)
single_shot(self.add_one)
def run(self):
count = 1
for mi, cover, formats in self.duplicates:
formats = [f for f in formats if not f.lower().endswith('.opf')]
id = self.db.create_book_entry(mi, cover=cover,
add_duplicates=True)
# here we add all the formats for dupe book record created above
self.db_adder.add_formats(id, formats)
self.db_adder.number_of_books_added += 1
self.emit(SIGNAL('added(PyQt_PyObject)'), count)
count += 1
self.emit(SIGNAL('adding_done()'))
# }}} # }}}
class RecursiveFind(QThread): # {{{ class RecursiveFind(QThread): # {{{
update = pyqtSignal(object)
found = pyqtSignal(object)
def __init__(self, parent, db, root, single): def __init__(self, parent, db, root, single):
QThread.__init__(self, parent) QThread.__init__(self, parent)
self.db = db self.db = db
@ -50,8 +64,8 @@ class RecursiveFind(QThread): # {{{
for dirpath in os.walk(root): for dirpath in os.walk(root):
if self.canceled: if self.canceled:
return return
self.emit(SIGNAL('update(PyQt_PyObject)'), self.update.emit(
_('Searching in')+' '+dirpath[0]) _('Searching in')+' '+dirpath[0])
self.books += list(self.db.find_books_in_directory(dirpath[0], self.books += list(self.db.find_books_in_directory(dirpath[0],
self.single_book_per_directory)) self.single_book_per_directory))
@ -71,46 +85,55 @@ class RecursiveFind(QThread): # {{{
msg = unicode(err) msg = unicode(err)
except: except:
msg = repr(err) msg = repr(err)
self.emit(SIGNAL('found(PyQt_PyObject)'), msg) self.found.emit(msg)
return return
self.books = [formats for formats in self.books if formats] self.books = [formats for formats in self.books if formats]
if not self.canceled: if not self.canceled:
self.emit(SIGNAL('found(PyQt_PyObject)'), self.books) self.found.emit(self.books)
# }}} # }}}
class DBAdder(Thread): # {{{ class DBAdder(QObject): # {{{
def __init__(self, parent, db, ids, nmap):
QObject.__init__(self, parent)
def __init__(self, db, ids, nmap):
self.db, self.ids, self.nmap = db, dict(**ids), dict(**nmap) self.db, self.ids, self.nmap = db, dict(**ids), dict(**nmap)
self.end = False
self.critical = {} self.critical = {}
self.number_of_books_added = 0 self.number_of_books_added = 0
self.duplicates = [] self.duplicates = []
self.names, self.paths, self.infos = [], [], [] self.names, self.paths, self.infos = [], [], []
Thread.__init__(self)
self.daemon = True
self.input_queue = Queue() self.input_queue = Queue()
self.output_queue = Queue() self.output_queue = Queue()
self.merged_books = set([]) self.merged_books = set([])
def run(self): def end(self):
while not self.end: self.input_queue.put((None, None, None))
try:
id, opf, cover = self.input_queue.get(True, 0.2) def start(self):
except Empty: try:
continue id, opf, cover = self.input_queue.get_nowait()
name = self.nmap.pop(id) except Empty:
title = None single_shot(self.start)
try: return
title = self.add(id, opf, cover, name) if id is None and opf is None and cover is None:
except: return
import traceback name = self.nmap.pop(id)
self.critical[name] = traceback.format_exc() title = None
title = name if DEBUG:
self.output_queue.put(title) st = time.time()
try:
title = self.add(id, opf, cover, name)
except:
import traceback
self.critical[name] = traceback.format_exc()
title = name
self.output_queue.put(title)
if DEBUG:
prints('Added', title, 'to db in:', time.time() - st, 'seconds')
single_shot(self.start)
def process_formats(self, opf, formats): def process_formats(self, opf, formats):
imp = opf[:-4]+'.import' imp = opf[:-4]+'.import'
@ -201,10 +224,10 @@ class Adder(QObject): # {{{
self.pd.setModal(True) self.pd.setModal(True)
self.pd.show() self.pd.show()
self._parent = parent self._parent = parent
self.rfind = self.worker = self.timer = None self.rfind = self.worker = None
self.callback = callback self.callback = callback
self.callback_called = False self.callback_called = False
self.connect(self.pd, SIGNAL('canceled()'), self.canceled) self.pd.canceled_signal.connect(self.canceled)
def add_recursive(self, root, single=True): def add_recursive(self, root, single=True):
self.path = root self.path = root
@ -213,10 +236,8 @@ class Adder(QObject): # {{{
self.pd.set_max(0) self.pd.set_max(0)
self.pd.value = 0 self.pd.value = 0
self.rfind = RecursiveFind(self, self.db, root, single) self.rfind = RecursiveFind(self, self.db, root, single)
self.connect(self.rfind, SIGNAL('update(PyQt_PyObject)'), self.rfind.update.connect(self.pd.set_msg, type=Qt.QueuedConnection)
self.pd.set_msg, Qt.QueuedConnection) self.rfind.found.connect(self.add, type=Qt.QueuedConnection)
self.connect(self.rfind, SIGNAL('found(PyQt_PyObject)'),
self.add, Qt.QueuedConnection)
self.rfind.start() self.rfind.start()
def add(self, books): def add(self, books):
@ -246,12 +267,12 @@ class Adder(QObject): # {{{
self.pd.set_min(0) self.pd.set_min(0)
self.pd.set_max(len(self.ids)) self.pd.set_max(len(self.ids))
self.pd.value = 0 self.pd.value = 0
self.db_adder = DBAdder(self.db, self.ids, self.nmap) self.db_adder = DBAdder(self, self.db, self.ids, self.nmap)
self.db_adder.start() self.db_adder.start()
self.last_added_at = time.time() self.last_added_at = time.time()
self.entry_count = len(self.ids) self.entry_count = len(self.ids)
self.continue_updating = True self.continue_updating = True
QTimer.singleShot(200, self.update) single_shot(self.update)
def canceled(self): def canceled(self):
self.continue_updating = False self.continue_updating = False
@ -260,14 +281,14 @@ class Adder(QObject): # {{{
if self.worker is not None: if self.worker is not None:
self.worker.canceled = True self.worker.canceled = True
if hasattr(self, 'db_adder'): if hasattr(self, 'db_adder'):
self.db_adder.end = True self.db_adder.end()
self.pd.hide() self.pd.hide()
if not self.callback_called: if not self.callback_called:
self.callback(self.paths, self.names, self.infos) self.callback(self.paths, self.names, self.infos)
self.callback_called = True self.callback_called = True
def duplicates_processed(self): def duplicates_processed(self):
self.db_adder.end = True self.db_adder.end()
if not self.callback_called: if not self.callback_called:
self.callback(self.paths, self.names, self.infos) self.callback(self.paths, self.names, self.infos)
self.callback_called = True self.callback_called = True
@ -300,7 +321,7 @@ class Adder(QObject): # {{{
if (time.time() - self.last_added_at) > self.ADD_TIMEOUT: if (time.time() - self.last_added_at) > self.ADD_TIMEOUT:
self.continue_updating = False self.continue_updating = False
self.pd.hide() self.pd.hide()
self.db_adder.end = True self.db_adder.end()
if not self.callback_called: if not self.callback_called:
self.callback([], [], []) self.callback([], [], [])
self.callback_called = True self.callback_called = True
@ -311,7 +332,7 @@ class Adder(QObject): # {{{
'find the problem book.'), show=True) 'find the problem book.'), show=True)
if self.continue_updating: if self.continue_updating:
QTimer.singleShot(200, self.update) single_shot(self.update)
def process_duplicates(self): def process_duplicates(self):
@ -332,11 +353,8 @@ class Adder(QObject): # {{{
self.__p_d = pd self.__p_d = pd
self.__d_a = DuplicatesAdder(self._parent, self.db, duplicates, self.__d_a = DuplicatesAdder(self._parent, self.db, duplicates,
self.db_adder) self.db_adder)
self.connect(self.__d_a, SIGNAL('added(PyQt_PyObject)'), self.__d_a.added.connect(pd.setValue)
pd.setValue) self.__d_a.adding_done.connect(self.duplicates_processed)
self.connect(self.__d_a, SIGNAL('adding_done()'),
self.duplicates_processed)
self.__d_a.start()
else: else:
return self.duplicates_processed() return self.duplicates_processed()
@ -407,14 +425,12 @@ class Saver(QObject): # {{{
self.worker = SaveWorker(self.rq, db, self.ids, path, self.opts, self.worker = SaveWorker(self.rq, db, self.ids, path, self.opts,
spare_server=self.spare_server) spare_server=self.spare_server)
self.pd.canceled_signal.connect(self.canceled) self.pd.canceled_signal.connect(self.canceled)
self.timer = QTimer(self) self.continue_updating = True
self.connect(self.timer, SIGNAL('timeout()'), self.update) single_shot(self.update)
self.timer.start(200)
def canceled(self): def canceled(self):
if self.timer is not None: self.continue_updating = False
self.timer.stop()
if self.worker is not None: if self.worker is not None:
self.worker.canceled = True self.worker.canceled = True
self.pd.hide() self.pd.hide()
@ -424,14 +440,38 @@ class Saver(QObject): # {{{
def update(self): def update(self):
if not self.ids or not self.worker.is_alive(): if not self.continue_updating:
self.timer.stop() return
if not self.worker.is_alive():
# Check that all ids were processed
while self.ids:
# Get all queued results since worker is dead
before = len(self.ids)
self.get_result()
if before == len(self.ids):
# No results available => worker died unexpectedly
for i in list(self.ids):
self.failures.add(('id:%d'%i, 'Unknown error'))
self.ids.remove(i)
if not self.ids:
self.continue_updating = False
self.pd.hide() self.pd.hide()
if not self.callback_called: if not self.callback_called:
self.callback(self.worker.path, self.failures, self.worker.error) try:
# Give the worker time to clean up and set worker.error
self.worker.join(2)
except:
pass # The worker was not yet started
self.callback_called = True self.callback_called = True
return self.callback(self.worker.path, self.failures, self.worker.error)
if self.continue_updating:
self.get_result()
single_shot(self.update)
def get_result(self):
try: try:
id, title, ok, tb = self.rq.get_nowait() id, title, ok, tb = self.rq.get_nowait()
except Empty: except Empty:
@ -441,6 +481,7 @@ class Saver(QObject): # {{{
if not isinstance(title, unicode): if not isinstance(title, unicode):
title = str(title).decode(preferred_encoding, 'replace') title = str(title).decode(preferred_encoding, 'replace')
self.pd.set_msg(_('Saved')+' '+title) self.pd.set_msg(_('Saved')+' '+title)
if not ok: if not ok:
self.failures.add((title, tb)) self.failures.add((title, tb))
# }}} # }}}

34
src/calibre/gui2/dialogs/fetch_metadata.py
View File

@ -9,7 +9,7 @@ from threading import Thread
from PyQt4.QtCore import Qt, QObject, SIGNAL, QVariant, pyqtSignal, \ from PyQt4.QtCore import Qt, QObject, SIGNAL, QVariant, pyqtSignal, \
QAbstractTableModel, QCoreApplication, QTimer QAbstractTableModel, QCoreApplication, QTimer
from PyQt4.QtGui import QDialog, QItemSelectionModel from PyQt4.QtGui import QDialog, QItemSelectionModel, QIcon
from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata
from calibre.gui2 import error_dialog, NONE, info_dialog, config from calibre.gui2 import error_dialog, NONE, info_dialog, config
@ -42,6 +42,7 @@ class Matches(QAbstractTableModel):
def __init__(self, matches): def __init__(self, matches):
self.matches = matches self.matches = matches
self.yes_icon = QVariant(QIcon(I('ok.png')))
QAbstractTableModel.__init__(self) QAbstractTableModel.__init__(self)
def rowCount(self, *args): def rowCount(self, *args):
@ -61,8 +62,8 @@ class Matches(QAbstractTableModel):
elif section == 3: text = _("Publisher") elif section == 3: text = _("Publisher")
elif section == 4: text = _("ISBN") elif section == 4: text = _("ISBN")
elif section == 5: text = _("Published") elif section == 5: text = _("Published")
elif section == 6: text = _("Cover?") elif section == 6: text = _("Has Cover")
elif section == 7: text = _("Summary?") elif section == 7: text = _("Has Summary")
return QVariant(text) return QVariant(text)
else: else:
@ -73,8 +74,8 @@ class Matches(QAbstractTableModel):
def data(self, index, role): def data(self, index, role):
row, col = index.row(), index.column() row, col = index.row(), index.column()
book = self.matches[row]
if role == Qt.DisplayRole: if role == Qt.DisplayRole:
book = self.matches[row]
res = None res = None
if col == 0: if col == 0:
res = book.title res = book.title
@ -89,13 +90,14 @@ class Matches(QAbstractTableModel):
elif col == 5: elif col == 5:
if hasattr(book.pubdate, 'timetuple'): if hasattr(book.pubdate, 'timetuple'):
res = strftime('%b %Y', book.pubdate.timetuple()) res = strftime('%b %Y', book.pubdate.timetuple())
elif col == 6 and book.has_cover:
res = 'OK'
elif col == 7 and book.comments:
res = 'OK'
if not res: if not res:
return NONE return NONE
return QVariant(res) return QVariant(res)
elif role == Qt.DecorationRole:
if col == 6 and book.has_cover:
return self.yes_icon
if col == 7 and book.comments:
return self.yes_icon
return NONE return NONE
class FetchMetadata(QDialog, Ui_FetchMetadata): class FetchMetadata(QDialog, Ui_FetchMetadata):
@ -137,8 +139,7 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
self.fetch_metadata() self.fetch_metadata()
self.opt_get_social_metadata.setChecked(config['get_social_metadata']) self.opt_get_social_metadata.setChecked(config['get_social_metadata'])
self.opt_overwrite_author_title_metadata.setChecked(config['overwrite_author_title_metadata']) self.opt_overwrite_author_title_metadata.setChecked(config['overwrite_author_title_metadata'])
self.opt_overwrite_cover_image.setChecked(config['overwrite_cover_image']) self.opt_auto_download_cover.setChecked(config['auto_download_cover'])
def show_summary(self, current, *args): def show_summary(self, current, *args):
row = current.row() row = current.row()
@ -220,13 +221,12 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
_hung_fetchers.add(self.fetcher) _hung_fetchers.add(self.fetcher)
if hasattr(self, '_hangcheck') and self._hangcheck.isActive(): if hasattr(self, '_hangcheck') and self._hangcheck.isActive():
self._hangcheck.stop() self._hangcheck.stop()
#option configure # Save value of auto_download_cover, since this is the only place it can
if self.opt_get_social_metadata.isChecked() != config['get_social_metadata']: # be set. The values of the other options can be set in
config.set('get_social_metadata', self.opt_get_social_metadata.isChecked()) # Preferences->Behavior and should not be set here as they affect bulk
if self.opt_overwrite_author_title_metadata.isChecked() != config['overwrite_author_title_metadata']: # downloading as well.
config.set('overwrite_author_title_metadata', self.opt_overwrite_author_title_metadata.isChecked()) if self.opt_auto_download_cover.isChecked() != config['auto_download_cover']:
if self.opt_overwrite_cover_image.isChecked() != config['overwrite_cover_image']: config.set('auto_download_cover', self.opt_auto_download_cover.isChecked())
config.set('overwrite_cover_image', self.opt_overwrite_cover_image.isChecked())
def __enter__(self, *args): def __enter__(self, *args):
return self return self

4
src/calibre/gui2/dialogs/fetch_metadata.ui
View File

@ -124,9 +124,9 @@
</widget> </widget>
</item> </item>
<item> <item>
<widget class="QCheckBox" name="opt_overwrite_cover_image"> <widget class="QCheckBox" name="opt_auto_download_cover">
<property name="text"> <property name="text">
<string>Overwrite cover image with downloaded cover if available for the selected book</string> <string>Automatically download the cover, if available</string>
</property> </property>
</widget> </widget>
</item> </item>

3
src/calibre/gui2/dialogs/metadata_bulk.py
View File

@ -17,7 +17,7 @@ from calibre.gui2 import error_dialog
from calibre.gui2.progress_indicator import ProgressIndicator from calibre.gui2.progress_indicator import ProgressIndicator
from calibre.utils.config import dynamic from calibre.utils.config import dynamic
from calibre.utils.titlecase import titlecase from calibre.utils.titlecase import titlecase
from calibre.utils.icu import sort_key from calibre.utils.icu import sort_key, capitalize
class MyBlockingBusy(QDialog): class MyBlockingBusy(QDialog):
@ -187,6 +187,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
_('Lower Case') : lambda x: icu_lower(x), _('Lower Case') : lambda x: icu_lower(x),
_('Upper Case') : lambda x: icu_upper(x), _('Upper Case') : lambda x: icu_upper(x),
_('Title Case') : lambda x: titlecase(x), _('Title Case') : lambda x: titlecase(x),
_('Capitalize') : lambda x: capitalize(x),
} }
s_r_match_modes = [ _('Character match'), s_r_match_modes = [ _('Character match'),

32
src/calibre/gui2/dialogs/metadata_single.py
View File

@ -8,8 +8,9 @@ add/remove formats
import os, re, time, traceback, textwrap import os, re, time, traceback, textwrap
from functools import partial from functools import partial
from threading import Thread
from PyQt4.Qt import SIGNAL, QObject, Qt, QTimer, QThread, QDate, \ from PyQt4.Qt import SIGNAL, QObject, Qt, QTimer, QDate, \
QPixmap, QListWidgetItem, QDialog, pyqtSignal, QMessageBox, QIcon, \ QPixmap, QListWidgetItem, QDialog, pyqtSignal, QMessageBox, QIcon, \
QPushButton QPushButton
@ -34,9 +35,12 @@ from calibre.gui2.preferences.social import SocialMetadata
from calibre.gui2.custom_column_widgets import populate_metadata_page from calibre.gui2.custom_column_widgets import populate_metadata_page
from calibre import strftime from calibre import strftime
class CoverFetcher(QThread): # {{{ class CoverFetcher(Thread): # {{{
def __init__(self, username, password, isbn, timeout, title, author): def __init__(self, username, password, isbn, timeout, title, author):
Thread.__init__(self)
self.daemon = True
self.username = username.strip() if username else username self.username = username.strip() if username else username
self.password = password.strip() if password else password self.password = password.strip() if password else password
self.timeout = timeout self.timeout = timeout
@ -44,8 +48,7 @@ class CoverFetcher(QThread): # {{{
self.title = title self.title = title
self.needs_isbn = False self.needs_isbn = False
self.author = author self.author = author
QThread.__init__(self) self.exception = self.traceback = self.cover_data = self.errors = None
self.exception = self.traceback = self.cover_data = None
def run(self): def run(self):
try: try:
@ -238,20 +241,20 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
self.timeout, title, author) self.timeout, title, author)
self.cover_fetcher.start() self.cover_fetcher.start()
self._hangcheck = QTimer(self) self._hangcheck = QTimer(self)
self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck) self._hangcheck.timeout.connect(self.hangcheck,
type=Qt.QueuedConnection)
self.cf_start_time = time.time() self.cf_start_time = time.time()
self.pi.start(_('Downloading cover...')) self.pi.start(_('Downloading cover...'))
self._hangcheck.start(100) self._hangcheck.start(100)
def hangcheck(self): def hangcheck(self):
if not self.cover_fetcher.isFinished() and \ if self.cover_fetcher.is_alive() and \
time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT: time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT:
return return
self._hangcheck.stop() self._hangcheck.stop()
try: try:
if self.cover_fetcher.isRunning(): if self.cover_fetcher.is_alive():
self.cover_fetcher.terminate()
error_dialog(self, _('Cannot fetch cover'), error_dialog(self, _('Cannot fetch cover'),
_('<b>Could not fetch cover.</b><br/>')+ _('<b>Could not fetch cover.</b><br/>')+
_('The download timed out.')).exec_() _('The download timed out.')).exec_()
@ -759,11 +762,9 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
if book.author_sort: self.author_sort.setText(book.author_sort) if book.author_sort: self.author_sort.setText(book.author_sort)
if book.publisher: self.publisher.setEditText(book.publisher) if book.publisher: self.publisher.setEditText(book.publisher)
if book.isbn: self.isbn.setText(book.isbn) if book.isbn: self.isbn.setText(book.isbn)
if d.opt_overwrite_cover_image.isChecked() and book.has_cover:
self.fetch_cover()
if book.pubdate: if book.pubdate:
d = book.pubdate dt = book.pubdate
self.pubdate.setDate(QDate(d.year, d.month, d.day)) self.pubdate.setDate(QDate(dt.year, dt.month, dt.day))
summ = book.comments summ = book.comments
if summ: if summ:
prefix = unicode(self.comments.toPlainText()) prefix = unicode(self.comments.toPlainText())
@ -779,8 +780,11 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
self.series.setText(book.series) self.series.setText(book.series)
if book.series_index is not None: if book.series_index is not None:
self.series_index.setValue(book.series_index) self.series_index.setValue(book.series_index)
# Needed because of Qt focus bug on OS X if book.has_cover:
self.fetch_cover_button.setFocus(Qt.OtherFocusReason) if d.opt_auto_download_cover.isChecked() and book.has_cover:
self.fetch_cover()
else:
self.fetch_cover_button.setFocus(Qt.OtherFocusReason)
else: else:
error_dialog(self, _('Cannot fetch metadata'), error_dialog(self, _('Cannot fetch metadata'),
_('You must specify at least one of ISBN, Title, ' _('You must specify at least one of ISBN, Title, '

3
src/calibre/gui2/library/models.py
View File

@ -1023,8 +1023,7 @@ class DeviceBooksModel(BooksModel): # {{{
x = '' x = ''
if y == None: if y == None:
y = '' y = ''
x, y = icu_lower(x.strip()), icu_lower(y.strip()) return icu_strcmp(x.strip(), y.strip())
return icu_strcmp(x, y)
return _strcmp return _strcmp
def datecmp(x, y): def datecmp(x, y):
x = self.db[x].datetime x = self.db[x].datetime

1
src/calibre/gui2/preferences/plugins.py
View File

@ -151,6 +151,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
self._plugin_model.populate() self._plugin_model.populate()
self._plugin_model.reset() self._plugin_model.reset()
self.changed_signal.emit() self.changed_signal.emit()
self.plugin_path.setText('')
else: else:
error_dialog(self, _('No valid plugin path'), error_dialog(self, _('No valid plugin path'),
_('%s is not a valid plugin path')%path).exec_() _('%s is not a valid plugin path')%path).exec_()

3
src/calibre/gui2/tag_view.py
View File

@ -73,6 +73,7 @@ class TagsView(QTreeView): # {{{
def __init__(self, parent=None): def __init__(self, parent=None):
QTreeView.__init__(self, parent=None) QTreeView.__init__(self, parent=None)
self.tag_match = None self.tag_match = None
self.disable_recounting = False
self.setUniformRowHeights(True) self.setUniformRowHeights(True)
self.setCursor(Qt.PointingHandCursor) self.setCursor(Qt.PointingHandCursor)
self.setIconSize(QSize(30, 30)) self.setIconSize(QSize(30, 30))
@ -299,6 +300,8 @@ class TagsView(QTreeView): # {{{
return self.isExpanded(idx) return self.isExpanded(idx)
def recount(self, *args): def recount(self, *args):
if self.disable_recounting:
return
self.refresh_signal_processed = True self.refresh_signal_processed = True
ci = self.currentIndex() ci = self.currentIndex()
if not ci.isValid(): if not ci.isValid():

111
src/calibre/library/save_to_disk.py
View File

@ -6,7 +6,7 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import os, traceback, cStringIO, re import os, traceback, cStringIO, re, shutil
from calibre.constants import DEBUG from calibre.constants import DEBUG
from calibre.utils.config import Config, StringConfig, tweaks from calibre.utils.config import Config, StringConfig, tweaks
@ -203,31 +203,49 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
return shorten_components_to(length, components) return shorten_components_to(length, components)
def save_book_to_disk(id, db, root, opts, length): def save_book_to_disk(id_, db, root, opts, length):
mi = db.get_metadata(id, index_is_id=True) mi = db.get_metadata(id_, index_is_id=True)
cover = db.cover(id_, index_is_id=True, as_path=True)
plugboards = db.prefs.get('plugboards', {})
available_formats = db.formats(id, index_is_id=True) available_formats = db.formats(id_, index_is_id=True)
if not available_formats: if not available_formats:
available_formats = [] available_formats = []
else: else:
available_formats = [x.lower().strip() for x in available_formats = [x.lower().strip() for x in
available_formats.split(',')] available_formats.split(',')]
formats = {}
fmts = db.formats(id_, index_is_id=True, verify_formats=False)
if fmts:
fmts = fmts.split(',')
for fmt in fmts:
fpath = db.format_abspath(id_, fmt, index_is_id=True)
if fpath is not None:
formats[fmt.lower()] = fpath
return do_save_book_to_disk(id_, mi, cover, plugboards,
formats, root, opts, length)
def do_save_book_to_disk(id_, mi, cover, plugboards,
format_map, root, opts, length):
available_formats = [x.lower().strip() for x in format_map.keys()]
if opts.formats == 'all': if opts.formats == 'all':
asked_formats = available_formats asked_formats = available_formats
else: else:
asked_formats = [x.lower().strip() for x in opts.formats.split(',')] asked_formats = [x.lower().strip() for x in opts.formats.split(',')]
formats = set(available_formats).intersection(set(asked_formats)) formats = set(available_formats).intersection(set(asked_formats))
if not formats: if not formats:
return True, id, mi.title return True, id_, mi.title
components = get_components(opts.template, mi, id, opts.timefmt, length, components = get_components(opts.template, mi, id_, opts.timefmt, length,
ascii_filename if opts.asciiize else sanitize_file_name, ascii_filename if opts.asciiize else sanitize_file_name,
to_lowercase=opts.to_lowercase, to_lowercase=opts.to_lowercase,
replace_whitespace=opts.replace_whitespace) replace_whitespace=opts.replace_whitespace)
base_path = os.path.join(root, *components) base_path = os.path.join(root, *components)
base_name = os.path.basename(base_path) base_name = os.path.basename(base_path)
dirpath = os.path.dirname(base_path) dirpath = os.path.dirname(base_path)
# Don't test for existence first are the test could fail but # Don't test for existence first as the test could fail but
# another worker process could create the directory before # another worker process could create the directory before
# the call to makedirs # the call to makedirs
try: try:
@ -236,29 +254,23 @@ def save_book_to_disk(id, db, root, opts, length):
if not os.path.exists(dirpath): if not os.path.exists(dirpath):
raise raise
cdata = db.cover(id, index_is_id=True) if opts.save_cover and cover and os.access(cover, os.R_OK):
if opts.save_cover: with open(base_path+'.jpg', 'wb') as f:
if cdata is not None: with open(cover, 'rb') as s:
with open(base_path+'.jpg', 'wb') as f: shutil.copyfileobj(s, f)
f.write(cdata) mi.cover = base_name+'.jpg'
mi.cover = base_name+'.jpg' else:
else: mi.cover = None
mi.cover = None
if opts.write_opf: if opts.write_opf:
opf = metadata_to_opf(mi) opf = metadata_to_opf(mi)
with open(base_path+'.opf', 'wb') as f: with open(base_path+'.opf', 'wb') as f:
f.write(opf) f.write(opf)
if cdata is not None:
mi.cover_data = ('jpg', cdata)
mi.cover = None
written = False written = False
for fmt in formats: for fmt in formats:
global plugboard_save_to_disk_value, plugboard_any_format_value global plugboard_save_to_disk_value, plugboard_any_format_value
dev_name = plugboard_save_to_disk_value dev_name = plugboard_save_to_disk_value
plugboards = db.prefs.get('plugboards', {})
cpb = None cpb = None
if fmt in plugboards: if fmt in plugboards:
cpb = plugboards[fmt] cpb = plugboards[fmt]
@ -275,11 +287,12 @@ def save_book_to_disk(id, db, root, opts, length):
# Leave this here for a while, in case problems arise. # Leave this here for a while, in case problems arise.
if cpb is not None: if cpb is not None:
prints('Save-to-disk using plugboard:', fmt, cpb) prints('Save-to-disk using plugboard:', fmt, cpb)
data = db.format(id, fmt, index_is_id=True) fp = format_map.get(fmt, None)
if data is None: if fp is None:
continue continue
else: with open(fp, 'rb') as f:
written = True data = f.read()
written = True
if opts.update_metadata: if opts.update_metadata:
stream = cStringIO.StringIO() stream = cStringIO.StringIO()
stream.write(data) stream.write(data)
@ -300,9 +313,21 @@ def save_book_to_disk(id, db, root, opts, length):
with open(fmt_path, 'wb') as f: with open(fmt_path, 'wb') as f:
f.write(data) f.write(data)
return not written, id, mi.title return not written, id_, mi.title
def _sanitize_args(root, opts):
if opts is None:
opts = config().parse()
if isinstance(root, unicode):
root = root.encode(filesystem_encoding)
root = os.path.abspath(root)
opts.template = preprocess_template(opts.template)
length = 1000 if supports_long_names(root) else 250
length -= len(root)
if length < 5:
raise ValueError('%r is too long.'%root)
return root, opts, length
def save_to_disk(db, ids, root, opts=None, callback=None): def save_to_disk(db, ids, root, opts=None, callback=None):
''' '''
@ -316,17 +341,7 @@ def save_to_disk(db, ids, root, opts=None, callback=None):
:return: A list of failures. Each element of the list is a tuple :return: A list of failures. Each element of the list is a tuple
(id, title, traceback) (id, title, traceback)
''' '''
if opts is None: root, opts, length = _sanitize_args(root, opts)
opts = config().parse()
if isinstance(root, unicode):
root = root.encode(filesystem_encoding)
root = os.path.abspath(root)
opts.template = preprocess_template(opts.template)
length = 1000 if supports_long_names(root) else 250
length -= len(root)
if length < 5:
raise ValueError('%r is too long.'%root)
failures = [] failures = []
for x in ids: for x in ids:
tb = '' tb = ''
@ -343,4 +358,28 @@ def save_to_disk(db, ids, root, opts=None, callback=None):
break break
return failures return failures
def save_serialized_to_disk(ids, data, plugboards, root, opts, callback):
from calibre.ebooks.metadata.opf2 import OPF
root, opts, length = _sanitize_args(root, opts)
failures = []
for x in ids:
opf, cover, format_map = data[x]
if isinstance(opf, unicode):
opf = opf.encode('utf-8')
mi = OPF(cStringIO.StringIO(opf)).to_book_metadata()
tb = ''
try:
failed, id, title = do_save_book_to_disk(x, mi, cover, plugboards,
format_map, root, opts, length)
tb = _('Requested formats not available')
except:
failed, id, title = True, x, mi.title
tb = traceback.format_exc()
if failed:
failures.append((id, title, tb))
if callable(callback):
if not callback(int(id), title, failed, tb):
break
return failures

10
src/calibre/manual/template_lang.rst
View File

@ -46,7 +46,6 @@ and if a book does not have a series::
(|app| automatically removes multiple slashes and leading or trailing spaces). (|app| automatically removes multiple slashes and leading or trailing spaces).
Advanced formatting Advanced formatting
---------------------- ----------------------
@ -80,6 +79,9 @@ For trailing zeros, use::
{series_index:0<3s} - Three digits with trailing zeros {series_index:0<3s} - Three digits with trailing zeros
If you use series indices with sub values (e.g., 1.1), you might want to ensure that the decimal points line up. For example, you might want the indices 1 and 2.5 to appear as 01.00 and 02.50 so that they will sort correctly. To do this, use::
{series_index:0>5.2f} - Five characters, consisting of two digits with leading zeros, a decimal point, then 2 digits after the decimal point
If you want only the first two letters of the data, use:: If you want only the first two letters of the data, use::
@ -115,15 +117,15 @@ The functions available are:
* ``lowercase()`` -- return value of the field in lower case. * ``lowercase()`` -- return value of the field in lower case.
* ``uppercase()`` -- return the value of the field in upper case. * ``uppercase()`` -- return the value of the field in upper case.
* ``titlecase()`` -- return the value of the field in title case. * ``titlecase()`` -- return the value of the field in title case.
* ``capitalize()`` -- return the value as capitalized. * ``capitalize()`` -- return the value with the first letter upper case and the rest lower case.
* ``ifempty(text)`` -- if the field is not empty, return the value of the field. Otherwise return `text`.
* ``test(text if not empty, text if empty)`` -- return `text if not empty` if the field is not empty, otherwise return `text if empty`.
* ``contains(pattern, text if match, text if not match`` -- checks if field contains matches for the regular expression `pattern`. Returns `text if match` if matches are found, otherwise it returns `text if no match`. * ``contains(pattern, text if match, text if not match`` -- checks if field contains matches for the regular expression `pattern`. Returns `text if match` if matches are found, otherwise it returns `text if no match`.
* ``count(separator)`` -- interprets the value as a list of items separated by `separator`, returning the number of items in the list. Most lists use a comma as the separator, but authors uses an ampersand. Examples: `{tags:count(,)}`, `{authors:count(&)}` * ``count(separator)`` -- interprets the value as a list of items separated by `separator`, returning the number of items in the list. Most lists use a comma as the separator, but authors uses an ampersand. Examples: `{tags:count(,)}`, `{authors:count(&)}`
* ``ifempty(text)`` -- if the field is not empty, return the value of the field. Otherwise return `text`.
* ``lookup(pattern, field, pattern, field, ..., else_field)`` -- like switch, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later). * ``lookup(pattern, field, pattern, field, ..., else_field)`` -- like switch, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later).
* ``re(pattern, replacement)`` -- return the field after applying the regular expression. All instances of `pattern` are replaced with `replacement`. As in all of |app|, these are python-compatible regular expressions. * ``re(pattern, replacement)`` -- return the field after applying the regular expression. All instances of `pattern` are replaced with `replacement`. As in all of |app|, these are python-compatible regular expressions.
* ``shorten(left chars, middle text, right chars)`` -- Return a shortened version of the field, consisting of `left chars` characters from the beginning of the field, followed by `middle text`, followed by `right chars` characters from the end of the string. `Left chars` and `right chars` must be integers. For example, assume the title of the book is `Ancient English Laws in the Times of Ivanhoe`, and you want it to fit in a space of at most 15 characters. If you use ``{title:shorten(9,-,5)}``, the result will be `Ancient E-nhoe`. If the field's length is less than ``left chars`` + ``right chars`` + the length of ``middle text``, then the field will be used intact. For example, the title `The Dome` would not be changed. * ``shorten(left chars, middle text, right chars)`` -- Return a shortened version of the field, consisting of `left chars` characters from the beginning of the field, followed by `middle text`, followed by `right chars` characters from the end of the string. `Left chars` and `right chars` must be integers. For example, assume the title of the book is `Ancient English Laws in the Times of Ivanhoe`, and you want it to fit in a space of at most 15 characters. If you use ``{title:shorten(9,-,5)}``, the result will be `Ancient E-nhoe`. If the field's length is less than ``left chars`` + ``right chars`` + the length of ``middle text``, then the field will be used intact. For example, the title `The Dome` would not be changed.
* ``switch(pattern, value, pattern, value, ..., else_value)`` -- for each ``pattern, value`` pair, checks if the field matches the regular expression ``pattern`` and if so, returns that ``value``. If no ``pattern`` matches, then ``else_value`` is returned. You can have as many ``pattern, value`` pairs as you want. * ``switch(pattern, value, pattern, value, ..., else_value)`` -- for each ``pattern, value`` pair, checks if the field matches the regular expression ``pattern`` and if so, returns that ``value``. If no ``pattern`` matches, then ``else_value`` is returned. You can have as many ``pattern, value`` pairs as you want.
* ``test(text if not empty, text if empty)`` -- return `text if not empty` if the field is not empty, otherwise return `text if empty`.
Now, about using functions and formatting in the same field. Suppose you have an integer custom column called ``#myint`` that you want to see with leading zeros, as in ``003``. To do this, you would use a format of ``0>3s``. However, by default, if a number (integer or float) equals zero then the field produces the empty value, so zero values will produce nothing, not ``000``. If you really want to see ``000`` values, then you use both the format string and the ``ifempty`` function to change the empty value back to a zero. The field reference would be:: Now, about using functions and formatting in the same field. Suppose you have an integer custom column called ``#myint`` that you want to see with leading zeros, as in ``003``. To do this, you would use a format of ``0>3s``. However, by default, if a number (integer or float) equals zero then the field produces the empty value, so zero values will produce nothing, not ``000``. If you really want to see ``000`` values, then you use both the format string and the ``ifempty`` function to change the empty value back to a zero. The field reference would be::

4
src/calibre/utils/bibtex.py
View File

@ -64,10 +64,6 @@ __author__ = 'sengian <sengian1 at gmail.com>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import re, string import re, string
from UserDict import UserDict
from calibre.constants import preferred_encoding
from calibre.utils.mreplace import MReplace
from calibre.constants import preferred_encoding from calibre.constants import preferred_encoding
from calibre.utils.mreplace import MReplace from calibre.utils.mreplace import MReplace

24
src/calibre/utils/cleantext.py
View File

@ -5,11 +5,19 @@ __docformat__ = 'restructuredtext en'
import re import re
def clean_ascii_char(txt, charlist = None): _ascii_pat = None
#remove ASCII invalid chars : 0 to 8 and 11-14 to 24-26-27 by default
chars = list(range(8)) + [0x0B, 0x0E, 0x0F] + list(range(0x10, 0x19)) \ def clean_ascii_chars(txt, charlist=None):
+ [0x1A, 0x1B] 'remove ASCII invalid chars : 0 to 8 and 11-14 to 24-26-27 by default'
if charlist is not None: global _ascii_pat
chars = charlist if _ascii_pat is None:
illegal_chars = re.compile(u'|'.join(map(unichr, chars))) chars = list(range(8)) + [0x0B, 0x0E, 0x0F] + list(range(0x10, 0x19)) \
return illegal_chars.sub('', txt) + [0x1A, 0x1B]
_ascii_pat = re.compile(u'|'.join(map(unichr, chars)))
if charlist is None:
pat = _ascii_pat
else:
pat = re.compile(u'|'.join(map(unichr, charlist)))
return pat.sub('', txt)

9
src/calibre/utils/date.py
View File

@ -152,7 +152,7 @@ def format_date(dt, format, assume_utc=False, as_utc=False):
format = re.sub('M{1,4}', format_month, format) format = re.sub('M{1,4}', format_month, format)
return re.sub('yyyy|yy', format_year, format) return re.sub('yyyy|yy', format_year, format)
def replace_months(datez, clang): def replace_months(datestr, clang):
# Replace months by english equivalent for parse_date # Replace months by english equivalent for parse_date
frtoen = { frtoen = {
u'[jJ]anvier': u'jan', u'[jJ]anvier': u'jan',
@ -186,9 +186,10 @@ def replace_months(datez, clang):
elif clang == 'de': elif clang == 'de':
dictoen = detoen dictoen = detoen
else: else:
return datez return datestr
for k in dictoen.iterkeys(): for k in dictoen.iterkeys():
tmp = re.sub(k, dictoen[k], datez) tmp = re.sub(k, dictoen[k], datestr)
if tmp != datez: break if tmp != datestr: break
return tmp return tmp

62
src/calibre/utils/formatter.py
View File

@ -8,12 +8,15 @@ import re, string, traceback
from calibre.constants import DEBUG from calibre.constants import DEBUG
from calibre.utils.titlecase import titlecase from calibre.utils.titlecase import titlecase
from calibre.utils.icu import capitalize
class TemplateFormatter(string.Formatter): class TemplateFormatter(string.Formatter):
''' '''
Provides a format function that substitutes '' for any missing value Provides a format function that substitutes '' for any missing value
''' '''
_validation_string = 'This Is Some Text THAT SHOULD be LONG Enough.%^&*'
# Dict to do recursion detection. It is up the the individual get_value # Dict to do recursion detection. It is up the the individual get_value
# method to use it. It is cleared when starting to format a template # method to use it. It is cleared when starting to format a template
composite_values = {} composite_values = {}
@ -86,7 +89,7 @@ class TemplateFormatter(string.Formatter):
'uppercase' : (0, lambda s,x: x.upper()), 'uppercase' : (0, lambda s,x: x.upper()),
'lowercase' : (0, lambda s,x: x.lower()), 'lowercase' : (0, lambda s,x: x.lower()),
'titlecase' : (0, lambda s,x: titlecase(x)), 'titlecase' : (0, lambda s,x: titlecase(x)),
'capitalize' : (0, lambda s,x: x.capitalize()), 'capitalize' : (0, lambda s,x: capitalize(x)),
'contains' : (3, _contains), 'contains' : (3, _contains),
'ifempty' : (1, _ifempty), 'ifempty' : (1, _ifempty),
'lookup' : (-1, _lookup), 'lookup' : (-1, _lookup),
@ -97,19 +100,29 @@ class TemplateFormatter(string.Formatter):
'count' : (1, _count), 'count' : (1, _count),
} }
format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$') def _do_format(self, val, fmt):
compress_spaces = re.compile(r'\s+') if not fmt or not val:
backslash_comma_to_comma = re.compile(r'\\,') return val
if val == self._validation_string:
arg_parser = re.Scanner([ val = '0'
(r',', lambda x,t: ''), typ = fmt[-1]
(r'.*?((?<!\\),)', lambda x,t: t[:-1]), if typ == 's':
(r'.*?\)', lambda x,t: t[:-1]), pass
]) elif 'bcdoxXn'.find(typ) >= 0:
try:
def get_value(self, key, args, kwargs): val = int(val)
raise Exception('get_value must be implemented in the subclass') except:
raise ValueError(
_('format: type {0} requires an integer value, got {1}').format(typ, val))
elif 'eEfFgGn%'.find(typ) >= 0:
try:
val = float(val)
except:
raise ValueError(
_('format: type {0} requires a decimal (float) value, got {1}').format(typ, val))
else:
raise ValueError(_('format: unknown format type letter {0}').format(typ))
return unicode(('{0:'+fmt+'}').format(val))
def _explode_format_string(self, fmt): def _explode_format_string(self, fmt):
try: try:
@ -122,6 +135,21 @@ class TemplateFormatter(string.Formatter):
traceback.print_exc() traceback.print_exc()
return fmt, '', '' return fmt, '', ''
format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
compress_spaces = re.compile(r'\s+')
backslash_comma_to_comma = re.compile(r'\\,')
arg_parser = re.Scanner([
(r',', lambda x,t: ''),
(r'.*?((?<!\\),)', lambda x,t: t[:-1]),
(r'.*?\)', lambda x,t: t[:-1]),
])
################## Override parent classes methods #####################
def get_value(self, key, args, kwargs):
raise Exception('get_value must be implemented in the subclass')
def format_field(self, val, fmt): def format_field(self, val, fmt):
# Handle conditional text # Handle conditional text
fmt, prefix, suffix = self._explode_format_string(fmt) fmt, prefix, suffix = self._explode_format_string(fmt)
@ -155,7 +183,7 @@ class TemplateFormatter(string.Formatter):
else: else:
val = func[1](self, val, *args).strip() val = func[1](self, val, *args).strip()
if val: if val:
val = string.Formatter.format_field(self, val, dispfmt) val = self._do_format(val, dispfmt)
if not val: if not val:
return '' return ''
return prefix + val + suffix return prefix + val + suffix
@ -164,6 +192,8 @@ class TemplateFormatter(string.Formatter):
ans = string.Formatter.vformat(self, fmt, args, kwargs) ans = string.Formatter.vformat(self, fmt, args, kwargs)
return self.compress_spaces.sub(' ', ans).strip() return self.compress_spaces.sub(' ', ans).strip()
########## a formatter guaranteed not to throw and exception ############
def safe_format(self, fmt, kwargs, error_value, book): def safe_format(self, fmt, kwargs, error_value, book):
self.kwargs = kwargs self.kwargs = kwargs
self.book = book self.book = book
@ -181,7 +211,7 @@ class ValidateFormat(TemplateFormatter):
Provides a format function that substitutes '' for any missing value Provides a format function that substitutes '' for any missing value
''' '''
def get_value(self, key, args, kwargs): def get_value(self, key, args, kwargs):
return 'this is some text that should be long enough' return self._validation_string
def validate(self, x): def validate(self, x):
return self.vformat(x, [], {}) return self.vformat(x, [], {})

4
src/calibre/utils/icu.c
View File

@ -237,8 +237,6 @@ static PyTypeObject icu_CollatorType = { // {{{
// }} // }}
// }}}
// }}} // }}}
// Module initialization {{{ // Module initialization {{{
@ -286,7 +284,7 @@ icu_upper(PyObject *self, PyObject *args) {
PyMem_Free(input); PyMem_Free(input);
return ret; return ret;
} } // }}}
// lower {{{ // lower {{{
static PyObject * static PyObject *

20
src/calibre/utils/icu.py
View File

@ -56,7 +56,7 @@ def py_sort_key(obj):
def icu_sort_key(collator, obj): def icu_sort_key(collator, obj):
if not obj: if not obj:
return _none2 return _none2
return collator.sort_key(obj.lower()) return collator.sort_key(lower(obj))
def py_case_sensitive_sort_key(obj): def py_case_sensitive_sort_key(obj):
if not obj: if not obj:
@ -69,7 +69,7 @@ def icu_case_sensitive_sort_key(collator, obj):
return collator.sort_key(obj) return collator.sort_key(obj)
def icu_strcmp(collator, a, b): def icu_strcmp(collator, a, b):
return collator.strcmp(a.lower(), b.lower()) return collator.strcmp(lower(a), lower(b))
def py_strcmp(a, b): def py_strcmp(a, b):
return cmp(a.lower(), b.lower()) return cmp(a.lower(), b.lower())
@ -104,6 +104,13 @@ lower = (lambda s: s.lower()) if _icu_not_ok else \
title_case = (lambda s: s.title()) if _icu_not_ok else \ title_case = (lambda s: s.title()) if _icu_not_ok else \
partial(_icu.title, get_locale()) partial(_icu.title, get_locale())
def icu_capitalize(s):
s = lower(s)
return s.replace(s[0], upper(s[0]))
capitalize = (lambda s: s.capitalize()) if _icu_not_ok else \
(lambda s: icu_capitalize(s))
################################################################################ ################################################################################
def test(): # {{{ def test(): # {{{
@ -215,14 +222,15 @@ pêché'''
print '\t', x.encode('utf-8') print '\t', x.encode('utf-8')
if fs != create(french_good): if fs != create(french_good):
print 'French failed (note that French fails with icu < 4.6 i.e. on windows and OS X)' print 'French failed (note that French fails with icu < 4.6 i.e. on windows and OS X)'
return # return
test_strcmp(german + french) test_strcmp(german + french)
print '\nTesting case transforms in current locale' print '\nTesting case transforms in current locale'
for x in ('a', 'Alice\'s code'): for x in ('a', 'Alice\'s code'):
print 'Upper:', x, '->', 'py:', x.upper().encode('utf-8'), 'icu:', upper(x).encode('utf-8') print 'Upper: ', x, '->', 'py:', x.upper().encode('utf-8'), 'icu:', upper(x).encode('utf-8')
print 'Lower:', x, '->', 'py:', x.lower().encode('utf-8'), 'icu:', lower(x).encode('utf-8') print 'Lower: ', x, '->', 'py:', x.lower().encode('utf-8'), 'icu:', lower(x).encode('utf-8')
print 'Title:', x, '->', 'py:', x.title().encode('utf-8'), 'icu:', title_case(x).encode('utf-8') print 'Title: ', x, '->', 'py:', x.title().encode('utf-8'), 'icu:', title_case(x).encode('utf-8')
print 'Capitalize:', x, '->', 'py:', x.capitalize().encode('utf-8'), 'icu:', capitalize(x).encode('utf-8')
print print
# }}} # }}}

4
src/calibre/utils/ipc/server.py
View File

@ -292,12 +292,12 @@ class Server(Thread):
except: except:
pass pass
time.sleep(0.2) time.sleep(0.2)
for worker in self.workers: for worker in list(self.workers):
try: try:
worker.kill() worker.kill()
except: except:
pass pass
for worker in self.pool: for worker in list(self.pool):
try: try:
worker.kill() worker.kill()
except: except:

7
src/calibre/utils/titlecase.py
View File

@ -9,6 +9,8 @@ License: http://www.opensource.org/licenses/mit-license.php
import re import re
from calibre.utils.icu import capitalize
__all__ = ['titlecase'] __all__ = ['titlecase']
__version__ = '0.5' __version__ = '0.5'
@ -40,11 +42,6 @@ def titlecase(text):
""" """
def capitalize(w):
w = icu_lower(w)
w = w.replace(w[0], icu_upper(w[0]))
return w
all_caps = ALL_CAPS.match(text) all_caps = ALL_CAPS.match(text)
words = re.split('\s', text) words = re.split('\s', text)

2
src/calibre/utils/zipfile.py
View File

@ -1227,7 +1227,7 @@ class ZipFile:
self.fp.flush() self.fp.flush()
if zinfo.flag_bits & 0x08: if zinfo.flag_bits & 0x08:
# Write CRC and file sizes after the file data # Write CRC and file sizes after the file data
self.fp.write(struct.pack("<lLL", zinfo.CRC, zinfo.compress_size, self.fp.write(struct.pack("<LLL", zinfo.CRC, zinfo.compress_size,
zinfo.file_size)) zinfo.file_size))
self.filelist.append(zinfo) self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo self.NameToInfo[zinfo.filename] = zinfo