diff --git a/resources/recipes/mainichi.recipe b/resources/recipes/mainichi.recipe index 2a44fa0980..baa7f409ec 100644 --- a/resources/recipes/mainichi.recipe +++ b/resources/recipes/mainichi.recipe @@ -4,6 +4,7 @@ __copyright__ = '2010, Hiroshi Miura ' www.mainichi.jp ''' +import re from calibre.web.feeds.news import BasicNewsRecipe class MainichiDailyNews(BasicNewsRecipe): @@ -22,3 +23,18 @@ class MainichiDailyNews(BasicNewsRecipe): remove_tags = [{'class':"RelatedArticle"}] remove_tags_after = {'class':"Credit"} + def parse_feeds(self): + + feeds = BasicNewsRecipe.parse_feeds(self) + + for curfeed in feeds: + delList = [] + for a,curarticle in enumerate(curfeed.articles): + if re.search(r'pheedo.jp', curarticle.url): + delList.append(curarticle) + if len(delList)>0: + for d in delList: + index = curfeed.articles.index(d) + curfeed.articles[index:index+1] = [] + + return feeds diff --git a/resources/recipes/mainichi_it_news.recipe b/resources/recipes/mainichi_it_news.recipe index 8e15496e57..4c285a2c01 100644 --- a/resources/recipes/mainichi_it_news.recipe +++ b/resources/recipes/mainichi_it_news.recipe @@ -14,5 +14,19 @@ class MainichiDailyITNews(BasicNewsRecipe): remove_tags_before = {'class':"NewsTitle"} remove_tags = [{'class':"RelatedArticle"}] - remove_tags_after = {'class':"Credit"} + def parse_feeds(self): + + feeds = BasicNewsRecipe.parse_feeds(self) + + for curfeed in feeds: + delList = [] + for a,curarticle in enumerate(curfeed.articles): + if re.search(r'pheedo.jp', curarticle.url): + delList.append(curarticle) + if len(delList)>0: + for d in delList: + index = curfeed.articles.index(d) + curfeed.articles[index:index+1] = [] + + return feeds remove_tags_after = {'class':"Credit"} diff --git a/resources/recipes/nikkei_sub_life.recipe b/resources/recipes/nikkei_sub_life.recipe index 1bfa08a55f..60e5b170ca 100644 --- a/resources/recipes/nikkei_sub_life.recipe +++ b/resources/recipes/nikkei_sub_life.recipe @@ -32,12 +32,9 @@ class NikkeiNet_sub_life(BasicNewsRecipe): remove_tags_after = {'class':"cmn-pr_list"} feeds = [ (u'\u304f\u3089\u3057', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kurashi'), - (u'\u30b9\u30dd\u30fc\u30c4', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=sports'), - (u'\u793e\u4f1a', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=shakai'), (u'\u30a8\u30b3', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=eco'), (u'\u5065\u5eb7', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kenkou'), - (u'\u7279\u96c6', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=special'), - (u'\u30e9\u30f3\u30ad\u30f3\u30b0', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=ranking') + (u'\u7279\u96c6', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=special') ] def get_browser(self): diff --git a/resources/recipes/nikkei_sub_shakai.recipe b/resources/recipes/nikkei_sub_shakai.recipe new file mode 100644 index 0000000000..ed86493265 --- /dev/null +++ b/resources/recipes/nikkei_sub_shakai.recipe @@ -0,0 +1,102 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +www.nikkei.com +''' + +import re +from calibre.web.feeds.recipes import BasicNewsRecipe +import mechanize +from calibre.ptempfile import PersistentTemporaryFile + + +class NikkeiNet_sub_life(BasicNewsRecipe): + title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(\u751f\u6d3b)' + __author__ = 'Hiroshi Miura' + description = 'News and current market affairs from Japan' + cover_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' + masthead_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' + needs_subscription = True + oldest_article = 2 + max_articles_per_feed = 20 + language = 'ja' + remove_javascript = False + temp_files = [] + + remove_tags_before = {'class':"cmn-section cmn-indent"} + remove_tags = [ + {'class':"JSID_basePageMove JSID_baseAsyncSubmit cmn-form_area JSID_optForm_utoken"}, + {'class':"cmn-article_keyword cmn-clearfix"}, + {'class':"cmn-print_headline cmn-clearfix"}, + ] + remove_tags_after = {'class':"cmn-pr_list"} + + feeds = [ + (u'\u793e\u4f1a', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=shakai') + ] + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + + cj = mechanize.LWPCookieJar() + br.set_cookiejar(cj) + + #br.set_debug_http(True) + #br.set_debug_redirects(True) + #br.set_debug_responses(True) + + if self.username is not None and self.password is not None: + #print "----------------------------get login form--------------------------------------------" + # open login form + br.open('https://id.nikkei.com/lounge/nl/base/LA0010.seam') + response = br.response() + #print "----------------------------get login form---------------------------------------------" + #print "----------------------------set login form---------------------------------------------" + # remove disabled input which brings error on mechanize + response.set_data(response.get_data().replace("", " -->")) + br.set_response(response) + br.select_form(name='LA0010Form01') + br['LA0010Form01:LA0010Email'] = self.username + br['LA0010Form01:LA0010Password'] = self.password + br.form.find_control(id='LA0010Form01:LA0010AutoLoginOn',type="checkbox").get(nr=0).selected = True + br.submit() + br.response() + #print "----------------------------send login form---------------------------------------------" + #print "----------------------------open news main page-----------------------------------------" + # open news site + br.open('http://www.nikkei.com/') + br.response() + #print "----------------------------www.nikkei.com BODY --------------------------------------" + #print response2.get_data() + #print "-------------------------^^-got auto redirect form----^^--------------------------------" + # forced redirect in default + br.select_form(nr=0) + br.submit() + response3 = br.response() + # return some cookie which should be set by Javascript + #print response3.geturl() + raw = response3.get_data() + #print "---------------------------response to form --------------------------------------------" + # grab cookie from JS and set it + redirectflag = re.search(r"var checkValue = '(\d+)';", raw, re.M).group(1) + br.select_form(nr=0) + + self.temp_files.append(PersistentTemporaryFile('_fa.html')) + self.temp_files[-1].write("#LWP-Cookies-2.0\n") + + self.temp_files[-1].write("Set-Cookie3: Cookie-dummy=Cookie-value; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") + self.temp_files[-1].write("Set-Cookie3: redirectFlag="+redirectflag+"; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") + self.temp_files[-1].close() + cj.load(self.temp_files[-1].name) + + br.submit() + + #br.set_debug_http(False) + #br.set_debug_redirects(False) + #br.set_debug_responses(False) + return br + + + + diff --git a/resources/recipes/st_louis_post_dispatch.recipe b/resources/recipes/st_louis_post_dispatch.recipe new file mode 100644 index 0000000000..3b7701cedc --- /dev/null +++ b/resources/recipes/st_louis_post_dispatch.recipe @@ -0,0 +1,70 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1282093204(BasicNewsRecipe): + title = u'St Louis Post-Dispatch' + __author__ = 'cisaak' + language = 'en' + + oldest_article = 1 + max_articles_per_feed = 15 + masthead_url = 'http://farm5.static.flickr.com/4118/4929686950_0e22e2c88a.jpg' + + feeds = [ + (u'News-Bill McClellan', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fcolumns%2Fbill-mclellan&f=rss&t=article'), + (u'News-Columns', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcolumns*&l=50&f=rss&t=article'), + (u'News-Crime & Courtshttp://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcrime-and-courts&l=50&f=rss&t=article'), + (u'News-Deb Peterson', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fcolumns%2Fdeb-peterson&f=rss&t=article'), + (u'News-Education', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2feducation&f=rss&t=article'), + (u'News-Government & Politics', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fgovt-and-politics&f=rss&t=article'), + (u'News-Local', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal&f=rss&t=article'), + (u'News-Metro', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fmetro&f=rss&t=article'), + (u'News-Metro East', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fillinois&f=rss&t=article'), + (u'News-Missouri Out State', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Fstate-and-regional%2FMissouri&l=50&f=rss&t=article'), + (u'Opinion-Colleen Carroll Campbell', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Fopinion%2fcolumns%2Fcolleen-carroll-campbell&f=rss&t=article'), + (u'Opinion-Editorial', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Fopinion%2feditorial&f=rss&t=article'), + (u'Opinion-Kevin Horrigan', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Fopinion%2fcolumns%2Fkevin-horrigan&f=rss&t=article'), + (u'Opinion-Mailbag', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Fopinion%2fmailbag&f=rss&t=article'), + (u'Business Columns-Savvy Consumer', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fsavvy-consumer&l=100&f=rss&t=article'), + (u'Business Columns-Lager Heads', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Flager-heads&l=100&f=rss&t=article'), + (u'Business Columns-Job Watch', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fjob-watch&l=100&f=rss&t=article'), + (u'Business Columns-Steve Geigerich', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fsteve-giegerich&l=100&f=rss&t=article'), + (u'Business Columns-David Nicklaus', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fdavid-nicklaus&l=100&f=rss&t=article'), + (u'Business Columns-Jim Gallagher', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fjim-gallagher&l=100&f=rss&t=article'), + (u'Business Columns-Building Blocks', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fcolumns%2Fbuilding-blocks&l=100&f=rss&t=article'), + (u'Business', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business*l&l=100&f=rss&t=article'), + (u'Business-Technology', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Ftechnology&l=50&f=rss&t=article'), + (u'Business-National', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=business%2Fnational-and-international&l=50&f=rss&t=article'), + (u'Travel', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=travel*&l=100&f=rss&t=article'), + (u'Sports', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports*&f=rss&t=article'), + (u'Sports-Baseball', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fbaseball%2Fprofessional&l=100&f=rss&t=article'), + (u'Sports-Bernie Miklasz', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcolumns%2Fbernie-miklasz&l=50&f=rss&t=article'), + (u'Sports-Bryan Burwell', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcolumns%2Fbryan-burwell&l=50&f=rss&t=article'), + (u'Sports-College', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcollege*&l=100&f=rss&t=article'), + (u'Sports-Dan Caesar', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcolumns%2Fdan-caesar&l=50&f=rss&t=article'), + (u'Sports-Football', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Ffootball%2Fprofessional&l=100&f=rss&t=article'), + (u'Sports-Hockey', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fhockey%2Fprofessional&l=100&f=rss&t=article'), + (u'Sports-Illini', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcollege%2Fillini&l=100&f=rss&t=article'), + (u'Sports-Jeff Gordon', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fcolumns%2Fjeff-gordon&l=100&f=rss&t=article'), + (u'Life & Style', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles&l=100&f=rss&t=article'), + (u'Life & Style-Debra Bass', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Ffashion-and-style%2Fdebra-bass&l=100&f=rss&t=article'), + (u'Life & Style-Food and Cooking', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Ffood-and-cooking&l=100&f=rss&t=article'), + (u'Life & Style-Health/Medicine/Fitness', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Fhealth-med-fit&l=100&f=rss&t=article'), + (u'Life & Style-Joe Holleman', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Fcolumns%2Fjoe-holleman&l=100&f=rss&t=article'), + (u'Life & Style-Steals-and-Deals', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Fcolumns%2Fsteals-and-deals&l=100&f=rss&t=article'), + (u'Life & Style-Tim Townsend', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=lifestyles%2Ffaith-and-values%2Ftim-townsend&l=100&f=rss&t=article'), + (u'Entertainment', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment&l=100&f=rss&t=article'), + (u'Entertainment-Arts & Theatre', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Farts-and-theatre&l=100&f=rss&t=article'), + (u'Entertainment-Books & Literature', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fbooks-and-literature&l=100&f=rss&t=article'), + (u'Entertainment-Dining', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=sports%2Fhockey%2Fprofessional&l=100&f=rss&t=article'), + (u'Entertainment-Events Calendar', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fevents-calendar&l=100&f=rss&t=article'), + (u'Entertainment-Gail Pennington', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Ftelevision%2Fgail-pennington&l=100&f=rss&t=article'), + (u'Entertainment-Hip Hops', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fdining%2Fbars-and-clubs-other%2Fhip-hops&l=100&f=rss&t=article'), + (u'Entertainment-House-O-Fun', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fhouse-o-fun&l=100&f=rss&t=article'), + (u'Entertainment-Kevin C. Johnson', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fmusic%2Fkevin-johnson&l=100&f=rss&t=article') + ] + remove_empty_feeds = True + remove_tags = [dict(name='div', attrs={'id':'blox-logo'}),dict(name='a')] + keep_only_tags = [dict(name='h1'), dict(name='p', attrs={'class':'byline'}), dict(name="div", attrs={'id':'blox-story-text'})] + extra_css = 'p {text-align: left;}' + + diff --git a/resources/recipes/yomiuri.recipe b/resources/recipes/yomiuri.recipe index d30aa9066f..fb17bb1210 100644 --- a/resources/recipes/yomiuri.recipe +++ b/resources/recipes/yomiuri.recipe @@ -21,7 +21,7 @@ class YOLNews(BasicNewsRecipe): remove_javascript = True masthead_title = u'YOMIURI ONLINE' - remove_tags_before = {'class':"article-def"} + keep_only_tags = [{'class':"article-def"}] remove_tags = [{'class':"RelatedArticle"}, {'class':"sbtns"} ] diff --git a/resources/recipes/yomiuri_world.recipe b/resources/recipes/yomiuri_world.recipe index f5f21c4aab..41ee4fd23d 100644 --- a/resources/recipes/yomiuri_world.recipe +++ b/resources/recipes/yomiuri_world.recipe @@ -21,7 +21,7 @@ class YOLNews(BasicNewsRecipe): remove_javascript = True masthead_title = u"YOMIURI ONLINE" - remove_tags_before = {'class':"article-def"} + keep_only_tags = [{'class':"article-def"}] remove_tags = [{'class':"RelatedArticle"}, {'class':"sbtns"} ] diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 0deef5eb92..8b30631528 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -21,7 +21,7 @@ class ANDROID(USBMS): # HTC 0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100, 0x0227], 0x0ff9 : [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226], - 0xc92 : [0x100]}, + 0xc92 : [0x100], 0xc97: [0x226]}, # Eken 0x040d : { 0x8510 : [0x0001] }, @@ -63,7 +63,7 @@ class ANDROID(USBMS): WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', - 'SCH-I500_CARD'] + 'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID'] diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index 23ce1716af..7a5e8c49b3 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -11,9 +11,9 @@ from calibre.ebooks.metadata.book.base import Metadata from calibre.devices.mime import mime_type_ext from calibre.devices.interface import BookList as _BookList from calibre.constants import preferred_encoding -from calibre import isbytestring +from calibre import isbytestring, force_unicode from calibre.utils.config import prefs, tweaks -from calibre.utils.icu import sort_key, strcmp as icu_strcmp +from calibre.utils.icu import strcmp class Book(Metadata): def __init__(self, prefix, lpath, size=None, other=None): @@ -241,7 +241,7 @@ class CollectionsBookList(BookList): if y is None: return -1 if isinstance(x, (unicode, str)): - c = strcmp(x, y) + c = strcmp(force_unicode(x), force_unicode(y)) else: c = cmp(x, y) if c != 0: diff --git a/src/calibre/ebooks/metadata/amazonfr.py b/src/calibre/ebooks/metadata/amazonfr.py index 5df962d8f5..156fff3d75 100644 --- a/src/calibre/ebooks/metadata/amazonfr.py +++ b/src/calibre/ebooks/metadata/amazonfr.py @@ -10,7 +10,7 @@ from lxml import html from lxml.html import soupparser from calibre.utils.date import parse_date, utcnow, replace_months -from calibre.utils.cleantext import clean_ascii_char +from calibre.utils.cleantext import clean_ascii_chars from calibre import browser, preferred_encoding from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.metadata import MetaInformation, check_isbn, \ @@ -22,8 +22,8 @@ from calibre.library.comments import sanitize_comments_html class AmazonFr(MetadataSource): - name = 'Amazon french' - description = _('Downloads social metadata from amazon.fr') + name = 'Amazon French' + description = _('Downloads metadata from amazon.fr') supported_platforms = ['windows', 'osx', 'linux'] author = 'Sengian' version = (1, 0, 0) @@ -39,8 +39,8 @@ class AmazonFr(MetadataSource): class AmazonEs(MetadataSource): - name = 'Amazon spanish' - description = _('Downloads social metadata from amazon.com in spanish') + name = 'Amazon Spanish' + description = _('Downloads metadata from amazon.com in spanish') supported_platforms = ['windows', 'osx', 'linux'] author = 'Sengian' version = (1, 0, 0) @@ -56,8 +56,8 @@ class AmazonEs(MetadataSource): class AmazonEn(MetadataSource): - name = 'Amazon english' - description = _('Downloads social metadata from amazon.com in english') + name = 'Amazon English' + description = _('Downloads metadata from amazon.com in english') supported_platforms = ['windows', 'osx', 'linux'] author = 'Sengian' version = (1, 0, 0) @@ -73,8 +73,8 @@ class AmazonEn(MetadataSource): class AmazonDe(MetadataSource): - name = 'Amazon german' - description = _('Downloads social metadata from amazon.de') + name = 'Amazon German' + description = _('Downloads metadata from amazon.de') supported_platforms = ['windows', 'osx', 'linux'] author = 'Sengian' version = (1, 0, 0) @@ -91,7 +91,7 @@ class AmazonDe(MetadataSource): class Amazon(MetadataSource): name = 'Amazon' - description = _('Downloads social metadata from amazon.com') + description = _('Downloads metadata from amazon.com') supported_platforms = ['windows', 'osx', 'linux'] author = 'Kovid Goyal & Sengian' version = (1, 1, 0) @@ -106,7 +106,7 @@ class Amazon(MetadataSource): except Exception, e: self.exception = e self.tb = traceback.format_exc() - + # @property # def string_customization_help(self): # return _('You can select here the language for metadata search with amazon.com') @@ -130,8 +130,8 @@ class Query(object): assert (max_results < 21) self.max_results = int(max_results) - self.renbres = re.compile(u'\s*(\d+)\s*') - + self.renbres = re.compile(u'\s*(\d+)\s*') + q = { 'search-alias' : 'stripbooks' , 'unfiltered' : '1', 'field-keywords' : '', @@ -151,7 +151,7 @@ class Query(object): # 'field-collection' : '', #many options available } - + if rlang =='all': q['sort'] = 'relevanceexprank' self.urldata = self.BASE_URL_ALL @@ -170,7 +170,7 @@ class Query(object): q['sort'] = 'relevancerank' self.urldata = self.BASE_URL_DE self.baseurl = self.urldata - + if isbn is not None: q['field-isbn'] = isbn.replace('-', '') else: @@ -184,13 +184,13 @@ class Query(object): q['field-keywords'] = keywords if isinstance(q, unicode): - q = q.encode('utf-8') + q = q.encode('utf-8') self.urldata += '/gp/search/ref=sr_adv_b/?' + urlencode(q) def __call__(self, browser, verbose, timeout = 5.): if verbose: print 'Query:', self.urldata - + try: raw = browser.open_novisit(self.urldata, timeout=timeout).read() except Exception, e: @@ -203,22 +203,22 @@ class Query(object): return raw = xml_to_unicode(raw, strip_encoding_pats=True, resolve_entities=True)[0] - + try: feed = soupparser.fromstring(raw) except: try: #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_char(raw)) + return soupparser.fromstring(clean_ascii_chars(raw)) except: return None, self.urldata - + #nb of page try: nbresults = self.renbres.findall(feed.xpath("//*[@class='resultCount']")[0].text) except: return None, self.urldata - + pages =[feed] if len(nbresults) > 1: nbpagetoquery = int(ceil(float(min(int(nbresults[2]), self.max_results))/ int(nbresults[1]))) @@ -237,11 +237,11 @@ class Query(object): except: try: #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_char(raw)) + return soupparser.fromstring(clean_ascii_chars(raw)) except: continue pages.append(feed) - + results = [] for x in pages: results.extend([i.getparent().get('href') \ @@ -429,7 +429,7 @@ class ResultList(list): except: try: #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_char(raw)) + return soupparser.fromstring(clean_ascii_chars(raw)) except: report(verbose) return @@ -438,7 +438,7 @@ class ResultList(list): for x in entries: try: entry = self.get_individual_metadata(browser, x, verbose) - # clean results + # clean results # inv_ids = ('divsinglecolumnminwidth', 'sims.purchase', 'AutoBuyXGetY', 'A9AdsMiddleBoxTop') # inv_class = ('buyingDetailsGrid', 'productImageGrid') # inv_tags ={'script': True, 'style': True, 'form': False} @@ -460,10 +460,10 @@ def search(title=None, author=None, publisher=None, isbn=None, br = browser() entries, baseurl = Query(title=title, author=author, isbn=isbn, publisher=publisher, keywords=keywords, max_results=max_results,rlang=lang)(br, verbose) - + if entries is None or len(entries) == 0: return - + #List of entry ans = ResultList(baseurl, lang) ans.populate(entries, br, verbose) @@ -513,4 +513,4 @@ def main(args=sys.argv): print if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/fictionwise.py b/src/calibre/ebooks/metadata/fictionwise.py index c4a8597dde..b780f2b39d 100644 --- a/src/calibre/ebooks/metadata/fictionwise.py +++ b/src/calibre/ebooks/metadata/fictionwise.py @@ -6,7 +6,6 @@ __docformat__ = 'restructuredtext en' import sys, textwrap, re, traceback, socket from urllib import urlencode -from lxml import html from lxml.html import soupparser, tostring from calibre import browser, preferred_encoding @@ -17,7 +16,7 @@ from calibre.library.comments import sanitize_comments_html from calibre.ebooks.metadata.fetch import MetadataSource from calibre.utils.config import OptionParser from calibre.utils.date import parse_date, utcnow -from calibre.utils.cleantext import clean_ascii_char +from calibre.utils.cleantext import clean_ascii_chars class Fictionwise(MetadataSource): # {{{ @@ -89,7 +88,7 @@ class Query(object): def __call__(self, browser, verbose, timeout = 5.): if verbose: print _('Query: %s') % self.BASE_URL+self.urldata - + try: raw = browser.open_novisit(self.BASE_URL, self.urldata, timeout=timeout).read() except Exception, e: @@ -109,7 +108,7 @@ class Query(object): except: try: #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_char(raw)) + feed = soupparser.fromstring(clean_ascii_chars(raw)) except: return None @@ -123,10 +122,10 @@ class Query(object): return results class ResultList(list): - + BASE_URL = 'http://www.fictionwise.com' COLOR_VALUES = {'BLUE': 4, 'GREEN': 3, 'YELLOW': 2, 'RED': 1, 'NA': 0} - + def __init__(self): self.retitle = re.compile(r'\[[^\[\]]+\]') self.rechkauth = re.compile(r'.*book\s*by', re.I) @@ -202,7 +201,7 @@ class ResultList(list): except: report(verbose) return None - hval = dict((self.COLOR_VALUES[self.recolor.search(image.get('src', default='NA.gif')).group("ncolor")], + hval = dict((self.COLOR_VALUES[self.recolor.search(image.get('src', default='NA.gif')).group("ncolor")], float(image.get('height', default=0))) \ for image in entrytable.getiterator('img')) #ratings as x/5 @@ -295,7 +294,7 @@ class ResultList(list): except: try: #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_char(raw)) + return soupparser.fromstring(clean_ascii_chars(raw)) except: return None @@ -343,7 +342,7 @@ def search(title=None, author=None, publisher=None, isbn=None, br = browser() entries = Query(title=title, author=author, publisher=publisher, keywords=keywords, max_results=max_results)(br, verbose, timeout = 15.) - + #List of entry ans = ResultList() ans.populate(entries, br, verbose) diff --git a/src/calibre/ebooks/metadata/isbndb.py b/src/calibre/ebooks/metadata/isbndb.py index cd39f8a0e1..9169227326 100644 --- a/src/calibre/ebooks/metadata/isbndb.py +++ b/src/calibre/ebooks/metadata/isbndb.py @@ -137,7 +137,7 @@ def create_books(opts, args, timeout=5.): if opts.verbose: print ('ISBNDB query: '+url) - + tans = [ISBNDBMetadata(book) for book in fetch_metadata(url, timeout=timeout)] #remove duplicates ISBN return list(dict((book.isbn, book) for book in tans).values()) diff --git a/src/calibre/ebooks/metadata/nicebooks.py b/src/calibre/ebooks/metadata/nicebooks.py index 01e20261b3..8914e2d985 100644 --- a/src/calibre/ebooks/metadata/nicebooks.py +++ b/src/calibre/ebooks/metadata/nicebooks.py @@ -11,7 +11,7 @@ from copy import deepcopy from lxml.html import soupparser from calibre.utils.date import parse_date, utcnow, replace_months -from calibre.utils.cleantext import clean_ascii_char +from calibre.utils.cleantext import clean_ascii_chars from calibre import browser, preferred_encoding from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.metadata import MetaInformation, check_isbn, \ @@ -126,7 +126,7 @@ class Query(object): except: try: #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_char(raw)) + feed = soupparser.fromstring(clean_ascii_chars(raw)) except: return None @@ -155,7 +155,7 @@ class Query(object): except: try: #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_char(raw)) + feed = soupparser.fromstring(clean_ascii_chars(raw)) except: continue pages.append(feed) @@ -251,7 +251,7 @@ class ResultList(list): except: try: #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_char(raw)) + feed = soupparser.fromstring(clean_ascii_chars(raw)) except: return None diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index 247050856d..d059d7e34c 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -8,12 +8,12 @@ __docformat__ = 'restructuredtext en' from threading import Thread from Queue import Empty -import os, time, sys, shutil +import os, time, sys, shutil, json from calibre.utils.ipc.job import ParallelJob from calibre.utils.ipc.server import Server from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory -from calibre import prints +from calibre import prints, isbytestring from calibre.constants import filesystem_encoding @@ -194,14 +194,42 @@ class SaveWorker(Thread): self.daemon = True self.path, self.opts = path, opts self.ids = ids - self.library_path = db.library_path + self.db = db self.canceled = False self.result_queue = result_queue self.error = None self.spare_server = spare_server self.start() + def collect_data(self, ids): + from calibre.ebooks.metadata.opf2 import metadata_to_opf + data = {} + for i in set(ids): + mi = self.db.get_metadata(i, index_is_id=True, get_cover=True) + opf = metadata_to_opf(mi) + if isbytestring(opf): + opf = opf.decode('utf-8') + cpath = None + if mi.cover: + cpath = mi.cover + if isbytestring(cpath): + cpath = cpath.decode(filesystem_encoding) + formats = {} + if mi.formats: + for fmt in mi.formats: + fpath = self.db.format_abspath(i, fmt, index_is_id=True) + if fpath is not None: + if isbytestring(fpath): + fpath = fpath.decode(filesystem_encoding) + formats[fmt.lower()] = fpath + data[i] = [opf, cpath, formats] + return data + def run(self): + with TemporaryDirectory('save_to_disk_data') as tdir: + self._run(tdir) + + def _run(self, tdir): from calibre.library.save_to_disk import config server = Server() if self.spare_server is None else self.spare_server ids = set(self.ids) @@ -212,12 +240,19 @@ class SaveWorker(Thread): for pref in c.preferences: recs[pref.name] = getattr(self.opts, pref.name) + plugboards = self.db.prefs.get('plugboards', {}) + for i, task in enumerate(tasks): tids = [x[-1] for x in task] + data = self.collect_data(tids) + dpath = os.path.join(tdir, '%d.json'%i) + with open(dpath, 'wb') as f: + f.write(json.dumps(data, ensure_ascii=False).encode('utf-8')) + job = ParallelJob('save_book', 'Save books (%d of %d)'%(i, len(tasks)), lambda x,y:x, - args=[tids, self.library_path, self.path, recs]) + args=[tids, dpath, plugboards, self.path, recs]) jobs.add(job) server.add_job(job) @@ -226,21 +261,21 @@ class SaveWorker(Thread): time.sleep(0.2) running = False for job in jobs: - job.update(consume_notifications=False) - while True: - try: - id, title, ok, tb = job.notifications.get_nowait()[0] - if id in ids: - self.result_queue.put((id, title, ok, tb)) - ids.remove(id) - except Empty: - break + self.get_notifications(job, ids) if not job.is_finished: running = True if not running: break + for job in jobs: + if not job.result: + continue + for id_, title, ok, tb in job.result: + if id_ in ids: + self.result_queue.put((id_, title, ok, tb)) + ids.remove(id_) + server.close() time.sleep(1) @@ -257,21 +292,39 @@ class SaveWorker(Thread): except: pass + def get_notifications(self, job, ids): + job.update(consume_notifications=False) + while True: + try: + id, title, ok, tb = job.notifications.get_nowait()[0] + if id in ids: + self.result_queue.put((id, title, ok, tb)) + ids.remove(id) + except Empty: + break -def save_book(task, library_path, path, recs, notification=lambda x,y:x): - from calibre.library.database2 import LibraryDatabase2 - db = LibraryDatabase2(library_path) - from calibre.library.save_to_disk import config, save_to_disk + +def save_book(ids, dpath, plugboards, path, recs, notification=lambda x,y:x): + from calibre.library.save_to_disk import config, save_serialized_to_disk from calibre.customize.ui import apply_null_metadata opts = config().parse() for name in recs: setattr(opts, name, recs[name]) + results = [] def callback(id, title, failed, tb): + results.append((id, title, not failed, tb)) notification((id, title, not failed, tb)) return True - with apply_null_metadata: - save_to_disk(db, task, path, opts, callback) + data_ = json.loads(open(dpath, 'rb').read().decode('utf-8')) + data = {} + for k, v in data_.iteritems(): + data[int(k)] = v + + with apply_null_metadata: + save_serialized_to_disk(ids, data, plugboards, path, opts, callback) + + return results diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index 0f61969373..8e11ac6498 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -544,7 +544,7 @@ class OEBReader(object): data = render_html_svg_workaround(path, self.logger) if not data: data = '' - id, href = self.oeb.manifest.generate('cover', 'cover.jpeg') + id, href = self.oeb.manifest.generate('cover', 'cover.jpg') item = self.oeb.manifest.add(id, href, JPEG_MIME, data=data) return item diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index b7cbc24d65..57ca2a1880 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -123,8 +123,8 @@ def _config(): help=_('Download social metadata (tags/rating/etc.)')) c.add_opt('overwrite_author_title_metadata', default=True, help=_('Overwrite author and title with new metadata')) - c.add_opt('overwrite_cover_image', default=False, - help=_('Overwrite cover with new new cover if existing')) + c.add_opt('auto_download_cover', default=False, + help=_('Automatically download the cover, if available')) c.add_opt('enforce_cpu_limit', default=True, help=_('Limit max simultaneous jobs to number of CPUs')) c.add_opt('tag_browser_hidden_categories', default=set(), diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 9b348d8285..014fa573d2 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -61,6 +61,7 @@ class AddAction(InterfaceAction): self._adder = Adder(self.gui, self.gui.library_view.model().db, self.Dispatcher(self._files_added), spare_server=self.gui.spare_server) + self.gui.tags_view.disable_recounting = True self._adder.add_recursive(root, single) def add_recursive_single(self, *args): @@ -201,9 +202,11 @@ class AddAction(InterfaceAction): self._adder = Adder(self.gui, None if to_device else self.gui.library_view.model().db, self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server) + self.gui.tags_view.disable_recounting = True self._adder.add(paths) def _files_added(self, paths=[], names=[], infos=[], on_card=None): + self.gui.tags_view.disable_recounting = False if paths: self.gui.upload_books(paths, list(map(ascii_filename, names)), @@ -214,6 +217,7 @@ class AddAction(InterfaceAction): self.gui.library_view.model().books_added(self._adder.number_of_books_added) if hasattr(self.gui, 'db_images'): self.gui.db_images.reset() + self.gui.tags_view.recount() if getattr(self._adder, 'merged_books', False): books = u'\n'.join([x if isinstance(x, unicode) else x.decode(preferred_encoding, 'replace') for x in diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 1339070446..5f555ef138 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -3,41 +3,55 @@ UI for adding books to the database and saving books to disk ''' import os, shutil, time from Queue import Queue, Empty -from threading import Thread +from functools import partial -from PyQt4.Qt import QThread, SIGNAL, QObject, QTimer, Qt, \ - QProgressDialog +from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer from calibre.gui2.dialogs.progress import ProgressDialog from calibre.gui2 import question_dialog, error_dialog, info_dialog from calibre.ebooks.metadata.opf2 import OPF from calibre.ebooks.metadata import MetaInformation -from calibre.constants import preferred_encoding, filesystem_encoding +from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG from calibre.utils.config import prefs +from calibre import prints + +single_shot = partial(QTimer.singleShot, 75) + +class DuplicatesAdder(QObject): # {{{ + + added = pyqtSignal(object) + adding_done = pyqtSignal() -class DuplicatesAdder(QThread): # {{{ - # Add duplicate books def __init__(self, parent, db, duplicates, db_adder): - QThread.__init__(self, parent) + QObject.__init__(self, parent) self.db, self.db_adder = db, db_adder - self.duplicates = duplicates + self.duplicates = list(duplicates) + self.count = 0 + single_shot(self.add_one) + + def add_one(self): + if not self.duplicates: + self.adding_done.emit() + return + + mi, cover, formats = self.duplicates.pop() + formats = [f for f in formats if not f.lower().endswith('.opf')] + id = self.db.create_book_entry(mi, cover=cover, + add_duplicates=True) + # here we add all the formats for dupe book record created above + self.db_adder.add_formats(id, formats) + self.db_adder.number_of_books_added += 1 + self.count += 1 + self.added.emit(self.count) + single_shot(self.add_one) - def run(self): - count = 1 - for mi, cover, formats in self.duplicates: - formats = [f for f in formats if not f.lower().endswith('.opf')] - id = self.db.create_book_entry(mi, cover=cover, - add_duplicates=True) - # here we add all the formats for dupe book record created above - self.db_adder.add_formats(id, formats) - self.db_adder.number_of_books_added += 1 - self.emit(SIGNAL('added(PyQt_PyObject)'), count) - count += 1 - self.emit(SIGNAL('adding_done()')) # }}} class RecursiveFind(QThread): # {{{ + update = pyqtSignal(object) + found = pyqtSignal(object) + def __init__(self, parent, db, root, single): QThread.__init__(self, parent) self.db = db @@ -50,8 +64,8 @@ class RecursiveFind(QThread): # {{{ for dirpath in os.walk(root): if self.canceled: return - self.emit(SIGNAL('update(PyQt_PyObject)'), - _('Searching in')+' '+dirpath[0]) + self.update.emit( + _('Searching in')+' '+dirpath[0]) self.books += list(self.db.find_books_in_directory(dirpath[0], self.single_book_per_directory)) @@ -71,46 +85,55 @@ class RecursiveFind(QThread): # {{{ msg = unicode(err) except: msg = repr(err) - self.emit(SIGNAL('found(PyQt_PyObject)'), msg) + self.found.emit(msg) return self.books = [formats for formats in self.books if formats] if not self.canceled: - self.emit(SIGNAL('found(PyQt_PyObject)'), self.books) + self.found.emit(self.books) # }}} -class DBAdder(Thread): # {{{ +class DBAdder(QObject): # {{{ + + def __init__(self, parent, db, ids, nmap): + QObject.__init__(self, parent) - def __init__(self, db, ids, nmap): self.db, self.ids, self.nmap = db, dict(**ids), dict(**nmap) - self.end = False self.critical = {} self.number_of_books_added = 0 self.duplicates = [] self.names, self.paths, self.infos = [], [], [] - Thread.__init__(self) - self.daemon = True self.input_queue = Queue() self.output_queue = Queue() self.merged_books = set([]) - def run(self): - while not self.end: - try: - id, opf, cover = self.input_queue.get(True, 0.2) - except Empty: - continue - name = self.nmap.pop(id) - title = None - try: - title = self.add(id, opf, cover, name) - except: - import traceback - self.critical[name] = traceback.format_exc() - title = name - self.output_queue.put(title) + def end(self): + self.input_queue.put((None, None, None)) + + def start(self): + try: + id, opf, cover = self.input_queue.get_nowait() + except Empty: + single_shot(self.start) + return + if id is None and opf is None and cover is None: + return + name = self.nmap.pop(id) + title = None + if DEBUG: + st = time.time() + try: + title = self.add(id, opf, cover, name) + except: + import traceback + self.critical[name] = traceback.format_exc() + title = name + self.output_queue.put(title) + if DEBUG: + prints('Added', title, 'to db in:', time.time() - st, 'seconds') + single_shot(self.start) def process_formats(self, opf, formats): imp = opf[:-4]+'.import' @@ -201,10 +224,10 @@ class Adder(QObject): # {{{ self.pd.setModal(True) self.pd.show() self._parent = parent - self.rfind = self.worker = self.timer = None + self.rfind = self.worker = None self.callback = callback self.callback_called = False - self.connect(self.pd, SIGNAL('canceled()'), self.canceled) + self.pd.canceled_signal.connect(self.canceled) def add_recursive(self, root, single=True): self.path = root @@ -213,10 +236,8 @@ class Adder(QObject): # {{{ self.pd.set_max(0) self.pd.value = 0 self.rfind = RecursiveFind(self, self.db, root, single) - self.connect(self.rfind, SIGNAL('update(PyQt_PyObject)'), - self.pd.set_msg, Qt.QueuedConnection) - self.connect(self.rfind, SIGNAL('found(PyQt_PyObject)'), - self.add, Qt.QueuedConnection) + self.rfind.update.connect(self.pd.set_msg, type=Qt.QueuedConnection) + self.rfind.found.connect(self.add, type=Qt.QueuedConnection) self.rfind.start() def add(self, books): @@ -246,12 +267,12 @@ class Adder(QObject): # {{{ self.pd.set_min(0) self.pd.set_max(len(self.ids)) self.pd.value = 0 - self.db_adder = DBAdder(self.db, self.ids, self.nmap) + self.db_adder = DBAdder(self, self.db, self.ids, self.nmap) self.db_adder.start() self.last_added_at = time.time() self.entry_count = len(self.ids) self.continue_updating = True - QTimer.singleShot(200, self.update) + single_shot(self.update) def canceled(self): self.continue_updating = False @@ -260,14 +281,14 @@ class Adder(QObject): # {{{ if self.worker is not None: self.worker.canceled = True if hasattr(self, 'db_adder'): - self.db_adder.end = True + self.db_adder.end() self.pd.hide() if not self.callback_called: self.callback(self.paths, self.names, self.infos) self.callback_called = True def duplicates_processed(self): - self.db_adder.end = True + self.db_adder.end() if not self.callback_called: self.callback(self.paths, self.names, self.infos) self.callback_called = True @@ -300,7 +321,7 @@ class Adder(QObject): # {{{ if (time.time() - self.last_added_at) > self.ADD_TIMEOUT: self.continue_updating = False self.pd.hide() - self.db_adder.end = True + self.db_adder.end() if not self.callback_called: self.callback([], [], []) self.callback_called = True @@ -311,7 +332,7 @@ class Adder(QObject): # {{{ 'find the problem book.'), show=True) if self.continue_updating: - QTimer.singleShot(200, self.update) + single_shot(self.update) def process_duplicates(self): @@ -332,11 +353,8 @@ class Adder(QObject): # {{{ self.__p_d = pd self.__d_a = DuplicatesAdder(self._parent, self.db, duplicates, self.db_adder) - self.connect(self.__d_a, SIGNAL('added(PyQt_PyObject)'), - pd.setValue) - self.connect(self.__d_a, SIGNAL('adding_done()'), - self.duplicates_processed) - self.__d_a.start() + self.__d_a.added.connect(pd.setValue) + self.__d_a.adding_done.connect(self.duplicates_processed) else: return self.duplicates_processed() @@ -407,14 +425,12 @@ class Saver(QObject): # {{{ self.worker = SaveWorker(self.rq, db, self.ids, path, self.opts, spare_server=self.spare_server) self.pd.canceled_signal.connect(self.canceled) - self.timer = QTimer(self) - self.connect(self.timer, SIGNAL('timeout()'), self.update) - self.timer.start(200) + self.continue_updating = True + single_shot(self.update) def canceled(self): - if self.timer is not None: - self.timer.stop() + self.continue_updating = False if self.worker is not None: self.worker.canceled = True self.pd.hide() @@ -424,14 +440,38 @@ class Saver(QObject): # {{{ def update(self): - if not self.ids or not self.worker.is_alive(): - self.timer.stop() + if not self.continue_updating: + return + if not self.worker.is_alive(): + # Check that all ids were processed + while self.ids: + # Get all queued results since worker is dead + before = len(self.ids) + self.get_result() + if before == len(self.ids): + # No results available => worker died unexpectedly + for i in list(self.ids): + self.failures.add(('id:%d'%i, 'Unknown error')) + self.ids.remove(i) + + if not self.ids: + self.continue_updating = False self.pd.hide() if not self.callback_called: - self.callback(self.worker.path, self.failures, self.worker.error) + try: + # Give the worker time to clean up and set worker.error + self.worker.join(2) + except: + pass # The worker was not yet started self.callback_called = True - return + self.callback(self.worker.path, self.failures, self.worker.error) + if self.continue_updating: + self.get_result() + single_shot(self.update) + + + def get_result(self): try: id, title, ok, tb = self.rq.get_nowait() except Empty: @@ -441,6 +481,7 @@ class Saver(QObject): # {{{ if not isinstance(title, unicode): title = str(title).decode(preferred_encoding, 'replace') self.pd.set_msg(_('Saved')+' '+title) + if not ok: self.failures.add((title, tb)) # }}} diff --git a/src/calibre/gui2/dialogs/fetch_metadata.py b/src/calibre/gui2/dialogs/fetch_metadata.py index f577632781..3da0e67e3d 100644 --- a/src/calibre/gui2/dialogs/fetch_metadata.py +++ b/src/calibre/gui2/dialogs/fetch_metadata.py @@ -9,7 +9,7 @@ from threading import Thread from PyQt4.QtCore import Qt, QObject, SIGNAL, QVariant, pyqtSignal, \ QAbstractTableModel, QCoreApplication, QTimer -from PyQt4.QtGui import QDialog, QItemSelectionModel +from PyQt4.QtGui import QDialog, QItemSelectionModel, QIcon from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata from calibre.gui2 import error_dialog, NONE, info_dialog, config @@ -42,6 +42,7 @@ class Matches(QAbstractTableModel): def __init__(self, matches): self.matches = matches + self.yes_icon = QVariant(QIcon(I('ok.png'))) QAbstractTableModel.__init__(self) def rowCount(self, *args): @@ -61,8 +62,8 @@ class Matches(QAbstractTableModel): elif section == 3: text = _("Publisher") elif section == 4: text = _("ISBN") elif section == 5: text = _("Published") - elif section == 6: text = _("Cover?") - elif section == 7: text = _("Summary?") + elif section == 6: text = _("Has Cover") + elif section == 7: text = _("Has Summary") return QVariant(text) else: @@ -73,8 +74,8 @@ class Matches(QAbstractTableModel): def data(self, index, role): row, col = index.row(), index.column() + book = self.matches[row] if role == Qt.DisplayRole: - book = self.matches[row] res = None if col == 0: res = book.title @@ -89,13 +90,14 @@ class Matches(QAbstractTableModel): elif col == 5: if hasattr(book.pubdate, 'timetuple'): res = strftime('%b %Y', book.pubdate.timetuple()) - elif col == 6 and book.has_cover: - res = 'OK' - elif col == 7 and book.comments: - res = 'OK' if not res: return NONE return QVariant(res) + elif role == Qt.DecorationRole: + if col == 6 and book.has_cover: + return self.yes_icon + if col == 7 and book.comments: + return self.yes_icon return NONE class FetchMetadata(QDialog, Ui_FetchMetadata): @@ -137,8 +139,7 @@ class FetchMetadata(QDialog, Ui_FetchMetadata): self.fetch_metadata() self.opt_get_social_metadata.setChecked(config['get_social_metadata']) self.opt_overwrite_author_title_metadata.setChecked(config['overwrite_author_title_metadata']) - self.opt_overwrite_cover_image.setChecked(config['overwrite_cover_image']) - + self.opt_auto_download_cover.setChecked(config['auto_download_cover']) def show_summary(self, current, *args): row = current.row() @@ -220,13 +221,12 @@ class FetchMetadata(QDialog, Ui_FetchMetadata): _hung_fetchers.add(self.fetcher) if hasattr(self, '_hangcheck') and self._hangcheck.isActive(): self._hangcheck.stop() - #option configure - if self.opt_get_social_metadata.isChecked() != config['get_social_metadata']: - config.set('get_social_metadata', self.opt_get_social_metadata.isChecked()) - if self.opt_overwrite_author_title_metadata.isChecked() != config['overwrite_author_title_metadata']: - config.set('overwrite_author_title_metadata', self.opt_overwrite_author_title_metadata.isChecked()) - if self.opt_overwrite_cover_image.isChecked() != config['overwrite_cover_image']: - config.set('overwrite_cover_image', self.opt_overwrite_cover_image.isChecked()) + # Save value of auto_download_cover, since this is the only place it can + # be set. The values of the other options can be set in + # Preferences->Behavior and should not be set here as they affect bulk + # downloading as well. + if self.opt_auto_download_cover.isChecked() != config['auto_download_cover']: + config.set('auto_download_cover', self.opt_auto_download_cover.isChecked()) def __enter__(self, *args): return self diff --git a/src/calibre/gui2/dialogs/fetch_metadata.ui b/src/calibre/gui2/dialogs/fetch_metadata.ui index 0b39089ee3..b140fa158d 100644 --- a/src/calibre/gui2/dialogs/fetch_metadata.ui +++ b/src/calibre/gui2/dialogs/fetch_metadata.ui @@ -124,9 +124,9 @@ - + - Overwrite cover image with downloaded cover if available for the selected book + Automatically download the cover, if available diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 4a44b0cefa..a640c50fb8 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -17,7 +17,7 @@ from calibre.gui2 import error_dialog from calibre.gui2.progress_indicator import ProgressIndicator from calibre.utils.config import dynamic from calibre.utils.titlecase import titlecase -from calibre.utils.icu import sort_key +from calibre.utils.icu import sort_key, capitalize class MyBlockingBusy(QDialog): @@ -187,6 +187,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): _('Lower Case') : lambda x: icu_lower(x), _('Upper Case') : lambda x: icu_upper(x), _('Title Case') : lambda x: titlecase(x), + _('Capitalize') : lambda x: capitalize(x), } s_r_match_modes = [ _('Character match'), diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index 5506f0e062..3205b1d23c 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -8,8 +8,9 @@ add/remove formats import os, re, time, traceback, textwrap from functools import partial +from threading import Thread -from PyQt4.Qt import SIGNAL, QObject, Qt, QTimer, QThread, QDate, \ +from PyQt4.Qt import SIGNAL, QObject, Qt, QTimer, QDate, \ QPixmap, QListWidgetItem, QDialog, pyqtSignal, QMessageBox, QIcon, \ QPushButton @@ -34,9 +35,12 @@ from calibre.gui2.preferences.social import SocialMetadata from calibre.gui2.custom_column_widgets import populate_metadata_page from calibre import strftime -class CoverFetcher(QThread): # {{{ +class CoverFetcher(Thread): # {{{ def __init__(self, username, password, isbn, timeout, title, author): + Thread.__init__(self) + self.daemon = True + self.username = username.strip() if username else username self.password = password.strip() if password else password self.timeout = timeout @@ -44,8 +48,7 @@ class CoverFetcher(QThread): # {{{ self.title = title self.needs_isbn = False self.author = author - QThread.__init__(self) - self.exception = self.traceback = self.cover_data = None + self.exception = self.traceback = self.cover_data = self.errors = None def run(self): try: @@ -238,20 +241,20 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.timeout, title, author) self.cover_fetcher.start() self._hangcheck = QTimer(self) - self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck) + self._hangcheck.timeout.connect(self.hangcheck, + type=Qt.QueuedConnection) self.cf_start_time = time.time() self.pi.start(_('Downloading cover...')) self._hangcheck.start(100) def hangcheck(self): - if not self.cover_fetcher.isFinished() and \ + if self.cover_fetcher.is_alive() and \ time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT: return self._hangcheck.stop() try: - if self.cover_fetcher.isRunning(): - self.cover_fetcher.terminate() + if self.cover_fetcher.is_alive(): error_dialog(self, _('Cannot fetch cover'), _('Could not fetch cover.
')+ _('The download timed out.')).exec_() @@ -759,11 +762,9 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): if book.author_sort: self.author_sort.setText(book.author_sort) if book.publisher: self.publisher.setEditText(book.publisher) if book.isbn: self.isbn.setText(book.isbn) - if d.opt_overwrite_cover_image.isChecked() and book.has_cover: - self.fetch_cover() if book.pubdate: - d = book.pubdate - self.pubdate.setDate(QDate(d.year, d.month, d.day)) + dt = book.pubdate + self.pubdate.setDate(QDate(dt.year, dt.month, dt.day)) summ = book.comments if summ: prefix = unicode(self.comments.toPlainText()) @@ -779,8 +780,11 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.series.setText(book.series) if book.series_index is not None: self.series_index.setValue(book.series_index) - # Needed because of Qt focus bug on OS X - self.fetch_cover_button.setFocus(Qt.OtherFocusReason) + if book.has_cover: + if d.opt_auto_download_cover.isChecked() and book.has_cover: + self.fetch_cover() + else: + self.fetch_cover_button.setFocus(Qt.OtherFocusReason) else: error_dialog(self, _('Cannot fetch metadata'), _('You must specify at least one of ISBN, Title, ' diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 311cbaf369..e82e1dddd4 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -1023,8 +1023,7 @@ class DeviceBooksModel(BooksModel): # {{{ x = '' if y == None: y = '' - x, y = icu_lower(x.strip()), icu_lower(y.strip()) - return icu_strcmp(x, y) + return icu_strcmp(x.strip(), y.strip()) return _strcmp def datecmp(x, y): x = self.db[x].datetime diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index 388227e438..d493b615b5 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -151,6 +151,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self._plugin_model.populate() self._plugin_model.reset() self.changed_signal.emit() + self.plugin_path.setText('') else: error_dialog(self, _('No valid plugin path'), _('%s is not a valid plugin path')%path).exec_() diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index fdae1bdbc9..2ede698c85 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -73,6 +73,7 @@ class TagsView(QTreeView): # {{{ def __init__(self, parent=None): QTreeView.__init__(self, parent=None) self.tag_match = None + self.disable_recounting = False self.setUniformRowHeights(True) self.setCursor(Qt.PointingHandCursor) self.setIconSize(QSize(30, 30)) @@ -299,6 +300,8 @@ class TagsView(QTreeView): # {{{ return self.isExpanded(idx) def recount(self, *args): + if self.disable_recounting: + return self.refresh_signal_processed = True ci = self.currentIndex() if not ci.isValid(): diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index c6cc12a978..af57d563ac 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback, cStringIO, re +import os, traceback, cStringIO, re, shutil from calibre.constants import DEBUG from calibre.utils.config import Config, StringConfig, tweaks @@ -203,31 +203,49 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, return shorten_components_to(length, components) -def save_book_to_disk(id, db, root, opts, length): - mi = db.get_metadata(id, index_is_id=True) +def save_book_to_disk(id_, db, root, opts, length): + mi = db.get_metadata(id_, index_is_id=True) + cover = db.cover(id_, index_is_id=True, as_path=True) + plugboards = db.prefs.get('plugboards', {}) - available_formats = db.formats(id, index_is_id=True) + available_formats = db.formats(id_, index_is_id=True) if not available_formats: available_formats = [] else: available_formats = [x.lower().strip() for x in available_formats.split(',')] + formats = {} + fmts = db.formats(id_, index_is_id=True, verify_formats=False) + if fmts: + fmts = fmts.split(',') + for fmt in fmts: + fpath = db.format_abspath(id_, fmt, index_is_id=True) + if fpath is not None: + formats[fmt.lower()] = fpath + + return do_save_book_to_disk(id_, mi, cover, plugboards, + formats, root, opts, length) + + +def do_save_book_to_disk(id_, mi, cover, plugboards, + format_map, root, opts, length): + available_formats = [x.lower().strip() for x in format_map.keys()] if opts.formats == 'all': asked_formats = available_formats else: asked_formats = [x.lower().strip() for x in opts.formats.split(',')] formats = set(available_formats).intersection(set(asked_formats)) if not formats: - return True, id, mi.title + return True, id_, mi.title - components = get_components(opts.template, mi, id, opts.timefmt, length, + components = get_components(opts.template, mi, id_, opts.timefmt, length, ascii_filename if opts.asciiize else sanitize_file_name, to_lowercase=opts.to_lowercase, replace_whitespace=opts.replace_whitespace) base_path = os.path.join(root, *components) base_name = os.path.basename(base_path) dirpath = os.path.dirname(base_path) - # Don't test for existence first are the test could fail but + # Don't test for existence first as the test could fail but # another worker process could create the directory before # the call to makedirs try: @@ -236,29 +254,23 @@ def save_book_to_disk(id, db, root, opts, length): if not os.path.exists(dirpath): raise - cdata = db.cover(id, index_is_id=True) - if opts.save_cover: - if cdata is not None: - with open(base_path+'.jpg', 'wb') as f: - f.write(cdata) - mi.cover = base_name+'.jpg' - else: - mi.cover = None + if opts.save_cover and cover and os.access(cover, os.R_OK): + with open(base_path+'.jpg', 'wb') as f: + with open(cover, 'rb') as s: + shutil.copyfileobj(s, f) + mi.cover = base_name+'.jpg' + else: + mi.cover = None if opts.write_opf: opf = metadata_to_opf(mi) with open(base_path+'.opf', 'wb') as f: f.write(opf) - if cdata is not None: - mi.cover_data = ('jpg', cdata) - mi.cover = None - written = False for fmt in formats: global plugboard_save_to_disk_value, plugboard_any_format_value dev_name = plugboard_save_to_disk_value - plugboards = db.prefs.get('plugboards', {}) cpb = None if fmt in plugboards: cpb = plugboards[fmt] @@ -275,11 +287,12 @@ def save_book_to_disk(id, db, root, opts, length): # Leave this here for a while, in case problems arise. if cpb is not None: prints('Save-to-disk using plugboard:', fmt, cpb) - data = db.format(id, fmt, index_is_id=True) - if data is None: + fp = format_map.get(fmt, None) + if fp is None: continue - else: - written = True + with open(fp, 'rb') as f: + data = f.read() + written = True if opts.update_metadata: stream = cStringIO.StringIO() stream.write(data) @@ -300,9 +313,21 @@ def save_book_to_disk(id, db, root, opts, length): with open(fmt_path, 'wb') as f: f.write(data) - return not written, id, mi.title + return not written, id_, mi.title +def _sanitize_args(root, opts): + if opts is None: + opts = config().parse() + if isinstance(root, unicode): + root = root.encode(filesystem_encoding) + root = os.path.abspath(root) + opts.template = preprocess_template(opts.template) + length = 1000 if supports_long_names(root) else 250 + length -= len(root) + if length < 5: + raise ValueError('%r is too long.'%root) + return root, opts, length def save_to_disk(db, ids, root, opts=None, callback=None): ''' @@ -316,17 +341,7 @@ def save_to_disk(db, ids, root, opts=None, callback=None): :return: A list of failures. Each element of the list is a tuple (id, title, traceback) ''' - if opts is None: - opts = config().parse() - if isinstance(root, unicode): - root = root.encode(filesystem_encoding) - root = os.path.abspath(root) - - opts.template = preprocess_template(opts.template) - length = 1000 if supports_long_names(root) else 250 - length -= len(root) - if length < 5: - raise ValueError('%r is too long.'%root) + root, opts, length = _sanitize_args(root, opts) failures = [] for x in ids: tb = '' @@ -343,4 +358,28 @@ def save_to_disk(db, ids, root, opts=None, callback=None): break return failures +def save_serialized_to_disk(ids, data, plugboards, root, opts, callback): + from calibre.ebooks.metadata.opf2 import OPF + root, opts, length = _sanitize_args(root, opts) + failures = [] + for x in ids: + opf, cover, format_map = data[x] + if isinstance(opf, unicode): + opf = opf.encode('utf-8') + mi = OPF(cStringIO.StringIO(opf)).to_book_metadata() + tb = '' + try: + failed, id, title = do_save_book_to_disk(x, mi, cover, plugboards, + format_map, root, opts, length) + tb = _('Requested formats not available') + except: + failed, id, title = True, x, mi.title + tb = traceback.format_exc() + if failed: + failures.append((id, title, tb)) + if callable(callback): + if not callback(int(id), title, failed, tb): + break + + return failures diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst index 1bef32fbd6..b2d32f0767 100644 --- a/src/calibre/manual/template_lang.rst +++ b/src/calibre/manual/template_lang.rst @@ -46,7 +46,6 @@ and if a book does not have a series:: (|app| automatically removes multiple slashes and leading or trailing spaces). - Advanced formatting ---------------------- @@ -80,6 +79,9 @@ For trailing zeros, use:: {series_index:0<3s} - Three digits with trailing zeros +If you use series indices with sub values (e.g., 1.1), you might want to ensure that the decimal points line up. For example, you might want the indices 1 and 2.5 to appear as 01.00 and 02.50 so that they will sort correctly. To do this, use:: + + {series_index:0>5.2f} - Five characters, consisting of two digits with leading zeros, a decimal point, then 2 digits after the decimal point If you want only the first two letters of the data, use:: @@ -115,15 +117,15 @@ The functions available are: * ``lowercase()`` -- return value of the field in lower case. * ``uppercase()`` -- return the value of the field in upper case. * ``titlecase()`` -- return the value of the field in title case. - * ``capitalize()`` -- return the value as capitalized. - * ``ifempty(text)`` -- if the field is not empty, return the value of the field. Otherwise return `text`. - * ``test(text if not empty, text if empty)`` -- return `text if not empty` if the field is not empty, otherwise return `text if empty`. + * ``capitalize()`` -- return the value with the first letter upper case and the rest lower case. * ``contains(pattern, text if match, text if not match`` -- checks if field contains matches for the regular expression `pattern`. Returns `text if match` if matches are found, otherwise it returns `text if no match`. * ``count(separator)`` -- interprets the value as a list of items separated by `separator`, returning the number of items in the list. Most lists use a comma as the separator, but authors uses an ampersand. Examples: `{tags:count(,)}`, `{authors:count(&)}` + * ``ifempty(text)`` -- if the field is not empty, return the value of the field. Otherwise return `text`. * ``lookup(pattern, field, pattern, field, ..., else_field)`` -- like switch, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later). * ``re(pattern, replacement)`` -- return the field after applying the regular expression. All instances of `pattern` are replaced with `replacement`. As in all of |app|, these are python-compatible regular expressions. * ``shorten(left chars, middle text, right chars)`` -- Return a shortened version of the field, consisting of `left chars` characters from the beginning of the field, followed by `middle text`, followed by `right chars` characters from the end of the string. `Left chars` and `right chars` must be integers. For example, assume the title of the book is `Ancient English Laws in the Times of Ivanhoe`, and you want it to fit in a space of at most 15 characters. If you use ``{title:shorten(9,-,5)}``, the result will be `Ancient E-nhoe`. If the field's length is less than ``left chars`` + ``right chars`` + the length of ``middle text``, then the field will be used intact. For example, the title `The Dome` would not be changed. * ``switch(pattern, value, pattern, value, ..., else_value)`` -- for each ``pattern, value`` pair, checks if the field matches the regular expression ``pattern`` and if so, returns that ``value``. If no ``pattern`` matches, then ``else_value`` is returned. You can have as many ``pattern, value`` pairs as you want. + * ``test(text if not empty, text if empty)`` -- return `text if not empty` if the field is not empty, otherwise return `text if empty`. Now, about using functions and formatting in the same field. Suppose you have an integer custom column called ``#myint`` that you want to see with leading zeros, as in ``003``. To do this, you would use a format of ``0>3s``. However, by default, if a number (integer or float) equals zero then the field produces the empty value, so zero values will produce nothing, not ``000``. If you really want to see ``000`` values, then you use both the format string and the ``ifempty`` function to change the empty value back to a zero. The field reference would be:: diff --git a/src/calibre/utils/bibtex.py b/src/calibre/utils/bibtex.py index b78e4cc002..d19a6b05fe 100644 --- a/src/calibre/utils/bibtex.py +++ b/src/calibre/utils/bibtex.py @@ -64,10 +64,6 @@ __author__ = 'sengian ' __docformat__ = 'restructuredtext en' import re, string -from UserDict import UserDict - -from calibre.constants import preferred_encoding -from calibre.utils.mreplace import MReplace from calibre.constants import preferred_encoding from calibre.utils.mreplace import MReplace diff --git a/src/calibre/utils/cleantext.py b/src/calibre/utils/cleantext.py index 6655129c15..b4afe7576d 100644 --- a/src/calibre/utils/cleantext.py +++ b/src/calibre/utils/cleantext.py @@ -5,11 +5,19 @@ __docformat__ = 'restructuredtext en' import re -def clean_ascii_char(txt, charlist = None): - #remove ASCII invalid chars : 0 to 8 and 11-14 to 24-26-27 by default - chars = list(range(8)) + [0x0B, 0x0E, 0x0F] + list(range(0x10, 0x19)) \ - + [0x1A, 0x1B] - if charlist is not None: - chars = charlist - illegal_chars = re.compile(u'|'.join(map(unichr, chars))) - return illegal_chars.sub('', txt) \ No newline at end of file +_ascii_pat = None + +def clean_ascii_chars(txt, charlist=None): + 'remove ASCII invalid chars : 0 to 8 and 11-14 to 24-26-27 by default' + global _ascii_pat + if _ascii_pat is None: + chars = list(range(8)) + [0x0B, 0x0E, 0x0F] + list(range(0x10, 0x19)) \ + + [0x1A, 0x1B] + _ascii_pat = re.compile(u'|'.join(map(unichr, chars))) + + if charlist is None: + pat = _ascii_pat + else: + pat = re.compile(u'|'.join(map(unichr, charlist))) + return pat.sub('', txt) + diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index 1ea8a2c4a0..f025a0c9bf 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -152,7 +152,7 @@ def format_date(dt, format, assume_utc=False, as_utc=False): format = re.sub('M{1,4}', format_month, format) return re.sub('yyyy|yy', format_year, format) -def replace_months(datez, clang): +def replace_months(datestr, clang): # Replace months by english equivalent for parse_date frtoen = { u'[jJ]anvier': u'jan', @@ -180,15 +180,16 @@ def replace_months(datez, clang): u'[Oo]ktober': u'oct', u'[nN]ovember': u'nov', u'[dD]ezember': u'dec' } - + if clang == 'fr': dictoen = frtoen elif clang == 'de': dictoen = detoen else: - return datez - + return datestr + for k in dictoen.iterkeys(): - tmp = re.sub(k, dictoen[k], datez) - if tmp != datez: break + tmp = re.sub(k, dictoen[k], datestr) + if tmp != datestr: break return tmp + diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index 15534a9c8a..a7fb3682aa 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -8,12 +8,15 @@ import re, string, traceback from calibre.constants import DEBUG from calibre.utils.titlecase import titlecase +from calibre.utils.icu import capitalize class TemplateFormatter(string.Formatter): ''' Provides a format function that substitutes '' for any missing value ''' + _validation_string = 'This Is Some Text THAT SHOULD be LONG Enough.%^&*' + # Dict to do recursion detection. It is up the the individual get_value # method to use it. It is cleared when starting to format a template composite_values = {} @@ -86,7 +89,7 @@ class TemplateFormatter(string.Formatter): 'uppercase' : (0, lambda s,x: x.upper()), 'lowercase' : (0, lambda s,x: x.lower()), 'titlecase' : (0, lambda s,x: titlecase(x)), - 'capitalize' : (0, lambda s,x: x.capitalize()), + 'capitalize' : (0, lambda s,x: capitalize(x)), 'contains' : (3, _contains), 'ifempty' : (1, _ifempty), 'lookup' : (-1, _lookup), @@ -97,19 +100,29 @@ class TemplateFormatter(string.Formatter): 'count' : (1, _count), } - format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$') - compress_spaces = re.compile(r'\s+') - backslash_comma_to_comma = re.compile(r'\\,') - - arg_parser = re.Scanner([ - (r',', lambda x,t: ''), - (r'.*?((?= 0: + try: + val = int(val) + except: + raise ValueError( + _('format: type {0} requires an integer value, got {1}').format(typ, val)) + elif 'eEfFgGn%'.find(typ) >= 0: + try: + val = float(val) + except: + raise ValueError( + _('format: type {0} requires a decimal (float) value, got {1}').format(typ, val)) + else: + raise ValueError(_('format: unknown format type letter {0}').format(typ)) + return unicode(('{0:'+fmt+'}').format(val)) def _explode_format_string(self, fmt): try: @@ -122,6 +135,21 @@ class TemplateFormatter(string.Formatter): traceback.print_exc() return fmt, '', '' + format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$') + compress_spaces = re.compile(r'\s+') + backslash_comma_to_comma = re.compile(r'\\,') + + arg_parser = re.Scanner([ + (r',', lambda x,t: ''), + (r'.*?((?', 'py:', x.upper().encode('utf-8'), 'icu:', upper(x).encode('utf-8') - print 'Lower:', x, '->', 'py:', x.lower().encode('utf-8'), 'icu:', lower(x).encode('utf-8') - print 'Title:', x, '->', 'py:', x.title().encode('utf-8'), 'icu:', title_case(x).encode('utf-8') + print 'Upper: ', x, '->', 'py:', x.upper().encode('utf-8'), 'icu:', upper(x).encode('utf-8') + print 'Lower: ', x, '->', 'py:', x.lower().encode('utf-8'), 'icu:', lower(x).encode('utf-8') + print 'Title: ', x, '->', 'py:', x.title().encode('utf-8'), 'icu:', title_case(x).encode('utf-8') + print 'Capitalize:', x, '->', 'py:', x.capitalize().encode('utf-8'), 'icu:', capitalize(x).encode('utf-8') print # }}} diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index 380e2e074b..4d35113d80 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -292,12 +292,12 @@ class Server(Thread): except: pass time.sleep(0.2) - for worker in self.workers: + for worker in list(self.workers): try: worker.kill() except: pass - for worker in self.pool: + for worker in list(self.pool): try: worker.kill() except: diff --git a/src/calibre/utils/titlecase.py b/src/calibre/utils/titlecase.py index b85670f038..bbc4c26688 100755 --- a/src/calibre/utils/titlecase.py +++ b/src/calibre/utils/titlecase.py @@ -9,6 +9,8 @@ License: http://www.opensource.org/licenses/mit-license.php import re +from calibre.utils.icu import capitalize + __all__ = ['titlecase'] __version__ = '0.5' @@ -40,11 +42,6 @@ def titlecase(text): """ - def capitalize(w): - w = icu_lower(w) - w = w.replace(w[0], icu_upper(w[0])) - return w - all_caps = ALL_CAPS.match(text) words = re.split('\s', text) diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py index dbcc125274..5c19444bd6 100644 --- a/src/calibre/utils/zipfile.py +++ b/src/calibre/utils/zipfile.py @@ -1227,7 +1227,7 @@ class ZipFile: self.fp.flush() if zinfo.flag_bits & 0x08: # Write CRC and file sizes after the file data - self.fp.write(struct.pack("