diff --git a/manual/faq.rst b/manual/faq.rst index ceaa2bc735..a73a380029 100644 --- a/manual/faq.rst +++ b/manual/faq.rst @@ -647,12 +647,17 @@ computers. Run |app| on a single computer and access it via the Content Server or a Remote Desktop solution. If you must share the actual library, use a file syncing tool like -DropBox or rsync or Microsoft SkyDrive instead of a networked drive. Even with -these tools there is danger of data corruption/loss, so only do this if you are -willing to live with that risk. In particular, be aware that **Google Drive** -is incompatible with |app|, if you put your |app| library in Google Drive, you -*will* suffer data loss. See -`this thread `_ for details. +DropBox or rsync or Microsoft SkyDrive instead of a networked drive. If you are +using a file-syncing tool it is **essential** that you make sure that both +|app| and the file syncing tool do not try to access the |app| library at the +same time. In other words, **do not** run the file syncing tool and |app| at +the same time. + +Even with these tools there is danger of data corruption/loss, so only do this +if you are willing to live with that risk. In particular, be aware that +**Google Drive** is incompatible with |app|, if you put your |app| library in +Google Drive, **you will suffer data loss**. See `this thread +`_ for details. Content From The Web --------------------- diff --git a/recipes/economia.recipe b/recipes/economia.recipe new file mode 100644 index 0000000000..249125b76f --- /dev/null +++ b/recipes/economia.recipe @@ -0,0 +1,17 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1314326622(BasicNewsRecipe): + title = u'Economia' + __author__ = 'Manish Bhattarai' + description = 'Economia - Intelligence & Insight for ICAEW Members' + language = 'en_GB' + oldest_article = 7 + max_articles_per_feed = 25 + masthead_url = 'http://economia.icaew.com/~/media/Images/Design%20Images/Economia_Red_website.ashx' + cover_url = 'http://economia.icaew.com/~/media/Images/Design%20Images/Economia_Red_website.ashx' + no_stylesheets = True + remove_empty_feeds = True + remove_tags_before = dict(id='content') + remove_tags_after = dict(id='stars-wrapper') + remove_tags = [dict(attrs={'class':['floatR', 'sharethis', 'rating clearfix']})] + feeds = [(u'News', u'http://feedity.com/icaew-com/VlNTVFRa.rss'),(u'Business', u'http://feedity.com/icaew-com/VlNTVFtS.rss'),(u'People', u'http://feedity.com/icaew-com/VlNTVFtX.rss'),(u'Opinion', u'http://feedity.com/icaew-com/VlNTVFtW.rss'),(u'Finance', u'http://feedity.com/icaew-com/VlNTVFtV.rss')] diff --git a/recipes/fronda.recipe b/recipes/fronda.recipe index 6ed5d052a3..8372bb4d81 100644 --- a/recipes/fronda.recipe +++ b/recipes/fronda.recipe @@ -6,6 +6,7 @@ __copyright__ = u'2010-2013, Tomasz Dlugosz ' fronda.pl ''' +import re from calibre.web.feeds.news import BasicNewsRecipe from datetime import timedelta, date @@ -23,6 +24,7 @@ class Fronda(BasicNewsRecipe): extra_css = ''' h1 {font-size:150%} .body {text-align:left;} + div#featured-image {font-style:italic; font-size:70%} ''' earliest_date = date.today() - timedelta(days=oldest_article) @@ -55,7 +57,10 @@ class Fronda(BasicNewsRecipe): articles = {} for url, genName in genres: - soup = self.index_to_soup('http://www.fronda.pl/c/'+ url) + try: + soup = self.index_to_soup('http://www.fronda.pl/c/'+ url) + except: + continue articles[genName] = [] for item in soup.findAll('li'): article_h = item.find('h2') @@ -77,16 +82,15 @@ class Fronda(BasicNewsRecipe): ] remove_tags = [ - dict(name='div', attrs={'class':['related-articles', - 'button right', - 'pagination']}), + dict(name='div', attrs={'class':['related-articles','button right','pagination','related-articles content']}), dict(name='h3', attrs={'class':'block-header article comments'}), - dict(name='ul', attrs={'class':'comment-list'}), - dict(name='ul', attrs={'class':'category'}), - dict(name='ul', attrs={'class':'tag-list'}), + dict(name='ul', attrs={'class':['comment-list','category','tag-list']}), dict(name='p', attrs={'id':'comments-disclaimer'}), dict(name='div', attrs={'style':'text-align: left; margin-bottom: 15px;'}), dict(name='div', attrs={'style':'text-align: left; margin-top: 15px; margin-bottom: 30px;'}), - dict(name='div', attrs={'class':'related-articles content'}), - dict(name='div', attrs={'id':'comment-form'}) + dict(name='div', attrs={'id':'comment-form'}), + dict(name='span', attrs={'class':'separator'}) ] + + preprocess_regexps = [ + (re.compile(r'komentarzy: .*?', re.IGNORECASE | re.DOTALL | re.M ), lambda match: '')] diff --git a/recipes/icons/newsweek_polska.png b/recipes/icons/newsweek_polska.png new file mode 100644 index 0000000000..83e84b89e2 Binary files /dev/null and b/recipes/icons/newsweek_polska.png differ diff --git a/recipes/new_yorker.recipe b/recipes/new_yorker.recipe index 2730b45d6d..93a231792c 100644 --- a/recipes/new_yorker.recipe +++ b/recipes/new_yorker.recipe @@ -1,64 +1,44 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- __license__ = 'GPL v3' -__copyright__ = '2008-2013, Darko Miletic ' -''' -newyorker.com -''' +''' +www.canada.com +''' +import re from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup + class NewYorker(BasicNewsRecipe): - title = 'The New Yorker' - __author__ = 'Darko Miletic' - description = 'The best of US journalism' - oldest_article = 15 - language = 'en' - max_articles_per_feed = 100 - no_stylesheets = True - use_embedded_content = False - publisher = 'Conde Nast Publications' - category = 'news, politics, USA' - encoding = 'cp1252' - publication_type = 'magazine' - masthead_url = 'http://www.newyorker.com/css/i/hed/logo.gif' - extra_css = """ - body {font-family: "Times New Roman",Times,serif} - .articleauthor{color: #9F9F9F; - font-family: Arial, sans-serif; - font-size: small; - text-transform: uppercase} - .rubric,.dd,h6#credit{color: #CD0021; - font-family: Arial, sans-serif; - font-size: small; - text-transform: uppercase} - .descender:first-letter{display: inline; font-size: xx-large; font-weight: bold} - .dd,h6#credit{color: gray} - .c{display: block} - .caption,h2#articleintro{font-style: italic} - .caption{font-size: small} - """ - conversion_options = { - 'comment' : description - , 'tags' : category - , 'publisher' : publisher - , 'language' : language - } - keep_only_tags = [dict(name='div', attrs={'id':'pagebody'})] - remove_tags = [ - dict(name=['meta','iframe','base','link','embed','object']) - ,dict(attrs={'class':['utils','socialUtils','articleRailLinks','icons','social-utils-top','entry-keywords','entry-categories','utilsPrintEmail'] }) - ,dict(attrs={'id':['show-header','show-footer'] }) - ] - remove_tags_after = dict(attrs={'class':'entry-content'}) - remove_attributes = ['lang'] - feeds = [(u'The New Yorker', u'http://www.newyorker.com/services/mrss/feeds/everything.xml')] + title = u'New Yorker Magazine' + newyorker_prefix = 'http://m.newyorker.com' + description = u'Content from the New Yorker website' + fp_tag = 'CAN_TC' - def print_version(self, url): - return url + '?printable=true¤tPage=all' + masthead_url = 'http://www.newyorker.com/images/elements/print/newyorker_printlogo.gif' - def image_url_processor(self, baseurl, url): - return url.strip() + compress_news_images = True + compress_news_images_auto_size = 8 + scale_news_images_to_device = False + scale_news_images = (768, 1024) + + url_list = [] + language = 'en' + __author__ = 'Nick Redding' + no_stylesheets = True + timefmt = ' [%b %d]' + encoding = 'utf-8' + extra_css = ''' + .byline { font-size:xx-small; font-weight: bold;} + h3 { margin-bottom: 6px; } + .caption { font-size: xx-small; font-style: italic; font-weight: normal; } + ''' + keep_only_tags = [dict(name='div', attrs={'id':re.compile('pagebody')})] + + remove_tags = [{'class':'socialUtils'},{'class':'entry-keywords'}] def get_cover_url(self): cover_url = "http://www.newyorker.com/images/covers/1925/1925_02_21_p233.jpg" @@ -68,13 +48,233 @@ class NewYorker(BasicNewsRecipe): cover_url = 'http://www.newyorker.com' + cover_item.div.img['src'].strip() return cover_url - def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - auth = soup.find(attrs={'id':'articleauthor'}) - if auth: - alink = auth.find('a') - if alink and alink.string is not None: - txt = alink.string - alink.replaceWith(txt) + def fixChars(self,string): + # Replace lsquo (\x91) + fixed = re.sub("\x91","‘",string) + # Replace rsquo (\x92) + fixed = re.sub("\x92","’",fixed) + # Replace ldquo (\x93) + fixed = re.sub("\x93","“",fixed) + # Replace rdquo (\x94) + fixed = re.sub("\x94","”",fixed) + # Replace ndash (\x96) + fixed = re.sub("\x96","–",fixed) + # Replace mdash (\x97) + fixed = re.sub("\x97","—",fixed) + fixed = re.sub("’","’",fixed) + return fixed + + def massageNCXText(self, description): + # Kindle TOC descriptions won't render certain characters + if description: + massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + # Replace '&' with '&' + massaged = re.sub("&","&", massaged) + return self.fixChars(massaged) + else: + return description + + def populate_article_metadata(self, article, soup, first): + if first: + picdiv = soup.find('body').find('img') + if picdiv is not None: + self.add_toc_thumbnail(article,re.sub(r'links\\link\d+\\','',picdiv['src'])) + xtitle = article.text_summary.strip() + if len(xtitle) == 0: + desc = soup.find('meta',attrs={'property':'og:description'}) + if desc is not None: + article.summary = article.text_summary = desc['content'] + shortparagraph = "" +## try: + if len(article.text_summary.strip()) == 0: + articlebodies = soup.findAll('div',attrs={'class':'entry-content'}) + if articlebodies: + for articlebody in articlebodies: + if articlebody: + paras = articlebody.findAll('p') + for p in paras: + refparagraph = self.massageNCXText(self.tag_to_string(p,use_alt=False)).strip() + #account for blank paragraphs and short paragraphs by appending them to longer ones + if len(refparagraph) > 0: + if len(refparagraph) > 70: #approximately one line of text + newpara = shortparagraph + refparagraph + article.summary = article.text_summary = newpara.strip() + return + else: + shortparagraph = refparagraph + " " + if shortparagraph.strip().find(" ") == -1 and not shortparagraph.strip().endswith(":"): + shortparagraph = shortparagraph + "- " + else: + article.summary = article.text_summary = self.massageNCXText(article.text_summary) +## except: +## self.log("Error creating article descriptions") +## return + + + def strip_anchors(self,soup): + paras = soup.findAll(True) + for para in paras: + aTags = para.findAll('a') + for a in aTags: + if a.img is None: + a.replaceWith(a.renderContents().decode('cp1252','replace')) return soup + + def preprocess_html(self,soup): + dateline = soup.find('div','published') + byline = soup.find('div','byline') + title = soup.find('h1','entry-title') + if title is None: + return self.strip_anchors(soup) + if byline is None: + title.append(dateline) + return self.strip_anchors(soup) + byline.append(dateline) + return self.strip_anchors(soup) + + def load_global_nav(self,soup): + seclist = [] + ul = soup.find('ul',attrs={'id':re.compile('global-nav-menu')}) + if ul is not None: + for li in ul.findAll('li'): + if li.a is not None: + securl = li.a['href'] + if securl != '/' and securl != '/magazine' and securl.startswith('/'): + seclist.append((self.tag_to_string(li.a),self.newyorker_prefix+securl)) + return seclist + + def exclude_url(self,url): + if url in self.url_list: + return True + if not url.endswith('html'): + return True + if 'goings-on-about-town-app' in url: + return True + if 'something-to-be-thankful-for' in url: + return True + if '/shouts/' in url: + return True + if 'out-loud' in url: + return True + if '/rss/' in url: + return True + if '/video-' in url: + return True + self.url_list.append(url) + return False + + + def load_index_page(self,soup): + article_list = [] + for div in soup.findAll('div',attrs={'class':re.compile('^rotator')}): + h2 = div.h2 + if h2 is not None: + a = h2.a + if a is not None: + url = a['href'] + if not self.exclude_url(url): + if url.startswith('/'): + url = self.newyorker_prefix+url + byline = h2.span + if byline is not None: + author = self.tag_to_string(byline) + if author.startswith('by '): + author.replace('by ','') + byline.extract() + else: + author = '' + if h2.br is not None: + h2.br.replaceWith(' ') + title = self.tag_to_string(h2) + desc = div.find(attrs={'class':['rotator-ad-body','feature-blurb-text']}) + if desc is not None: + description = self.tag_to_string(desc) + else: + description = '' + article_list.append(dict(title=title,url=url,date='',description=description,author=author,content='')) + ul = div.find('ul','feature-blurb-links') + if ul is not None: + for li in ul.findAll('li'): + a = li.a + if a is not None: + url = a['href'] + if not self.exclude_url(url): + if url.startswith('/'): + url = self.newyorker_prefix+url + if a.br is not None: + a.br.replaceWith(' ') + title = '>>'+self.tag_to_string(a) + article_list.append(dict(title=title,url=url,date='',description='',author='',content='')) + for h3 in soup.findAll('h3','header'): + a = h3.a + if a is not None: + url = a['href'] + if not self.exclude_url(url): + if url.startswith('/'): + url = self.newyorker_prefix+url + byline = h3.span + if byline is not None: + author = self.tag_to_string(byline) + if author.startswith('by '): + author = author.replace('by ','') + byline.extract() + else: + author = '' + if h3.br is not None: + h3.br.replaceWith(' ') + title = self.tag_to_string(h3).strip() + article_list.append(dict(title=title,url=url,date='',description='',author=author,content='')) + return article_list + + def load_global_section(self,securl): + article_list = [] + try: + soup = self.index_to_soup(securl) + except: + return article_list + if '/blogs/' not in securl: + return self.load_index_page(soup) + for div in soup.findAll('div',attrs={'id':re.compile('^entry')}): + h3 = div.h3 + if h3 is not None: + a = h3.a + if a is not None: + url = a['href'] + if not self.exclude_url(url): + if url.startswith('/'): + url = self.newyorker_prefix+url + if h3.br is not None: + h3.br.replaceWith(' ') + title = self.tag_to_string(h3) + article_list.append(dict(title=title,url=url,date='',description='',author='',content='')) + return article_list + + def filter_ans(self, ans) : + total_article_count = 0 + idx = 0 + idx_max = len(ans)-1 + while idx <= idx_max: + if True: #self.verbose + self.log("Section %s: %d articles" % (ans[idx][0], len(ans[idx][1])) ) + for article in ans[idx][1]: + total_article_count += 1 + if True: #self.verbose + self.log("\t%-40.40s... \t%-60.60s..." % (article['title'].encode('cp1252','replace'), + article['url'].replace('http://m.newyorker.com','').encode('cp1252','replace'))) + idx = idx+1 + self.log( "Queued %d articles" % total_article_count ) + return ans + + + def parse_index(self): + ans = [] + try: + soup = self.index_to_soup(self.newyorker_prefix) + except: + return ans + seclist = self.load_global_nav(soup) + ans.append(('Front Page',self.load_index_page(soup))) + for (sectitle,securl) in seclist: + ans.append((sectitle,self.load_global_section(securl))) + return self.filter_ans(ans) + diff --git a/recipes/san_fran_chronicle.recipe b/recipes/san_fran_chronicle.recipe index cfdf0b07f0..37f15dbe87 100644 --- a/recipes/san_fran_chronicle.recipe +++ b/recipes/san_fran_chronicle.recipe @@ -7,7 +7,6 @@ sfgate.com ''' from calibre.web.feeds.news import BasicNewsRecipe -import re class SanFranciscoChronicle(BasicNewsRecipe): title = u'San Francisco Chronicle' @@ -19,16 +18,7 @@ class SanFranciscoChronicle(BasicNewsRecipe): max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False - - - - remove_tags_before = {'id':'printheader'} - - remove_tags = [ - dict(name='div',attrs={'id':'printheader'}) - ,dict(name='a', attrs={'href':re.compile('http://ads\.pheedo\.com.*')}) - ,dict(name='div',attrs={'id':'footer'}) - ] + auto_cleanup = True extra_css = ''' h1{font-family :Arial,Helvetica,sans-serif; font-size:large;} @@ -43,33 +33,13 @@ class SanFranciscoChronicle(BasicNewsRecipe): ''' feeds = [ - (u'Top News Stories', u'http://www.sfgate.com/rss/feeds/news.xml') + (u'Bay Area News', u'http://www.sfgate.com/bayarea/feed/Bay-Area-News-429.php'), + (u'City Insider', u'http://www.sfgate.com/default/feed/City-Insider-Blog-573.php'), + (u'Crime Scene', u'http://www.sfgate.com/rss/feed/Crime-Scene-Blog-599.php'), + (u'Education News', u'http://www.sfgate.com/education/feed/Education-News-from-SFGate-430.php'), + (u'National News', u'http://www.sfgate.com/rss/feed/National-News-RSS-Feed-435.php'), + (u'Weird News', u'http://www.sfgate.com/weird/feed/Weird-News-RSS-Feed-433.php'), + (u'World News', u'http://www.sfgate.com/rss/feed/World-News-From-SFGate-432.php'), ] - def print_version(self,url): - url= url +"&type=printable" - return url - - def get_article_url(self, article): - print str(article['title_detail']['value']) - url = article.get('guid',None) - url = "http://www.sfgate.com/cgi-bin/article.cgi?f="+url - if "Presented By:" in str(article['title_detail']['value']): - url = '' - return url - - - - - - - - - - - - - - - diff --git a/recipes/theonion.recipe b/recipes/theonion.recipe index b0eacbb5e0..6cb19cee94 100644 --- a/recipes/theonion.recipe +++ b/recipes/theonion.recipe @@ -1,5 +1,5 @@ __license__ = 'GPL v3' -__copyright__ = '2009-2011, Darko Miletic ' +__copyright__ = '2009-2013, Darko Miletic ' ''' theonion.com @@ -10,7 +10,7 @@ from calibre.web.feeds.news import BasicNewsRecipe class TheOnion(BasicNewsRecipe): title = 'The Onion' __author__ = 'Darko Miletic' - description = "America's finest news source" + description = "The Onion, America's Finest News Source, is an award-winning publication covering world, national, and * local issues. It is updated daily online and distributed weekly in select American cities." oldest_article = 2 max_articles_per_feed = 100 publisher = 'Onion, Inc.' @@ -20,7 +20,8 @@ class TheOnion(BasicNewsRecipe): use_embedded_content = False encoding = 'utf-8' publication_type = 'newsportal' - masthead_url = 'http://o.onionstatic.com/img/headers/onion_190.png' + needs_subscription = 'optional' + masthead_url = 'http://www.theonion.com/static/onion/img/logo_1x.png' extra_css = """ body{font-family: Helvetica,Arial,sans-serif} .section_title{color: gray; text-transform: uppercase} @@ -37,18 +38,12 @@ class TheOnion(BasicNewsRecipe): , 'language' : language } - keep_only_tags = [ - dict(name='h2', attrs={'class':['section_title','title']}) - ,dict(attrs={'class':['main_image','meta','article_photo_lead','article_body']}) - ,dict(attrs={'id':['entries']}) - ] - remove_attributes=['lang','rel'] - remove_tags_after = dict(attrs={'class':['article_body','feature_content']}) - remove_tags = [ - dict(name=['object','link','iframe','base','meta']) - ,dict(name='div', attrs={'class':['toolbar_side','graphical_feature','toolbar_bottom']}) - ,dict(name='div', attrs={'id':['recent_slider','sidebar','pagination','related_media']}) - ] + keep_only_tags = [dict(attrs={'class':'full-article'})] + remove_attributes = ['lang','rel'] + remove_tags = [ + dict(name=['object','link','iframe','base','meta']) + ,dict(attrs={'class':lambda x: x and 'share-tools' in x.split()}) + ] feeds = [ @@ -56,6 +51,17 @@ class TheOnion(BasicNewsRecipe): ,(u'Sports' , u'http://feeds.theonion.com/theonion/sports' ) ] + def get_browser(self): + br = BasicNewsRecipe.get_browser(self) + br.open('http://www.theonion.com/') + if self.username is not None and self.password is not None: + br.open('https://ui.ppjol.com/login/onion/u/j_spring_security_check') + br.select_form(name='f') + br['j_username'] = self.username + br['j_password'] = self.password + br.submit() + return br + def get_article_url(self, article): artl = BasicNewsRecipe.get_article_url(self, article) if artl.startswith('http://www.theonion.com/audio/'): @@ -79,4 +85,8 @@ class TheOnion(BasicNewsRecipe): else: str = self.tag_to_string(item) item.replaceWith(str) + for item in soup.findAll('img'): + if item.has_key('data-src'): + item['src'] = item['data-src'] return soup + diff --git a/recipes/tomshardware.recipe b/recipes/tomshardware.recipe index c75b19189d..e0f21ae2a4 100644 --- a/recipes/tomshardware.recipe +++ b/recipes/tomshardware.recipe @@ -1,7 +1,5 @@ -#!/usr/bin/env python - __license__ = 'GPL v3' -__copyright__ = '2008-2009, Darko Miletic ' +__copyright__ = '2008-2013, Darko Miletic ' ''' tomshardware.com/us ''' @@ -16,22 +14,20 @@ class Tomshardware(BasicNewsRecipe): publisher = "Tom's Hardware" category = 'news, IT, hardware, USA' no_stylesheets = True - needs_subscription = True - language = 'en' - + needs_subscription = 'optional' + language = 'en' INDEX = 'http://www.tomshardware.com' LOGIN = INDEX + '/membres/' remove_javascript = True use_embedded_content= False - html2lrf_options = [ - '--comment', description - , '--category', category - , '--publisher', publisher - ] - - html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' - + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + } + def get_browser(self): br = BasicNewsRecipe.get_browser(self) br.open(self.INDEX+'/us/') @@ -50,8 +46,8 @@ class Tomshardware(BasicNewsRecipe): ] feeds = [ - (u'Latest Articles', u'http://www.tomshardware.com/feeds/atom/tom-s-hardware-us,18-2.xml' ) - ,(u'Latest News' , u'http://www.tomshardware.com/feeds/atom/tom-s-hardware-us,18-1.xml') + (u'Reviews', u'http://www.tomshardware.com/feeds/rss2/tom-s-hardware-us,18-2.xml') + ,(u'News' , u'http://www.tomshardware.com/feeds/rss2/tom-s-hardware-us,18-1.xml') ] def print_version(self, url): diff --git a/recipes/universe_today.recipe b/recipes/universe_today.recipe new file mode 100644 index 0000000000..65aefc231f --- /dev/null +++ b/recipes/universe_today.recipe @@ -0,0 +1,17 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class UniverseToday(BasicNewsRecipe): + title = u'Universe Today' + language = 'en' + description = u'Space and astronomy news.' + __author__ = 'seird' + publisher = u'universetoday.com' + category = 'science, astronomy, news, rss' + oldest_article = 7 + max_articles_per_feed = 40 + auto_cleanup = True + no_stylesheets = True + use_embedded_content = False + remove_empty_feeds = True + + feeds = [(u'Universe Today', u'http://feeds.feedburner.com/universetoday/pYdq')] diff --git a/session.vim b/session.vim index 5e127428cf..a67c5ed8e6 100644 --- a/session.vim +++ b/session.vim @@ -1,6 +1,3 @@ -" Project wide builtins -let $PYFLAKES_BUILTINS = "_,dynamic_property,__,P,I,lopen,icu_lower,icu_upper,icu_title,ngettext" - " Include directories for C++ modules let g:syntastic_cpp_include_dirs = [ \'/usr/include/python2.7', diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..5a0ae0e629 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 160 +builtins = _,dynamic_property,__,P,I,lopen,icu_lower,icu_upper,icu_title,ngettext +ignore = E12,E22,E231,E301,E302,E304,E401,W391 diff --git a/setup/check.py b/setup/check.py index 0baec38a50..281527e51a 100644 --- a/setup/check.py +++ b/setup/check.py @@ -22,40 +22,12 @@ class Message: self.filename, self.lineno, self.msg = filename, lineno, msg def __str__(self): - return '%s:%s: %s'%(self.filename, self.lineno, self.msg) - -def check_for_python_errors(code_string, filename): - import _ast - # First, compile into an AST and handle syntax errors. - try: - tree = compile(code_string, filename, "exec", _ast.PyCF_ONLY_AST) - except (SyntaxError, IndentationError) as value: - msg = value.args[0] - - (lineno, offset, text) = value.lineno, value.offset, value.text - - # If there's an encoding problem with the file, the text is None. - if text is None: - # Avoid using msg, since for the only known case, it contains a - # bogus message that claims the encoding the file declared was - # unknown. - msg = "%s: problem decoding source" % filename - - return [Message(filename, lineno, msg)] - else: - checker = __import__('pyflakes.checker').checker - # Okay, it's syntactically valid. Now check it. - w = checker.Checker(tree, filename) - w.messages.sort(lambda a, b: cmp(a.lineno, b.lineno)) - return [Message(x.filename, x.lineno, x.message%x.message_args) for x in - w.messages] + return '%s:%s: %s' % (self.filename, self.lineno, self.msg) class Check(Command): description = 'Check for errors in the calibre source code' - BUILTINS = ['_', '__', 'dynamic_property', 'I', 'P', 'lopen', 'icu_lower', - 'icu_upper', 'icu_title', 'ngettext'] CACHE = '.check-cache.pickle' def get_files(self, cache): @@ -65,10 +37,10 @@ class Check(Command): mtime = os.stat(y).st_mtime if cache.get(y, 0) == mtime: continue - if (f.endswith('.py') and f not in ('feedparser.py', - 'pyparsing.py', 'markdown.py') and - 'prs500/driver.py' not in y): - yield y, mtime + if (f.endswith('.py') and f not in ( + 'feedparser.py', 'pyparsing.py', 'markdown.py') and + 'prs500/driver.py' not in y): + yield y, mtime if f.endswith('.coffee'): yield y, mtime @@ -79,25 +51,22 @@ class Check(Command): if f.endswith('.recipe') and cache.get(f, 0) != mtime: yield f, mtime - def run(self, opts): cache = {} if os.path.exists(self.CACHE): cache = cPickle.load(open(self.CACHE, 'rb')) - builtins = list(set_builtins(self.BUILTINS)) for f, mtime in self.get_files(cache): self.info('\tChecking', f) errors = False ext = os.path.splitext(f)[1] if ext in {'.py', '.recipe'}: - w = check_for_python_errors(open(f, 'rb').read(), f) - if w: + p = subprocess.Popen(['flake8', '--ignore=E,W', f]) + if p.wait() != 0: errors = True - self.report_errors(w) else: from calibre.utils.serve_coffee import check_coffeescript try: - check_coffeescript(f) + check_coffeescript(f) except: errors = True if errors: @@ -106,8 +75,6 @@ class Check(Command): self.j(self.SRC, '../session.vim'), '-f', f]) raise SystemExit(1) cache[f] = mtime - for x in builtins: - delattr(__builtin__, x) cPickle.dump(cache, open(self.CACHE, 'wb'), -1) wn_path = os.path.expanduser('~/work/servers/src/calibre_servers/main') if os.path.exists(wn_path): diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 5d12bdc686..b0916ebf73 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -68,4 +68,5 @@ Various things that require other things before they can be migrated: libraries/switching/on calibre startup. 3. From refresh in the legacy interface: Rember to flush the composite column template cache. + 4. Replace the metadatabackup thread with the new implementation when using the new backend. ''' diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 9259fc628b..0194cfc2ae 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -41,8 +41,7 @@ Differences in semantics from pysqlite: ''' - -class DynamicFilter(object): # {{{ +class DynamicFilter(object): # {{{ 'No longer used, present for legacy compatibility' @@ -57,7 +56,7 @@ class DynamicFilter(object): # {{{ self.ids = frozenset(ids) # }}} -class DBPrefs(dict): # {{{ +class DBPrefs(dict): # {{{ 'Store preferences as key:value pairs in the db' @@ -114,9 +113,10 @@ class DBPrefs(dict): # {{{ return default def set_namespaced(self, namespace, key, val): - if u':' in key: raise KeyError('Colons are not allowed in keys') - if u':' in namespace: raise KeyError('Colons are not allowed in' - ' the namespace') + if u':' in key: + raise KeyError('Colons are not allowed in keys') + if u':' in namespace: + raise KeyError('Colons are not allowed in the namespace') key = u'namespaced:%s:%s'%(namespace, key) self[key] = val @@ -170,7 +170,8 @@ def pynocase(one, two, encoding='utf-8'): return cmp(one.lower(), two.lower()) def _author_to_author_sort(x): - if not x: return '' + if not x: + return '' return author_to_author_sort(x.replace('|', ',')) def icu_collator(s1, s2): @@ -239,9 +240,9 @@ def AumSortedConcatenate(): # }}} -class Connection(apsw.Connection): # {{{ +class Connection(apsw.Connection): # {{{ - BUSY_TIMEOUT = 2000 # milliseconds + BUSY_TIMEOUT = 2000 # milliseconds def __init__(self, path): apsw.Connection.__init__(self, path) @@ -257,7 +258,7 @@ class Connection(apsw.Connection): # {{{ self.createscalarfunction('title_sort', title_sort, 1) self.createscalarfunction('author_to_author_sort', _author_to_author_sort, 1) - self.createscalarfunction('uuid4', lambda : str(uuid.uuid4()), + self.createscalarfunction('uuid4', lambda: str(uuid.uuid4()), 0) # Dummy functions for dynamically created filters @@ -380,7 +381,7 @@ class DB(object): self.initialize_custom_columns() self.initialize_tables() - def initialize_prefs(self, default_prefs): # {{{ + def initialize_prefs(self, default_prefs): # {{{ self.prefs = DBPrefs(self) if default_prefs is not None and not self._exists: @@ -493,7 +494,7 @@ class DB(object): self.prefs.set('user_categories', user_cats) # }}} - def initialize_custom_columns(self): # {{{ + def initialize_custom_columns(self): # {{{ with self.conn: # Delete previously marked custom columns for record in self.conn.get( @@ -634,11 +635,11 @@ class DB(object): self.custom_data_adapters = { 'float': adapt_number, - 'int': adapt_number, - 'rating':lambda x,d : x if x is None else min(10., max(0., float(x))), - 'bool': adapt_bool, + 'int': adapt_number, + 'rating':lambda x,d: x if x is None else min(10., max(0., float(x))), + 'bool': adapt_bool, 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}), - 'datetime' : adapt_datetime, + 'datetime': adapt_datetime, 'text':adapt_text, 'series':adapt_text, 'enumeration': adapt_enum @@ -661,7 +662,7 @@ class DB(object): # }}} - def initialize_tables(self): # {{{ + def initialize_tables(self): # {{{ tables = self.tables = {} for col in ('title', 'sort', 'author_sort', 'series_index', 'comments', 'timestamp', 'pubdate', 'uuid', 'path', 'cover', @@ -866,8 +867,8 @@ class DB(object): Read all data from the db into the python in-memory tables ''' - with self.conn: # Use a single transaction, to ensure nothing modifies - # the db while we are reading + with self.conn: # Use a single transaction, to ensure nothing modifies + # the db while we are reading for table in self.tables.itervalues(): try: table.read(self) @@ -885,7 +886,7 @@ class DB(object): return fmt_path try: candidates = glob.glob(os.path.join(path, '*'+fmt)) - except: # If path contains strange characters this throws an exc + except: # If path contains strange characters this throws an exc candidates = [] if fmt and candidates and os.path.exists(candidates[0]): shutil.copyfile(candidates[0], fmt_path) @@ -954,7 +955,7 @@ class DB(object): if path != dest: os.rename(path, dest) except: - pass # Nothing too catastrophic happened, the cases mismatch, that's all + pass # Nothing too catastrophic happened, the cases mismatch, that's all else: windows_atomic_move.copy_path_to(path, dest) else: @@ -970,7 +971,7 @@ class DB(object): try: os.rename(path, dest) except: - pass # Nothing too catastrophic happened, the cases mismatch, that's all + pass # Nothing too catastrophic happened, the cases mismatch, that's all else: if use_hardlink: try: @@ -1021,7 +1022,7 @@ class DB(object): if not os.path.exists(tpath): os.makedirs(tpath) - if source_ok: # Migrate existing files + if source_ok: # Migrate existing files dest = os.path.join(tpath, 'cover.jpg') self.copy_cover_to(current_path, dest, windows_atomic_move=wam, use_hardlink=True) @@ -1064,8 +1065,18 @@ class DB(object): os.rename(os.path.join(curpath, oldseg), os.path.join(curpath, newseg)) except: - break # Fail silently since nothing catastrophic has happened + break # Fail silently since nothing catastrophic has happened curpath = os.path.join(curpath, newseg) + def write_backup(self, path, raw): + path = os.path.abspath(os.path.join(self.library_path, path, 'metadata.opf')) + with lopen(path, 'wb') as f: + f.write(raw) + + def read_backup(self, path): + path = os.path.abspath(os.path.join(self.library_path, path, 'metadata.opf')) + with lopen(path, 'rb') as f: + return f.read() + # }}} diff --git a/src/calibre/db/backup.py b/src/calibre/db/backup.py new file mode 100644 index 0000000000..6410a347c6 --- /dev/null +++ b/src/calibre/db/backup.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import weakref, traceback +from threading import Thread, Event + +from calibre import prints +from calibre.ebooks.metadata.opf2 import metadata_to_opf + +class Abort(Exception): + pass + +class MetadataBackup(Thread): + ''' + Continuously backup changed metadata into OPF files + in the book directory. This class runs in its own + thread. + ''' + + def __init__(self, db, interval=2, scheduling_interval=0.1): + Thread.__init__(self) + self.daemon = True + self._db = weakref.ref(db) + self.stop_running = Event() + self.interval = interval + self.scheduling_interval = scheduling_interval + + @property + def db(self): + ans = self._db() + if ans is None: + raise Abort() + return ans + + def stop(self): + self.stop_running.set() + + def wait(self, interval): + if self.stop_running.wait(interval): + raise Abort() + + def run(self): + while not self.stop_running.is_set(): + try: + self.wait(self.interval) + self.do_one() + except Abort: + break + + def do_one(self): + try: + book_id = self.db.get_a_dirtied_book() + if book_id is None: + return + except Abort: + raise + except: + # Happens during interpreter shutdown + return + + self.wait(0) + + try: + mi, sequence = self.db.get_metadata_for_dump(book_id) + except: + prints('Failed to get backup metadata for id:', book_id, 'once') + traceback.print_exc() + self.wait(self.interval) + try: + mi, sequence = self.db.get_metadata_for_dump(book_id) + except: + prints('Failed to get backup metadata for id:', book_id, 'again, giving up') + traceback.print_exc() + return + + if mi is None: + self.db.clear_dirtied(book_id, sequence) + + # Give the GUI thread a chance to do something. Python threads don't + # have priorities, so this thread would naturally keep the processor + # until some scheduling event happens. The wait makes such an event + self.wait(self.scheduling_interval) + + try: + raw = metadata_to_opf(mi) + except: + prints('Failed to convert to opf for id:', book_id) + traceback.print_exc() + return + + self.wait(self.scheduling_interval) + + try: + self.db.write_backup(book_id, raw) + except: + prints('Failed to write backup metadata for id:', book_id, 'once') + self.wait(self.interval) + try: + self.db.write_backup(book_id, raw) + except: + prints('Failed to write backup metadata for id:', book_id, 'again, giving up') + return + + self.db.clear_dirtied(book_id, sequence) + + def break_cycles(self): + # Legacy compatibility + pass + diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 0f648e96dd..0fa280d997 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback +import os, traceback, random from io import BytesIO from collections import defaultdict from functools import wraps, partial @@ -15,7 +15,7 @@ from functools import wraps, partial from calibre.constants import iswindows from calibre.db import SPOOL_SIZE from calibre.db.categories import get_categories -from calibre.db.locking import create_locks, RecordLock +from calibre.db.locking import create_locks from calibre.db.errors import NoSuchFormat from calibre.db.fields import create_field from calibre.db.search import Search @@ -23,9 +23,10 @@ from calibre.db.tables import VirtualTable from calibre.db.write import get_series_values from calibre.db.lazy import FormatMetadata, FormatsList from calibre.ebooks.metadata.book.base import Metadata +from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ptempfile import (base_dir, PersistentTemporaryFile, SpooledTemporaryFile) -from calibre.utils.date import now +from calibre.utils.date import now as nowf from calibre.utils.icu import sort_key def api(f): @@ -57,9 +58,10 @@ class Cache(object): self.fields = {} self.composites = set() self.read_lock, self.write_lock = create_locks() - self.record_lock = RecordLock(self.read_lock) self.format_metadata_cache = defaultdict(dict) self.formatter_template_cache = {} + self.dirtied_cache = {} + self.dirtied_sequence = 0 self._search_api = Search(self.field_metadata.get_search_terms()) # Implement locking for all simple read/write API methods @@ -78,17 +80,18 @@ class Cache(object): self.initialize_dynamic() + @write_api def initialize_dynamic(self): # Reconstruct the user categories, putting them into field_metadata # Assumption is that someone else will fix them if they change. self.field_metadata.remove_dynamic_categories() - for user_cat in sorted(self.pref('user_categories', {}).iterkeys(), key=sort_key): - cat_name = '@' + user_cat # add the '@' to avoid name collision + for user_cat in sorted(self._pref('user_categories', {}).iterkeys(), key=sort_key): + cat_name = '@' + user_cat # add the '@' to avoid name collision self.field_metadata.add_user_category(label=cat_name, name=user_cat) # add grouped search term user categories - muc = frozenset(self.pref('grouped_search_make_user_categories', [])) - for cat in sorted(self.pref('grouped_search_terms', {}).iterkeys(), key=sort_key): + muc = frozenset(self._pref('grouped_search_make_user_categories', [])) + for cat in sorted(self._pref('grouped_search_terms', {}).iterkeys(), key=sort_key): if cat in muc: # There is a chance that these can be duplicates of an existing # user category. Print the exception and continue. @@ -102,15 +105,20 @@ class Cache(object): # self.field_metadata.add_search_category(label='search', name=_('Searches')) self.field_metadata.add_grouped_search_terms( - self.pref('grouped_search_terms', {})) + self._pref('grouped_search_terms', {})) self._search_api.change_locations(self.field_metadata.get_search_terms()) + self.dirtied_cache = {x:i for i, (x,) in enumerate( + self.backend.conn.execute('SELECT book FROM metadata_dirtied'))} + if self.dirtied_cache: + self.dirtied_sequence = max(self.dirtied_cache.itervalues())+1 + @property def field_metadata(self): return self.backend.field_metadata - def _get_metadata(self, book_id, get_user_categories=True): # {{{ + def _get_metadata(self, book_id, get_user_categories=True): # {{{ mi = Metadata(None, template_cache=self.formatter_template_cache) author_ids = self._field_ids_for('authors', book_id) aut_list = [self._author_data(i) for i in author_ids] @@ -131,7 +139,7 @@ class Cache(object): mi.author_link_map = aul mi.comments = self._field_for('comments', book_id) mi.publisher = self._field_for('publisher', book_id) - n = now() + n = nowf() mi.timestamp = self._field_for('timestamp', book_id, default_value=n) mi.pubdate = self._field_for('pubdate', book_id, default_value=n) mi.uuid = self._field_for('uuid', book_id, @@ -395,16 +403,19 @@ class Cache(object): ''' if as_file: ret = SpooledTemporaryFile(SPOOL_SIZE) - if not self.copy_cover_to(book_id, ret): return + if not self.copy_cover_to(book_id, ret): + return ret.seek(0) elif as_path: pt = PersistentTemporaryFile('_dbcover.jpg') with pt: - if not self.copy_cover_to(book_id, pt): return + if not self.copy_cover_to(book_id, pt): + return ret = pt.name else: buf = BytesIO() - if not self.copy_cover_to(book_id, buf): return + if not self.copy_cover_to(book_id, buf): + return ret = buf.getvalue() if as_image: from PyQt4.Qt import QImage @@ -413,7 +424,7 @@ class Cache(object): ret = i return ret - @api + @read_api def copy_cover_to(self, book_id, dest, use_hardlink=False): ''' Copy the cover to the file like object ``dest``. Returns False @@ -422,17 +433,15 @@ class Cache(object): copied to it iff the path is different from the current path (taking case sensitivity into account). ''' - with self.read_lock: - try: - path = self._field_for('path', book_id).replace('/', os.sep) - except: - return False + try: + path = self._field_for('path', book_id).replace('/', os.sep) + except AttributeError: + return False - with self.record_lock.lock(book_id): - return self.backend.copy_cover_to(path, dest, + return self.backend.copy_cover_to(path, dest, use_hardlink=use_hardlink) - @api + @read_api def copy_format_to(self, book_id, fmt, dest, use_hardlink=False): ''' Copy the format ``fmt`` to the file like object ``dest``. If the @@ -441,15 +450,13 @@ class Cache(object): the path is different from the current path (taking case sensitivity into account). ''' - with self.read_lock: - try: - name = self.fields['formats'].format_fname(book_id, fmt) - path = self._field_for('path', book_id).replace('/', os.sep) - except: - raise NoSuchFormat('Record %d has no %s file'%(book_id, fmt)) + try: + name = self.fields['formats'].format_fname(book_id, fmt) + path = self._field_for('path', book_id).replace('/', os.sep) + except (KeyError, AttributeError): + raise NoSuchFormat('Record %d has no %s file'%(book_id, fmt)) - with self.record_lock.lock(book_id): - return self.backend.copy_format_to(book_id, fmt, name, path, dest, + return self.backend.copy_format_to(book_id, fmt, name, path, dest, use_hardlink=use_hardlink) @read_api @@ -520,16 +527,16 @@ class Cache(object): this means that repeated calls yield the same temp file (which is re-created each time) ''' - with self.read_lock: - ext = ('.'+fmt.lower()) if fmt else '' - try: - fname = self.fields['formats'].format_fname(book_id, fmt) - except: - return None - fname += ext - + ext = ('.'+fmt.lower()) if fmt else '' if as_path: if preserve_filename: + with self.read_lock: + try: + fname = self.fields['formats'].format_fname(book_id, fmt) + except: + return None + fname += ext + bd = base_dir() d = os.path.join(bd, 'format_abspath') try: @@ -537,36 +544,40 @@ class Cache(object): except: pass ret = os.path.join(d, fname) - with self.record_lock.lock(book_id): - try: - self.copy_format_to(book_id, fmt, ret) - except NoSuchFormat: - return None + try: + self.copy_format_to(book_id, fmt, ret) + except NoSuchFormat: + return None else: - with PersistentTemporaryFile(ext) as pt, self.record_lock.lock(book_id): + with PersistentTemporaryFile(ext) as pt: try: self.copy_format_to(book_id, fmt, pt) except NoSuchFormat: return None ret = pt.name elif as_file: - ret = SpooledTemporaryFile(SPOOL_SIZE) - with self.record_lock.lock(book_id): + with self.read_lock: try: - self.copy_format_to(book_id, fmt, ret) - except NoSuchFormat: + fname = self.fields['formats'].format_fname(book_id, fmt) + except: return None + fname += ext + + ret = SpooledTemporaryFile(SPOOL_SIZE) + try: + self.copy_format_to(book_id, fmt, ret) + except NoSuchFormat: + return None ret.seek(0) # Various bits of code try to use the name as the default # title when reading metadata, so set it ret.name = fname else: buf = BytesIO() - with self.record_lock.lock(book_id): - try: - self.copy_format_to(book_id, fmt, buf) - except NoSuchFormat: - return None + try: + self.copy_format_to(book_id, fmt, buf) + except NoSuchFormat: + return None ret = buf.getvalue() @@ -620,6 +631,30 @@ class Cache(object): return get_categories(self, sort=sort, book_ids=book_ids, icon_map=icon_map) + @write_api + def update_last_modified(self, book_ids, now=None): + if now is None: + now = nowf() + if book_ids: + f = self.fields['last_modified'] + f.writer.set_books({book_id:now for book_id in book_ids}, self.backend) + + @write_api + def mark_as_dirty(self, book_ids): + self._update_last_modified(book_ids) + already_dirtied = set(self.dirtied_cache).intersection(book_ids) + new_dirtied = book_ids - already_dirtied + already_dirtied = {book_id:self.dirtied_sequence+i for i, book_id in enumerate(already_dirtied)} + if already_dirtied: + self.dirtied_sequence = max(already_dirtied.itervalues()) + 1 + self.dirtied_cache.update(already_dirtied) + if new_dirtied: + self.backend.conn.executemany('INSERT OR IGNORE INTO metadata_dirtied (book) VALUES (?)', + ((x,) for x in new_dirtied)) + new_dirtied = {book_id:self.dirtied_sequence+i for i, book_id in enumerate(new_dirtied)} + self.dirtied_sequence = max(new_dirtied.itervalues()) + 1 + self.dirtied_cache.update(new_dirtied) + @write_api def set_field(self, name, book_id_to_val_map, allow_case_change=True): f = self.fields[name] @@ -637,7 +672,7 @@ class Cache(object): else: v = sid = None if name.startswith('#') and sid is None: - sid = 1.0 # The value will be set to 1.0 in the db table + sid = 1.0 # The value will be set to 1.0 in the db table bimap[k] = v if sid is not None: simap[k] = sid @@ -657,7 +692,7 @@ class Cache(object): if dirtied and update_path: self._update_path(dirtied, mark_as_dirtied=False) - # TODO: Mark these as dirtied so that the opf is regenerated + self._mark_as_dirty(dirtied) return dirtied @@ -668,13 +703,115 @@ class Cache(object): author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0] self.backend.update_path(book_id, title, author, self.fields['path'], self.fields['formats']) if mark_as_dirtied: + self._mark_as_dirty(book_ids) + + @read_api + def get_a_dirtied_book(self): + if self.dirtied_cache: + return random.choice(tuple(self.dirtied_cache.iterkeys())) + return None + + @read_api + def get_metadata_for_dump(self, book_id): + mi = None + # get the current sequence number for this book to pass back to the + # backup thread. This will avoid double calls in the case where the + # thread has not done the work between the put and the get_metadata + sequence = self.dirtied_cache.get(book_id, None) + if sequence is not None: + try: + # While a book is being created, the path is empty. Don't bother to + # try to write the opf, because it will go to the wrong folder. + if self._field_for('path', book_id): + mi = self._get_metadata(book_id) + # Always set cover to cover.jpg. Even if cover doesn't exist, + # no harm done. This way no need to call dirtied when + # cover is set/removed + mi.cover = 'cover.jpg' + except: + # This almost certainly means that the book has been deleted while + # the backup operation sat in the queue. pass - # TODO: Mark these books as dirtied so that metadata.opf is - # re-created + return mi, sequence + + @write_api + def clear_dirtied(self, book_id, sequence): + ''' + Clear the dirtied indicator for the books. This is used when fetching + metadata, creating an OPF, and writing a file are separated into steps. + The last step is clearing the indicator + ''' + dc_sequence = self.dirtied_cache.get(book_id, None) + if dc_sequence is None or sequence is None or dc_sequence == sequence: + self.backend.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', + (book_id,)) + self.dirtied_cache.pop(book_id, None) + + @write_api + def write_backup(self, book_id, raw): + try: + path = self._field_for('path', book_id).replace('/', os.sep) + except: + return + + self.backend.write_backup(path, raw) + + @read_api + def dirty_queue_length(self): + return len(self.dirtied_cache) + + @read_api + def read_backup(self, book_id): + ''' Return the OPF metadata backup for the book as a bytestring or None + if no such backup exists. ''' + try: + path = self._field_for('path', book_id).replace('/', os.sep) + except: + return + + try: + return self.backend.read_backup(path) + except EnvironmentError: + return None + + @write_api + def dump_metadata(self, book_ids=None, remove_from_dirtied=True, + callback=None): + ''' + Write metadata for each record to an individual OPF file. If callback + is not None, it is called once at the start with the number of book_ids + being processed. And once for every book_id, with arguments (book_id, + mi, ok). + ''' + if book_ids is None: + book_ids = set(self.dirtied_cache) + + if callback is not None: + callback(len(book_ids), True, False) + + for book_id in book_ids: + if self._field_for('path', book_id) is None: + if callback is not None: + callback(book_id, None, False) + continue + mi, sequence = self._get_metadata_for_dump(book_id) + if mi is None: + if callback is not None: + callback(book_id, mi, False) + continue + try: + raw = metadata_to_opf(mi) + self._write_backup(book_id, raw) + if remove_from_dirtied: + self._clear_dirtied(book_id, sequence) + except: + pass + if callback is not None: + callback(book_id, mi, True) # }}} -class SortKey(object): # {{{ +class SortKey(object): # {{{ def __init__(self, fields, sort_keys, book_id): self.orders = tuple(1 if f[1] else -1 for f in fields) diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py index f49789e16c..3f7bbb9e61 100644 --- a/src/calibre/db/categories.py +++ b/src/calibre/db/categories.py @@ -18,7 +18,7 @@ from calibre.utils.config_base import tweaks from calibre.utils.icu import sort_key from calibre.utils.search_query_parser import saved_searches -CATEGORY_SORTS = ('name', 'popularity', 'rating') # This has to be a tuple not a set +CATEGORY_SORTS = ('name', 'popularity', 'rating') # This has to be a tuple not a set class Tag(object): @@ -218,7 +218,7 @@ def get_categories(dbcache, sort='name', book_ids=None, icon_map=None): else: items.append(taglist[label][n]) # else: do nothing, to not include nodes w zero counts - cat_name = '@' + user_cat # add the '@' to avoid name collision + cat_name = '@' + user_cat # add the '@' to avoid name collision # Not a problem if we accumulate entries in the icon map if icon_map is not None: icon_map[cat_name] = icon_map['user:'] diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index e0074de7d1..20d0d75ff4 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -31,7 +31,7 @@ class Field(object): self.table_type = self.table.table_type self._sort_key = (sort_key if dt in ('text', 'series', 'enumeration') else lambda x: x) self._default_sort_key = '' - if dt in { 'int', 'float', 'rating' }: + if dt in {'int', 'float', 'rating'}: self._default_sort_key = 0 elif dt == 'bool': self._default_sort_key = None @@ -138,7 +138,7 @@ class OneToOneField(Field): return self.table.book_col_map.iterkeys() def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): - return {id_ : self._sort_key(self.table.book_col_map.get(id_, + return {id_: self._sort_key(self.table.book_col_map.get(id_, self._default_sort_key)) for id_ in all_book_ids} def iter_searchable_values(self, get_metadata, candidates, default_value=None): @@ -183,7 +183,7 @@ class CompositeField(OneToOneField): return ans def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): - return {id_ : sort_key(self.get_value_with_cache(id_, get_metadata)) for id_ in + return {id_: sort_key(self.get_value_with_cache(id_, get_metadata)) for id_ in all_book_ids} def iter_searchable_values(self, get_metadata, candidates, default_value=None): @@ -245,7 +245,7 @@ class OnDeviceField(OneToOneField): return iter(()) def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): - return {id_ : self.for_book(id_) for id_ in + return {id_: self.for_book(id_) for id_ in all_book_ids} def iter_searchable_values(self, get_metadata, candidates, default_value=None): @@ -280,12 +280,12 @@ class ManyToOneField(Field): return self.table.id_map.iterkeys() def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): - ans = {id_ : self.table.book_col_map.get(id_, None) + ans = {id_: self.table.book_col_map.get(id_, None) for id_ in all_book_ids} - sk_map = {cid : (self._default_sort_key if cid is None else + sk_map = {cid: (self._default_sort_key if cid is None else self._sort_key(self.table.id_map[cid])) for cid in ans.itervalues()} - return {id_ : sk_map[cid] for id_, cid in ans.iteritems()} + return {id_: sk_map[cid] for id_, cid in ans.iteritems()} def iter_searchable_values(self, get_metadata, candidates, default_value=None): cbm = self.table.col_book_map @@ -327,14 +327,14 @@ class ManyToManyField(Field): return self.table.id_map.iterkeys() def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): - ans = {id_ : self.table.book_col_map.get(id_, ()) + ans = {id_: self.table.book_col_map.get(id_, ()) for id_ in all_book_ids} all_cids = set() for cids in ans.itervalues(): all_cids = all_cids.union(set(cids)) - sk_map = {cid : self._sort_key(self.table.id_map[cid]) + sk_map = {cid: self._sort_key(self.table.id_map[cid]) for cid in all_cids} - return {id_ : (tuple(sk_map[cid] for cid in cids) if cids else + return {id_: (tuple(sk_map[cid] for cid in cids) if cids else (self._default_sort_key,)) for id_, cids in ans.iteritems()} @@ -369,9 +369,9 @@ class IdentifiersField(ManyToManyField): def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): 'Sort by identifier keys' - ans = {id_ : self.table.book_col_map.get(id_, ()) + ans = {id_: self.table.book_col_map.get(id_, ()) for id_ in all_book_ids} - return {id_ : (tuple(sorted(cids.iterkeys())) if cids else + return {id_: (tuple(sorted(cids.iterkeys())) if cids else (self._default_sort_key,)) for id_, cids in ans.iteritems()} @@ -397,9 +397,9 @@ class AuthorsField(ManyToManyField): def author_data(self, author_id): return { - 'name' : self.table.id_map[author_id], - 'sort' : self.table.asort_map[author_id], - 'link' : self.table.alink_map[author_id], + 'name': self.table.id_map[author_id], + 'sort': self.table.asort_map[author_id], + 'link': self.table.alink_map[author_id], } def category_sort_value(self, item_id, book_ids, lang_map): @@ -505,9 +505,9 @@ class TagsField(ManyToManyField): def create_field(name, table): cls = { - ONE_ONE : OneToOneField, - MANY_ONE : ManyToOneField, - MANY_MANY : ManyToManyField, + ONE_ONE: OneToOneField, + MANY_ONE: ManyToOneField, + MANY_MANY: ManyToManyField, }[table.table_type] if name == 'authors': cls = AuthorsField diff --git a/src/calibre/db/locking.py b/src/calibre/db/locking.py index d08c7b99fe..0791a5ac07 100644 --- a/src/calibre/db/locking.py +++ b/src/calibre/db/locking.py @@ -39,7 +39,7 @@ def create_locks(): l = SHLock() return RWLockWrapper(l), RWLockWrapper(l, is_shared=False) -class SHLock(object): # {{{ +class SHLock(object): # {{{ ''' Shareable lock class. Used to implement the Multiple readers-single writer paradigm. As best as I can tell, neither writer nor reader starvation @@ -191,7 +191,7 @@ class SHLock(object): # {{{ try: return self._free_waiters.pop() except IndexError: - return Condition(self._lock)#, verbose=True) + return Condition(self._lock) def _return_waiter(self, waiter): self._free_waiters.append(waiter) diff --git a/src/calibre/db/schema_upgrades.py b/src/calibre/db/schema_upgrades.py index f3ca6f9852..c8eaa748c7 100644 --- a/src/calibre/db/schema_upgrades.py +++ b/src/calibre/db/schema_upgrades.py @@ -172,7 +172,6 @@ class SchemaUpgrade(object): ''' ) - def upgrade_version_6(self): 'Show authors in order' self.conn.execute(''' @@ -337,7 +336,7 @@ class SchemaUpgrade(object): FROM {tn}; '''.format(tn=table_name, cn=column_name, - vcn=view_column_name, scn= sort_column_name)) + vcn=view_column_name, scn=sort_column_name)) self.conn.execute(script) def create_cust_tag_browser_view(table_name, link_table_name): diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 57039e191d..c7fed18f9d 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -64,7 +64,7 @@ def _match(query, value, matchkind, use_primary_find_in_search=True): else: internal_match_ok = False for t in value: - try: ### ignore regexp exceptions, required because search-ahead tries before typing is finished + try: # ignore regexp exceptions, required because search-ahead tries before typing is finished t = icu_lower(t) if (matchkind == EQUALS_MATCH): if internal_match_ok: @@ -95,20 +95,20 @@ def _match(query, value, matchkind, use_primary_find_in_search=True): return False # }}} -class DateSearch(object): # {{{ +class DateSearch(object): # {{{ def __init__(self): self.operators = { - '=' : (1, self.eq), - '!=' : (2, self.ne), - '>' : (1, self.gt), - '>=' : (2, self.ge), - '<' : (1, self.lt), - '<=' : (2, self.le), + '=': (1, self.eq), + '!=': (2, self.ne), + '>': (1, self.gt), + '>=': (2, self.ge), + '<': (1, self.lt), + '<=': (2, self.le), } - self.local_today = { '_today', 'today', icu_lower(_('today')) } - self.local_yesterday = { '_yesterday', 'yesterday', icu_lower(_('yesterday')) } - self.local_thismonth = { '_thismonth', 'thismonth', icu_lower(_('thismonth')) } + self.local_today = {'_today', 'today', icu_lower(_('today'))} + self.local_yesterday = {'_yesterday', 'yesterday', icu_lower(_('yesterday'))} + self.local_thismonth = {'_thismonth', 'thismonth', icu_lower(_('thismonth'))} self.daysago_pat = re.compile(r'(%s|daysago|_daysago)$'%_('daysago')) def eq(self, dbdate, query, field_count): @@ -216,16 +216,16 @@ class DateSearch(object): # {{{ return matches # }}} -class NumericSearch(object): # {{{ +class NumericSearch(object): # {{{ def __init__(self): self.operators = { - '=':( 1, lambda r, q: r == q ), - '>':( 1, lambda r, q: r is not None and r > q ), - '<':( 1, lambda r, q: r is not None and r < q ), - '!=':( 2, lambda r, q: r != q ), - '>=':( 2, lambda r, q: r is not None and r >= q ), - '<=':( 2, lambda r, q: r is not None and r <= q ) + '=':(1, lambda r, q: r == q), + '>':(1, lambda r, q: r is not None and r > q), + '<':(1, lambda r, q: r is not None and r < q), + '!=':(2, lambda r, q: r != q), + '>=':(2, lambda r, q: r is not None and r >= q), + '<=':(2, lambda r, q: r is not None and r <= q) } def __call__(self, query, field_iter, location, datatype, candidates, is_many=False): @@ -267,7 +267,7 @@ class NumericSearch(object): # {{{ p, relop = self.operators['='] cast = int - if dt == 'rating': + if dt == 'rating': cast = lambda x: 0 if x is None else int(x) adjust = lambda x: x/2 elif dt in ('float', 'composite'): @@ -303,7 +303,7 @@ class NumericSearch(object): # {{{ # }}} -class BooleanSearch(object): # {{{ +class BooleanSearch(object): # {{{ def __init__(self): self.local_no = icu_lower(_('no')) @@ -324,27 +324,27 @@ class BooleanSearch(object): # {{{ for val, book_ids in field_iter(): val = force_to_bool(val) if not bools_are_tristate: - if val is None or not val: # item is None or set to false - if query in { self.local_no, self.local_unchecked, 'no', '_no', 'false' }: + if val is None or not val: # item is None or set to false + if query in {self.local_no, self.local_unchecked, 'no', '_no', 'false'}: matches |= book_ids - else: # item is explicitly set to true - if query in { self.local_yes, self.local_checked, 'yes', '_yes', 'true' }: + else: # item is explicitly set to true + if query in {self.local_yes, self.local_checked, 'yes', '_yes', 'true'}: matches |= book_ids else: if val is None: - if query in { self.local_empty, self.local_blank, 'empty', '_empty', 'false' }: + if query in {self.local_empty, self.local_blank, 'empty', '_empty', 'false'}: matches |= book_ids - elif not val: # is not None and false - if query in { self.local_no, self.local_unchecked, 'no', '_no', 'true' }: + elif not val: # is not None and false + if query in {self.local_no, self.local_unchecked, 'no', '_no', 'true'}: matches |= book_ids - else: # item is not None and true - if query in { self.local_yes, self.local_checked, 'yes', '_yes', 'true' }: + else: # item is not None and true + if query in {self.local_yes, self.local_checked, 'yes', '_yes', 'true'}: matches |= book_ids return matches # }}} -class KeyPairSearch(object): # {{{ +class KeyPairSearch(object): # {{{ def __call__(self, query, field_iter, candidates, use_primary_find): matches = set() @@ -547,11 +547,12 @@ class Parser(SearchQueryParser): field_metadata = {} for x, fm in self.field_metadata.iteritems(): - if x.startswith('@'): continue + if x.startswith('@'): + continue if fm['search_terms'] and x != 'series_sort': all_locs.add(x) field_metadata[x] = fm - if fm['datatype'] in { 'composite', 'text', 'comments', 'series', 'enumeration' }: + if fm['datatype'] in {'composite', 'text', 'comments', 'series', 'enumeration'}: text_fields.add(x) locations = all_locs if location == 'all' else {location} @@ -687,8 +688,8 @@ class Search(object): dbcache, all_book_ids, dbcache.pref('grouped_search_terms'), self.date_search, self.num_search, self.bool_search, self.keypair_search, - prefs[ 'limit_search_columns' ], - prefs[ 'limit_search_columns_to' ], self.all_search_locations, + prefs['limit_search_columns'], + prefs['limit_search_columns_to'], self.all_search_locations, virtual_fields) try: diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index bbc5e3bdef..fc62fbe951 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -82,7 +82,7 @@ class OneToOneTable(Table): self.metadata['column'], self.metadata['table'])): self.book_col_map[row[0]] = self.unserialize(row[1]) -class PathTable(OneToOneTable): +class PathTable(OneToOneTable): def set_path(self, book_id, path, db): self.book_col_map[book_id] = path diff --git a/src/calibre/db/tests/main.py b/src/calibre/db/tests/main.py index c4bb058b7e..bdc9561ec5 100644 --- a/src/calibre/db/tests/main.py +++ b/src/calibre/db/tests/main.py @@ -9,15 +9,32 @@ __docformat__ = 'restructuredtext en' import unittest, os, argparse +try: + import init_calibre # noqa +except ImportError: + pass + def find_tests(): return unittest.defaultTestLoader.discover(os.path.dirname(os.path.abspath(__file__)), pattern='*.py') if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('name', nargs='?', default=None, help='The name of the test to run, for e.g. writing.WritingTest.many_many_basic') + parser.add_argument('name', nargs='?', default=None, + help='The name of the test to run, for e.g. writing.WritingTest.many_many_basic or .many_many_basic for a shortcut') args = parser.parse_args() - if args.name: - unittest.TextTestRunner(verbosity=4).run(unittest.defaultTestLoader.loadTestsFromName(args.name)) + if args.name and args.name.startswith('.'): + tests = find_tests() + ans = None + try: + for suite in tests: + for test in suite._tests: + for s in test: + if s._testMethodName == args.name[1:]: + tests = s + raise StopIteration() + except StopIteration: + pass else: - unittest.TextTestRunner(verbosity=4).run(find_tests()) + tests = unittest.defaultTestLoader.loadTestsFromName(args.name) if args.name else find_tests() + unittest.TextTestRunner(verbosity=4).run(tests) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index bf691a3b03..979e2e9247 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -8,6 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' import datetime +from io import BytesIO from calibre.utils.date import utc_tz from calibre.db.tests.base import BaseTest @@ -205,6 +206,9 @@ class ReadingTest(BaseTest): else: self.assertEqual(cdata, cache.cover(book_id, as_path=True), 'Reading of null cover as path failed') + buf = BytesIO() + self.assertFalse(cache.copy_cover_to(99999, buf), 'copy_cover_to() did not return False for non-existent book_id') + self.assertFalse(cache.copy_cover_to(3, buf), 'copy_cover_to() did not return False for non-existent cover') # }}} @@ -305,6 +309,7 @@ class ReadingTest(BaseTest): def test_get_formats(self): # {{{ 'Test reading ebook formats using the format() method' from calibre.library.database2 import LibraryDatabase2 + from calibre.db.cache import NoSuchFormat old = LibraryDatabase2(self.library_path) ids = old.all_ids() lf = {i:set(old.formats(i, index_is_id=True).split(',')) if old.formats( @@ -332,6 +337,9 @@ class ReadingTest(BaseTest): self.assertEqual(old, f.read(), 'Failed to read format as path') + buf = BytesIO() + self.assertRaises(NoSuchFormat, cache.copy_format_to, 99999, 'X', buf, 'copy_format_to() failed to raise an exception for non-existent book') + self.assertRaises(NoSuchFormat, cache.copy_format_to, 1, 'X', buf, 'copy_format_to() failed to raise an exception for non-existent format') # }}} diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 127bcd3609..c37a173ae4 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en' from collections import namedtuple from functools import partial +from io import BytesIO from calibre.ebooks.metadata import author_to_author_sort from calibre.utils.date import UNDEFINED_DATE @@ -35,7 +36,7 @@ class WritingTest(BaseTest): ans = lambda db:partial(getattr(db, setter), commit=True) return ans - def create_test(self, name, vals, getter=None, setter=None ): + def create_test(self, name, vals, getter=None, setter=None): T = namedtuple('Test', 'name vals getter setter') return T(name, vals, self.create_getter(name, getter), self.create_setter(name, setter)) @@ -71,7 +72,7 @@ class WritingTest(BaseTest): test.name, old_sqlite_res, sqlite_res)) del db - def test_one_one(self): # {{{ + def test_one_one(self): # {{{ 'Test setting of values in one-one fields' tests = [self.create_test('#yesno', (True, False, 'true', 'false', None))] for name, getter, setter in ( @@ -112,7 +113,7 @@ class WritingTest(BaseTest): self.run_tests(tests) # }}} - def test_many_one_basic(self): # {{{ + def test_many_one_basic(self): # {{{ 'Test the different code paths for writing to a many-one field' cl = self.cloned_library cache = self.init_cache(cl) @@ -199,7 +200,7 @@ class WritingTest(BaseTest): # }}} - def test_many_many_basic(self): # {{{ + def test_many_many_basic(self): # {{{ 'Test the different code paths for writing to a many-many field' cl = self.cloned_library cache = self.init_cache(cl) @@ -289,6 +290,67 @@ class WritingTest(BaseTest): ae(c.field_for('sort', 1), 'Moose, The') ae(c.field_for('sort', 2), 'Cat') - # }}} + def test_dirtied(self): # {{{ + 'Test the setting of the dirtied flag and the last_modified column' + cl = self.cloned_library + cache = self.init_cache(cl) + ae, af, sf = self.assertEqual, self.assertFalse, cache.set_field + # First empty dirtied + cache.dump_metadata() + af(cache.dirtied_cache) + af(self.init_cache(cl).dirtied_cache) + + prev = cache.field_for('last_modified', 3) + import calibre.db.cache as c + from datetime import timedelta + utime = prev+timedelta(days=1) + onowf = c.nowf + c.nowf = lambda: utime + try: + ae(sf('title', {3:'xxx'}), set([3])) + self.assertTrue(3 in cache.dirtied_cache) + ae(cache.field_for('last_modified', 3), utime) + cache.dump_metadata() + raw = cache.read_backup(3) + from calibre.ebooks.metadata.opf2 import OPF + opf = OPF(BytesIO(raw)) + ae(opf.title, 'xxx') + finally: + c.nowf = onowf + # }}} + + def test_backup(self): # {{{ + 'Test the automatic backup of changed metadata' + cl = self.cloned_library + cache = self.init_cache(cl) + ae, af, sf, ff = self.assertEqual, self.assertFalse, cache.set_field, cache.field_for + # First empty dirtied + cache.dump_metadata() + af(cache.dirtied_cache) + from calibre.db.backup import MetadataBackup + interval = 0.01 + mb = MetadataBackup(cache, interval=interval, scheduling_interval=0) + mb.start() + try: + ae(sf('title', {1:'title1', 2:'title2', 3:'title3'}), {1,2,3}) + ae(sf('authors', {1:'author1 & author2', 2:'author1 & author2', 3:'author1 & author2'}), {1,2,3}) + count = 6 + while cache.dirty_queue_length() and count > 0: + mb.join(interval) + count -= 1 + af(cache.dirty_queue_length()) + finally: + mb.stop() + mb.join(interval) + af(mb.is_alive()) + from calibre.ebooks.metadata.opf2 import OPF + for book_id in (1, 2, 3): + raw = cache.read_backup(book_id) + opf = OPF(BytesIO(raw)) + ae(opf.title, 'title%d'%book_id) + ae(opf.authors, ['author1', 'author2']) + # }}} + + diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index e0f99eede0..633fc6d9f9 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -60,10 +60,10 @@ class View(object): else: try: self._field_getters[idx] = { - 'id' : self._get_id, - 'au_map' : self.get_author_data, + 'id': self._get_id, + 'au_map': self.get_author_data, 'ondevice': self.get_ondevice, - 'marked' : self.get_marked, + 'marked': self.get_marked, }[col] except KeyError: self._field_getters[idx] = partial(self.get, col) diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py index 29a27e16bf..87e7179661 100644 --- a/src/calibre/db/write.py +++ b/src/calibre/db/write.py @@ -417,7 +417,7 @@ def many_many(book_id_val_map, db, field, allow_case_change, *args): # }}} -def identifiers(book_id_val_map, db, field, *args): # {{{ +def identifiers(book_id_val_map, db, field, *args): # {{{ table = field.table updates = set() for book_id, identifiers in book_id_val_map.iteritems(): diff --git a/src/calibre/ebooks/conversion/plugins/txt_input.py b/src/calibre/ebooks/conversion/plugins/txt_input.py index e916b30c29..50f1409ea6 100644 --- a/src/calibre/ebooks/conversion/plugins/txt_input.py +++ b/src/calibre/ebooks/conversion/plugins/txt_input.py @@ -97,6 +97,12 @@ class TXTInput(InputFormatPlugin): if not ienc: ienc = 'utf-8' log.debug('No input encoding specified and could not auto detect using %s' % ienc) + # Remove BOM from start of txt as its presence can confuse markdown + import codecs + for bom in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE, codecs.BOM_UTF8, codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE): + if txt.startswith(bom): + txt = txt[len(bom):] + break txt = txt.decode(ienc, 'replace') # Replace entities diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index 3fefe2d886..fe39c3cd16 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -132,7 +132,7 @@ class Worker(Thread): # Get details {{{ text()="Détails sur le produit" or \ text()="Detalles del producto" or \ text()="Detalhes do produto" or \ - text()="登録情報"]/../div[@class="content"] + starts-with(text(), "登録情報")]/../div[@class="content"] ''' # Editor: is for Spanish self.publisher_xpath = ''' @@ -235,6 +235,12 @@ class Worker(Thread): # Get details {{{ msg = 'Failed to parse amazon details page: %r'%self.url self.log.exception(msg) return + if self.domain == 'jp': + for a in root.xpath('//a[@href]'): + if 'black-curtain-redirect.html' in a.get('href'): + self.url = 'http://amazon.co.jp'+a.get('href') + self.log('Black curtain redirect found, following') + return self.get_details() errmsg = root.xpath('//*[@id="errorMessage"]') if errmsg: @@ -252,8 +258,8 @@ class Worker(Thread): # Get details {{{ self.log.exception('Error parsing asin for url: %r'%self.url) asin = None if self.testing: - import tempfile - with tempfile.NamedTemporaryFile(prefix=asin + '_', + import tempfile, uuid + with tempfile.NamedTemporaryFile(prefix=(asin or str(uuid.uuid4()))+ '_', suffix='.html', delete=False) as f: f.write(raw) print ('Downloaded html for', asin, 'saved in', f.name) @@ -499,7 +505,7 @@ class Worker(Thread): # Get details {{{ def parse_language(self, pd): for x in reversed(pd.xpath(self.language_xpath)): if x.tail: - raw = x.tail.strip() + raw = x.tail.strip().partition(',')[0].strip() ans = self.lang_map.get(raw, None) if ans: return ans @@ -1004,6 +1010,11 @@ if __name__ == '__main__': # tests {{{ ] # }}} jp_tests = [ # {{{ + ( # Adult filtering test + {'identifiers':{'isbn':'4799500066'}}, + [title_test(u'Ｂｉｔｃｈ Ｔｒａｐ'),] + ), + ( # isbn -> title, authors {'identifiers':{'isbn': '9784101302720' }}, [title_test(u'精霊の守り人', diff --git a/src/calibre/ebooks/metadata/sources/edelweiss.py b/src/calibre/ebooks/metadata/sources/edelweiss.py index 53ae6c6ee3..27fd296503 100644 --- a/src/calibre/ebooks/metadata/sources/edelweiss.py +++ b/src/calibre/ebooks/metadata/sources/edelweiss.py @@ -106,6 +106,8 @@ class Worker(Thread): # {{{ parts = pub.partition(':')[0::2] pub = parts[1] or parts[0] try: + if ', Ship Date:' in pub: + pub = pub.partition(', Ship Date:')[0] q = parse_only_date(pub, assume_utc=True) if q.year != UNDEFINED_DATE: mi.pubdate = q diff --git a/src/calibre/ebooks/oeb/polish/main.py b/src/calibre/ebooks/oeb/polish/main.py index c04686ed6c..7ec34b5174 100644 --- a/src/calibre/ebooks/oeb/polish/main.py +++ b/src/calibre/ebooks/oeb/polish/main.py @@ -43,8 +43,8 @@ sizes, adjust margins, etc. Every action performs only the minimum set of changes needed for the desired effect.

You should use this tool as the last step in your ebook creation process.

- -

Note that polishing only works on files in the %s formats.

+{0} +

Note that polishing only works on files in the %s formats.

\ ''')%_(' or ').join('%s'%x for x in SUPPORTED), 'subset': _('''\ @@ -69,7 +69,7 @@ text might not be covered by the subset font.

'jacket': _('''\

Insert a "book jacket" page at the start of the book that contains all the book metadata such as title, tags, authors, series, comments, -etc.

'''), +etc. Any previous book jacket will be replaced.

'''), 'remove_jacket': _('''\

Remove a previous inserted book jacket page.

@@ -85,7 +85,7 @@ when single quotes at the start of contractions are involved.

def hfix(name, raw): if name == 'about': - return raw + return raw.format('') raw = raw.replace('\n\n', '__XX__') raw = raw.replace('\n', ' ') raw = raw.replace('__XX__', '\n') diff --git a/src/calibre/ebooks/rtf2xml/border_parse.py b/src/calibre/ebooks/rtf2xml/border_parse.py index 910d877135..f6cb2ee507 100755 --- a/src/calibre/ebooks/rtf2xml/border_parse.py +++ b/src/calibre/ebooks/rtf2xml/border_parse.py @@ -180,5 +180,6 @@ class BorderParse: elif 'single' in border_style_list: new_border_dict[att] = 'single' else: - new_border_dict[att] = border_style_list[0] + if border_style_list: + new_border_dict[att] = border_style_list[0] return new_border_dict diff --git a/src/calibre/gui2/actions/delete.py b/src/calibre/gui2/actions/delete.py index 7bdcb18644..178d94a477 100644 --- a/src/calibre/gui2/actions/delete.py +++ b/src/calibre/gui2/actions/delete.py @@ -180,6 +180,13 @@ class DeleteAction(InterfaceAction): self.gui.library_view.currentIndex()) self.gui.tags_view.recount() + def restore_format(self, book_id, original_fmt): + self.gui.current_db.restore_original_format(book_id, original_fmt) + self.gui.library_view.model().refresh_ids([book_id]) + self.gui.library_view.model().current_changed(self.gui.library_view.currentIndex(), + self.gui.library_view.currentIndex()) + self.gui.tags_view.recount() + def delete_selected_formats(self, *args): ids = self._get_selected_ids() if not ids: diff --git a/src/calibre/gui2/actions/polish.py b/src/calibre/gui2/actions/polish.py index 127749cc51..5aecbd2d87 100644 --- a/src/calibre/gui2/actions/polish.py +++ b/src/calibre/gui2/actions/polish.py @@ -37,7 +37,13 @@ class Polish(QDialog): # {{{ self.setWindowTitle(title) self.help_text = { - 'polish': _('

About Polishing books

%s')%HELP['about'], + 'polish': _('

About Polishing books

%s')%HELP['about'].format( + _('''

If you have both EPUB and ORIGINAL_EPUB in your book, + then polishing will run on ORIGINAL_EPUB (the same for other + ORIGINAL_* formats). So if you + want Polishing to not run on the ORIGINAL_* format, delete the + ORIGINAL_* format before running it.

''') + ), 'subset':_('

Subsetting fonts

%s')%HELP['subset'], diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index 896516c665..cbe44e8f82 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -88,9 +88,7 @@ class StoreAction(InterfaceAction): if row == None: error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True) return - - query = 'author:"%s"' % self._get_author(row) - self.search(query) + self.search({ 'author': self._get_author(row) }) def _get_title(self, row): title = '' @@ -107,18 +105,14 @@ class StoreAction(InterfaceAction): if row == None: error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True) return - - query = 'title:"%s"' % self._get_title(row) - self.search(query) + self.search({ 'title': self._get_title(row) }) def search_author_title(self): row = self._get_selected_row() if row == None: error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True) return - - query = 'author:"%s" title:"%s"' % (self._get_author(row), self._get_title(row)) - self.search(query) + self.search({ 'author': self._get_author(row), 'title': self._get_title(row) }) def choose(self): from calibre.gui2.store.config.chooser.chooser_dialog import StoreChooserDialog diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 4d00d282d5..45430da6f4 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -405,6 +405,7 @@ class BookInfo(QWebView): link_clicked = pyqtSignal(object) remove_format = pyqtSignal(int, object) save_format = pyqtSignal(int, object) + restore_format = pyqtSignal(int, object) def __init__(self, vertical, parent=None): QWebView.__init__(self, parent) @@ -418,7 +419,7 @@ class BookInfo(QWebView): palette.setBrush(QPalette.Base, Qt.transparent) self.page().setPalette(palette) self.css = P('templates/book_details.css', data=True).decode('utf-8') - for x, icon in [('remove', 'trash.png'), ('save', 'save.png')]: + for x, icon in [('remove', 'trash.png'), ('save', 'save.png'), ('restore', 'edit-undo.png')]: ac = QAction(QIcon(I(icon)), '', self) ac.current_fmt = None ac.triggered.connect(getattr(self, '%s_format_triggerred'%x)) @@ -436,6 +437,9 @@ class BookInfo(QWebView): def save_format_triggerred(self): self.context_action_triggered('save') + def restore_format_triggerred(self): + self.context_action_triggered('restore') + def link_activated(self, link): self._link_clicked = True if unicode(link.scheme()) in ('http', 'https'): @@ -479,7 +483,11 @@ class BookInfo(QWebView): traceback.print_exc() else: for a, t in [('remove', _('Delete the %s format')), - ('save', _('Save the %s format to disk'))]: + ('save', _('Save the %s format to disk')), + ('restore', _('Restore the %s format')), + ]: + if a == 'restore' and not fmt.upper().startswith('ORIGINAL_'): + continue ac = getattr(self, '%s_format_action'%a) ac.current_fmt = (book_id, fmt) ac.setText(t%parts[2]) @@ -585,6 +593,7 @@ class BookDetails(QWidget): # {{{ view_specific_format = pyqtSignal(int, object) remove_specific_format = pyqtSignal(int, object) save_specific_format = pyqtSignal(int, object) + restore_specific_format = pyqtSignal(int, object) remote_file_dropped = pyqtSignal(object, object) files_dropped = pyqtSignal(object, object) cover_changed = pyqtSignal(object, object) @@ -654,6 +663,7 @@ class BookDetails(QWidget): # {{{ self.book_info.link_clicked.connect(self.handle_click) self.book_info.remove_format.connect(self.remove_specific_format) self.book_info.save_format.connect(self.save_specific_format) + self.book_info.restore_format.connect(self.restore_specific_format) self.setCursor(Qt.PointingHandCursor) def handle_click(self, link): diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index eff36e865b..2a5b061819 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -272,6 +272,8 @@ class LayoutMixin(object): # {{{ self.iactions['Remove Books'].remove_format_by_id) self.book_details.save_specific_format.connect( self.iactions['Save To Disk'].save_library_format_by_ids) + self.book_details.restore_specific_format.connect( + self.iactions['Remove Books'].restore_format) self.book_details.view_device_book.connect( self.iactions['View'].view_device_book) diff --git a/src/calibre/gui2/proceed.py b/src/calibre/gui2/proceed.py index 9bdf48e086..d09c247bd0 100644 --- a/src/calibre/gui2/proceed.py +++ b/src/calibre/gui2/proceed.py @@ -123,7 +123,8 @@ class ProceedQuestion(QDialog): self.resize(sz) def show_question(self): - if self.isVisible(): return + if self.isVisible(): + return if self.questions: question = self.questions[0] self.msg_label.setText(question.msg) diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index 20c6c09a03..c001bfbe5c 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -62,16 +62,20 @@ class SearchDialog(QDialog, Ui_Dialog): self.setup_store_checks() # Set the search query + if isinstance(query, (str, unicode)): + self.search_edit.setText(query) + elif isinstance(query, dict): + if 'author' in query: + self.search_author.setText(query['author']) + if 'title' in query: + self.search_title.setText(query['title']) # Title - self.search_title.setText(query) self.search_title.setSizeAdjustPolicy(QComboBox.AdjustToMinimumContentsLengthWithIcon) self.search_title.setMinimumContentsLength(25) # Author - self.search_author.setText(query) self.search_author.setSizeAdjustPolicy(QComboBox.AdjustToMinimumContentsLengthWithIcon) self.search_author.setMinimumContentsLength(25) # Keyword - self.search_edit.setText(query) self.search_edit.setSizeAdjustPolicy(QComboBox.AdjustToMinimumContentsLengthWithIcon) self.search_edit.setMinimumContentsLength(25) @@ -408,7 +412,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.save_state() def exec_(self): - if unicode(self.search_edit.text()).strip(): + if unicode(self.search_edit.text()).strip() or unicode(self.search_title.text()).strip() or unicode(self.search_author.text()).strip(): self.do_search() return QDialog.exec_(self) diff --git a/src/calibre/gui2/store/stores/nexto_plugin.py b/src/calibre/gui2/store/stores/nexto_plugin.py index df3867ec1a..813a31b4c2 100644 --- a/src/calibre/gui2/store/stores/nexto_plugin.py +++ b/src/calibre/gui2/store/stores/nexto_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 2 # Needed for dynamic plugin loading +store_version = 3 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2011-2013, Tomasz Długosz ' @@ -67,7 +67,7 @@ class NextoStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//img[@class="cover"]/@src')) cover_url = re.sub(r'%2F', '/', cover_url) - cover_url = re.sub(r'\widthMax=120&heightMax=200', 'widthMax=64&heightMax=64', cover_url) + cover_url = re.sub(r'widthMax=120&heightMax=200', 'widthMax=64&heightMax=64', cover_url) title = ''.join(data.xpath('.//a[@class="title"]/text()')) title = re.sub(r' - ebook$', '', title) formats = ', '.join(data.xpath('.//ul[@class="formats_available"]/li//b/text()')) @@ -82,7 +82,7 @@ class NextoStore(BasicStoreConfig, StorePlugin): counter -= 1 s = SearchResult() - s.cover_url = 'http://www.nexto.pl' + cover_url + s.cover_url = cover_url if cover_url[:4] == 'http' else 'http://www.nexto.pl' + cover_url s.title = title.strip() s.author = author.strip() s.price = price diff --git a/src/calibre/gui2/store/stores/virtualo_plugin.py b/src/calibre/gui2/store/stores/virtualo_plugin.py index 567da2df4e..86824b5542 100644 --- a/src/calibre/gui2/store/stores/virtualo_plugin.py +++ b/src/calibre/gui2/store/stores/virtualo_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 2 # Needed for dynamic plugin loading +store_version = 3 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2011-2013, Tomasz Długosz ' @@ -41,7 +41,7 @@ class VirtualoStore(BasicStoreConfig, StorePlugin): url = 'http://virtualo.pl/?q=' + urllib.quote(query) + '&f=format_id:4,6,3' br = browser() - no_drm_pattern = re.compile("Znak wodny") + no_drm_pattern = re.compile(r'Znak wodny|Brak') counter = max_results with closing(br.open(url, timeout=timeout)) as f: @@ -58,8 +58,8 @@ class VirtualoStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//div[@class="list_middle_left"]//a//img/@src')) title = ''.join(data.xpath('.//div[@class="list_title list_text_left"]/a/text()')) author = ', '.join(data.xpath('.//div[@class="list_authors list_text_left"]/a/text()')) - formats = [ form.split('_')[-1].replace('.png', '') for form in data.xpath('.//div[@style="width:55%;float:left;text-align:left;height:18px;"]//a/img/@src')] - nodrm = no_drm_pattern.search(''.join(data.xpath('.//div[@style="width:45%;float:right;text-align:right;height:18px;"]/div/div/text()'))) + formats = [ form.split('_')[-1].replace('.png', '') for form in data.xpath('.//div[@style="width:55%;float:left;text-align:left;height:18px;"]//a/span/img/@src')] + nodrm = no_drm_pattern.search(''.join(data.xpath('.//div[@style="width:45%;float:right;text-align:right;height:18px;"]//span[@class="prompt_preview"]/text()'))) counter -= 1 @@ -70,6 +70,6 @@ class VirtualoStore(BasicStoreConfig, StorePlugin): s.price = price + ' zł' s.detail_item = 'http://virtualo.pl' + id.strip().split('http://')[0] s.formats = ', '.join(formats).upper() - s.drm = SearchResult.DRM_UNLOCKED if nodrm else SearchResult.DRM_UNKNOWN + s.drm = SearchResult.DRM_UNLOCKED if nodrm else SearchResult.DRM_LOCKED yield s diff --git a/src/calibre/gui2/toc/main.py b/src/calibre/gui2/toc/main.py index 7cb4f9b462..90d9a8f4a8 100644 --- a/src/calibre/gui2/toc/main.py +++ b/src/calibre/gui2/toc/main.py @@ -559,11 +559,11 @@ class TOCView(QWidget): # {{{ b.setToolTip(_('Remove all selected entries')) b.clicked.connect(self.del_items) - self.left_button = b = QToolButton(self) + self.right_button = b = QToolButton(self) b.setIcon(QIcon(I('forward.png'))) b.setIconSize(QSize(ICON_SIZE, ICON_SIZE)) l.addWidget(b, 4, 3) - b.setToolTip(_('Unindent the current entry [Ctrl+Left]')) + b.setToolTip(_('Indent the current entry [Ctrl+Right]')) b.clicked.connect(self.tocw.move_right) self.down_button = b = QToolButton(self) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 2fdec62ff0..b3737d15fa 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -88,7 +88,7 @@ def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, se for f in data: fmts = [x for x in f['formats'] if x is not None] f['formats'] = u'[%s]'%u','.join(fmts) - widths = list(map(lambda x : 0, fields)) + widths = list(map(lambda x: 0, fields)) for record in data: for f in record.keys(): if hasattr(record[f], 'isoformat'): @@ -164,7 +164,8 @@ List the books available in the calibre database. parser.add_option('--ascending', default=False, action='store_true', help=_('Sort results in ascending order')) parser.add_option('-s', '--search', default=None, - help=_('Filter the results by the search query. For the format of the search query, please see the search related documentation in the User Manual. Default is to do no filtering.')) + help=_('Filter the results by the search query. For the format of the search query,' + ' please see the search related documentation in the User Manual. Default is to do no filtering.')) parser.add_option('-w', '--line-width', default=-1, type=int, help=_('The maximum width of a single line in the output. Defaults to detecting screen size.')) parser.add_option('--separator', default=' ', help=_('The string used to separate fields. Default is a space.')) @@ -244,7 +245,8 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates, otitle, mi.authors = [_('Unknown')] for x in ('title', 'authors', 'isbn', 'tags', 'series'): val = locals()['o'+x] - if val: setattr(mi, x, val) + if val: + setattr(mi, x, val) if oseries: mi.series_index = oseries_index if ocover: @@ -425,18 +427,26 @@ def command_remove(args, dbpath): return 0 -def do_add_format(db, id, fmt, path): - db.add_format_with_hooks(id, fmt.upper(), path, index_is_id=True) - send_message() +def do_add_format(db, id, fmt, path, opts): + done = db.add_format_with_hooks(id, fmt.upper(), path, index_is_id=True, + replace=opts.replace) + if not done and not opts.replace: + prints(_('A %s file already exists for book: %d, not replacing')%(fmt.upper(), id)) + else: + send_message() def add_format_option_parser(): - return get_parser(_( + parser = get_parser(_( '''\ %prog add_format [options] id ebook_file Add the ebook in ebook_file to the available formats for the logical book identified \ -by id. You can get id by using the list command. If the format already exists, it is replaced. +by id. You can get id by using the list command. If the format already exists, \ +it is replaced, unless the do not replace option is specified.\ ''')) + parser.add_option('--dont-replace', dest='replace', default=True, action='store_false', + help=_('Do not replace the format if it already exists')) + return parser def command_add_format(args, dbpath): @@ -451,7 +461,7 @@ def command_add_format(args, dbpath): id, path, fmt = int(args[1]), args[2], os.path.splitext(args[2])[-1] if not fmt: print _('ebook file must have an extension') - do_add_format(get_db(dbpath, opts), id, fmt[1:], path) + do_add_format(get_db(dbpath, opts), id, fmt[1:], path, opts) return 0 def do_remove_format(db, id, fmt): @@ -791,7 +801,7 @@ def catalog_option_parser(args): if not file_extension in available_catalog_formats(): print_help(parser, log) log.error("No catalog plugin available for extension '%s'.\n" % file_extension + - "Catalog plugins available for %s\n" % ', '.join(available_catalog_formats()) ) + "Catalog plugins available for %s\n" % ', '.join(available_catalog_formats())) raise SystemExit(1) return output, file_extension @@ -1214,7 +1224,8 @@ def command_restore_database(args, dbpath): dbpath = dbpath.decode(preferred_encoding) class Progress(object): - def __init__(self): self.total = 1 + def __init__(self): + self.total = 1 def __call__(self, msg, step): if msg is None: @@ -1308,7 +1319,7 @@ def command_list_categories(args, dbpath): from calibre.utils.terminal import geometry, ColoredStream separator = ' ' - widths = list(map(lambda x : 0, fields)) + widths = list(map(lambda x: 0, fields)) for i in data: for j, field in enumerate(fields): widths[j] = max(widths[j], max(len(field), len(unicode(i[field])))) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 14c71d5918..ccb614fbce 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -205,7 +205,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return row[loc] def initialize_dynamic(self): - self.field_metadata = FieldMetadata() #Ensure we start with a clean copy + self.field_metadata = FieldMetadata() # Ensure we start with a clean copy self.prefs = DBPrefs(self) defs = self.prefs.defaults defs['gui_restriction'] = defs['cs_restriction'] = '' @@ -372,7 +372,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): '''.format(_('News'))) self.conn.commit() - CustomColumns.__init__(self) template = '''\ (SELECT {query} FROM books_{table}_link AS link INNER JOIN @@ -464,7 +463,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # Assumption is that someone else will fix them if they change. self.field_metadata.remove_dynamic_categories() for user_cat in sorted(self.prefs.get('user_categories', {}).keys(), key=sort_key): - cat_name = '@' + user_cat # add the '@' to avoid name collision + cat_name = '@' + user_cat # add the '@' to avoid name collision self.field_metadata.add_user_category(label=cat_name, name=user_cat) # add grouped search term user categories @@ -616,7 +615,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): current title and author. If there was a previous directory, its contents are copied and it is deleted. ''' - id = index if index_is_id else self.id(index) + id = index if index_is_id else self.id(index) path = self.construct_path_name(id) current_path = self.path(id, index_is_id=True).replace(os.sep, '/') formats = self.formats(id, index_is_id=True) @@ -640,7 +639,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if not os.path.exists(tpath): os.makedirs(tpath) - if source_ok: # Migrate existing files + if source_ok: # Migrate existing files self.copy_cover_to(id, os.path.join(tpath, 'cover.jpg'), index_is_id=True, windows_atomic_move=wam, use_hardlink=True) @@ -688,7 +687,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): os.rename(os.path.join(curpath, oldseg), os.path.join(curpath, newseg)) except: - break # Fail silently since nothing catastrophic has happened + break # Fail silently since nothing catastrophic has happened curpath = os.path.join(curpath, newseg) def add_listener(self, listener): @@ -747,7 +746,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return ret def cover_last_modified(self, index, index_is_id=False): - id = index if index_is_id else self.id(index) + id = index if index_is_id else self.id(index) path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') try: return utcfromtimestamp(os.stat(path).st_mtime) @@ -1094,8 +1093,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): identical_book_ids = set([]) if mi.authors: try: - quathors = mi.authors[:10] # Too many authors causes parsing of - # the search expression to fail + quathors = mi.authors[:10] # Too many authors causes parsing of + # the search expression to fail query = u' and '.join([u'author:"=%s"'%(a.replace('"', '')) for a in quathors]) qauthors = mi.authors[10:] @@ -1327,7 +1326,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return fmt_path try: candidates = glob.glob(os.path.join(path, '*'+format)) - except: # If path contains strange characters this throws an exc + except: # If path contains strange characters this throws an exc candidates = [] if format and candidates and os.path.exists(candidates[0]): try: @@ -1370,7 +1369,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if path != dest: os.rename(path, dest) except: - pass # Nothing too catastrophic happened, the cases mismatch, that's all + pass # Nothing too catastrophic happened, the cases mismatch, that's all else: windows_atomic_move.copy_path_to(path, dest) else: @@ -1386,7 +1385,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): try: os.rename(path, dest) except: - pass # Nothing too catastrophic happened, the cases mismatch, that's all + pass # Nothing too catastrophic happened, the cases mismatch, that's all else: if use_hardlink: try: @@ -1496,12 +1495,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return ret def add_format_with_hooks(self, index, format, fpath, index_is_id=False, - path=None, notify=True): + path=None, notify=True, replace=True): npath = self.run_import_plugins(fpath, format) format = os.path.splitext(npath)[-1].lower().replace('.', '').upper() stream = lopen(npath, 'rb') format = check_ebook_format(stream, format) - retval = self.add_format(index, format, stream, + retval = self.add_format(index, format, stream, replace=replace, index_is_id=index_is_id, path=path, notify=notify) run_plugins_on_postimport(self, id, format) return retval @@ -1509,7 +1508,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def add_format(self, index, format, stream, index_is_id=False, path=None, notify=True, replace=True, copy_function=None): id = index if index_is_id else self.id(index) - if not format: format = '' + if not format: + format = '' self.format_metadata_cache[id].pop(format.upper(), None) name = self.format_filename_cache[id].get(format.upper(), None) if path is None: @@ -1561,6 +1561,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): opath = self.format_abspath(book_id, nfmt, index_is_id=True) return fmt if opath is None else nfmt + def restore_original_format(self, book_id, original_fmt, notify=True): + opath = self.format_abspath(book_id, original_fmt, index_is_id=True) + if opath is not None: + fmt = original_fmt.partition('_')[2] + with lopen(opath, 'rb') as f: + self.add_format(book_id, fmt, f, index_is_id=True, notify=False) + self.remove_format(book_id, original_fmt, index_is_id=True, notify=notify) + def delete_book(self, id, notify=True, commit=True, permanent=False, do_clean=True): ''' @@ -1588,7 +1596,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def remove_format(self, index, format, index_is_id=False, notify=True, commit=True, db_only=False): id = index if index_is_id else self.id(index) - if not format: format = '' + if not format: + format = '' self.format_metadata_cache[id].pop(format.upper(), None) name = self.format_filename_cache[id].get(format.upper(), None) if name: @@ -1757,12 +1766,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # Get the ids for the item values if not cat['is_custom']: funcs = { - 'authors' : self.get_authors_with_ids, - 'series' : self.get_series_with_ids, + 'authors': self.get_authors_with_ids, + 'series': self.get_series_with_ids, 'publisher': self.get_publishers_with_ids, - 'tags' : self.get_tags_with_ids, + 'tags': self.get_tags_with_ids, 'languages': self.get_languages_with_ids, - 'rating' : self.get_ratings_with_ids, + 'rating': self.get_ratings_with_ids, } func = funcs.get(category, None) if func: @@ -1845,7 +1854,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): item.rc += 1 continue try: - (item_id, sort_val) = tid_cat[val] # let exceptions fly + (item_id, sort_val) = tid_cat[val] # let exceptions fly item = tcats_cat.get(val, None) if not item: item = tag_class(val, sort_val) @@ -1867,7 +1876,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tid_cat[val] = (val, val) for val in vals: try: - (item_id, sort_val) = tid_cat[val] # let exceptions fly + (item_id, sort_val) = tid_cat[val] # let exceptions fly item = tcats_cat.get(val, None) if not item: item = tag_class(val, sort_val) @@ -1935,7 +1944,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # in the main Tag loop. Saves a few % if datatype == 'rating': formatter = (lambda x:u'\u2605'*int(x/2)) - avgr = lambda x : x.n + avgr = lambda x: x.n # eliminate the zero ratings line as well as count == 0 items = [v for v in tcategories[category].values() if v.c > 0 and v.n != 0] elif category == 'authors': @@ -1952,7 +1961,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # sort the list if sort == 'name': - kf = lambda x :sort_key(x.s) + kf = lambda x:sort_key(x.s) reverse=False elif sort == 'popularity': kf = lambda x: x.c @@ -2017,9 +2026,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if sort == 'popularity': categories['formats'].sort(key=lambda x: x.count, reverse=True) - else: # no ratings exist to sort on + else: # no ratings exist to sort on # No need for ICU here. - categories['formats'].sort(key = lambda x:x.name) + categories['formats'].sort(key=lambda x:x.name) # Now do identifiers. This works like formats categories['identifiers'] = [] @@ -2046,9 +2055,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if sort == 'popularity': categories['identifiers'].sort(key=lambda x: x.count, reverse=True) - else: # no ratings exist to sort on + else: # no ratings exist to sort on # No need for ICU here. - categories['identifiers'].sort(key = lambda x:x.name) + categories['identifiers'].sort(key=lambda x:x.name) #### Now do the user-defined categories. #### user_categories = dict.copy(self.clean_user_categories()) @@ -2095,7 +2104,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: items.append(taglist[label][n]) # else: do nothing, to not include nodes w zero counts - cat_name = '@' + user_cat # add the '@' to avoid name collision + cat_name = '@' + user_cat # add the '@' to avoid name collision # Not a problem if we accumulate entries in the icon map if icon_map is not None: icon_map[cat_name] = icon_map['user:'] @@ -2343,11 +2352,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): elif mi_idents: identifiers = self.get_identifiers(id, index_is_id=True) for key, val in mi_idents.iteritems(): - if val and val.strip(): # Don't delete an existing identifier + if val and val.strip(): # Don't delete an existing identifier identifiers[icu_lower(key)] = val self.set_identifiers(id, identifiers, notify=False, commit=False) - user_mi = mi.get_all_user_metadata(make_copy=False) for key in user_mi.iterkeys(): if key in self.field_metadata and \ @@ -2467,7 +2475,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): try: self.conn.execute('''INSERT INTO books_authors_link(book, author) VALUES (?,?)''', (id, aid)) - except IntegrityError: # Sometimes books specify the same author twice in their metadata + except IntegrityError: # Sometimes books specify the same author twice in their metadata pass if case_change: bks = self.conn.get('''SELECT book FROM books_authors_link @@ -2626,7 +2634,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if notify: self.notify('metadata', [id]) - def set_publisher(self, id, publisher, notify=True, commit=True, allow_case_change=False): self.conn.execute('DELETE FROM books_publishers_link WHERE book=?',(id,)) @@ -2832,7 +2839,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if new_id is None or old_id == new_id: new_id = old_id # New name doesn't exist. Simply change the old name - self.conn.execute('UPDATE publishers SET name=? WHERE id=?', \ + self.conn.execute('UPDATE publishers SET name=? WHERE id=?', (new_name, old_id)) else: # Change the link table to point at the new one @@ -2872,7 +2879,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.commit() def set_sort_field_for_author(self, old_id, new_sort, commit=True, notify=False): - self.conn.execute('UPDATE authors SET sort=? WHERE id=?', \ + self.conn.execute('UPDATE authors SET sort=? WHERE id=?', (new_sort.strip(), old_id)) if commit: self.conn.commit() @@ -2971,7 +2978,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): @classmethod def cleanup_tags(cls, tags): tags = [x.strip().replace(',', ';') for x in tags if x.strip()] - tags = [x.decode(preferred_encoding, 'replace') \ + tags = [x.decode(preferred_encoding, 'replace') if isbytestring(x) else x for x in tags] tags = [u' '.join(x.split()) for x in tags] ans, seen = [], set([]) @@ -3372,10 +3379,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.add_format(db_id, format, stream, index_is_id=True) self.conn.commit() - self.data.refresh_ids(self, [db_id]) # Needed to update format list and size + self.data.refresh_ids(self, [db_id]) # Needed to update format list and size return db_id - def add_news(self, path, arg): from calibre.ebooks.metadata.meta import get_metadata @@ -3411,7 +3417,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if not hasattr(path, 'read'): stream.close() self.conn.commit() - self.data.refresh_ids(self, [id]) # Needed to update format list and size + self.data.refresh_ids(self, [id]) # Needed to update format list and size return id def run_import_plugins(self, path_or_stream, format): @@ -3475,7 +3481,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): traceback.print_exc() return id - def add_books(self, paths, formats, metadata, add_duplicates=True, return_ids=False): ''' @@ -3519,7 +3524,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): stream.close() postimport.append((id, format)) self.conn.commit() - self.data.refresh_ids(self, ids) # Needed to update format list and size + self.data.refresh_ids(self, ids) # Needed to update format list and size for book_id, fmt in postimport: run_plugins_on_postimport(self, book_id, fmt) if duplicates: @@ -3569,7 +3574,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # set_metadata, but probably isn't good enough self.dirtied([id], commit=False) self.conn.commit() - self.data.refresh_ids(self, [id]) # Needed to update format list and size + self.data.refresh_ids(self, [id]) # Needed to update format list and size if notify: self.notify('add', [id]) return id @@ -3663,7 +3668,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): FIELDS.add('%d_index'%x) data = [] for record in self.data: - if record is None: continue + if record is None: + continue db_id = record[self.FIELD_MAP['id']] if ids is not None and db_id not in ids: continue @@ -3706,8 +3712,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): progress.setValue(0) progress.setLabelText(header) QCoreApplication.processEvents() - db.conn.row_factory = lambda cursor, row : tuple(row) - db.conn.text_factory = lambda x : unicode(x, 'utf-8', 'replace') + db.conn.row_factory = lambda cursor, row: tuple(row) + db.conn.text_factory = lambda x: unicode(x, 'utf-8', 'replace') books = db.conn.get('SELECT id, title, sort, timestamp, series_index, author_sort, isbn FROM books ORDER BY id ASC') progress.setAutoReset(False) progress.setRange(0, len(books)) @@ -3783,7 +3789,7 @@ books_series_link feeds continue key = os.path.splitext(path)[0] - if not books.has_key(key): + if key not in books: books[key] = [] books[key].append(path) diff --git a/src/calibre/linux.py b/src/calibre/linux.py index 395831fa8f..d2b0c941a6 100644 --- a/src/calibre/linux.py +++ b/src/calibre/linux.py @@ -33,7 +33,7 @@ entry_points = { 'fetch-ebook-metadata = calibre.ebooks.metadata.sources.cli:main', 'calibre-smtp = calibre.utils.smtp:main', ], - 'gui_scripts' : [ + 'gui_scripts' : [ __appname__+' = calibre.gui2.main:main', 'lrfviewer = calibre.gui2.lrf_renderer.main:main', 'ebook-viewer = calibre.gui2.viewer.main:main', diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py index d9c0c2d412..fbd4c47a35 100644 --- a/src/calibre/utils/config.py +++ b/src/calibre/utils/config.py @@ -88,7 +88,7 @@ class OptionParser(_OptionParser): if epilog is None: epilog = _('Created by ')+colored(__author__, fg='cyan') usage += '\n\n'+_('''Whenever you pass arguments to %prog that have spaces in them, ''' - '''enclose the arguments in quotation marks.''') + '''enclose the arguments in quotation marks.''')+'\n' _OptionParser.__init__(self, usage=usage, version=version, epilog=epilog, formatter=CustomHelpFormatter(), conflict_handler=conflict_handler, **kwds) @@ -171,7 +171,7 @@ class OptionParser(_OptionParser): non default values in lower. ''' for dest in lower.__dict__.keys(): - if not upper.__dict__.has_key(dest): + if not dest in upper.__dict__: continue opt = self.option_by_dest(dest) if lower.__dict__[dest] != opt.default and \ @@ -319,12 +319,16 @@ class XMLConfig(dict): self.__setitem__(key, val) def __delitem__(self, key): - if dict.has_key(self, key): + try: dict.__delitem__(self, key) + except KeyError: + pass # ignore missing keys + else: self.commit() def commit(self): - if self.no_commit: return + if self.no_commit: + return if hasattr(self, 'file_path') and self.file_path: dpath = os.path.dirname(self.file_path) if not os.path.exists(dpath): diff --git a/src/calibre/utils/localunzip.py b/src/calibre/utils/localunzip.py index 4fd3006076..289b9d46c0 100644 --- a/src/calibre/utils/localunzip.py +++ b/src/calibre/utils/localunzip.py @@ -174,7 +174,13 @@ def _extractall(f, path=None, file_info=None): has_data_descriptors = header.flags & (1 << 3) seekval = header.compressed_size + (16 if has_data_descriptors else 0) found = True - parts = header.filename.split('/') + # Sanitize path changing absolute to relative paths and removing .. and + # . + fname = header.filename.replace(os.sep, '/') + fname = os.path.splitdrive(fname)[1] + parts = [x for x in fname.split('/') if x not in {'', os.path.pardir, os.path.curdir}] + if not parts: + continue if header.uncompressed_size == 0: # Directory f.seek(f.tell()+seekval) diff --git a/src/calibre/utils/monotonic.py b/src/calibre/utils/monotonic.py new file mode 100644 index 0000000000..2bda006929 --- /dev/null +++ b/src/calibre/utils/monotonic.py @@ -0,0 +1,104 @@ +# vim:fileencoding=utf-8 + +from __future__ import division, absolute_import + +try: + try: + # >=python-3.3, Unix + from time import clock_gettime + try: + # >={kernel}-sources-2.6.28 + from time import CLOCK_MONOTONIC_RAW as CLOCK_ID + except ImportError: + from time import CLOCK_MONOTONIC as CLOCK_ID # NOQA + + monotonic = lambda: clock_gettime(CLOCK_ID) + + except ImportError: + # >=python-3.3 + from time import monotonic # NOQA + +except ImportError: + import ctypes + import sys + + try: + if sys.platform == 'win32': + # Windows only + GetTickCount64 = ctypes.windll.kernel32.GetTickCount64 + GetTickCount64.restype = ctypes.c_ulonglong + + def monotonic(): # NOQA + return GetTickCount64() / 1000 + + elif sys.platform == 'darwin': + # Mac OS X + from ctypes.util import find_library + + libc_name = find_library('c') + if not libc_name: + raise OSError + + libc = ctypes.CDLL(libc_name, use_errno=True) + + mach_absolute_time = libc.mach_absolute_time + mach_absolute_time.argtypes = () + mach_absolute_time.restype = ctypes.c_uint64 + + class mach_timebase_info_data_t(ctypes.Structure): + _fields_ = ( + ('numer', ctypes.c_uint32), + ('denom', ctypes.c_uint32), + ) + mach_timebase_info_data_p = ctypes.POINTER(mach_timebase_info_data_t) + + _mach_timebase_info = libc.mach_timebase_info + _mach_timebase_info.argtypes = (mach_timebase_info_data_p,) + _mach_timebase_info.restype = ctypes.c_int + + def mach_timebase_info(): + timebase = mach_timebase_info_data_t() + _mach_timebase_info(ctypes.byref(timebase)) + return (timebase.numer, timebase.denom) + + timebase = mach_timebase_info() + factor = timebase[0] / timebase[1] * 1e-9 + + def monotonic(): # NOQA + return mach_absolute_time() * factor + else: + # linux only (no librt on OS X) + import os + + # See + CLOCK_MONOTONIC = 1 + CLOCK_MONOTONIC_RAW = 4 + + class timespec(ctypes.Structure): + _fields_ = ( + ('tv_sec', ctypes.c_long), + ('tv_nsec', ctypes.c_long) + ) + tspec = timespec() + + librt = ctypes.CDLL('librt.so.1', use_errno=True) + clock_gettime = librt.clock_gettime + clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] + + if clock_gettime(CLOCK_MONOTONIC_RAW, ctypes.pointer(tspec)) == 0: + # >={kernel}-sources-2.6.28 + clock_id = CLOCK_MONOTONIC_RAW + elif clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(tspec)) == 0: + clock_id = CLOCK_MONOTONIC + else: + raise OSError + + def monotonic(): # NOQA + if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(tspec)) != 0: + errno_ = ctypes.get_errno() + raise OSError(errno_, os.strerror(errno_)) + return tspec.tv_sec + tspec.tv_nsec / 1e9 + + except: + from time import time as monotonic # NOQA + monotonic diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py index 95e556418c..566823b639 100644 --- a/src/calibre/utils/zipfile.py +++ b/src/calibre/utils/zipfile.py @@ -1099,10 +1099,13 @@ class ZipFile: base_target = targetpath # Added by Kovid - # don't include leading "/" from file name if present - fname = member.filename - if fname.startswith('/'): - fname = fname[1:] + # Sanitize path, changing absolute paths to relative paths + # and removing .. and . (changed by Kovid) + fname = member.filename.replace(os.sep, '/') + fname = os.path.splitdrive(fname)[1] + fname = '/'.join(x for x in fname.split('/') if x not in {'', os.path.curdir, os.path.pardir}) + if not fname: + raise BadZipfile('The member %r has an invalid name'%member.filename) targetpath = os.path.normpath(os.path.join(base_target, fname)) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 0e0afe2bc4..e0b31c7fcf 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -219,7 +219,7 @@ class BasicNewsRecipe(Recipe): #: } #: #: All keys are optional. For a full explanantion of the search criteria, see - #: `Beautiful Soup `_ + #: `Beautiful Soup `_ #: A common example:: #: #: remove_tags = [dict(name='div', attrs={'class':'advert'})] @@ -556,7 +556,7 @@ class BasicNewsRecipe(Recipe): url = article[key] if url and url.startswith('http://'): return url - ans = article.get('link', None) + ans = article.get('link', None) if not ans and getattr(article, 'links', None): for item in article.links: if item.get('rel', 'alternate') == 'alternate': @@ -725,7 +725,7 @@ class BasicNewsRecipe(Recipe): `weights`: A dictionary that maps weights to titles. If any titles in index are not in weights, they are assumed to have a weight of 0. ''' - weights = defaultdict(lambda : 0, weights) + weights = defaultdict(lambda: 0, weights) index.sort(cmp=lambda x, y: cmp(weights[x], weights[y])) return index @@ -860,8 +860,8 @@ class BasicNewsRecipe(Recipe): if isinstance(self.feeds, basestring): self.feeds = [self.feeds] - if self.needs_subscription and (\ - self.username is None or self.password is None or \ + if self.needs_subscription and ( + self.username is None or self.password is None or (not self.username and not self.password)): if self.needs_subscription != 'optional': raise ValueError(_('The "%s" recipe needs a username and password.')%self.title) @@ -870,7 +870,7 @@ class BasicNewsRecipe(Recipe): self.image_map, self.image_counter = {}, 1 self.css_map = {} - web2disk_cmdline = [ 'web2disk', + web2disk_cmdline = ['web2disk', '--timeout', str(self.timeout), '--max-recursions', str(self.recursions), '--delay', str(self.delay), @@ -913,7 +913,6 @@ class BasicNewsRecipe(Recipe): self.failed_downloads = [] self.partial_failures = [] - def _postprocess_html(self, soup, first_fetch, job_info): if self.no_stylesheets: for link in list(soup.findAll('link', type=re.compile('css')))+list(soup.findAll('style')): @@ -923,7 +922,8 @@ class BasicNewsRecipe(Recipe): head = soup.find('body') if not head: head = soup.find(True) - style = BeautifulSoup(u''%(self.template_css +'\n\n'+(self.extra_css if self.extra_css else ''))).find('style') + style = BeautifulSoup(u''%( + self.template_css +'\n\n'+(self.extra_css if self.extra_css else ''))).find('style') head.insert(len(head.contents), style) if first_fetch and job_info: url, f, a, feed_len = job_info @@ -969,7 +969,6 @@ class BasicNewsRecipe(Recipe): self.populate_article_metadata(article, ans, first_fetch) return ans - def download(self): ''' Download and pre-process all articles from the feeds in this recipe. @@ -1041,12 +1040,12 @@ class BasicNewsRecipe(Recipe): def feed2index(self, f, feeds): feed = feeds[f] - if feed.image_url is not None: # Download feed image + if feed.image_url is not None: # Download feed image imgdir = os.path.join(self.output_dir, 'images') if not os.path.isdir(imgdir): os.makedirs(imgdir) - if self.image_map.has_key(feed.image_url): + if feed.image_url in self.image_map: feed.image_url = self.image_map[feed.image_url] else: bn = urlparse.urlsplit(feed.image_url).path @@ -1065,7 +1064,6 @@ class BasicNewsRecipe(Recipe): if isinstance(feed.image_url, str): feed.image_url = feed.image_url.decode(sys.getfilesystemencoding(), 'strict') - templ = (templates.TouchscreenFeedTemplate if self.touchscreen else templates.FeedTemplate) templ = templ(lang=self.lang_for_html) @@ -1074,7 +1072,6 @@ class BasicNewsRecipe(Recipe): return templ.generate(f, feeds, self.description_limiter, extra_css=css).render(doctype='xhtml') - def _fetch_article(self, url, dir_, f, a, num_of_feeds): br = self.browser if self.get_browser.im_func is BasicNewsRecipe.get_browser.im_func: @@ -1117,7 +1114,7 @@ class BasicNewsRecipe(Recipe): with PersistentTemporaryFile('_feeds2disk.html') as pt: pt.write(raw) url = ('file:'+pt.name) if iswindows else ('file://'+pt.name) - return self._fetch_article(url, dir, f, a, num_of_feeds) + return self._fetch_article(url, dir, f, a, num_of_feeds) def remove_duplicate_articles(self, feeds): seen_keys = defaultdict(set) @@ -1223,9 +1220,9 @@ class BasicNewsRecipe(Recipe): if not url: continue func, arg = (self.fetch_embedded_article, article) \ - if self.use_embedded_content or (self.use_embedded_content == None and feed.has_embedded_content()) \ + if self.use_embedded_content or (self.use_embedded_content is None and feed.has_embedded_content()) \ else \ - ((self.fetch_obfuscated_article if self.articles_are_obfuscated \ + ((self.fetch_obfuscated_article if self.articles_are_obfuscated else self.fetch_article), url) req = WorkRequest(func, (arg, art_dir, f, a, len(feed)), {}, (f, a), self.article_downloaded, @@ -1235,13 +1232,11 @@ class BasicNewsRecipe(Recipe): req.feed_dir = feed_dir self.jobs.append(req) - self.jobs_done = 0 tp = ThreadPool(self.simultaneous_downloads) for req in self.jobs: tp.putRequest(req, block=True, timeout=0) - self.report_progress(0, _('Starting download [%d thread(s)]...')%self.simultaneous_downloads) while True: try: @@ -1328,7 +1323,6 @@ class BasicNewsRecipe(Recipe): if os.path.exists(mpath): os.remove(mpath) - def download_masthead(self, url): try: self._download_masthead(url) @@ -1455,7 +1449,6 @@ class BasicNewsRecipe(Recipe): self.play_order_counter = 0 self.play_order_map = {} - def feed_index(num, parent): f = feeds[num] for j, a in enumerate(f): @@ -1595,7 +1588,6 @@ class BasicNewsRecipe(Recipe): parsed_feeds.append(feed) self.log.exception(msg) - remove = [f for f in parsed_feeds if len(f) == 0 and self.remove_empty_feeds] for f in remove: @@ -1629,8 +1621,11 @@ class BasicNewsRecipe(Recipe): res = self.tag_to_string(item) if res: strings.append(res) - elif use_alt and item.has_key('alt'): - strings.append(item['alt']) + elif use_alt: + try: + strings.append(item['alt']) + except KeyError: + pass ans = u''.join(strings) if normalize_whitespace: ans = re.sub(r'\s+', ' ', ans) @@ -1646,24 +1641,26 @@ class BasicNewsRecipe(Recipe): @classmethod def adeify_images(cls, soup): - ''' - If your recipe when converted to EPUB has problems with images when - viewed in Adobe Digital Editions, call this method from within - :meth:`postprocess_html`. - ''' - for item in soup.findAll('img'): - for attrib in ['height','width','border','align','style']: - if item.has_key(attrib): + ''' + If your recipe when converted to EPUB has problems with images when + viewed in Adobe Digital Editions, call this method from within + :meth:`postprocess_html`. + ''' + for item in soup.findAll('img'): + for attrib in ['height','width','border','align','style']: + try: del item[attrib] - oldParent = item.parent - myIndex = oldParent.contents.index(item) - item.extract() - divtag = Tag(soup,'div') - brtag = Tag(soup,'br') - oldParent.insert(myIndex,divtag) - divtag.append(item) - divtag.append(brtag) - return soup + except KeyError: + pass + oldParent = item.parent + myIndex = oldParent.contents.index(item) + item.extract() + divtag = Tag(soup,'div') + brtag = Tag(soup,'br') + oldParent.insert(myIndex,divtag) + divtag.append(item) + divtag.append(brtag) + return soup class CustomIndexRecipe(BasicNewsRecipe):