diff --git a/icons/book.icns b/icons/book.icns new file mode 100644 index 0000000000..ee305dc9dd Binary files /dev/null and b/icons/book.icns differ diff --git a/resources/catalog/stylesheet.css b/resources/catalog/stylesheet.css index 458d1a9bf0..b5770599e6 100644 --- a/resources/catalog/stylesheet.css +++ b/resources/catalog/stylesheet.css @@ -8,7 +8,7 @@ p.title { font-size:xx-large; border-bottom: solid black 4px; } - + p.author { margin-top:0em; margin-bottom:0em; @@ -31,25 +31,34 @@ p.description { margin-top: 0em; } +p.date_index { + font-size:x-large; + text-align:center; + font-weight:bold; + margin-top:1em; + margin-bottom:0px; + } + p.letter_index { font-size:x-large; - text-align:left; - margin-top:0px; - margin-bottom:0px; + text-align:center; + font-weight:bold; + margin-top:1em; + margin-bottom:0px; } p.author_index { font-size:large; text-align:left; margin-top:0px; - margin-bottom:0px; + margin-bottom:0px; text-indent: 0em; } p.read_book { text-align:left; margin-top:0px; - margin-bottom:0px; + margin-bottom:0px; margin-left:2em; text-indent:-2em; } @@ -57,8 +66,8 @@ p.read_book { p.unread_book { text-align:left; margin-top:0px; - margin-bottom:0px; + margin-bottom:0px; margin-left:2em; text-indent:-2em; } - + diff --git a/resources/images/news/information_dk.png b/resources/images/news/information_dk.png new file mode 100644 index 0000000000..301e2992c7 Binary files /dev/null and b/resources/images/news/information_dk.png differ diff --git a/resources/images/news/jp_dk.png b/resources/images/news/jp_dk.png new file mode 100644 index 0000000000..c9553659aa Binary files /dev/null and b/resources/images/news/jp_dk.png differ diff --git a/resources/images/news/michellemalkin_icon.png b/resources/images/news/michellemalkin_icon.png new file mode 100644 index 0000000000..76842ec642 Binary files /dev/null and b/resources/images/news/michellemalkin_icon.png differ diff --git a/resources/images/news/politiken_dk.png b/resources/images/news/politiken_dk.png new file mode 100644 index 0000000000..66f324a8c7 Binary files /dev/null and b/resources/images/news/politiken_dk.png differ diff --git a/resources/recipes/heraldo.recipe b/resources/recipes/heraldo.recipe new file mode 100644 index 0000000000..381e97b9ce --- /dev/null +++ b/resources/recipes/heraldo.recipe @@ -0,0 +1,50 @@ +#!/usr/bin/env python +__license__ = 'GPL v3' +__author__ = 'Lorenzo Vigentini' +__copyright__ = '2009, Lorenzo Vigentini ' +__description__ = 'Daily newspaper from Aragon' +__version__ = 'v1.01' +__date__ = '30, January 2010' + +''' +http://www.heraldo.es/ +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class heraldo(BasicNewsRecipe): + author = 'Lorenzo Vigentini' + description = 'Daily newspaper from Aragon' + + cover_url = 'http://www.heraldo.es/MODULOS/global/publico/interfaces/img/logo.gif' + title = u'Heraldo de Aragon' + publisher = 'OJD Nielsen' + category = 'News, politics, culture, economy, general interest' + + language = 'es' + timefmt = '[%a, %d %b, %Y]' + + oldest_article = 1 + max_articles_per_feed = 25 + + use_embedded_content = False + recursion = 10 + + remove_javascript = True + no_stylesheets = True + + keep_only_tags = [ + dict(name='div', attrs={'class':['titularNoticiaNN','textoGrisVerdanaContenidos']}) + ] + + feeds = [ + (u'Portadas ', u'http://www.heraldo.es/index.php/mod.portadas/mem.rss') + ] + extra_css = ''' + .articledate {color: gray;font-family: monospace;} + .articledescription {display: block;font-family: sans;font-size: 0.7em; text-indent: 0;} + .firma {color: #666;display: block;font-family: verdana, arial, helvetica;font-size: 1em;margin-bottom: 8px;} + .textoGrisVerdanaContenidos {color: #56595c;display: block;font-family: Verdana;font-size: 1.28571em;padding-bottom: 10px} + .titularNoticiaNN {display: block;padding-bottom: 10px;padding-left: 0;padding-right: 0;padding-top: 4px} + .titulo {color: #003066;font-family: Tahoma;font-size: 1.92857em;font-weight: bold;line-height: 1.2em} + ''' diff --git a/resources/recipes/information_dk.recipe b/resources/recipes/information_dk.recipe new file mode 100644 index 0000000000..1db5a7c47e --- /dev/null +++ b/resources/recipes/information_dk.recipe @@ -0,0 +1,50 @@ + +__license__ = 'GPL v3' +__copyright__ = '2010, Darko Miletic ' +''' +information.dk +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Information_dk(BasicNewsRecipe): + title = 'Information - Denmark' + __author__ = 'Darko Miletic' + description = 'News from Denmark' + publisher = 'information.dk' + category = 'news, politics, Denmark' + oldest_article = 2 + max_articles_per_feed = 100 + no_stylesheets = True + remove_empty_feeds = True + use_embedded_content = False + encoding = 'utf8' + language = 'da' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher': publisher + , 'language' : language + } + + feeds = [ + (u'Nyheder fra' , u'http://www.information.dk/feed') + ,(u'Bedst lige nu' , u'http://www.information.dk/bedstligenu/feed') + ,(u'Politik og internationalt' , u'http://www.information.dk/politik/feed') + ,(u'Kunst og kultur' , u'http://www.information.dk/kultur/feed') + ,(u'Moderne Tider' , u'http://www.information.dk/modernetider/feed') + ,(u'Klima' , u'http://www.information.dk/klima/feed') + ,(u'Opinion' , u'http://www.information.dk/opinion/feed') + ,(u'Literatur' , u'http://www.information.dk/litteratur/feed') + ,(u'Film' , u'http://www.information.dk/film/feed') + ,(u'Kunst' , u'http://www.information.dk/kunst/feed') + ] + + remove_tags_before = dict(name='h1',attrs={'class':'print-title'}) + remove_tags_after = dict(name='div',attrs={'class':'print-footer'}) + remove_tags = [dict(name=['object','link'])] + + def print_version(self, url): + return url.replace('information.dk/','information.dk/print/') + diff --git a/resources/recipes/jp_dk.recipe b/resources/recipes/jp_dk.recipe new file mode 100644 index 0000000000..cdf10f29da --- /dev/null +++ b/resources/recipes/jp_dk.recipe @@ -0,0 +1,50 @@ + +__license__ = 'GPL v3' +__copyright__ = '2010, Darko Miletic ' +''' +jp.dk +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class JP_dk(BasicNewsRecipe): + title = 'Jyllands-Posten' + __author__ = 'Darko Miletic' + description = 'News from Denmark' + publisher = 'jp.dk' + category = 'news, politics, Denmark' + oldest_article = 2 + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + encoding = 'cp1252' + language = 'da' + + extra_css = ' body{font-family: Arial,Verdana,Helvetica,Geneva,sans-serif } h1{font-family: Times,Georgia,Verdana,serif } ' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher': publisher + , 'language' : language + } + + feeds = [ + (u'Tophistorier', u'http://www.jp.dk/rss/topnyheder.jsp') + ,(u'Seneste nyt' , u'http://jp.dk/index.jsp?service=rssfeed&submode=seneste') + ,(u'Indland' , u'http://www.jp.dk/rss/indland.jsp') + ,(u'Udland' , u'http://www.jp.dk/rss/udland.jsp') + ,(u'Ny viden' , u'http://www.jp.dk/rss/nyviden.jsp') + ,(u'Timeout' , u'http://www.jp.dk/rss/timeout.jsp') + ,(u'Kultur' , u'http://www.jp.dk/rss/kultur.jsp') + ,(u'Sport' , u'http://www.jp.dk/rss/sport.jsp') + ] + + remove_tags = [ + dict(name=['object','link']) + ,dict(name='p',attrs={'class':'artByline'}) + ] + + def print_version(self, url): + return url + '?service=printversion' + diff --git a/resources/recipes/metro_montreal.recipe b/resources/recipes/metro_montreal.recipe index 9c308a91d8..094f00316f 100644 --- a/resources/recipes/metro_montreal.recipe +++ b/resources/recipes/metro_montreal.recipe @@ -4,7 +4,7 @@ class Metro_Montreal(BasicNewsRecipe): title = u'M\xe9tro Montr\xe9al' __author__ = 'Jerry Clapperton' - description = 'Le quotidien le plus branché sur le monde' + description = u'Le quotidien le plus branch\xe9 sur le monde' language = 'fr' oldest_article = 7 diff --git a/resources/recipes/michellemalkin.recipe b/resources/recipes/michellemalkin.recipe new file mode 100644 index 0000000000..e933ed8f1c --- /dev/null +++ b/resources/recipes/michellemalkin.recipe @@ -0,0 +1,49 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Walt Anthony ' +''' +www.michellemalkin.com +''' +from calibre.web.feeds.news import BasicNewsRecipe + +class MichelleMalkin(BasicNewsRecipe): + title = u'Michelle Malkin' + description = "Michelle Malkin's take on events, a mother, wife, blogger, conservative syndicated columnist, author, and Fox News Channel contributor." + __author__ = 'Walt Anthony' + publisher = 'Michelle Malkin LLC' + category = 'news, politics, USA' + oldest_article = 7 #days + max_articles_per_feed = 50 + summary_length = 150 + language = 'en' + + remove_javascript = True + no_stylesheets = True + + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'linearize_tables' : True + } + + + keep_only_tags = [ + dict(name='div', attrs={'class':'article'}) + ] + + remove_tags = [ + dict(name=['iframe', 'embed', 'object']), + dict(name='div', attrs={'id':['comments', 'commentForm']}), + dict(name='div', attrs={'class':['postCategories', 'comments', 'blogInfo', 'postInfo']}) + + ] + + + feeds = [(u'http://feeds.feedburner.com/michellemalkin/posts')] + + + + def print_version(self, url): + return url + '?print=1' diff --git a/resources/recipes/nin.recipe b/resources/recipes/nin.recipe index 535652b6a0..0872467d2f 100644 --- a/resources/recipes/nin.recipe +++ b/resources/recipes/nin.recipe @@ -1,46 +1,42 @@ -#!/usr/bin/env python __license__ = 'GPL v3' -__copyright__ = '2008-2009, Darko Miletic ' +__copyright__ = '2008-2010, Darko Miletic ' ''' -nin.co.rs +www.nin.co.rs ''' import re, urllib from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe -from calibre.ebooks.BeautifulSoup import Tag class Nin(BasicNewsRecipe): title = 'NIN online' __author__ = 'Darko Miletic' - description = 'Nedeljne informativne novine' - publisher = 'NIN D.O.O.' + description = 'Nedeljne Informativne Novine' + publisher = 'NIN d.o.o.' category = 'news, politics, Serbia' no_stylesheets = True oldest_article = 15 - simultaneous_downloads = 1 - delay = 1 encoding = 'utf-8' needs_subscription = True + remove_empty_feeds = True PREFIX = 'http://www.nin.co.rs' INDEX = PREFIX + '/?change_lang=ls' LOGIN = PREFIX + '/?logout=true' use_embedded_content = False language = 'sr' - lang = 'sr-Latn-RS' - direction = 'ltr' - extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif} .artTitle{font-size: x-large; font-weight: bold} .columnhead{font-size: small; font-weight: bold}' + extra_css = ' @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: Verdana, Lucida, sans1, sans-serif} .article_description{font-family: Verdana, Lucida, sans1, sans-serif} .artTitle{font-size: x-large; font-weight: bold; color: #900} .izjava{font-size: x-large; font-weight: bold} .columnhead{font-size: small; font-weight: bold;} img{margin-top:0.5em; margin-bottom: 0.7em} b{margin-top: 1em} ' conversion_options = { 'comment' : description , 'tags' : category , 'publisher' : publisher , 'language' : language - , 'pretty_print' : True + , 'linearize_tables' : True } preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')] + remove_attributes = ['height','width'] def get_browser(self): br = BasicNewsRecipe.get_browser() @@ -65,35 +61,20 @@ class Nin(BasicNewsRecipe): cover_url = self.PREFIX + link_item['src'] return cover_url - def preprocess_html(self, soup): - soup.html['lang'] = self.lang - soup.html['dir' ] = self.direction - mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)]) - mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=utf-8")]) - soup.head.insert(0,mlang) - soup.head.insert(1,mcharset) - attribs = [ 'style','font','valign' - ,'colspan','width','height' - ,'rowspan','summary','align' - ,'cellspacing','cellpadding' - ,'frames','rules','border' - ] - for item in soup.body.findAll(name=['table','td','tr','th','caption','thead','tfoot','tbody','colgroup','col']): - item.name = 'div' - for attrib in attribs: - if item.has_key(attrib): - del item[attrib] - return soup - def parse_index(self): articles = [] + count = 0 soup = self.index_to_soup(self.PREFIX) for item in soup.findAll('a',attrs={'class':'lmeninavFont'}): + count = count +1 + if self.test and count > 2: + return articles section = self.tag_to_string(item) feedlink = self.PREFIX + item['href'] feedpage = self.index_to_soup(feedlink) self.report_progress(0, _('Fetching feed')+' %s...'%(section)) inarts = [] + count2 = 0 for art in feedpage.findAll('span',attrs={'class':'artTitle'}): alink = art.parent url = self.PREFIX + alink['href'] @@ -110,3 +91,4 @@ class Nin(BasicNewsRecipe): }) articles.append((section,inarts)) return articles + diff --git a/resources/recipes/oc_register.recipe b/resources/recipes/oc_register.recipe new file mode 100644 index 0000000000..9a04585a3c --- /dev/null +++ b/resources/recipes/oc_register.recipe @@ -0,0 +1,73 @@ +#!/usr/bin/env python +__license__ = 'GPL v3' +__author__ = 'Lorenzo Vigentini' +__copyright__ = '2009, Lorenzo Vigentini ' +description = 'News from the Orange county - v1.01 (29, January 2010)' + +''' +http://www.ocregister.com/ +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class ocRegister(BasicNewsRecipe): + author = 'Lorenzo Vigentini' + description = 'News from the Orange county' + + cover_url = 'http://images.onset.freedom.com/ocregister/logo.gif' + title = u'Orange County Register' + publisher = 'Orange County Register Communication' + category = 'News, finance, economy, politics' + + language = 'en' + timefmt = '[%a, %d %b, %Y]' + + oldest_article = 1 + max_articles_per_feed = 25 + use_embedded_content = False + recursion = 10 + + remove_javascript = True + no_stylesheets = True + + def print_version(self,url): + printUrl = 'http://www.ocregister.com/common/printer/view.php?db=ocregister&id=' + segments = url.split('/') + subSegments = (segments[4]).split('.') + myArticle = (subSegments[0]).replace('-', '') + myURL= printUrl + myArticle + return myURL + + keep_only_tags = [ + dict(name='div', attrs={'id':'ArticleContentWrap'}) + ] + + remove_tags = [ + dict(name='div', attrs={'class':'hideForPrint'}), + dict(name='div', attrs={'id':'ContentFooter'}) + ] + + feeds = [ + (u'News', u'http://www.ocregister.com/common/rss/rss.php?catID=18800'), + (u'Today paper', u'http://www.ocregister.com/common/rss/rss.php?catID=18976'), + (u'Business', u'http://www.ocregister.com/common/rss/rss.php?catID=18909'), + (u'Cars', u'http://www.ocregister.com/common/rss/rss.php?catID=20128'), + (u'Entertainment', u'http://www.ocregister.com/common/rss/rss.php?catID=18926'), + (u'Home', u'http://www.ocregister.com/common/rss/rss.php?catID=19142'), + (u'Life', u'http://www.ocregister.com/common/rss/rss.php?catID=18936'), + (u'Opinion', u'http://www.ocregister.com/common/rss/rss.php?catID=18963'), + (u'Sports', u'http://www.ocregister.com/common/rss/rss.php?catID=18901'), + (u'Travel', u'http://www.ocregister.com/common/rss/rss.php?catID=18959') + ] + + extra_css = ''' + h1 {color:#ff6600;font-family:Arial,Helvetica,sans-serif; font-size:20px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:20px;} + h2 {color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:16px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:16px; } + h3 {color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:15px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:15px;} + h4 {color:#333333; font-family:Arial,Helvetica,sans-serif;font-size:13px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:13px; } + h5 {color:#333333; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:11px; text-transform:uppercase;} + #articledate {color:#333333;font-family:Arial,Helvetica,sans-serif;font-size:10px; font-size-adjust:none; font-stretch:normal; font-style:italic; font-variant:normal; font-weight:bold; line-height:10px; text-decoration:none;} + #articlebyline {color:#4D4D4D;font-family:Arial,Helvetica,sans-serif;font-size:10px; font-size-adjust:none; font-stretch:normal; font-style:bold; font-variant:normal; font-weight:bold; line-height:10px; text-decoration:none;} + img {align:left;} + #topstoryhead {color:#ff6600;font-family:Arial,Helvetica,sans-serif; font-size:22px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:20px;} + ''' diff --git a/resources/recipes/open_left.recipe b/resources/recipes/open_left.recipe new file mode 100644 index 0000000000..148bb07f13 --- /dev/null +++ b/resources/recipes/open_left.recipe @@ -0,0 +1,22 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class OpenLeft(BasicNewsRecipe): + # Information about the recipe + + title = 'Open Left' + description = 'Progressive American commentary on current events' + category = 'news, commentary' + language = 'en' + __author__ = 'Xanthan Gum' + + # Fetch no article older than seven days + + oldest_article = 7 + + # Fetch no more than 100 articles + + max_articles_per_feed = 100 + + # Fetch the articles from the RSS feed + + feeds = [(u'Articles', u'http://www.openleft.com/rss/rss2.xml')] diff --git a/resources/recipes/politiken_dk.recipe b/resources/recipes/politiken_dk.recipe new file mode 100644 index 0000000000..aa117fda8d --- /dev/null +++ b/resources/recipes/politiken_dk.recipe @@ -0,0 +1,55 @@ + +__license__ = 'GPL v3' +__copyright__ = '2010, Darko Miletic ' +''' +politiken.dk +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Politiken_dk(BasicNewsRecipe): + title = 'Politiken.dk' + __author__ = 'Darko Miletic' + description = 'News from Denmark' + publisher = 'politiken.dk' + category = 'news, politics, Denmark' + oldest_article = 2 + max_articles_per_feed = 100 + no_stylesheets = True + remove_empty_feeds = True + use_embedded_content = False + encoding = 'cp1252' + language = 'da' + + extra_css = ' body{font-family: Arial,Helvetica,sans-serif } h1{font-family: Georgia,"Times New Roman",Times,serif } ' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher': publisher + , 'language' : language + } + + feeds = [ + (u'Tophistorier' , u'http://politiken.dk/rss/tophistorier.rss') + ,(u'Seneste nyt' , u'http://politiken.dk/rss/senestenyt.rss') + ,(u'Mest laeste' , u'http://politiken.dk/rss/mestlaeste.rss') + ,(u'Danmark' , u'http://politiken.dk/rss/indland.rss') + ,(u'Politik' , u'http://politiken.dk/rss/politik.rss') + ,(u'Klima' , u'http://politiken.dk/rss/klima.rss') + ,(u'Internationalt' , u'http://politiken.dk/rss/udland.rss') + ,(u'Erhverv' , u'http://politiken.dk/rss/erhverv.rss') + ,(u'Kultur' , u'http://politiken.dk/rss/kultur.rss') + ,(u'Sport' , u'http://politiken.dk/rss/sport.rss') + ,(u'Uddannelse' , u'http://politiken.dk/rss/uddannelse.rss') + ,(u'Videnskab' , u'http://politiken.dk/rss/videnskab.rss') + ] + remove_tags_before = dict(name='h1') + remove_tags = [ + dict(name=['object','link']) + ,dict(name='div',attrs={'class':'footer'}) + ] + + def print_version(self, url): + return url + '?service=print' + diff --git a/resources/recipes/wash_post.recipe b/resources/recipes/wash_post.recipe index 5e62aa753c..a917371cec 100644 --- a/resources/recipes/wash_post.recipe +++ b/resources/recipes/wash_post.recipe @@ -46,3 +46,10 @@ class WashingtonPost(BasicNewsRecipe): div['style'] = '' return soup + def preprocess_html(self, soup): + for tag in soup.findAll('font'): + if tag.has_key('size'): + if tag['size'] == '+2': + if tag.b: + return soup + return None diff --git a/resources/viewer/images.js b/resources/viewer/images.js index ea68009254..7b10f6169a 100644 --- a/resources/viewer/images.js +++ b/resources/viewer/images.js @@ -20,4 +20,20 @@ function setup_image_scaling_handlers() { }); } +function extract_svged_images() { + $("svg").each(function() { + var children = $(this).children("img"); + if (children.length == 1) { + var img = $(children[0]); + var href = img.attr('xlink:href'); + if (href != undefined) { + $(this).replaceWith('
SVG Image
'); + } + } + }); +} + +$(document).ready(function() { + //extract_svged_images(); +}); diff --git a/setup/installer/osx/app/main.py b/setup/installer/osx/app/main.py index 85533717b4..a473281e61 100644 --- a/setup/installer/osx/app/main.py +++ b/setup/installer/osx/app/main.py @@ -266,6 +266,7 @@ class Py2App(object): def get_local_dependencies(self, path_to_lib): for x in self.get_dependencies(path_to_lib): for y in (SW+'/lib/', '/usr/local/lib/', SW+'/qt/lib/', + '/opt/local/lib/', '/Library/Frameworks/Python.framework/', SW+'/freetype/lib/'): if x.startswith(y): if y == '/Library/Frameworks/Python.framework/': @@ -338,8 +339,8 @@ class Py2App(object): c = join(self.build_dir, 'Contents') for x in ('Frameworks', 'MacOS', 'Resources'): os.makedirs(join(c, x)) - x = 'library.icns' - shutil.copyfile(join('icons', x), join(self.resources_dir, x)) + for x in ('library.icns', 'book.icns'): + shutil.copyfile(join('icons', x), join(self.resources_dir, x)) @flush def add_calibre_plugins(self): @@ -355,8 +356,13 @@ class Py2App(object): @flush def create_plist(self): + from calibre.ebooks import BOOK_EXTENSIONS env = dict(**ENV) env['CALIBRE_LAUNCHED_FROM_BUNDLE']='1'; + docs = [{'CFBundleTypeName':'E-book', + 'CFBundleTypeExtensions':list(BOOK_EXTENSIONS), + 'CFBundleTypeRole':'Viewer', + }] pl = dict( CFBundleDevelopmentRegion='English', @@ -367,10 +373,11 @@ class Py2App(object): CFBundlePackageType='APPL', CFBundleSignature='????', CFBundleExecutable='calibre', + CFBundleDocumentTypes=docs, LSMinimumSystemVersion='10.4.2', LSRequiresNativeExecution=True, NSAppleScriptEnabled=False, - NSHumanReadableCopyright='Copyright 2008, Kovid Goyal', + NSHumanReadableCopyright='Copyright 2010, Kovid Goyal', CFBundleGetInfoString=('calibre, an E-book management ' 'application. Visit http://calibre-ebook.com for details.'), CFBundleIconFile='library.icns', @@ -594,6 +601,7 @@ class Py2App(object): if x == 'Info.plist': plist = plistlib.readPlist(join(self.contents_dir, x)) plist['LSUIElement'] = '1' + plist.pop('CFBundleDocumentTypes') plistlib.writePlist(plist, join(cc_dir, x)) else: os.symlink(join('../..', x), diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index e32c03fe13..e5e284fb5b 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -132,9 +132,12 @@ def prints(*args, **kwargs): try: arg = arg.encode(enc) except UnicodeEncodeError: - if not safe_encode: - raise - arg = repr(arg) + try: + arg = arg.encode('utf-8') + except: + if not safe_encode: + raise + arg = repr(arg) file.write(arg) if i != len(args)-1: diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 68d94bf178..42b379fa0b 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -288,7 +288,7 @@ class CatalogPlugin(Plugin): fields = list(all_fields) fields.sort() - if opts.sort_by: + if opts.sort_by and opts.sort_by in fields: fields.insert(0,fields.pop(int(fields.index(opts.sort_by)))) return fields diff --git a/src/calibre/devices/prs500/books.py b/src/calibre/devices/prs500/books.py index 07f9310e87..91fcb3255f 100644 --- a/src/calibre/devices/prs500/books.py +++ b/src/calibre/devices/prs500/books.py @@ -274,7 +274,7 @@ class BookList(_BookList): node.setAttribute(attr, attrs[attr]) try: w, h, data = mi.thumbnail - except TypeError: + except: w, h, data = None, None, None if data: diff --git a/src/calibre/ebooks/comic/input.py b/src/calibre/ebooks/comic/input.py index 122f61e45a..6fba918888 100755 --- a/src/calibre/ebooks/comic/input.py +++ b/src/calibre/ebooks/comic/input.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' Based on ideas from comiclrf created by FangornUK. ''' -import os, shutil, traceback, textwrap, time +import os, shutil, traceback, textwrap, time, codecs from ctypes import byref from Queue import Empty @@ -338,8 +338,9 @@ class ComicInput(InputFormatPlugin): if not os.path.exists('comics.txt'): raise ValueError('%s is not a valid comic collection' %stream.name) - for line in open('comics.txt', - 'rb').read().decode('utf-8').splitlines(): + raw = open('comics.txt', 'rb').read().decode('utf-8') + raw.lstrip(unicode(codecs.BOM_UTF8, "utf8" )) + for line in raw.splitlines(): line = line.strip() if not line: continue diff --git a/src/calibre/ebooks/epub/output.py b/src/calibre/ebooks/epub/output.py index 8e9c9efea9..6e381d5237 100644 --- a/src/calibre/ebooks/epub/output.py +++ b/src/calibre/ebooks/epub/output.py @@ -269,7 +269,7 @@ class EPUBOutput(OutputFormatPlugin): bad = [] for x in XPath('//h:img')(body): src = x.get('src', '').strip() - if src in ('', '#'): + if src in ('', '#') or src.startswith('http:'): bad.append(x) for img in bad: img.getparent().remove(img) diff --git a/src/calibre/ebooks/mobi/output.py b/src/calibre/ebooks/mobi/output.py index 7e4643dac1..e3a1da34cc 100644 --- a/src/calibre/ebooks/mobi/output.py +++ b/src/calibre/ebooks/mobi/output.py @@ -50,7 +50,7 @@ class MOBIOutput(OutputFormatPlugin): def check_for_masthead(self): found = 'masthead' in self.oeb.guide if not found: - self.oeb.log.debug('No masthead found, generating default one...') + self.oeb.log.debug('No masthead found in manifest, generating default mastheadImage...') try: from PIL import Image as PILImage PILImage @@ -65,6 +65,9 @@ class MOBIOutput(OutputFormatPlugin): id, href = self.oeb.manifest.generate('masthead', 'masthead') self.oeb.manifest.add(id, href, 'image/gif', data=raw) self.oeb.guide.add('masthead', 'Masthead Image', href) + else: + self.oeb.log.debug('Using mastheadImage supplied in manifest...') + def dump_toc(self, toc) : self.log( "\n >>> TOC contents <<<") diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 4f894ce088..4aac84e599 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -573,6 +573,8 @@ class MobiReader(object): attrib[attr] = "%dpx"%int(nval) except: del attrib[attr] + elif val.lower().endswith('%'): + del attrib[attr] elif tag.tag == 'pre': if not tag.text: tag.tag = 'div' diff --git a/src/calibre/ebooks/pdf/reflow.py b/src/calibre/ebooks/pdf/reflow.py index bf2d921a10..42c16225d2 100644 --- a/src/calibre/ebooks/pdf/reflow.py +++ b/src/calibre/ebooks/pdf/reflow.py @@ -256,11 +256,16 @@ class Region(object): return len(self.columns) == 0 @property - def is_small(self): + def line_count(self): max_lines = 0 for c in self.columns: max_lines = max(max_lines, len(c)) - return max_lines > 2 + return max_lines + + + @property + def is_small(self): + return self.line_count < 3 def absorb(self, singleton): @@ -431,7 +436,7 @@ class Page(object): def coalesce_regions(self): # find contiguous sets of small regions # absorb into a neighboring region (prefer the one with number of cols - # closer to the avg number of cols in the set, if equal use large + # closer to the avg number of cols in the set, if equal use larger # region) # merge contiguous regions that can contain each other absorbed = set([]) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 34f9f57161..fed22f87e2 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -3,7 +3,8 @@ __copyright__ = '2008, Kovid Goyal ' """ The GUI """ import os from PyQt4.QtCore import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, QSize, \ - QByteArray, QTranslator, QCoreApplication, QThread + QByteArray, QTranslator, QCoreApplication, QThread, \ + QEvent from PyQt4.QtGui import QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \ QIcon, QTableView, QApplication, QDialog, QPushButton @@ -88,6 +89,8 @@ def _config(): help=_('Maximum number of waiting worker processes')) c.add_opt('get_social_metadata', default=True, help=_('Download social metadata (tags/rating/etc.)')) + c.add_opt('overwrite_author_title_metadata', default=True, + help=_('Overwrite author and title with new metadata')) c.add_opt('enforce_cpu_limit', default=True, help=_('Limit max simultaneous jobs to number of CPUs')) @@ -524,6 +527,7 @@ class Application(QApplication): def __init__(self, args): qargs = [i.encode('utf-8') if isinstance(i, unicode) else i for i in args] QApplication.__init__(self, qargs) + self.file_event_hook = None global gui_thread, qt_app gui_thread = QThread.currentThread() self._translator = None @@ -549,6 +553,15 @@ class Application(QApplication): if set_qt_translator(self._translator): self.installTranslator(self._translator) + def event(self, e): + if callable(self.file_event_hook) and e.type() == QEvent.FileOpen: + path = unicode(e.file()) + if os.access(path, os.R_OK): + self.file_event_hook(path) + return True + else: + return QApplication.event(self, e) + _store_app = None def is_ok_to_use_qt(): diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 5419596334..ace2ac5c7e 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -12,7 +12,7 @@ from calibre.gui2.dialogs.progress import ProgressDialog from calibre.gui2 import question_dialog, error_dialog, info_dialog from calibre.ebooks.metadata.opf2 import OPF from calibre.ebooks.metadata import MetaInformation -from calibre.constants import preferred_encoding +from calibre.constants import preferred_encoding, filesystem_encoding class DuplicatesAdder(QThread): @@ -46,6 +46,8 @@ class RecursiveFind(QThread): def run(self): root = os.path.abspath(self.path) self.books = [] + if isinstance(root, unicode): + root = root.encode(filesystem_encoding) try: for dirpath in os.walk(root): if self.canceled: @@ -55,6 +57,8 @@ class RecursiveFind(QThread): self.books += list(self.db.find_books_in_directory(dirpath[0], self.single_book_per_directory)) except Exception, err: + import traceback + traceback.print_exc() try: msg = unicode(err) except: diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.ui b/src/calibre/gui2/catalog/catalog_epub_mobi.ui index 858aec429f..044ecdaaec 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.ui +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.ui @@ -14,19 +14,6 @@ Form - - - - Tags to exclude as genres (regex): - - - Qt::LogText - - - true - - - @@ -37,7 +24,7 @@ - + @@ -51,7 +38,7 @@ - + @@ -65,18 +52,67 @@ - + - + + + + Sort numbers as text + + + + - + - + + + + Regex pattern describing tags to exclude as genres: + + + Qt::LogText + + + true + + + + + + + + 14 + 75 + true + + + + Special marker tags for catalog generation + + + Qt::AlignCenter + + + + + + + Regex tips: +- The default regex of '\[[\w]*\]' ignores tags of the form '[tag]', e.g., '[Amazon Freebie]' +- A regex of '.' ignores all tags, generating no genre categories in the catalog + + + true + + + + Qt::Vertical @@ -89,13 +125,6 @@ - - - - Sort numbers as text - - - diff --git a/src/calibre/gui2/convert/gui_conversion.py b/src/calibre/gui2/convert/gui_conversion.py index 67c50758f8..70321b049b 100644 --- a/src/calibre/gui2/convert/gui_conversion.py +++ b/src/calibre/gui2/convert/gui_conversion.py @@ -23,7 +23,7 @@ def gui_convert(input, output, recommendations, notification=DummyReporter(), plumber.run() -def gui_catalog(fmt, title, dbspec, ids, out_file_name, fmt_options, +def gui_catalog(fmt, title, dbspec, ids, out_file_name, sync, fmt_options, notification=DummyReporter(), log=None): if log is None: log = Log() @@ -46,6 +46,7 @@ def gui_catalog(fmt, title, dbspec, ids, out_file_name, fmt_options, opts.ids = ids opts.search_text = None opts.sort_by = None + opts.sync = sync # Extract the option dictionary to comma-separated lists for option in fmt_options: diff --git a/src/calibre/gui2/dialogs/catalog.ui b/src/calibre/gui2/dialogs/catalog.ui index c18e08ef65..3d62f36e85 100644 --- a/src/calibre/gui2/dialogs/catalog.ui +++ b/src/calibre/gui2/dialogs/catalog.ui @@ -107,7 +107,7 @@ 12 12 - 205 + 301 17 diff --git a/src/calibre/gui2/dialogs/config/__init__.py b/src/calibre/gui2/dialogs/config/__init__.py index 156faec7ce..88697e55bb 100644 --- a/src/calibre/gui2/dialogs/config/__init__.py +++ b/src/calibre/gui2/dialogs/config/__init__.py @@ -458,6 +458,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): self.connect(self.button_open_config_dir, SIGNAL('clicked()'), self.open_config_dir) self.opt_get_social_metadata.setChecked(config['get_social_metadata']) + self.opt_overwrite_author_title_metadata.setChecked(config['overwrite_author_title_metadata']) self.opt_enforce_cpu_limit.setChecked(config['enforce_cpu_limit']) self.device_detection_button.clicked.connect(self.debug_device_detection) @@ -751,6 +752,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): config['upload_news_to_device'] = self.sync_news.isChecked() config['search_as_you_type'] = self.search_as_you_type.isChecked() config['get_social_metadata'] = self.opt_get_social_metadata.isChecked() + config['overwrite_author_title_metadata'] = self.opt_overwrite_author_title_metadata.isChecked() config['enforce_cpu_limit'] = bool(self.opt_enforce_cpu_limit.isChecked()) fmts = [] for i in range(self.viewer.count()): diff --git a/src/calibre/gui2/dialogs/config/config.ui b/src/calibre/gui2/dialogs/config/config.ui index b9306b0f10..6da5362248 100644 --- a/src/calibre/gui2/dialogs/config/config.ui +++ b/src/calibre/gui2/dialogs/config/config.ui @@ -171,6 +171,13 @@ + + + + Overwrite & author/title by default when fetching metadata + + + diff --git a/src/calibre/gui2/dialogs/fetch_metadata.py b/src/calibre/gui2/dialogs/fetch_metadata.py index b021a2470d..5a0957be31 100644 --- a/src/calibre/gui2/dialogs/fetch_metadata.py +++ b/src/calibre/gui2/dialogs/fetch_metadata.py @@ -119,6 +119,7 @@ class FetchMetadata(QDialog, Ui_FetchMetadata): self.matches.setMouseTracking(True) self.fetch_metadata() self.opt_get_social_metadata.setChecked(config['get_social_metadata']) + self.opt_overwrite_author_title_metadata.setChecked(config['overwrite_author_title_metadata']) def show_summary(self, current, *args): @@ -149,7 +150,8 @@ class FetchMetadata(QDialog, Ui_FetchMetadata): self.fetcher.start() self.pi.start(_('Finding metadata...')) self._hangcheck = QTimer(self) - self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck) + self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck, + Qt.QueuedConnection) self.start_time = time.time() self._hangcheck.start(100) diff --git a/src/calibre/gui2/dialogs/fetch_metadata.ui b/src/calibre/gui2/dialogs/fetch_metadata.ui index fe97b32f28..f14d402e11 100644 --- a/src/calibre/gui2/dialogs/fetch_metadata.ui +++ b/src/calibre/gui2/dialogs/fetch_metadata.ui @@ -116,6 +116,13 @@ + + + + Overwrite &author/title with author/title of selected book + + + diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index d76d8136db..46342c8a88 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -20,10 +20,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): self.db = db self.ids = [ db.id(r) for r in rows] self.write_series = False - self.write_rating = False self.changed = False QObject.connect(self.button_box, SIGNAL("accepted()"), self.sync) - QObject.connect(self.rating, SIGNAL('valueChanged(int)'), self.rating_changed) self.tags.update_tags_cache(self.db.all_tags()) self.remove_tags.update_tags_cache(self.db.all_tags()) @@ -99,7 +97,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): aus = unicode(self.author_sort.text()) if aus and self.author_sort.isEnabled(): self.db.set_author_sort(id, aus, notify=False) - if self.write_rating: + if self.rating.value() != -1: self.db.set_rating(id, 2*self.rating.value(), notify=False) pub = unicode(self.publisher.text()) if pub: @@ -134,5 +132,3 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): def series_changed(self): self.write_series = True - def rating_changed(self): - self.write_rating = True diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 0fdb36b717..1a38568b60 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -96,12 +96,21 @@ QAbstractSpinBox::PlusMinus + + No change + stars + + -1 + 5 + + -1 + diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index 89b7c92125..846851fd21 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -574,9 +574,10 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): det_msg=det, show=True) else: book.tags = [] - self.title.setText(book.title) - self.authors.setText(authors_to_string(book.authors)) - if book.author_sort: self.author_sort.setText(book.author_sort) + if d.opt_overwrite_author_title_metadata.isChecked(): + self.title.setText(book.title) + self.authors.setText(authors_to_string(book.authors)) + if book.author_sort: self.author_sort.setText(book.author_sort) if book.publisher: self.publisher.setEditText(book.publisher) if book.isbn: self.isbn.setText(book.isbn) if book.pubdate: diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index cf62508751..fef0853a23 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -2,6 +2,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' import sys, os, time, socket, traceback +from functools import partial from PyQt4.Qt import QCoreApplication, QIcon, QMessageBox @@ -52,10 +53,12 @@ def run_gui(opts, args, actions, listener, app): wizard().exec_() dynamic.set('welcome_wizard_was_run', True) main = Main(listener, opts, actions) + add_filesystem_book = partial(main.add_filesystem_book, allow_device=False) sys.excepthook = main.unhandled_exception if len(args) > 1: args[1] = os.path.abspath(args[1]) - main.add_filesystem_book(args[1]) + add_filesystem_book(args[1]) + app.file_event_hook = add_filesystem_book ret = app.exec_() if getattr(main, 'run_wizard_b4_shutdown', False): from calibre.gui2.wizard import wizard diff --git a/src/calibre/gui2/metadata.py b/src/calibre/gui2/metadata.py index ecdca29422..d63e9648cc 100644 --- a/src/calibre/gui2/metadata.py +++ b/src/calibre/gui2/metadata.py @@ -12,6 +12,7 @@ from Queue import Queue, Empty from calibre.ebooks.metadata.fetch import search, get_social_metadata +from calibre.gui2 import config from calibre.ebooks.metadata.library_thing import cover_from_isbn from calibre.customize.ui import get_isbndb_key @@ -98,6 +99,10 @@ class DownloadMetadata(Thread): self.fetched_metadata[id] = fmi if fmi.isbn and self.get_covers: self.worker.jobs.put(fmi.isbn) + if (not config['overwrite_author_title_metadata']): + fmi.authors = mi.authors + fmi.author_sort = mi.author_sort + fmi.title = mi.title mi.smart_update(fmi) if mi.isbn and self.get_social_metadata: self.social_metadata_exceptions = get_social_metadata(mi) diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 7dc83baf32..bd34f84821 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -254,11 +254,13 @@ def generate_catalog(parent, dbspec, ids): dbspec, ids, out.name, + d.catalog_sync, d.fmt_options ] out.close() - # This calls gui2.convert.gui_conversion:gui_catalog() + # This returns to gui2.ui:generate_catalog() + # Which then calls gui2.convert.gui_conversion:gui_catalog() return 'gui_catalog', args, _('Generate catalog'), out.name, d.catalog_sync, \ d.catalog_title diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 714b2c3a27..7f3ca297fd 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -987,10 +987,10 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.cover_cache.refresh([cid]) self.library_view.model().current_changed(current_idx, current_idx) - def add_filesystem_book(self, path): + def add_filesystem_book(self, path, allow_device=True): if os.access(path, os.R_OK): books = [os.path.abspath(path)] - to_device = self.stack.currentIndex() != 0 + to_device = allow_device and self.stack.currentIndex() != 0 self._add_books(books, to_device) if to_device: self.status_bar.showMessage(\ @@ -1048,6 +1048,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): if self._adder.critical: det_msg = [] for name, log in self._adder.critical.items(): + if isinstance(name, str): + name = name.decode(filesystem_encoding, 'replace') det_msg.append(name+'\n'+log) warning_dialog(self, _('Failed to read metadata'), _('Failed to read metadata from the following')+':', diff --git a/src/calibre/gui2/viewer/documentview.py b/src/calibre/gui2/viewer/documentview.py index f6fce62eac..9b911754c8 100644 --- a/src/calibre/gui2/viewer/documentview.py +++ b/src/calibre/gui2/viewer/documentview.py @@ -10,7 +10,7 @@ from base64 import b64encode from PyQt4.Qt import QSize, QSizePolicy, QUrl, SIGNAL, Qt, QTimer, \ QPainter, QPalette, QBrush, QFontDatabase, QDialog, \ QColor, QPoint, QImage, QRegion, QVariant, QIcon, \ - QFont, pyqtSignature, QAction + QFont, pyqtSignature, QAction, QByteArray from PyQt4.QtWebKit import QWebPage, QWebView, QWebSettings from calibre.utils.config import Config, StringConfig @@ -514,14 +514,18 @@ class DocumentView(QWebView): mt = guess_type(path)[0] html = open(path, 'rb').read().decode(path.encoding, 'replace') html = EntityDeclarationProcessor(html).processed_html + has_svg = re.search(r'<[:a-zA-Z]*svg', html) is not None + if 'xhtml' in mt: html = self.self_closing_pat.sub(self.self_closing_sub, html) if self.manager is not None: self.manager.load_started() self.loading_url = QUrl.fromLocalFile(path) - #self.setContent(QByteArray(html.encode(path.encoding)), mt, QUrl.fromLocalFile(path)) - #open('/tmp/t.html', 'wb').write(html.encode(path.encoding)) - self.setHtml(html, self.loading_url) + if has_svg: + prints('Rendering as XHTML...') + self.setContent(QByteArray(html.encode(path.encoding)), mt, QUrl.fromLocalFile(path)) + else: + self.setHtml(html, self.loading_url) self.turn_off_internal_scrollbars() def initialize_scrollbar(self): diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 98ab27c3c7..67e360da68 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -1,9 +1,10 @@ import os, re, shutil, htmlentitydefs from collections import namedtuple +from datetime import date from xml.sax.saxutils import escape -from calibre import filesystem_encoding, prints +from calibre import filesystem_encoding, prints, strftime from calibre.customize import CatalogPlugin from calibre.customize.conversion import OptionRecommendation, DummyReporter from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString @@ -50,18 +51,23 @@ class CSV_XML(CatalogPlugin): self.fmt = path_to_output.rpartition('.')[2] self.notification = notification - if False and opts.verbose: - log("%s:run" % self.name) - log(" path_to_output: %s" % path_to_output) - log(" Output format: %s" % self.fmt) - - # Display opts + if opts.verbose: opts_dict = vars(opts) - keys = opts_dict.keys() - keys.sort() - log(" opts:") - for key in keys: - log(" %s: %s" % (key, opts_dict[key])) + log("%s(): Generating %s" % (self.name,self.fmt)) + if opts_dict['search_text']: + log(" --search='%s'" % opts_dict['search_text']) + + if opts_dict['ids']: + log(" Book count: %d" % len(opts_dict['ids'])) + if opts_dict['search_text']: + log(" (--search ignored when a subset of the database is specified)") + + if opts_dict['fields']: + if opts_dict['fields'] == 'all': + log(" Fields: %s" % ', '.join(FIELDS[1:])) + else: + log(" Fields: %s" % opts_dict['fields']) + # If a list of ids are provided, don't use search_text if opts.ids: @@ -94,6 +100,10 @@ class CSV_XML(CatalogPlugin): item = ', '.join(fmt_list) elif field in ['authors','tags']: item = ', '.join(item) + elif field == 'isbn': + # Could be 9, 10 or 13 digits + field = u'%s' % re.sub(r'[\D]','',field) + if x < len(fields) - 1: if item is not None: outstr += u'"%s",' % unicode(item).replace('"','""') @@ -481,11 +491,8 @@ class EPUB_MOBI(CatalogPlugin): # Number of discrete steps to catalog creation current_step = 0.0 - total_steps = 13.0 + total_steps = 14.0 - # Used to xlate pubdate to friendly format - MONTHS = ['January', 'February','March','April','May','June', - 'July','August','September','October','November','December'] THUMB_WIDTH = 75 THUMB_HEIGHT = 100 @@ -500,7 +507,7 @@ class EPUB_MOBI(CatalogPlugin): # verbosity level of diagnostic printout def __init__(self, db, opts, plugin, - notification=DummyReporter(), + report_progress=DummyReporter(), stylesheet="content/stylesheet.css"): self.__opts = opts self.__authors = None @@ -525,16 +532,12 @@ class EPUB_MOBI(CatalogPlugin): self.__plugin_path = opts.plugin_path self.__progressInt = 0.0 self.__progressString = '' - self.__reporter = notification + self.__reporter = report_progress self.__stylesheet = stylesheet self.__thumbs = None self.__title = opts.catalog_title self.__verbose = opts.verbose - self.opts.log.info("CatalogBuilder(): Generating %s %s"% \ - (self.opts.fmt, - "for %s" % self.opts.output_profile if self.opts.output_profile \ - else '')) # Accessors ''' @dynamic_property @@ -753,53 +756,27 @@ class EPUB_MOBI(CatalogPlugin): # Methods def buildSources(self): - if getattr(self.reporter, 'cancel_requested', False): return 1 - if not self.booksByTitle: - self.fetchBooksByTitle() - - if getattr(self.reporter, 'cancel_requested', False): return 1 + self.fetchBooksByTitle() self.fetchBooksByAuthor() - - if getattr(self.reporter, 'cancel_requested', False): return 1 self.generateHTMLDescriptions() - - if getattr(self.reporter, 'cancel_requested', False): return 1 - self.generateHTMLByTitle() - - if getattr(self.reporter, 'cancel_requested', False): return 1 self.generateHTMLByAuthor() - - if getattr(self.reporter, 'cancel_requested', False): return 1 + self.generateHTMLByTitle() + self.generateHTMLByDateAdded() self.generateHTMLByTags() - if getattr(self.reporter, 'cancel_requested', False): return 1 from calibre.utils.PythonMagickWand import ImageMagick with ImageMagick(): self.generateThumbnails() - if getattr(self.reporter, 'cancel_requested', False): return 1 self.generateOPF() - - if getattr(self.reporter, 'cancel_requested', False): return 1 self.generateNCXHeader() - - if getattr(self.reporter, 'cancel_requested', False): return 1 self.generateNCXDescriptions("Descriptions") - - if getattr(self.reporter, 'cancel_requested', False): return 1 - self.generateNCXByTitle("Titles") - - if getattr(self.reporter, 'cancel_requested', False): return 1 self.generateNCXByAuthor("Authors") - - if getattr(self.reporter, 'cancel_requested', False): return 1 - self.generateNCXByTags("Genres") - - if getattr(self.reporter, 'cancel_requested', False): return 1 + self.generateNCXByTitle("Titles") + self.generateNCXByDateAdded("Recently Added") + self.generateNCXByGenre("Genres") self.writeNCX() - return 0 - def cleanUp(self): pass @@ -819,8 +796,14 @@ class EPUB_MOBI(CatalogPlugin): shutil.copy(os.path.join(catalog_resources,file[1]), os.path.join(self.catalogPath, file[0])) + # Create the custom masthead image overwriting default + try: + self.generate_masthead_image(os.path.join(self.catalogPath, 'images/mastheadImage.gif')) + except: + pass + def fetchBooksByTitle(self): - self.opts.log.info(self.updateProgressFullStep("fetchBooksByTitle()")) + self.updateProgressFullStep("Fetching database") # Get the database as a dictionary # Sort by title @@ -867,10 +850,8 @@ class EPUB_MOBI(CatalogPlugin): this_title['publisher'] = re.sub('&', '&', record['publisher']) this_title['rating'] = record['rating'] if record['rating'] else 0 - # 2009-11-05 09:29:37 - date_strings = str(record['pubdate']).split("-") - this_title['date'] = '%s %s' % (self.MONTHS[int(date_strings[1])-1], date_strings[0]) - + this_title['date'] = strftime(u'%B %Y', record['pubdate'].timetuple()) + this_title['timestamp'] = record['timestamp'] if record['comments']: this_title['description'] = re.sub('&', '&', record['comments']) this_title['short_description'] = self.generateShortDescription(this_title['description']) @@ -906,7 +887,7 @@ class EPUB_MOBI(CatalogPlugin): def fetchBooksByAuthor(self): # Generate a list of titles sorted by author from the database - self.opts.log.info(self.updateProgressFullStep("fetchBooksByAuthor()")) + self.updateProgressFullStep("Sorting database") # Sort titles case-insensitive self.booksByAuthor = sorted(self.booksByTitle, @@ -959,14 +940,15 @@ class EPUB_MOBI(CatalogPlugin): def generateHTMLDescriptions(self): # Write each title to a separate HTML file in contentdir - self.opts.log.info(self.updateProgressFullStep("generateHTMLDescriptions()")) + self.updateProgressFullStep("'Descriptions'") for (title_num, title) in enumerate(self.booksByTitle): if False: self.opts.log.info("%3s: %s - %s" % (title['id'], title['title'], title['author'])) - self.updateProgressMicroStep("generating book descriptions ...", - float(title_num*100/len(self.booksByTitle))/100) + self.updateProgressMicroStep("Description %d of %d" % \ + (title_num, len(self.booksByTitle)), + float(title_num*100/len(self.booksByTitle))/100) # Generate the header soup = self.generateHTMLDescriptionHeader("%s" % title['title']) @@ -1087,7 +1069,7 @@ class EPUB_MOBI(CatalogPlugin): def generateHTMLByTitle(self): # Write books by title A-Z to HTML file - self.opts.log.info(self.updateProgressFullStep("generateHTMLByTitle()")) + self.updateProgressFullStep("'Titles'") soup = self.generateHTMLEmptyHeader("Books By Alpha Title") body = soup.find('body') @@ -1189,7 +1171,7 @@ class EPUB_MOBI(CatalogPlugin): def generateHTMLByAuthor(self): # Write books by author A-Z - self.opts.log.info(self.updateProgressFullStep("generateHTMLByAuthor()")) + self.updateProgressFullStep("'Authors'") friendly_name = "By Author" @@ -1317,11 +1299,141 @@ class EPUB_MOBI(CatalogPlugin): outfile.close() self.htmlFileList.append("content/ByAlphaAuthor.html") + def generateHTMLByDateAdded(self): + # Write books by reverse chronological order + self.updateProgressFullStep("'Recently Added'") + + def add_books_to_HTML(this_months_list, dtc): + if len(this_months_list): + date_string = strftime(u'%B %Y', current_date.timetuple()) + this_months_list = sorted(this_months_list, + key=lambda x:(x['title_sort'], x['title_sort'])) + this_months_list = sorted(this_months_list, + key=lambda x:(x['author_sort'], x['author_sort'])) + # Create a new month anchor + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "date_index" + aTag = Tag(soup, "a") + aTag['name'] = "%s-%s" % (current_date.year, current_date.month) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(date_string)) + divTag.insert(dtc,pIndexTag) + dtc += 1 + current_author = None + + for new_entry in this_months_list: + if new_entry['author'] != current_author: + # Start a new author + current_author = new_entry['author'] + pAuthorTag = Tag(soup, "p") + pAuthorTag['class'] = "author_index" + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) + aTag.insert(0,NavigableString(current_author)) + emTag.insert(0,aTag) + pAuthorTag.insert(0,emTag) + divTag.insert(dtc,pAuthorTag) + dtc += 1 + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # Prefix book with read/unread symbol + if new_entry['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + aTag = Tag(soup, "a") + aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) + aTag.insert(0,escape(new_entry['title'])) + pBookTag.insert(ptc, aTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + return dtc + + + # Sort titles case-insensitive + self.booksByDate = sorted(self.booksByTitle, + key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) + + friendly_name = "Recently Added" + + soup = self.generateHTMLEmptyHeader(friendly_name) + body = soup.find('body') + + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + body.insert(btc, aTag) + btc += 1 + ''' + # We don't need this because the kindle inserts section titles + #

By Author

+ h2Tag = Tag(soup, "h2") + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + h2Tag.insert(0,aTag) + h2Tag.insert(1,NavigableString('%s' % friendly_name)) + body.insert(btc,h2Tag) + btc += 1 + ''' + + #

+ #

+ divTag = Tag(soup, "div") + dtc = 0 + + current_date = date.fromordinal(1) + + # Loop through books by date + this_months_list = [] + for book in self.booksByDate: + if book['timestamp'].month != current_date.month or \ + book['timestamp'].year != current_date.year: + dtc = add_books_to_HTML(this_months_list, dtc) + this_months_list = [] + current_date = book['timestamp'].date() + this_months_list.append(book) + + # Add the last month's list + add_books_to_HTML(this_months_list, dtc) + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to contentdir + outfile_spec = "%s/ByDateAdded.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList.append("content/ByDateAdded.html") + def generateHTMLByTags(self): # Generate individual HTML files for each tag, e.g. Fiction, Nonfiction ... # Note that special tags - ~+*[] - have already been filtered from books[] - self.opts.log.info(self.updateProgressFullStep("generateHTMLByTags()")) + self.updateProgressFullStep("'Genres'") # Filter out REMOVE_TAGS, sort filtered_tags = self.filterDbTags(self.db.all_tags()) @@ -1402,7 +1514,8 @@ class EPUB_MOBI(CatalogPlugin): for (i,title) in enumerate(self.booksByTitle): # Update status - self.updateProgressMicroStep("generating thumbnails ...", + self.updateProgressMicroStep("Thumbnail %d of %d" % \ + (i,len(self.booksByTitle)), i/float(len(self.booksByTitle))) # Check to see if source file exists if 'cover' in title and os.path.isfile(title['cover']): @@ -1424,7 +1537,7 @@ class EPUB_MOBI(CatalogPlugin): self.generateThumbnail(title, image_dir, thumb_file) else: # Use default cover - if self.verbose: + if False and self.verbose: self.opts.log.warn(" using default cover for '%s'" % \ (title['title'])) # Check to make sure default is current @@ -1457,13 +1570,13 @@ class EPUB_MOBI(CatalogPlugin): cover_timestamp = os.path.getmtime(cover) thumb_timestamp = os.path.getmtime(thumb_fp) if thumb_timestamp < cover_timestamp: - if self.verbose: + if False and self.verbose: self.opts.log.warn("updating thumbnail_default for %s" % title['title']) #title['cover'] = "%s/DefaultCover.jpg" % self.catalogPath title['cover'] = cover self.generateThumbnail(title, image_dir, "thumbnail_default.jpg") else: - if self.verbose: + if False and self.verbose: self.opts.log.warn(" generating new thumbnail_default.jpg") #title['cover'] = "%s/DefaultCover.jpg" % self.catalogPath title['cover'] = cover @@ -1473,7 +1586,7 @@ class EPUB_MOBI(CatalogPlugin): def generateOPF(self): - self.opts.log.info(self.updateProgressFullStep("generateOPF()")) + self.updateProgressFullStep("Saving OPF") header = ''' @@ -1605,7 +1718,7 @@ class EPUB_MOBI(CatalogPlugin): def generateNCXHeader(self): - self.opts.log.info(self.updateProgressFullStep("generateNCXHeader()")) + self.updateProgressFullStep("NCX header") header = ''' @@ -1641,7 +1754,7 @@ class EPUB_MOBI(CatalogPlugin): def generateNCXDescriptions(self, tocTitle): - self.opts.log.info(self.updateProgressFullStep("generateNCXDescriptions()")) + self.updateProgressFullStep("NCX 'Descriptions'") # --- Construct the 'Books by Title' section --- ncx_soup = self.ncxSoup @@ -1687,7 +1800,11 @@ class EPUB_MOBI(CatalogPlugin): # Add the author tag cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') cmTag['name'] = "author" - cmTag.insert(0, NavigableString(self.formatNCXText(book['author']))) + navStr = '%s | %s' % (self.formatNCXText(book['author']), + book['date'].split()[1]) + if 'tags' in book: + navStr += ' | %s' % self.formatNCXText(' · '.join(sorted(book['tags']))) + cmTag.insert(0, NavigableString(navStr)) navPointVolumeTag.insert(2, cmTag) # Add the description tag @@ -1708,8 +1825,12 @@ class EPUB_MOBI(CatalogPlugin): self.ncxSoup = ncx_soup def generateNCXByTitle(self, tocTitle): + self.updateProgressFullStep("NCX 'Titles'") - self.opts.log.info(self.updateProgressFullStep("generateNCXByTitle()")) + def add_to_books_by_letter(current_book_list): + current_book_list = " • ".join(current_book_list) + current_book_list = self.generateShortDescription(self.formatNCXText(current_book_list)) + books_by_letter.append(current_book_list) soup = self.ncxSoup output = "ByAlphaTitle" @@ -1744,9 +1865,7 @@ class EPUB_MOBI(CatalogPlugin): for book in self.booksByTitle: if self.letter_or_symbol(book['title_sort'][0]) != current_letter: # Save the old list - book_list = " • ".join(current_book_list) - short_description = self.generateShortDescription(self.formatNCXText(book_list)) - books_by_letter.append(short_description) + add_to_books_by_letter(current_book_list) # Start the new list current_letter = self.letter_or_symbol(book['title_sort'][0]) @@ -1760,9 +1879,7 @@ class EPUB_MOBI(CatalogPlugin): current_book_list.append(book['title']) # Add the last book list - book_list = " • ".join(current_book_list) - short_description = self.generateShortDescription(self.formatNCXText(book_list)) - books_by_letter.append(short_description) + add_to_books_by_letter(current_book_list) # Add *article* entries for each populated title letter for (i,books) in enumerate(books_by_letter): @@ -1773,7 +1890,8 @@ class EPUB_MOBI(CatalogPlugin): self.playOrder += 1 navLabelTag = Tag(soup, 'navLabel') textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString("Titles beginning with %s" % (title_letters[i]))) + textTag.insert(0, NavigableString(u"Titles beginning with %s" % \ + (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) navLabelTag.insert(0, textTag) navPointByLetterTag.insert(0,navLabelTag) contentTag = Tag(soup, 'content') @@ -1796,8 +1914,12 @@ class EPUB_MOBI(CatalogPlugin): self.ncxSoup = soup def generateNCXByAuthor(self, tocTitle): + self.updateProgressFullStep("NCX 'Authors'") - self.opts.log.info(self.updateProgressFullStep("generateNCXByAuthor()")) + def add_to_author_list(current_author_list, current_letter): + current_author_list = " • ".join(current_author_list) + current_author_list = self.generateShortDescription(self.formatNCXText(current_author_list)) + master_author_list.append((current_author_list, current_letter)) soup = self.ncxSoup HTML_file = "content/ByAlphaAuthor.html" @@ -1835,14 +1957,7 @@ class EPUB_MOBI(CatalogPlugin): for author in self.authors: if author[1][0] != current_letter: # Save the old list - author_list = " • ".join(current_author_list) - if len(current_author_list) == self.descriptionClip: - author_list += " …" - - author_list = self.formatNCXText(author_list) - if False and self.verbose: - self.opts.log.info(" adding '%s' to master_author_list" % current_letter) - master_author_list.append((author_list, current_letter)) + add_to_author_list(current_author_list, current_letter) # Start the new list current_letter = author[1][0] @@ -1852,13 +1967,7 @@ class EPUB_MOBI(CatalogPlugin): current_author_list.append(author[0]) # Add the last author list - author_list = " • ".join(current_author_list) - if len(current_author_list) == self.descriptionClip: - author_list += " …" - author_list = self.formatNCXText(author_list) - if False and self.verbose: - self.opts.log.info(" adding '%s' to master_author_list" % current_letter) - master_author_list.append((author_list, current_letter)) + add_to_author_list(current_author_list, current_letter) # Add *article* entries for each populated author initial letter # master_author_list{}: [0]:author list [1]:Initial letter @@ -1893,12 +2002,113 @@ class EPUB_MOBI(CatalogPlugin): self.ncxSoup = soup - def generateNCXByTags(self, tocTitle): + def generateNCXByDateAdded(self, tocTitle): + self.updateProgressFullStep("NCX 'Recently Added'") + + def add_to_master_month_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.generateShortDescription(self.formatNCXText(current_titles_list)) + master_month_list.append((current_titles_list, current_date, book_count)) + + soup = self.ncxSoup + HTML_file = "content/ByDateAdded.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Recently Added' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each populated month + # Loop over the booksByDate list, find start of each month, + # add description_preview_count titles + # master_month_list(list,date,count) + current_titles_list = [] + master_month_list = [] + current_date = self.booksByDate[0]['timestamp'] + + for book in self.booksByDate: + if book['timestamp'].month != current_date.month or \ + book['timestamp'].year != current_date.year: + # Save the old lists + add_to_master_month_list(current_titles_list) + + # Start the new list + current_date = book['timestamp'].date() + current_titles_list = [book['title']] + else: + current_titles_list.append(book['title']) + + # Add the last month list + add_to_master_month_list(current_titles_list) + + # Add *article* entries for each populated month + # master_months_list{}: [0]:titles list [1]:date + for books_by_month in master_month_list: + datestr = strftime(u'%B %Y', books_by_month[1].timetuple()) + navPointByMonthTag = Tag(soup, 'navPoint') + navPointByMonthTag['class'] = "article" + navPointByMonthTag['id'] = "%s-%s-ID" % (books_by_month[1].year,books_by_month[1].month ) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(datestr)) + navLabelTag.insert(0, textTag) + navPointByMonthTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#%s-%s" % (HTML_file, + books_by_month[1].year,books_by_month[1].month) + + navPointByMonthTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_month[0])) + navPointByMonthTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_month[2] if books_by_month[2] > 1 else \ + '%d title' % books_by_month[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByMonthTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByMonthTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncxSoup = soup + + def generateNCXByGenre(self, tocTitle): # Create an NCX section for 'By Genre' # Add each genre as an article # 'tag', 'file', 'authors' - self.opts.log.info(self.updateProgressFullStep("generateNCXByTags()")) + self.updateProgressFullStep("NCX 'Genres'") + + + if not len(self.genres): self.opts.log.warn(" No genres found in tags.\n" @@ -1980,7 +2190,7 @@ class EPUB_MOBI(CatalogPlugin): for title in genre['books']: titles.append(title['title']) titles = sorted(titles, key=lambda x:(self.generateSortTitle(x),self.generateSortTitle(x))) - titles_list = self.generateShortDescription(" • ".join(titles)) + titles_list = self.generateShortDescription(u" • ".join(titles)) cmTag.insert(0, NavigableString(self.formatNCXText(titles_list))) navPointVolumeTag.insert(3, cmTag) @@ -1996,8 +2206,7 @@ class EPUB_MOBI(CatalogPlugin): self.ncxSoup = ncx_soup def writeNCX(self): - - self.opts.log.info(self.updateProgressFullStep("writeNCX()")) + self.updateProgressFullStep("Saving NCX") outfile = open("%s/%s.ncx" % (self.catalogPath, self.basename), 'w') outfile.write(self.ncxSoup.prettify()) @@ -2061,6 +2270,13 @@ class EPUB_MOBI(CatalogPlugin): # Remove the special marker tags from the database's tag list, # return sorted list of tags representing valid genres + def next_tag(tags): + for (i, tag) in enumerate(tags): + if i < len(tags) - 1: + yield tag + ", " + else: + yield tag + filtered_tags = [] for tag in tags: if tag[0] in self.markerTags: @@ -2084,9 +2300,16 @@ class EPUB_MOBI(CatalogPlugin): else: continue if self.verbose: - self.opts.log.info(' %d Genre tags in database (exclude_genre: %s):' % \ + self.opts.log.info(u' %d Genre tags in database (exclude_genre: %s):' % \ (len(filtered_tags), self.opts.exclude_genre)) - self.opts.log.info(' %s' % ', '.join(filtered_tags)) + out_buf = '' + + for tag in next_tag(filtered_tags): + out_buf += tag + if len(out_buf) > 72: + self.opts.log(u' %s' % out_buf.rstrip()) + out_buf = '' + self.opts.log(u' %s' % out_buf) return filtered_tags @@ -2288,9 +2511,28 @@ class EPUB_MOBI(CatalogPlugin): titleTag.insert(0,escape(NavigableString(title))) return soup + def generate_masthead_image(self, out_path): + MI_WIDTH = 600 + MI_HEIGHT = 60 + + try: + from PIL import Image, ImageDraw, ImageFont + Image, ImageDraw, ImageFont + except ImportError: + import Image, ImageDraw, ImageFont + + img = Image.new('RGB', (MI_WIDTH, MI_HEIGHT), 'white') + draw = ImageDraw.Draw(img) + font = ImageFont.truetype(P('fonts/liberation/LiberationSerif-Bold.ttf'), 48) + text = self.title.encode('utf-8') + width, height = draw.textsize(text, font=font) + left = max(int((MI_WIDTH - width)/2.), 0) + top = max(int((MI_HEIGHT - height)/2.), 0) + draw.text((left, top), text, fill=(0,0,0), font=font) + img.save(open(out_path, 'wb'), 'GIF') + def generateShortDescription(self, description): # Truncate the description to description_clip, on word boundaries if necessary - if not description: return None @@ -2302,7 +2544,7 @@ class EPUB_MOBI(CatalogPlugin): # Start adding words until we reach description_clip short_description = "" - words = description.split(" ") + words = description.split() for word in words: short_description += word + " " if len(short_description) > self.descriptionClip: @@ -2322,6 +2564,7 @@ class EPUB_MOBI(CatalogPlugin): for (i,word) in enumerate(title_words): # Leading numbers optionally translated to text equivalent + # Capitalize leading sort word if i==0: if self.opts.numbers_as_text and re.search('[0-9]+',word): translated.append(EPUB_MOBI.NumberToText(word).text.capitalize()) @@ -2339,7 +2582,7 @@ class EPUB_MOBI(CatalogPlugin): word = '%10.2f' % float(re.sub('[^\d\.]','.',word)) except: word = '%10.2f' % float(EPUB_MOBI.NumberToText(word).number_as_float) - translated.append(word) + translated.append(word.capitalize()) else: if re.search('[0-9]+',word): # Coerce standard-width strings for numbers @@ -2400,12 +2643,12 @@ class EPUB_MOBI(CatalogPlugin): self.opts.log.info('%s not implemented' % self.error) def updateProgressFullStep(self, description): - self.current_step += 1 self.progressString = description self.progressInt = float((self.current_step-1)/self.total_steps) - self.reporter(self.progressInt/100., self.progressString) - return u"%.2f%% %s" % (self.progressInt, self.progressString) + self.reporter(self.progressInt, self.progressString) + if self.opts.cli_environment: + self.opts.log(u"%3.0f%% %s" % (self.progressInt*100, self.progressString)) def updateProgressMicroStep(self, description, micro_step_pct): step_range = 100/self.total_steps @@ -2413,45 +2656,49 @@ class EPUB_MOBI(CatalogPlugin): coarse_progress = float((self.current_step-1)/self.total_steps) fine_progress = float((micro_step_pct*step_range)/100) self.progressInt = coarse_progress + fine_progress - self.reporter(self.progressInt/100., self.progressString) - return u"%.2f%% %s" % (self.progressInt, self.progressString) + self.reporter(self.progressInt, self.progressString) def run(self, path_to_output, opts, db, notification=DummyReporter()): - opts.log = log = Log() opts.fmt = self.fmt = path_to_output.rpartition('.')[2] self.opts = opts # Add local options opts.creator = "calibre" - opts.descriptionClip = 250 + op = self.opts.output_profile + if op is None: + op = 'default' + opts.descriptionClip = 380 if op.endswith('dx') or 'kindle' not in op else 90 opts.basename = "Catalog" opts.plugin_path = self.plugin_path + opts.cli_environment = getattr(opts,'sync',True) if opts.verbose: opts_dict = vars(opts) - log("%s:run" % self.name) - log(" path_to_output: %s" % path_to_output) - log(" Output format: %s" % self.fmt) + log("%s(): Generating %s for %s in %s environment" % + (self.name,self.fmt,opts.output_profile, + 'CLI' if opts.cli_environment else 'GUI')) if opts_dict['ids']: log(" Book count: %d" % len(opts_dict['ids'])) # Display opts keys = opts_dict.keys() keys.sort() log(" opts:") + for key in keys: - if key == 'ids': - if opts_dict[key]: - continue - else: - log(" %s: (all)" % key) - log(" %s: %s" % (key, opts_dict[key])) + if key in ['catalog_title','exclude_genre','exclude_tags','note_tag', + 'numbers_as_text','read_tag','search_text','sort_by','sync']: + log(" %s: %s" % (key, opts_dict[key])) # Launch the Catalog builder - catalog = self.CatalogBuilder(db, opts, self, notification=notification) + if opts.verbose: + log.info("Begin generating catalog source") + catalog = self.CatalogBuilder(db, opts, self, report_progress=notification) catalog.createDirectoryStructure() catalog.copyResources() catalog.buildSources() + if opts.verbose: + log.info("Finished generating catalog source\n") recommendations = [] diff --git a/src/calibre/startup.py b/src/calibre/startup.py index 3a761cca10..3e33757f92 100644 --- a/src/calibre/startup.py +++ b/src/calibre/startup.py @@ -84,6 +84,7 @@ if not _run_once: return res os.path.abspath = my_abspath + _join = os.path.join def my_join(a, *p): encoding=sys.getfilesystemencoding() diff --git a/src/calibre/utils/ipc/job.py b/src/calibre/utils/ipc/job.py index 458d5adb8a..a6c39ffc6b 100644 --- a/src/calibre/utils/ipc/job.py +++ b/src/calibre/utils/ipc/job.py @@ -52,10 +52,13 @@ class BaseJob(object): else: self._status_text = _('Error') if self.failed else _('Finished') if DEBUG: - prints('Job:', self.id, self.description, 'finished', + try: + prints('Job:', self.id, self.description, 'finished', safe_encode=True) - prints('\t'.join(self.details.splitlines(True)), + prints('\t'.join(self.details.splitlines(True)), safe_encode=True) + except: + pass if not self._done_called: self._done_called = True try: diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index e76d8aaa8f..d182d856d8 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -274,6 +274,10 @@ class BasicNewsRecipe(Recipe): } ''' + #: By default, calibre will use a default image for the masthead (Kindle only). + #: Override this in your recipe to provide a url to use as a masthead. + masthead_url = None + #: Set to a non empty string to disable this recipe #: The string will be used as the disabled message recipe_disabled = None @@ -294,6 +298,17 @@ class BasicNewsRecipe(Recipe): ''' return getattr(self, 'cover_url', None) + def get_masthead_url(self): + ''' + Return a :term:`URL` to the masthead image for this issue or `None`. + By default it returns the value of the member `self.masthead_url` which + is normally `None`. If you want your recipe to download a masthead for the e-book + override this method in your subclass, or set the member variable `self.masthead_url` + before this method is called. + Masthead images are used in Kindle MOBI files. + ''' + return getattr(self, 'masthead_url', None) + def get_feeds(self): ''' Return a list of :term:`RSS` feeds to fetch for this profile. Each element of the list @@ -543,8 +558,6 @@ class BasicNewsRecipe(Recipe): '--max-recursions', str(self.recursions), '--delay', str(self.delay), ] - if self.encoding is not None: - web2disk_cmdline.extend(['--encoding', self.encoding]) if self.verbose: web2disk_cmdline.append('--verbose') @@ -563,6 +576,7 @@ class BasicNewsRecipe(Recipe): 'preprocess_html', 'remove_tags_after', 'remove_tags_before'): setattr(self.web2disk_options, extra, getattr(self, extra)) self.web2disk_options.postprocess_html = self._postprocess_html + self.web2disk_options.encoding = self.encoding if self.delay > 0: self.simultaneous_downloads = 1 @@ -745,6 +759,23 @@ class BasicNewsRecipe(Recipe): self.report_progress(0, _('Trying to download cover...')) self.download_cover() + self.report_progress(0, _('Generating masthead...')) + self.masthead_path = None + try: + murl = self.get_masthead_url() + except: + self.log.exception('Failed to get masthead url') + murl = None + if murl is not None: + self.download_masthead(murl) + if self.masthead_path is None: + self.masthead_path = os.path.join(self.output_dir, 'mastheadImage.jpg') + try: + self.default_masthead_image(self.masthead_path) + except: + self.log.exception('Failed to generate default masthead image') + self.masthead_path = None + if self.test: feeds = feeds[:2] self.has_single_feed = len(feeds) == 1 @@ -861,6 +892,32 @@ class BasicNewsRecipe(Recipe): self.log.exception('Failed to download cover') self.cover_path = None + def _download_masthead(self, mu): + ext = mu.rpartition('.')[-1] + if '?' in ext: + ext = '' + ext = ext.lower() if ext else 'jpg' + mpath = os.path.join(self.output_dir, 'masthead_source.'+ext) + outfile = os.path.join(self.output_dir, 'mastheadImage.jpg') + if os.access(mu, os.R_OK): + with open(mpath, 'wb') as mfile: + mfile.write(open(mu, 'rb').read()) + else: + with nested(open(mpath, 'wb'), closing(self.browser.open(mu))) as (mfile, r): + mfile.write(r.read()) + self.report_progress(1, _('Masthead image downloaded')) + self.prepare_masthead_image(mpath, outfile) + self.masthead_path = outfile + if os.path.exists(mpath): + os.remove(mpath) + + + def download_masthead(self, url): + try: + self._download_masthead(url) + except: + self.log.exception("Failed to download supplied masthead_url, synthesizing") + def default_cover(self, cover_file): ''' Create a generic cover for recipes that dont have a cover @@ -928,6 +985,9 @@ class BasicNewsRecipe(Recipe): 'Override in subclass to use something other than the recipe title' return self.title + MI_WIDTH = 600 + MI_HEIGHT = 60 + def default_masthead_image(self, out_path): try: from PIL import Image, ImageDraw, ImageFont @@ -935,14 +995,13 @@ class BasicNewsRecipe(Recipe): except ImportError: import Image, ImageDraw, ImageFont - - img = Image.new('RGB', (600, 100), 'white') + img = Image.new('RGB', (self.MI_WIDTH, self.MI_HEIGHT), 'white') draw = ImageDraw.Draw(img) font = ImageFont.truetype(P('fonts/liberation/LiberationSerif-Bold.ttf'), 48) text = self.get_masthead_title().encode('utf-8') width, height = draw.textsize(text, font=font) - left = max(int((600 - width)/2.), 0) - top = max(int((100 - height)/2.), 0) + left = max(int((self.MI_WIDTH - width)/2.), 0) + top = max(int((self.MI_HEIGHT - height)/2.), 0) draw.text((left, top), text, fill=(0,0,0), font=font) img.save(open(out_path, 'wb'), 'JPEG') @@ -964,11 +1023,11 @@ class BasicNewsRecipe(Recipe): raise IOError('Failed to read image from: %s: %s' %(path_to_image, msg)) pw.PixelSetColor(p, 'white') - width, height = pw.MagickGetImageWidth(img),pw.MagickGetImageHeight(img)[1:] - scaled, nwidth, nheight = fit_image(width, height, 600, 100) + width, height = pw.MagickGetImageWidth(img),pw.MagickGetImageHeight(img) + scaled, nwidth, nheight = fit_image(width, height, self.MI_WIDTH, self.MI_HEIGHT) if not pw.MagickNewImage(img2, width, height, p): raise RuntimeError('Out of memory') - if not pw.MagickNewImage(frame, 600, 100, p): + if not pw.MagickNewImage(frame, self.MI_WIDTH, self.MI_HEIGHT, p): raise RuntimeError('Out of memory') if not pw.MagickCompositeImage(img2, img, pw.OverCompositeOp, 0, 0): raise RuntimeError('Out of memory') @@ -976,8 +1035,8 @@ class BasicNewsRecipe(Recipe): if not pw.MagickResizeImage(img2, nwidth, nheight, pw.LanczosFilter, 0.5): raise RuntimeError('Out of memory') - left = int((600 - nwidth)/2.0) - top = int((100 - nheight)/2.0) + left = int((self.MI_WIDTH - nwidth)/2.0) + top = int((self.MI_HEIGHT - nheight)/2.0) if not pw.MagickCompositeImage(frame, img2, pw.OverCompositeOp, left, top): raise RuntimeError('Out of memory') @@ -988,7 +1047,6 @@ class BasicNewsRecipe(Recipe): for x in (img, img2, frame): pw.DestroyMagickWand(x) - def create_opf(self, feeds, dir=None): if dir is None: dir = self.output_dir @@ -1003,11 +1061,22 @@ class BasicNewsRecipe(Recipe): mi.pubdate = datetime.now() opf_path = os.path.join(dir, 'index.opf') ncx_path = os.path.join(dir, 'index.ncx') + opf = OPFCreator(dir, mi) + # Add mastheadImage entry to section + mp = getattr(self, 'masthead_path', None) + if mp is not None and os.access(mp, os.R_OK): + from calibre.ebooks.metadata.opf2 import Guide + ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu()) + ref.type = 'masthead' + ref.title = 'Masthead Image' + opf.guide.append(ref) manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))] manifest.append(os.path.join(dir, 'index.html')) manifest.append(os.path.join(dir, 'index.ncx')) + + # Get cover cpath = getattr(self, 'cover_path', None) if cpath is None: pf = open(os.path.join(dir, 'cover.jpg'), 'wb') @@ -1016,10 +1085,18 @@ class BasicNewsRecipe(Recipe): if cpath is not None and os.access(cpath, os.R_OK): opf.cover = cpath manifest.append(cpath) + + # Get masthead + mpath = getattr(self, 'masthead_path', None) + if mpath is not None and os.access(mpath, os.R_OK): + manifest.append(mpath) + opf.create_manifest_from_files_in(manifest) for mani in opf.manifest: if mani.path.endswith('.ncx'): mani.id = 'ncx' + if mani.path.endswith('mastheadImage.jpg'): + mani.id = 'masthead-image' entries = ['index.html'] toc = TOC(base_path=dir)