From 04d8e251c5bfb9763f75e529a62e75f18551712f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 May 2009 07:25:58 -0700 Subject: [PATCH 01/15] Fix for bug in windows detection of BeBook --- src/calibre/devices/scanner.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index a93ef54c32..c73b1cd2f6 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -47,8 +47,8 @@ class DeviceScanner(object): rev = ('rev_%4.4x'%c).replace('a', ':') if rev in device_id: return True - return False - + return False + def test_bcd(self, bcdDevice, bcd): if bcd is None or len(bcd) == 0: return True @@ -56,19 +56,20 @@ class DeviceScanner(object): if c == bcdDevice: return True return False - + def is_device_connected(self, device): vendor_ids = device.VENDOR_ID if hasattr(device.VENDOR_ID, '__len__') else [device.VENDOR_ID] product_ids = device.PRODUCT_ID if hasattr(device.PRODUCT_ID, '__len__') else [device.PRODUCT_ID] if iswindows: - for vendor_id, product_id in zip(vendor_ids, product_ids): - vid, pid = 'vid_%4.4x'%vendor_id, 'pid_%4.4x'%product_id - vidd, pidd = 'vid_%i'%vendor_id, 'pid_%i'%product_id - for device_id in self.devices: - if (vid in device_id or vidd in device_id) and (pid in device_id or pidd in device_id): - if self.test_bcd_windows(device_id, getattr(device, 'BCD', None)): - if device.can_handle(device_id): - return True + for vendor_id in vendor_ids: + for product_id in product_ids: + vid, pid = 'vid_%4.4x'%vendor_id, 'pid_%4.4x'%product_id + vidd, pidd = 'vid_%i'%vendor_id, 'pid_%i'%product_id + for device_id in self.devices: + if (vid in device_id or vidd in device_id) and (pid in device_id or pidd in device_id): + if self.test_bcd_windows(device_id, getattr(device, 'BCD', None)): + if device.can_handle(device_id): + return True else: for vendor, product, bcdDevice in self.devices: if vendor in vendor_ids and product in product_ids: From 5ff51fc3f9c97ce1403d4cf51cebd308e81dc5b6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 May 2009 12:16:14 -0700 Subject: [PATCH 02/15] New recipes for index.hu and pcworld.hu by Ezmegaz --- src/calibre/trac/plugins/download.py | 2 +- src/calibre/trac/plugins/templates/linux.html | 2 +- src/calibre/web/feeds/recipes/__init__.py | 2 +- src/calibre/web/feeds/recipes/recipe_blic.py | 28 ++---- .../web/feeds/recipes/recipe_index_hu.py | 20 +++++ src/calibre/web/feeds/recipes/recipe_nin.py | 14 +-- .../web/feeds/recipes/recipe_pcworld_hu.py | 22 +++++ .../web/feeds/recipes/recipe_pobjeda.py | 18 ++-- .../recipes/recipe_st_petersburg_times.py | 87 ++++++++++--------- .../web/feeds/recipes/recipe_vijesti.py | 29 ++++--- src/calibre/web/feeds/recipes/recipe_vreme.py | 10 ++- 11 files changed, 144 insertions(+), 90 deletions(-) create mode 100644 src/calibre/web/feeds/recipes/recipe_index_hu.py create mode 100644 src/calibre/web/feeds/recipes/recipe_pcworld_hu.py diff --git a/src/calibre/trac/plugins/download.py b/src/calibre/trac/plugins/download.py index 03a6676e7b..dd25279071 100644 --- a/src/calibre/trac/plugins/download.py +++ b/src/calibre/trac/plugins/download.py @@ -20,7 +20,7 @@ DEPENDENCIES = [ ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'), ('dnspython', '1.6.0', 'dnspython', 'dnspython', 'dnspython', 'dnspython'), ('poppler', '0.10.5', 'poppler', 'poppler', 'poppler', 'poppler'), - ('pdftk', '1.12', 'pdftk', 'pdftk', 'pdftk', 'pdftk'), + ('podofo', '0.7', 'podofo', 'podofo', 'podofo', 'podofo'), ] diff --git a/src/calibre/trac/plugins/templates/linux.html b/src/calibre/trac/plugins/templates/linux.html index 96881aa108..ffaa1d8394 100644 --- a/src/calibre/trac/plugins/templates/linux.html +++ b/src/calibre/trac/plugins/templates/linux.html @@ -49,7 +49,7 @@

${app} is available in the software repositories of the following - linux distributions: + supported linux distributions: diff --git a/src/calibre/web/feeds/recipes/__init__.py b/src/calibre/web/feeds/recipes/__init__.py index 48e5d9e720..4d2adfb1c0 100644 --- a/src/calibre/web/feeds/recipes/__init__.py +++ b/src/calibre/web/feeds/recipes/__init__.py @@ -42,7 +42,7 @@ recipe_modules = ['recipe_' + r for r in ( 'moneynews', 'der_standard', 'diepresse', 'nzz_ger', 'hna', 'seattle_times', 'scott_hanselman', 'coding_horror', 'twitchfilms', 'stackoverflow', 'telepolis_artikel', 'zaobao', 'usnews', - 'straitstimes', + 'straitstimes', 'index_hu', 'pcworld_hu', )] import re, imp, inspect, time, os diff --git a/src/calibre/web/feeds/recipes/recipe_blic.py b/src/calibre/web/feeds/recipes/recipe_blic.py index e4e4987dec..e212e73218 100644 --- a/src/calibre/web/feeds/recipes/recipe_blic.py +++ b/src/calibre/web/feeds/recipes/recipe_blic.py @@ -16,12 +16,14 @@ class Blic(BasicNewsRecipe): description = 'Blic.co.yu online verzija najtiraznije novine u Srbiji donosi najnovije vesti iz Srbije i sveta, komentare, politicke analize, poslovne i ekonomske vesti, vesti iz regiona, intervjue, informacije iz kulture, reportaze, pokriva sve sportske dogadjaje, detaljan tv program, nagradne igre, zabavu, fenomenalni Blic strip, dnevni horoskop, arhivu svih dogadjaja' publisher = 'RINGIER d.o.o.' category = 'news, politics, Serbia' + delay = 1 oldest_article = 2 max_articles_per_feed = 100 remove_javascript = True no_stylesheets = True use_embedded_content = False language = _('Serbian') + lang = 'sr-Latn-RS' extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif} ' html2lrf_options = [ @@ -45,26 +47,14 @@ class Blic(BasicNewsRecipe): start_url, question, rest_url = url.partition('?') return u'http://www.blic.rs/_print.php?' + rest_url - def cleanup_image_tags(self,soup): - for item in soup.findAll('img'): - for attrib in ['height','width','border','align']: - if item.has_key(attrib): - del item[attrib] - oldParent = item.parent - myIndex = oldParent.contents.index(item) - item.extract() - divtag = Tag(soup,'div') - brtag = Tag(soup,'br') - oldParent.insert(myIndex,divtag) - divtag.append(item) - divtag.append(brtag) - return soup - - def preprocess_html(self, soup): - mtag = '' - soup.head.insert(0,mtag) + mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)]) + soup.head.insert(0,mlang) for item in soup.findAll(style=True): del item['style'] - return self.cleanup_image_tags(soup) + return self.adeify_images(soup) + + def get_article_url(self, article): + raw = article.get('link', None) + return raw.replace('.co.yu','.rs') \ No newline at end of file diff --git a/src/calibre/web/feeds/recipes/recipe_index_hu.py b/src/calibre/web/feeds/recipes/recipe_index_hu.py new file mode 100644 index 0000000000..8b36500e5c --- /dev/null +++ b/src/calibre/web/feeds/recipes/recipe_index_hu.py @@ -0,0 +1,20 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class Index(BasicNewsRecipe): + + title = u'INDEX.HU' + oldest_article = 3 + max_articles_per_feed = 50 + language = _('Hungarian') + __author__ = 'Ezmegaz' + + feeds = [(u'ALL', u'http://index.hu/24ora/rss/'), + (u'BELF\xd6LD', u'http://index.hu/belfold/rss/default/'), + (u'K\xdcLF\xd6LD', u'http://index.hu/kulfold/rss/default/'), + (u'BULV\xc1R', u'http://index.hu/bulvar/rss/default/'), + (u'GAZDAS\xc1G', u'http://index.hu/gazdasag/rss/default/'), + (u'TECH', u'http://index.hu/tech/rss/main/'), + (u'KULT\xdaRA', u'http://index.hu/kultur/rss/main/'), + (u'TUDOM\xc1NY', u'http://index.hu/tudomany/rss/main/'), + (u'V\xc9LEM\xc9NY', u'http://index.hu/velemeny/rss/default/')] + diff --git a/src/calibre/web/feeds/recipes/recipe_nin.py b/src/calibre/web/feeds/recipes/recipe_nin.py index fe1e97e8b8..4de53a1049 100644 --- a/src/calibre/web/feeds/recipes/recipe_nin.py +++ b/src/calibre/web/feeds/recipes/recipe_nin.py @@ -8,12 +8,13 @@ nin.co.rs import re, urllib from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class Nin(BasicNewsRecipe): title = 'NIN online' __author__ = 'Darko Miletic' description = 'Nedeljne informativne novine' - publisher = 'NIN' + publisher = 'NIN D.O.O.' category = 'news, politics, Serbia' no_stylesheets = True oldest_article = 15 @@ -28,9 +29,9 @@ class Nin(BasicNewsRecipe): remove_javascript = True use_embedded_content = False language = _('Serbian') - lang = 'sr-RS' + lang = 'sr-Latn-RS' direction = 'ltr' - extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}' + extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif} .artTitle{font-size: x-large; font-weight: bold} .columnhead{font-size: small; font-weight: bold}' html2lrf_options = [ '--comment' , description @@ -70,9 +71,10 @@ class Nin(BasicNewsRecipe): def preprocess_html(self, soup): soup.html['lang'] = self.lang soup.html['dir' ] = self.direction - mtag = '' - mtag += '\n' - soup.head.insert(0,mtag) + mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)]) + mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=UTF-8")]) + soup.head.insert(0,mlang) + soup.head.insert(1,mcharset) for item in soup.findAll(style=True): del item['style'] return soup diff --git a/src/calibre/web/feeds/recipes/recipe_pcworld_hu.py b/src/calibre/web/feeds/recipes/recipe_pcworld_hu.py new file mode 100644 index 0000000000..ad1f1df72a --- /dev/null +++ b/src/calibre/web/feeds/recipes/recipe_pcworld_hu.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Index(BasicNewsRecipe): + + + title = u'PCWORLD.HU' + oldest_article = 3 + max_articles_per_feed = 50 + language = _('Hungarian') + __author__ = 'Ezmegaz' + + + feeds = [(u'H\xedrek', u'http://pcworld.hu/rss/rss.xml'), (u'Hardver h\xedrek', u'http://www.pcworld.hu/rss/rss_hardverhirek.xml'), (u'Szoftver h\xedrek', u'http://www.pcworld.hu/rss/rss_szoftverhirek.xml'), (u'Hardver cikkek', u'http://www.pcworld.hu/rss/rss_hardvercikkek.xml'), (u'Szoftver cikkek', u'http://www.pcworld.hu/rss/rss_szoftvercikkek.xml'), (u'Mobil h\xedrek', u'http://www.pcworld.hu/rss/rss_mobil.xml'), (u'\xdczleti h\xedrek', u'http://www.pcworld.hu/rss/rss_uzlet.xml'), (u'Let\xf6lt\xe9sek', u'http://www.pcworld.hu/rss/rss_letoltes.xml'), (u'PC World TV', u'http://tv.pcworld.hu/rss/rss_hun_pcw.xml'), (u'Tudta-e...?', u'http://pcworld.hu/rss/rss_tudtae.xml')] + diff --git a/src/calibre/web/feeds/recipes/recipe_pobjeda.py b/src/calibre/web/feeds/recipes/recipe_pobjeda.py index 5afb2b3f6a..6078e6ba0a 100644 --- a/src/calibre/web/feeds/recipes/recipe_pobjeda.py +++ b/src/calibre/web/feeds/recipes/recipe_pobjeda.py @@ -10,6 +10,7 @@ pobjeda.co.me import re from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class Pobjeda(BasicNewsRecipe): title = 'Pobjeda Online' @@ -22,12 +23,13 @@ class Pobjeda(BasicNewsRecipe): encoding = 'utf8' remove_javascript = True use_embedded_content = False + language = _('Serbian') + lang = 'sr-Latn-Me' INDEX = u'http://www.pobjeda.co.me' - extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: serif1, serif}' + extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif}' html2lrf_options = [ '--comment', description - , '--base-font-size', '10' , '--category', category , '--publisher', publisher ] @@ -59,11 +61,13 @@ class Pobjeda(BasicNewsRecipe): ] def preprocess_html(self, soup): - soup.html['xml:lang'] = 'sr-Latn-ME' - soup.html['lang'] = 'sr-Latn-ME' - mtag = '' - soup.head.insert(0,mtag) - return soup + soup.html['xml:lang'] = self.lang + soup.html['lang'] = self.lang + mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)]) + mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=UTF-8")]) + soup.head.insert(0,mlang) + soup.head.insert(1,mcharset) + return self.adeify_images(soup) def get_cover_url(self): cover_url = None diff --git a/src/calibre/web/feeds/recipes/recipe_st_petersburg_times.py b/src/calibre/web/feeds/recipes/recipe_st_petersburg_times.py index 8c22262904..cc023448c7 100644 --- a/src/calibre/web/feeds/recipes/recipe_st_petersburg_times.py +++ b/src/calibre/web/feeds/recipes/recipe_st_petersburg_times.py @@ -1,39 +1,48 @@ -#!/usr/bin/env python - -__license__ = 'GPL v3' -__copyright__ = '2008, Darko Miletic ' -''' -sptimes.ru -''' - -from calibre import strftime -from calibre.web.feeds.news import BasicNewsRecipe - -class PetersburgTimes(BasicNewsRecipe): - title = u'The St. Petersburg Times' - __author__ = 'Darko Miletic' - description = 'News from Russia' - oldest_article = 7 - max_articles_per_feed = 100 - no_stylesheets = True - use_embedded_content = False - language = _('English') - INDEX = 'http://www.sptimes.ru' - - def parse_index(self): - articles = [] - soup = self.index_to_soup(self.INDEX) - - for item in soup.findAll('a', attrs={'class':'story_link_o'}): - if item.has_key('href'): - url = self.INDEX + item['href'].replace('action_id=2','action_id=100') - title = self.tag_to_string(item) - c_date = strftime('%A, %d %B, %Y') - description = '' - articles.append({ - 'title':title, - 'date':c_date, - 'url':url, - 'description':description - }) - return [(soup.head.title.string, articles)] +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2009, Darko Miletic ' + +''' +sptimes.ru +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class PetersburgTimes(BasicNewsRecipe): + title = 'The St. Petersburg Times' + __author__ = 'Darko Miletic' + description = 'News from Russia' + publisher = 'sptimes.ru' + category = 'news, politics, Russia' + max_articles_per_feed = 100 + no_stylesheets = True + remove_javascript = True + encoding = 'cp1251' + use_embedded_content = False + language = _('English') + + html2lrf_options = [ + '--comment', description + , '--category', category + , '--publisher', publisher + , '--ignore-tables' + ] + + html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' + + remove_tags = [dict(name=['object','link','embed'])] + + feeds = [(u'Headlines', u'http://sptimes.ru/headlines.php' )] + + def preprocess_html(self, soup): + return self.adeify_images(soup) + + def get_article_url(self, article): + raw = article.get('guid', None) + return raw + + def print_version(self, url): + start_url, question, article_id = url.rpartition('/') + return u'http://www.sptimes.ru/index.php?action_id=100&story_id=' + article_id + \ No newline at end of file diff --git a/src/calibre/web/feeds/recipes/recipe_vijesti.py b/src/calibre/web/feeds/recipes/recipe_vijesti.py index 9923193d7b..9ef32e636c 100644 --- a/src/calibre/web/feeds/recipes/recipe_vijesti.py +++ b/src/calibre/web/feeds/recipes/recipe_vijesti.py @@ -9,6 +9,7 @@ vijesti.me import re from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class Vijesti(BasicNewsRecipe): title = 'Vijesti' @@ -16,8 +17,8 @@ class Vijesti(BasicNewsRecipe): description = 'News from Montenegro' publisher = 'Daily Press Vijesti' category = 'news, politics, Montenegro' - oldest_article = 1 - max_articles_per_feed = 100 + oldest_article = 2 + max_articles_per_feed = 150 no_stylesheets = True remove_javascript = True encoding = 'cp1250' @@ -25,7 +26,8 @@ class Vijesti(BasicNewsRecipe): remove_javascript = True use_embedded_content = False language = _('Serbian') - extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}' + lang ='sr-Latn-Me' + extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}' html2lrf_options = [ '--comment', description @@ -44,12 +46,15 @@ class Vijesti(BasicNewsRecipe): feeds = [(u'Sve vijesti', u'http://www.vijesti.me/rss.php' )] def preprocess_html(self, soup): - soup.html['xml:lang'] = 'sr-Latn-ME' - soup.html['lang'] = 'sr-Latn-ME' - mtag = '' - soup.head.insert(0,mtag) - for item in soup.findAll('img'): - if item.has_key('align'): - del item['align'] - item.insert(0,'

') - return soup + soup.html['xml:lang'] = self.lang + soup.html['lang'] = self.lang + mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)]) + mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=UTF-8")]) + soup.head.insert(0,mlang) + soup.head.insert(1,mcharset) + return self.adeify_images(soup) + + def get_article_url(self, article): + raw = article.get('link', None) + return raw.replace('.cg.yu','.me') + \ No newline at end of file diff --git a/src/calibre/web/feeds/recipes/recipe_vreme.py b/src/calibre/web/feeds/recipes/recipe_vreme.py index 1df953cae3..bcc7a14407 100644 --- a/src/calibre/web/feeds/recipes/recipe_vreme.py +++ b/src/calibre/web/feeds/recipes/recipe_vreme.py @@ -9,6 +9,7 @@ vreme.com import re from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class Vreme(BasicNewsRecipe): title = 'Vreme' @@ -27,7 +28,7 @@ class Vreme(BasicNewsRecipe): language = _('Serbian') lang = 'sr-Latn-RS' direction = 'ltr' - extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: serif1, serif}' + extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif} .heading1{font-size: x-large; font-weight: bold} .heading2{font-size: large; font-weight: bold} .toc-heading{font-size: small}' html2lrf_options = [ '--comment' , description @@ -89,9 +90,10 @@ class Vreme(BasicNewsRecipe): del item['size'] soup.html['lang'] = self.lang soup.html['dir' ] = self.direction - mtag = '' - mtag += '\n' - soup.head.insert(0,mtag) + mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)]) + mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=UTF-8")]) + soup.head.insert(0,mlang) + soup.head.insert(1,mcharset) return soup def get_cover_url(self): From 1a1cf7f1b98d3fb8a11725ff23b11d7499681294 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 May 2009 22:13:29 -0700 Subject: [PATCH 03/15] Update cssutils to latest --- src/cssutils/__init__.py | 9 +- src/cssutils/css/cssmediarule.py | 13 +- src/cssutils/css/cssstyledeclaration.py | 4 +- src/cssutils/css/cssvalue.py | 9 +- src/cssutils/css/property.py | 122 +++-- src/cssutils/css/selector.py | 13 +- src/cssutils/cssproductions.py | 6 +- src/cssutils/errorhandler.py | 8 +- src/cssutils/helper.py | 5 +- src/cssutils/profiles.py | 636 +++++++++++++----------- src/cssutils/serialize.py | 24 +- src/cssutils/stylesheets/mediaquery.py | 10 +- src/cssutils/util.py | 3 +- 13 files changed, 503 insertions(+), 359 deletions(-) diff --git a/src/cssutils/__init__.py b/src/cssutils/__init__.py index 846ab3b397..467d17238b 100644 --- a/src/cssutils/__init__.py +++ b/src/cssutils/__init__.py @@ -70,11 +70,11 @@ Usage may be:: __all__ = ['css', 'stylesheets', 'CSSParser', 'CSSSerializer'] __docformat__ = 'restructuredtext' __author__ = 'Christof Hoeke with contributions by Walter Doerwald' -__date__ = '$LastChangedDate:: 2009-02-16 12:05:02 -0800 #$:' +__date__ = '$LastChangedDate:: 2009-05-09 13:59:54 -0700 #$:' -VERSION = '0.9.6a1' +VERSION = '0.9.6a5' -__version__ = '%s $Id: __init__.py 1669 2009-02-16 20:05:02Z cthedot $' % VERSION +__version__ = '%s $Id: __init__.py 1747 2009-05-09 20:59:54Z cthedot $' % VERSION import codec import xml.dom @@ -92,6 +92,9 @@ from parse import CSSParser from serialize import CSSSerializer ser = CSSSerializer() +from profiles import Profiles +profile = Profiles(log=log) + # used by Selector defining namespace prefix '*' _ANYNS = -1 diff --git a/src/cssutils/css/cssmediarule.py b/src/cssutils/css/cssmediarule.py index 3867f99c16..d4b82af600 100644 --- a/src/cssutils/css/cssmediarule.py +++ b/src/cssutils/css/cssmediarule.py @@ -1,7 +1,7 @@ """CSSMediaRule implements DOM Level 2 CSS CSSMediaRule.""" __all__ = ['CSSMediaRule'] __docformat__ = 'restructuredtext' -__version__ = '$Id: cssmediarule.py 1641 2009-01-13 21:05:37Z cthedot $' +__version__ = '$Id: cssmediarule.py 1743 2009-05-09 20:33:15Z cthedot $' import cssrule import cssutils @@ -131,8 +131,15 @@ class CSSMediaRule(cssrule.CSSRule): mediaendonly=True, separateEnd=True) nonetoken = self._nexttoken(tokenizer, None) - if (u'}' != self._tokenvalue(braceOrEOF) and - 'EOF' != self._type(braceOrEOF)): + if 'EOF' == self._type(braceOrEOF): + # HACK!!! + # TODO: Not complete, add EOF to rule and } to @media + cssrulestokens.append(braceOrEOF) + braceOrEOF = ('CHAR', '}', 0, 0) + self._log.debug(u'CSSMediaRule: Incomplete, adding "}".', + token=braceOrEOF, neverraise=True) + + if u'}' != self._tokenvalue(braceOrEOF): self._log.error(u'CSSMediaRule: No "}" found.', token=braceOrEOF) elif nonetoken: diff --git a/src/cssutils/css/cssstyledeclaration.py b/src/cssutils/css/cssstyledeclaration.py index c766151116..ae106341bb 100644 --- a/src/cssutils/css/cssstyledeclaration.py +++ b/src/cssutils/css/cssstyledeclaration.py @@ -51,7 +51,7 @@ TODO: """ __all__ = ['CSSStyleDeclaration', 'Property'] __docformat__ = 'restructuredtext' -__version__ = '$Id: cssstyledeclaration.py 1658 2009-02-07 18:24:40Z cthedot $' +__version__ = '$Id: cssstyledeclaration.py 1710 2009-04-18 15:46:20Z cthedot $' from cssproperties import CSS2Properties from property import Property @@ -613,7 +613,7 @@ class CSSStyleDeclaration(CSS2Properties, cssutils.util.Base2): except IndexError: return u'' - length = property(lambda self: len(self.__nnames()), + length = property(lambda self: len(list(self.__nnames())), doc="(DOM) The number of distinct properties that have been explicitly " "in this declaration block. The range of valid indices is 0 to " "length-1 inclusive. These are properties with a different ``name`` " diff --git a/src/cssutils/css/cssvalue.py b/src/cssutils/css/cssvalue.py index 856e42e5c1..9b5a8a1aef 100644 --- a/src/cssutils/css/cssvalue.py +++ b/src/cssutils/css/cssvalue.py @@ -7,10 +7,9 @@ """ __all__ = ['CSSValue', 'CSSPrimitiveValue', 'CSSValueList', 'RGBColor'] __docformat__ = 'restructuredtext' -__version__ = '$Id: cssvalue.py 1638 2009-01-13 20:39:33Z cthedot $' +__version__ = '$Id: cssvalue.py 1684 2009-03-01 18:26:21Z cthedot $' from cssutils.prodparser import * -from cssutils.profiles import profiles import cssutils import cssutils.helper import re @@ -121,7 +120,8 @@ class CSSValue(cssutils.util._NewBase): # special case IE only expression Prod(name='expression', match=lambda t, v: t == self._prods.FUNCTION and - cssutils.helper.normalize(v) == 'expression(', + cssutils.helper.normalize(v) in (u'expression(', + u'alpha('), nextSor=nextSor, toSeq=lambda t, tokens: (ExpressionValue.name, ExpressionValue(cssutils.helper.pushtoken(t, @@ -968,7 +968,8 @@ class RGBColor(CSSPrimitiveValue): class ExpressionValue(CSSFunction): - """Special IE only CSSFunction which may contain *anything*.""" + """Special IE only CSSFunction which may contain *anything*. + Used for expressions and ``alpha(opacity=100)`` currently""" name = u'Expression (IE only)' def _productiondefinition(self): diff --git a/src/cssutils/css/property.py b/src/cssutils/css/property.py index 04a5e3c0eb..c096fa767d 100644 --- a/src/cssutils/css/property.py +++ b/src/cssutils/css/property.py @@ -1,10 +1,9 @@ """Property is a single CSS property in a CSSStyleDeclaration.""" __all__ = ['Property'] __docformat__ = 'restructuredtext' -__version__ = '$Id: property.py 1664 2009-02-07 22:47:09Z cthedot $' +__version__ = '$Id: property.py 1685 2009-03-01 18:26:48Z cthedot $' from cssutils.helper import Deprecated -from cssutils.profiles import profiles from cssvalue import CSSValue import cssutils import xml.dom @@ -67,6 +66,7 @@ class Property(cssutils.util.Base): self._mediaQuery = _mediaQuery self._parent = _parent + self.__nametoken = None self._name = u'' self._literalname = u'' if name: @@ -193,6 +193,7 @@ class Property(cssutils.util.Base): # define a token for error logging if isinstance(name, list): token = name[0] + self.__nametoken = token else: token = None @@ -208,9 +209,9 @@ class Property(cssutils.util.Base): self.seqs[0] = newseq # # validate - if self._name not in profiles.knownnames: + if self._name not in cssutils.profile.knownNames: # self.valid = False - self._log.warn(u'Property: Unknown Property.', + self._log.warn(u'Property: Unknown Property name.', token=token, neverraise=True) else: pass @@ -354,7 +355,7 @@ class Property(cssutils.util.Base): # validate priority if self._priority not in (u'', u'important'): self._log.error(u'Property: No CSS priority value: %r.' % - self._priority) + self._priority) priority = property(lambda self: self._priority, _setPriority, doc="Priority of this property.") @@ -362,42 +363,101 @@ class Property(cssutils.util.Base): literalpriority = property(lambda self: self._literalpriority, doc="Readonly literal (not normalized) priority of this property") - def validate(self, profile=None): - """Validate value against `profile`. + def validate(self, profiles=None): + """Validate value against `profiles`. - :param profile: - A profile name used for validating. If no `profile` is given - ``Property.profiles + :param profiles: + A list of profile names used for validating. If no `profiles` + is given ``cssutils.profile.defaultProfiles`` is used + + For each of the following cases a message is reported: + + - INVALID (so the property is known but not valid) + ``ERROR Property: Invalid value for "{PROFILE-1[/PROFILE-2...]" + property: ...`` + + - VALID but not in given profiles or defaultProfiles + ``WARNING Property: Not valid for profile "{PROFILE-X}" but valid + "{PROFILE-Y}" property: ...`` + + - VALID in current profile + ``DEBUG Found valid "{PROFILE-1[/PROFILE-2...]" property...`` + + - UNKNOWN property + ``WARNING Unknown Property name...`` is issued + + so for example:: + + cssutils.log.setLevel(logging.DEBUG) + parser = cssutils.CSSParser() + s = parser.parseString('''body { + unknown-property: x; + color: 4; + color: rgba(1,2,3,4); + color: red + }''') + + # Log output: + + WARNING Property: Unknown Property name. [2:9: unknown-property] + ERROR Property: Invalid value for "CSS Color Module Level 3/CSS Level 2.1" property: 4 [3:9: color] + DEBUG Property: Found valid "CSS Color Module Level 3" value: rgba(1, 2, 3, 4) [4:9: color] + DEBUG Property: Found valid "CSS Level 2.1" value: red [5:9: color] + + + and when setting an explicit default profile:: + + cssutils.profile.defaultProfiles = cssutils.profile.CSS_LEVEL_2 + s = parser.parseString('''body { + unknown-property: x; + color: 4; + color: rgba(1,2,3,4); + color: red + }''') + + # Log output: + + WARNING Property: Unknown Property name. [2:9: unknown-property] + ERROR Property: Invalid value for "CSS Color Module Level 3/CSS Level 2.1" property: 4 [3:9: color] + WARNING Property: Not valid for profile "CSS Level 2.1" but valid "CSS Color Module Level 3" value: rgba(1, 2, 3, 4) [4:9: color] + DEBUG Property: Found valid "CSS Level 2.1" value: red [5:9: color] """ valid = False if self.name and self.value: - if profile is None: - usedprofile = cssutils.profiles.defaultprofile - else: - usedprofile = profile - - if self.name in profiles.knownnames: - valid, validprofiles = profiles.validateWithProfile(self.name, - self.value, - usedprofile) + if self.name in cssutils.profile.knownNames: + # add valid, matching, validprofiles... + valid, matching, validprofiles = \ + cssutils.profile.validateWithProfile(self.name, + self.value, + profiles) + if not valid: - self._log.error(u'Property: Invalid value for "%s" property: %s: %s' - % (u'/'.join(validprofiles), - self.name, + self._log.error(u'Property: Invalid value for ' + u'"%s" property: %s' + % (u'/'.join(validprofiles), self.value), + token=self.__nametoken, + neverraise=True) + + # TODO: remove logic to profiles! + elif valid and not matching:#(profiles and profiles not in validprofiles): + if not profiles: + notvalidprofiles = u'/'.join(cssutils.profile.defaultProfiles) + else: + notvalidprofiles = profiles + self._log.warn(u'Property: Not valid for profile "%s" ' + u'but valid "%s" value: %s ' + % (notvalidprofiles, u'/'.join(validprofiles), self.value), + token = self.__nametoken, neverraise=True) - elif valid and (usedprofile and usedprofile not in validprofiles): - self._log.warn(u'Property: Not valid for profile "%s": %s: %s' - % (usedprofile, self.name, self.value), - neverraise=True) + valid = False - if valid: - self._log.info(u'Property: Found valid "%s" property: %s: %s' - % (u'/'.join(validprofiles), - self.name, - self.value), + elif valid: + self._log.debug(u'Property: Found valid "%s" value: %s' + % (u'/'.join(validprofiles), self.value), + token = self.__nametoken, neverraise=True) if self._priority not in (u'', u'important'): diff --git a/src/cssutils/css/selector.py b/src/cssutils/css/selector.py index c3120f29d2..a2191e548d 100644 --- a/src/cssutils/css/selector.py +++ b/src/cssutils/css/selector.py @@ -7,7 +7,7 @@ TODO """ __all__ = ['Selector'] __docformat__ = 'restructuredtext' -__version__ = '$Id: selector.py 1638 2009-01-13 20:39:33Z cthedot $' +__version__ = '$Id: selector.py 1741 2009-05-09 18:20:20Z cthedot $' from cssutils.util import _SimpleNamespaces import cssutils @@ -701,6 +701,14 @@ class Selector(cssutils.util.Base2): u'Selector: Unexpected negation.', token=token) return expected + def _atkeyword(expected, seq, token, tokenizer=None): + "invalidates selector" + new['wellformed'] = False + self._log.error( + u'Selector: Unexpected ATKEYWORD.', token=token) + return expected + + # expected: only|not or mediatype, mediatype, feature, and newseq = self._tempSeq() @@ -727,7 +735,8 @@ class Selector(cssutils.util.Base2): 'INCLUDES': _attcombinator, 'S': _S, - 'COMMENT': _COMMENT}) + 'COMMENT': _COMMENT, + 'ATKEYWORD': _atkeyword}) wellformed = wellformed and new['wellformed'] # post condition diff --git a/src/cssutils/cssproductions.py b/src/cssutils/cssproductions.py index 53cb0e0b31..90155539a0 100644 --- a/src/cssutils/cssproductions.py +++ b/src/cssutils/cssproductions.py @@ -12,14 +12,14 @@ open issues """ __all__ = ['CSSProductions', 'MACROS', 'PRODUCTIONS'] __docformat__ = 'restructuredtext' -__version__ = '$Id: cssproductions.py 1537 2008-12-03 14:37:10Z cthedot $' +__version__ = '$Id: cssproductions.py 1738 2009-05-02 13:03:28Z cthedot $' # a complete list of css3 macros MACROS = { 'nonascii': r'[^\0-\177]', 'unicode': r'\\[0-9a-f]{1,6}(?:{nl}|{s})?', - # 'escape': r'{unicode}|\\[ -~\200-\4177777]', - 'escape': r'{unicode}|\\[ -~\200-\777]', + #'escape': r'{unicode}|\\[ -~\200-\777]', + 'escape': r'{unicode}|\\[^\n\r\f0-9a-f]', 'nmstart': r'[_a-zA-Z]|{nonascii}|{escape}', 'nmchar': r'[-_a-zA-Z0-9]|{nonascii}|{escape}', 'string1': r'"([^\n\r\f\\"]|\\{nl}|{escape})*"', diff --git a/src/cssutils/errorhandler.py b/src/cssutils/errorhandler.py index 5e7a2f83f7..aecf3e5fb1 100644 --- a/src/cssutils/errorhandler.py +++ b/src/cssutils/errorhandler.py @@ -16,7 +16,7 @@ log """ __all__ = ['ErrorHandler'] __docformat__ = 'restructuredtext' -__version__ = '$Id: errorhandler.py 1560 2008-12-14 16:13:16Z cthedot $' +__version__ = '$Id: errorhandler.py 1728 2009-05-01 20:35:25Z cthedot $' from helper import Deprecated import logging @@ -89,9 +89,9 @@ class _ErrorHandler(object): elif issubclass(error, xml.dom.DOMException): error.line = line error.col = col - raise error(msg) - else: - raise error(msg) +# raise error(msg, line, col) +# else: + raise error(msg) else: self._logcall(msg) diff --git a/src/cssutils/helper.py b/src/cssutils/helper.py index 19d77ed27a..912d65d5e9 100644 --- a/src/cssutils/helper.py +++ b/src/cssutils/helper.py @@ -68,6 +68,9 @@ def string(value): u'\f', u'\\c ').replace( u'"', u'\\"') + if value.endswith(u'\\'): + value = value[:-1] + u'\\\\' + return u'"%s"' % value def stringvalue(string): @@ -77,7 +80,7 @@ def stringvalue(string): ``'a \'string'`` => ``a 'string`` """ - return string.replace('\\'+string[0], string[0])[1:-1] + return string.replace(u'\\'+string[0], string[0])[1:-1] _match_forbidden_in_uri = re.compile(ur'''.*?[\(\)\s\;,'"]''', re.U).match def uri(value): diff --git a/src/cssutils/profiles.py b/src/cssutils/profiles.py index 2392da6161..78fb43468d 100644 --- a/src/cssutils/profiles.py +++ b/src/cssutils/profiles.py @@ -1,41 +1,340 @@ -"""CSS profiles. - -css2 is based on cssvalues - contributed by Kevin D. Smith, thanks! - - "cssvalues" is used as a property validator. - it is an importable object that contains a dictionary of compiled regular - expressions. The keys of this dictionary are all of the valid CSS property - names. The values are compiled regular expressions that can be used to - validate the values for that property. (Actually, the values are references - to the 'match' method of a compiled regular expression, so that they are - simply called like functions.) +"""CSS profiles. +Profiles is based on code by Kevin D. Smith, orginally used as cssvalues, +thanks! """ -__all__ = ['profiles'] +__all__ = ['Profiles'] __docformat__ = 'restructuredtext' __version__ = '$Id: cssproperties.py 1116 2008-03-05 13:52:23Z cthedot $' -import cssutils import re +class NoSuchProfileException(Exception): + """Raised if no profile with given name is found""" + pass + + +class Profiles(object): + """ + All profiles used for validation. ``cssutils.profile`` is a + preset object of this class and used by all properties for validation. + + Predefined profiles are (use + :meth:`~cssutils.profiles.Profiles.propertiesByProfile` to + get a list of defined properties): + + :attr:`~cssutils.profiles.Profiles.CSS_LEVEL_2` + Properties defined by CSS2.1 + :attr:`~cssutils.profiles.Profiles.CSS3_COLOR` + CSS 3 color properties + :attr:`~cssutils.profiles.Profiles.CSS3_BOX` + Currently overflow related properties only + :attr:`~cssutils.profiles.Profiles.CSS3_PAGED_MEDIA` + As defined at http://www.w3.org/TR/css3-page/ (at 090307) + + Predefined macros are: + + :attr:`~cssutils.profiles.Profiles._TOKEN_MACROS` + Macros containing the token values as defined to CSS2 + :attr:`~cssutils.profiles.Profiles._MACROS` + Additional general macros. + + If you want to redefine any of these macros do this in your custom + macros. + """ + CSS_LEVEL_2 = 'CSS Level 2.1' + CSS3_COLOR = CSS_COLOR_LEVEL_3 = 'CSS Color Module Level 3' + CSS3_BOX = CSS_BOX_LEVEL_3 = 'CSS Box Module Level 3' + CSS3_PAGED_MEDIA = 'CSS3 Paged Media Module' + + _TOKEN_MACROS = { + 'ident': r'[-]?{nmstart}{nmchar}*', + 'name': r'{nmchar}+', + 'nmstart': r'[_a-z]|{nonascii}|{escape}', + 'nonascii': r'[^\0-\177]', + 'unicode': r'\\[0-9a-f]{1,6}(\r\n|[ \n\r\t\f])?', + 'escape': r'{unicode}|\\[ -~\200-\777]', + # 'escape': r'{unicode}|\\[ -~\200-\4177777]', + 'int': r'[-]?\d+', + 'nmchar': r'[\w-]|{nonascii}|{escape}', + 'num': r'[-]?\d+|[-]?\d*\.\d+', + 'number': r'{num}', + 'string': r'{string1}|{string2}', + 'string1': r'"(\\\"|[^\"])*"', + 'uri': r'url\({w}({string}|(\\\)|[^\)])+){w}\)', + 'string2': r"'(\\\'|[^\'])*'", + 'nl': r'\n|\r\n|\r|\f', + 'w': r'\s*', + } + _MACROS = { + 'hexcolor': r'#[0-9a-f]{3}|#[0-9a-f]{6}', + 'rgbcolor': r'rgb\({w}{int}{w},{w}{int}{w},{w}{int}{w}\)|rgb\({w}{num}%{w},{w}{num}%{w},{w}{num}%{w}\)', + 'namedcolor': r'(transparent|orange|maroon|red|orange|yellow|olive|purple|fuchsia|white|lime|green|navy|blue|aqua|teal|black|silver|gray)', + 'uicolor': r'(ActiveBorder|ActiveCaption|AppWorkspace|Background|ButtonFace|ButtonHighlight|ButtonShadow|ButtonText|CaptionText|GrayText|Highlight|HighlightText|InactiveBorder|InactiveCaption|InactiveCaptionText|InfoBackground|InfoText|Menu|MenuText|Scrollbar|ThreeDDarkShadow|ThreeDFace|ThreeDHighlight|ThreeDLightShadow|ThreeDShadow|Window|WindowFrame|WindowText)', + 'color': r'{namedcolor}|{hexcolor}|{rgbcolor}|{uicolor}', + #'color': r'(maroon|red|orange|yellow|olive|purple|fuchsia|white|lime|green|navy|blue|aqua|teal|black|silver|gray|ActiveBorder|ActiveCaption|AppWorkspace|Background|ButtonFace|ButtonHighlight|ButtonShadow|ButtonText|CaptionText|GrayText|Highlight|HighlightText|InactiveBorder|InactiveCaption|InactiveCaptionText|InfoBackground|InfoText|Menu|MenuText|Scrollbar|ThreeDDarkShadow|ThreeDFace|ThreeDHighlight|ThreeDLightShadow|ThreeDShadow|Window|WindowFrame|WindowText)|#[0-9a-f]{3}|#[0-9a-f]{6}|rgb\({w}{int}{w},{w}{int}{w},{w}{int}{w}\)|rgb\({w}{num}%{w},{w}{num}%{w},{w}{num}%{w}\)', + 'integer': r'{int}', + 'length': r'0|{num}(em|ex|px|in|cm|mm|pt|pc)', + 'angle': r'0|{num}(deg|grad|rad)', + 'time': r'0|{num}m?s', + 'frequency': r'0|{num}k?Hz', + 'percentage': r'{num}%', + } + + def __init__(self, log=None): + """A few known profiles are predefined.""" + self._log = log + self._profileNames = [] # to keep order, REFACTOR! + self._profiles = {} + self._defaultProfiles = None + + self.addProfile(self.CSS_LEVEL_2, + properties[self.CSS_LEVEL_2], + macros[self.CSS_LEVEL_2]) + self.addProfile(self.CSS3_BOX, + properties[self.CSS3_BOX], + macros[self.CSS3_BOX]) + self.addProfile(self.CSS3_COLOR, + properties[self.CSS3_COLOR], + macros[self.CSS3_COLOR]) + self.addProfile(self.CSS3_PAGED_MEDIA, + properties[self.CSS3_PAGED_MEDIA], + macros[self.CSS3_PAGED_MEDIA]) + + self.__update_knownNames() + + def _expand_macros(self, dictionary, macros): + """Expand macros in token dictionary""" + def macro_value(m): + return '(?:%s)' % macros[m.groupdict()['macro']] + for key, value in dictionary.items(): + if not hasattr(value, '__call__'): + while re.search(r'{[a-z][a-z0-9-]*}', value): + value = re.sub(r'{(?P[a-z][a-z0-9-]*)}', + macro_value, value) + dictionary[key] = value + return dictionary + + def _compile_regexes(self, dictionary): + """Compile all regular expressions into callable objects""" + for key, value in dictionary.items(): + if not hasattr(value, '__call__'): + value = re.compile('^(?:%s)$' % value, re.I).match + dictionary[key] = value + + return dictionary + + def __update_knownNames(self): + self._knownNames = [] + for properties in self._profiles.values(): + self._knownNames.extend(properties.keys()) + + def _getDefaultProfiles(self): + "If not explicitly set same as Profiles.profiles but in reverse order." + if not self._defaultProfiles: + return self.profiles#list(reversed(self.profiles)) + else: + return self._defaultProfiles + + def _setDefaultProfiles(self, profiles): + "profiles may be a single or a list of profile names" + if isinstance(profiles, basestring): + self._defaultProfiles = (profiles,) + else: + self._defaultProfiles = profiles + + defaultProfiles = property(_getDefaultProfiles, + _setDefaultProfiles, + doc=u"Names of profiles to use for validation." + u"To use e.g. the CSS2 profile set " + u"``cssutils.profile.defaultProfiles = " + u"cssutils.profile.CSS_LEVEL_2``") + + profiles = property(lambda self: self._profileNames, + doc=u'Names of all profiles in order as defined.') + + knownNames = property(lambda self: self._knownNames, + doc="All known property names of all profiles.") + + def addProfile(self, profile, properties, macros=None): + """Add a new profile with name `profile` (e.g. 'CSS level 2') + and the given `properties`. + + :param profile: + the new `profile`'s name + :param properties: + a dictionary of ``{ property-name: propery-value }`` items where + property-value is a regex which may use macros defined in given + ``macros`` or the standard macros Profiles.tokens and + Profiles.generalvalues. + + ``propery-value`` may also be a function which takes a single + argument which is the value to validate and which should return + True or False. + Any exceptions which may be raised during this custom validation + are reported or raised as all other cssutils exceptions depending + on cssutils.log.raiseExceptions which e.g during parsing normally + is False so the exceptions would be logged only. + :param macros: + may be used in the given properties definitions. There are some + predefined basic macros which may always be used in + :attr:`Profiles._TOKEN_MACROS` and :attr:`Profiles._MACROS`. + """ + if not macros: + macros = {} + m = Profiles._TOKEN_MACROS.copy() + m.update(Profiles._MACROS) + m.update(macros) + properties = self._expand_macros(properties, m) + self._profileNames.append(profile) + self._profiles[profile] = self._compile_regexes(properties) + + self.__update_knownNames() + + def removeProfile(self, profile=None, all=False): + """Remove `profile` or remove `all` profiles. + + :param profile: + profile name to remove + :param all: + if ``True`` removes all profiles to start with a clean state + :exceptions: + - :exc:`cssutils.profiles.NoSuchProfileException`: + If given `profile` cannot be found. + """ + if all: + self._profiles.clear() + del self._profileNames[:] + else: + try: + del self._profiles[profile] + del self._profileNames[self._profileNames.index(profile)] + except KeyError: + raise NoSuchProfileException(u'No profile %r.' % profile) + + self.__update_knownNames() + + def propertiesByProfile(self, profiles=None): + """Generator: Yield property names, if no `profiles` is given all + profile's properties are used. + + :param profiles: + a single profile name or a list of names. + """ + if not profiles: + profiles = self.profiles + elif isinstance(profiles, basestring): + profiles = (profiles, ) + try: + for profile in sorted(profiles): + for name in sorted(self._profiles[profile].keys()): + yield name + except KeyError, e: + raise NoSuchProfileException(e) + + def validate(self, name, value): + """Check if `value` is valid for given property `name` using **any** + profile. + + :param name: + a property name + :param value: + a CSS value (string) + :returns: + if the `value` is valid for the given property `name` in any + profile + """ + for profile in self.profiles: + if name in self._profiles[profile]: + try: + # custom validation errors are caught + r = bool(self._profiles[profile][name](value)) + except Exception, e: + self._log.error(e, error=Exception) + return False + if r: + return r + return False + + def validateWithProfile(self, name, value, profiles=None): + """Check if `value` is valid for given property `name` returning + ``(valid, profile)``. + + :param name: + a property name + :param value: + a CSS value (string) + :param profiles: + internal parameter used by Property.validate only + :returns: + ``valid, matching, profiles`` where ``valid`` is if the `value` + is valid for the given property `name` in any profile, + ``matching==True`` if it is valid in the given `profiles` + and ``profiles`` the profile names for which the value is valid + (or ``[]`` if not valid at all) + + Example:: + + >>> cssutils.profile.defaultProfiles = cssutils.profile.CSS_LEVEL_2 + >>> print cssutils.profile.validateWithProfile('color', 'rgba(1,1,1,1)') + (True, False, Profiles.CSS3_COLOR) + """ + if name not in self.knownNames: + return False, False, [] + else: + if not profiles: + profiles = self.defaultProfiles + elif isinstance(profiles, basestring): + profiles = (profiles, ) + + for profilename in profiles: + # check given profiles + if name in self._profiles[profilename]: + validate = self._profiles[profilename][name] + try: + if validate(value): + return True, True, [profilename] + except Exception, e: + self._log.error(e, error=Exception) + + for profilename in (p for p in self._profileNames + if p not in profiles): + # check remaining profiles as well + if name in self._profiles[profilename]: + validate = self._profiles[profilename][name] + try: + if validate(value): + return True, False, [profilename] + except Exception, e: + self._log.error(e, error=Exception) + + names = [] + for profilename, properties in self._profiles.items(): + # return profile to which name belongs + if name in properties.keys(): + names.append(profilename) + names.sort() + return False, False, names + + properties = {} +macros = {} """ Define some regular expression fragments that will be used as macros within the CSS property value regular expressions. """ -css2macros = { +macros[Profiles.CSS_LEVEL_2] = { 'border-style': 'none|hidden|dotted|dashed|solid|double|groove|ridge|inset|outset', 'border-color': '{color}', 'border-width': '{length}|thin|medium|thick', 'background-color': r'{color}|transparent|inherit', 'background-image': r'{uri}|none|inherit', - 'background-position': r'({percentage}|{length})(\s*({percentage}|{length}))?|((top|center|bottom)\s*(left|center|right))|((left|center|right)\s*(top|center|bottom))|inherit', + #'background-position': r'({percentage}|{length})(\s*({percentage}|{length}))?|((top|center|bottom)\s*(left|center|right)?)|((left|center|right)\s*(top|center|bottom)?)|inherit', + 'background-position': r'({percentage}|{length}|left|center|right)(\s*({percentage}|{length}|top|center|bottom))?|((top|center|bottom)\s*(left|center|right)?)|((left|center|right)\s*(top|center|bottom)?)|inherit', 'background-repeat': r'repeat|repeat-x|repeat-y|no-repeat|inherit', 'background-attachment': r'scroll|fixed|inherit', - 'shape': r'rect\(({w}({length}|auto}){w},){3}{w}({length}|auto){w}\)', 'counter': r'counter\({w}{identifier}{w}(?:,{w}{list-style-type}{w})?\)', 'identifier': r'{ident}', @@ -72,7 +371,7 @@ css2macros = { """ Define the regular expressions for validation all CSS values """ -properties['css2'] = { +properties[Profiles.CSS_LEVEL_2] = { 'azimuth': r'{angle}|(behind\s+)?(left-side|far-left|left|center-left|center|center-right|right|far-right|right-side)(\s+behind)?|behind|leftwards|rightwards|inherit', 'background-attachment': r'{background-attachment}', 'background-color': r'{background-color}', @@ -108,7 +407,7 @@ properties['css2'] = { 'clear': r'none|left|right|both|inherit', 'clip': r'{shape}|auto|inherit', 'color': r'{color}|inherit', - 'content': r'normal|{content}(\s+{content})*|inherit', + 'content': r'none|normal|{content}(\s+{content})*|inherit', 'counter-increment': r'({identifier}(\s+{integer})?)(\s+({identifier}(\s+{integer})))*|none|inherit', 'counter-reset': r'({identifier}(\s+{integer})?)(\s+({identifier}(\s+{integer})))*|none|inherit', 'cue-after': r'{uri}|none|inherit', @@ -191,288 +490,47 @@ properties['css2'] = { 'z-index': r'auto|{integer}|inherit', } +# CSS Box Module Level 3 +macros[Profiles.CSS3_BOX] = { + 'overflow': macros[Profiles.CSS_LEVEL_2]['overflow'] + } +properties[Profiles.CSS3_BOX] = { + 'overflow': '{overflow}\s?{overflow}?|inherit', + 'overflow-x': '{overflow}|inherit', + 'overflow-y': '{overflow}|inherit' + } + # CSS Color Module Level 3 -css3colormacros = { +macros[Profiles.CSS3_COLOR] = { # orange and transparent in CSS 2.1 - 'namedcolor': r'(currentcolor|transparent|orange|black|green|silver|lime|gray|olive|white|yellow|maroon|navy|red|blue|purple|teal|fuchsia|aqua)', + 'namedcolor': r'(currentcolor|transparent|aqua|black|blue|fuchsia|gray|green|lime|maroon|navy|olive|orange|purple|red|silver|teal|white|yellow)', # orange? 'rgbacolor': r'rgba\({w}{int}{w},{w}{int}{w},{w}{int}{w},{w}{int}{w}\)|rgba\({w}{num}%{w},{w}{num}%{w},{w}{num}%{w},{w}{num}{w}\)', 'hslcolor': r'hsl\({w}{int}{w},{w}{num}%{w},{w}{num}%{w}\)|hsla\({w}{int}{w},{w}{num}%{w},{w}{num}%{w},{w}{num}{w}\)', 'x11color': r'aliceblue|antiquewhite|aqua|aquamarine|azure|beige|bisque|black|blanchedalmond|blue|blueviolet|brown|burlywood|cadetblue|chartreuse|chocolate|coral|cornflowerblue|cornsilk|crimson|cyan|darkblue|darkcyan|darkgoldenrod|darkgray|darkgreen|darkgrey|darkkhaki|darkmagenta|darkolivegreen|darkorange|darkorchid|darkred|darksalmon|darkseagreen|darkslateblue|darkslategray|darkslategrey|darkturquoise|darkviolet|deeppink|deepskyblue|dimgray|dimgrey|dodgerblue|firebrick|floralwhite|forestgreen|fuchsia|gainsboro|ghostwhite|gold|goldenrod|gray|green|greenyellow|grey|honeydew|hotpink|indianred|indigo|ivory|khaki|lavender|lavenderblush|lawngreen|lemonchiffon|lightblue|lightcoral|lightcyan|lightgoldenrodyellow|lightgray|lightgreen|lightgrey|lightpink|lightsalmon|lightseagreen|lightskyblue|lightslategray|lightslategrey|lightsteelblue|lightyellow|lime|limegreen|linen|magenta|maroon|mediumaquamarine|mediumblue|mediumorchid|mediumpurple|mediumseagreen|mediumslateblue|mediumspringgreen|mediumturquoise|mediumvioletred|midnightblue|mintcream|mistyrose|moccasin|navajowhite|navy|oldlace|olive|olivedrab|orange|orangered|orchid|palegoldenrod|palegreen|paleturquoise|palevioletred|papayawhip|peachpuff|peru|pink|plum|powderblue|purple|red|rosybrown|royalblue|saddlebrown|salmon|sandybrown|seagreen|seashell|sienna|silver|skyblue|slateblue|slategray|slategrey|snow|springgreen|steelblue|tan|teal|thistle|tomato|turquoise|violet|wheat|white|whitesmoke|yellow|yellowgreen', 'uicolor': r'(ActiveBorder|ActiveCaption|AppWorkspace|Background|ButtonFace|ButtonHighlight|ButtonShadow|ButtonText|CaptionText|GrayText|Highlight|HighlightText|InactiveBorder|InactiveCaption|InactiveCaptionText|InfoBackground|InfoText|Menu|MenuText|Scrollbar|ThreeDDarkShadow|ThreeDFace|ThreeDHighlight|ThreeDLightShadow|ThreeDShadow|Window|WindowFrame|WindowText)', - } -properties['css3color'] = { +properties[Profiles.CSS3_COLOR] = { 'color': r'{namedcolor}|{hexcolor}|{rgbcolor}|{rgbacolor}|{hslcolor}|inherit', 'opacity': r'{num}|inherit' } -# CSS Box Module Level 3 -properties['css3box'] = { - 'overflow': '{overflow}\s?{overflow}?', - 'overflow-x': '{overflow}', - 'overflow-y': '{overflow}' +# CSS3 Paged Media +macros[Profiles.CSS3_PAGED_MEDIA] = { + 'pagesize': 'a5|a4|a3|b5|b4|letter|legal|ledger', + 'pagebreak': 'auto|always|avoid|left|right' + } +properties[Profiles.CSS3_PAGED_MEDIA] = { + 'fit': 'fill|hidden|meet|slice', + 'fit-position': r'auto|(({percentage}|{length})(\s*({percentage}|{length}))?|((top|center|bottom)\s*(left|center|right)?)|((left|center|right)\s*(top|center|bottom)?))', + 'image-orientation': 'auto|{angle}', + 'orphans': r'{integer}|inherit', + 'page': 'auto|{ident}', + 'page-break-before': '{pagebreak}|inherit', + 'page-break-after': '{pagebreak}|inherit', + 'page-break-inside': 'auto|avoid|inherit', + 'size': '({length}{w}){1,2}|auto|{pagesize}{w}(?:portrait|landscape)', + 'widows': r'{integer}|inherit' } -class NoSuchProfileException(Exception): - """Raised if no profile with given name is found""" - pass - - -class Profiles(object): - """ - All profiles used for validation. ``cssutils.profiles.profiles`` is a - preset object of this class and used by all properties for validation. - - Predefined profiles are (use - :meth:`~cssutils.profiles.Profiles.propertiesByProfile` to - get a list of defined properties): - - :attr:`~cssutils.profiles.Profiles.Profiles.CSS_LEVEL_2` - Properties defined by CSS2.1 - :attr:`~cssutils.profiles.Profiles.Profiles.CSS_COLOR_LEVEL_3` - CSS 3 color properties - :attr:`~cssutils.profiles.Profiles.Profiles.CSS_BOX_LEVEL_3` - Currently overflow related properties only - - """ - CSS_LEVEL_2 = 'CSS Level 2.1' - CSS_COLOR_LEVEL_3 = 'CSS Color Module Level 3' - CSS_BOX_LEVEL_3 = 'CSS Box Module Level 3' - - basicmacros = { - 'ident': r'[-]?{nmstart}{nmchar}*', - 'name': r'{nmchar}+', - 'nmstart': r'[_a-z]|{nonascii}|{escape}', - 'nonascii': r'[^\0-\177]', - 'unicode': r'\\[0-9a-f]{1,6}(\r\n|[ \n\r\t\f])?', - 'escape': r'{unicode}|\\[ -~\200-\777]', - # 'escape': r'{unicode}|\\[ -~\200-\4177777]', - 'int': r'[-]?\d+', - 'nmchar': r'[\w-]|{nonascii}|{escape}', - 'num': r'[-]?\d+|[-]?\d*\.\d+', - 'number': r'{num}', - 'string': r'{string1}|{string2}', - 'string1': r'"(\\\"|[^\"])*"', - 'uri': r'url\({w}({string}|(\\\)|[^\)])+){w}\)', - 'string2': r"'(\\\'|[^\'])*'", - 'nl': r'\n|\r\n|\r|\f', - 'w': r'\s*', - } - generalmacros = { - 'hexcolor': r'#[0-9a-f]{3}|#[0-9a-f]{6}', - 'rgbcolor': r'rgb\({w}{int}{w},{w}{int}{w},{w}{int}{w}\)|rgb\({w}{num}%{w},{w}{num}%{w},{w}{num}%{w}\)', - 'namedcolor': r'(transparent|orange|maroon|red|orange|yellow|olive|purple|fuchsia|white|lime|green|navy|blue|aqua|teal|black|silver|gray)', - 'uicolor': r'(ActiveBorder|ActiveCaption|AppWorkspace|Background|ButtonFace|ButtonHighlight|ButtonShadow|ButtonText|CaptionText|GrayText|Highlight|HighlightText|InactiveBorder|InactiveCaption|InactiveCaptionText|InfoBackground|InfoText|Menu|MenuText|Scrollbar|ThreeDDarkShadow|ThreeDFace|ThreeDHighlight|ThreeDLightShadow|ThreeDShadow|Window|WindowFrame|WindowText)', - 'color': r'{namedcolor}|{hexcolor}|{rgbcolor}|{uicolor}', - #'color': r'(maroon|red|orange|yellow|olive|purple|fuchsia|white|lime|green|navy|blue|aqua|teal|black|silver|gray|ActiveBorder|ActiveCaption|AppWorkspace|Background|ButtonFace|ButtonHighlight|ButtonShadow|ButtonText|CaptionText|GrayText|Highlight|HighlightText|InactiveBorder|InactiveCaption|InactiveCaptionText|InfoBackground|InfoText|Menu|MenuText|Scrollbar|ThreeDDarkShadow|ThreeDFace|ThreeDHighlight|ThreeDLightShadow|ThreeDShadow|Window|WindowFrame|WindowText)|#[0-9a-f]{3}|#[0-9a-f]{6}|rgb\({w}{int}{w},{w}{int}{w},{w}{int}{w}\)|rgb\({w}{num}%{w},{w}{num}%{w},{w}{num}%{w}\)', - 'integer': r'{int}', - 'length': r'0|{num}(em|ex|px|in|cm|mm|pt|pc)', - 'angle': r'0|{num}(deg|grad|rad)', - 'time': r'0|{num}m?s', - 'frequency': r'0|{num}k?Hz', - 'percentage': r'{num}%', - } - - def __init__(self): - """A few known profiles are predefined.""" - self._log = cssutils.log - self._profilenames = [] # to keep order, REFACTOR! - self._profiles = {} - - self.addProfile(self.CSS_LEVEL_2, properties['css2'], css2macros) - self.addProfile(self.CSS_COLOR_LEVEL_3, properties['css3color'], css3colormacros) - self.addProfile(self.CSS_BOX_LEVEL_3, properties['css3box']) - - self.__update_knownnames() - - def _expand_macros(self, dictionary, macros): - """Expand macros in token dictionary""" - def macro_value(m): - return '(?:%s)' % macros[m.groupdict()['macro']] - for key, value in dictionary.items(): - if not hasattr(value, '__call__'): - while re.search(r'{[a-z][a-z0-9-]*}', value): - value = re.sub(r'{(?P[a-z][a-z0-9-]*)}', - macro_value, value) - dictionary[key] = value - return dictionary - - def _compile_regexes(self, dictionary): - """Compile all regular expressions into callable objects""" - for key, value in dictionary.items(): - if not hasattr(value, '__call__'): - value = re.compile('^(?:%s)$' % value, re.I).match - dictionary[key] = value - - return dictionary - - def __update_knownnames(self): - self._knownnames = [] - for properties in self._profiles.values(): - self._knownnames.extend(properties.keys()) - - profiles = property(lambda self: sorted(self._profiles.keys()), - doc=u'Names of all profiles.') - - knownnames = property(lambda self: self._knownnames, - doc="All known property names of all profiles.") - - def addProfile(self, profile, properties, macros=None): - """Add a new profile with name `profile` (e.g. 'CSS level 2') - and the given `properties`. - - :param profile: - the new `profile`'s name - :param properties: - a dictionary of ``{ property-name: propery-value }`` items where - property-value is a regex which may use macros defined in given - ``macros`` or the standard macros Profiles.tokens and - Profiles.generalvalues. - - ``propery-value`` may also be a function which takes a single - argument which is the value to validate and which should return - True or False. - Any exceptions which may be raised during this custom validation - are reported or raised as all other cssutils exceptions depending - on cssutils.log.raiseExceptions which e.g during parsing normally - is False so the exceptions would be logged only. - :param macros: - may be used in the given properties definitions. There are some - predefined basic macros which may always be used in - :attr:`Profiles.basicmacros` and :attr:`Profiles.generalmacros`. - """ - if not macros: - macros = {} - m = self.basicmacros - m.update(self.generalmacros) - m.update(macros) - properties = self._expand_macros(properties, m) - self._profilenames.append(profile) - self._profiles[profile] = self._compile_regexes(properties) - - self.__update_knownnames() - - def removeProfile(self, profile=None, all=False): - """Remove `profile` or remove `all` profiles. - - :param profile: - profile name to remove - :param all: - if ``True`` removes all profiles to start with a clean state - :exceptions: - - :exc:`cssutils.profiles.NoSuchProfileException`: - If given `profile` cannot be found. - """ - if all: - self._profiles.clear() - else: - try: - del self._profiles[profile] - except KeyError: - raise NoSuchProfileException(u'No profile %r.' % profile) - - self.__update_knownnames() - - def propertiesByProfile(self, profiles=None): - """Generator: Yield property names, if no `profiles` is given all - profile's properties are used. - - :param profiles: - a single profile name or a list of names. - """ - if not profiles: - profiles = self.profiles - elif isinstance(profiles, basestring): - profiles = (profiles, ) - try: - for profile in sorted(profiles): - for name in sorted(self._profiles[profile].keys()): - yield name - except KeyError, e: - raise NoSuchProfileException(e) - - def validate(self, name, value): - """Check if `value` is valid for given property `name` using **any** - profile. - - :param name: - a property name - :param value: - a CSS value (string) - :returns: - if the `value` is valid for the given property `name` in any - profile - """ - for profile in self.profiles: - if name in self._profiles[profile]: - try: - # custom validation errors are caught - r = bool(self._profiles[profile][name](value)) - except Exception, e: - self._log.error(e, error=Exception) - return False - if r: - return r - return False - - def validateWithProfile(self, name, value, profiles=None): - """Check if `value` is valid for given property `name` returning - ``(valid, profile)``. - - :param name: - a property name - :param value: - a CSS value (string) - :returns: - ``valid, profiles`` where ``valid`` is if the `value` is valid for - the given property `name` in any profile of given `profiles` - and ``profiles`` the profile names for which the value is valid - (or ``[]`` if not valid at all) - - Example: You might expect a valid Profiles.CSS_LEVEL_2 value but - e.g. ``validateWithProfile('color', 'rgba(1,1,1,1)')`` returns - (True, Profiles.CSS_COLOR_LEVEL_3) - """ - if name not in self.knownnames: - return False, [] - else: - if not profiles: - profiles = self._profilenames - elif isinstance(profiles, basestring): - profiles = (profiles, ) - - for profilename in profiles: - # check given profiles - if name in self._profiles[profilename]: - validate = self._profiles[profilename][name] - try: - if validate(value): - return True, [profilename] - except Exception, e: - self._log.error(e, error=Exception) - - for profilename in (p for p in self._profilenames if p not in profiles): - # check remaining profiles as well - if name in self._profiles[profilename]: - validate = self._profiles[profilename][name] - try: - if validate(value): - return True, [profilename] - except Exception, e: - self._log.error(e, error=Exception) - - names = [] - for profilename, properties in self._profiles.items(): - # return profile to which name belongs - if name in properties.keys(): - names.append(profilename) - names.sort() - return False, names - -# used by -profiles = Profiles() - -# set for validation to e.g.``Profiles.CSS_LEVEL_2`` -defaultprofile = None diff --git a/src/cssutils/serialize.py b/src/cssutils/serialize.py index 0533901a05..ac6539ed29 100644 --- a/src/cssutils/serialize.py +++ b/src/cssutils/serialize.py @@ -3,7 +3,7 @@ """cssutils serializer""" __all__ = ['CSSSerializer', 'Preferences'] __docformat__ = 'restructuredtext' -__version__ = '$Id: serialize.py 1606 2009-01-03 20:32:17Z cthedot $' +__version__ = '$Id: serialize.py 1741 2009-05-09 18:20:20Z cthedot $' import codecs import cssutils @@ -58,6 +58,9 @@ class Preferences(object): keepEmptyRules = False defines if empty rules like e.g. ``a {}`` are kept in the resulting serialized sheet + keepUnkownAtRules = True + defines if unknown @rules like e.g. ``@three-dee {}`` are kept in the + serialized sheet keepUsedNamespaceRulesOnly = False if True only namespace rules which are actually used are kept @@ -82,12 +85,10 @@ class Preferences(object): spacer = u' ' general spacer, used e.g. by CSSUnknownRule - validOnly = False **DO NOT CHANGE YET** - if True only valid (currently Properties) are kept + validOnly = False + if True only valid (Properties) are output A Property is valid if it is a known Property with a valid value. - Currently CSS 2.1 values as defined in cssproperties.py would be - valid. """ def __init__(self, **initials): """Always use named instead of positional parameters.""" @@ -118,6 +119,7 @@ class Preferences(object): self.keepAllProperties = True self.keepComments = True self.keepEmptyRules = False + self.keepUnkownAtRules = True self.keepUsedNamespaceRulesOnly = False self.lineNumbers = False self.lineSeparator = u'\n' @@ -139,6 +141,7 @@ class Preferences(object): self.indent = u'' self.keepComments = False self.keepEmptyRules = False + self.keepUnkownAtRules = False self.keepUsedNamespaceRulesOnly = True self.lineNumbers = False self.lineSeparator = u'' @@ -563,7 +566,7 @@ class CSSSerializer(object): anything until ";" or "{...}" + CSSComments """ - if rule.wellformed: + if rule.wellformed and self.prefs.keepUnkownAtRules: out = Out(self) out.append(rule.atkeyword) @@ -741,10 +744,11 @@ class CSSSerializer(object): out.append(separator) elif isinstance(val, cssutils.css.Property): # PropertySimilarNameList - out.append(val.cssText) - if not (self.prefs.omitLastSemicolon and i==len(seq)-1): - out.append(u';') - out.append(separator) + if val.cssText: + out.append(val.cssText) + if not (self.prefs.omitLastSemicolon and i==len(seq)-1): + out.append(u';') + out.append(separator) elif isinstance(val, cssutils.css.CSSUnknownRule): # @rule out.append(val.cssText) diff --git a/src/cssutils/stylesheets/mediaquery.py b/src/cssutils/stylesheets/mediaquery.py index b75ec285cf..347ae8b0b0 100644 --- a/src/cssutils/stylesheets/mediaquery.py +++ b/src/cssutils/stylesheets/mediaquery.py @@ -5,7 +5,7 @@ A cssutils implementation, not defined in official DOM. """ __all__ = ['MediaQuery'] __docformat__ = 'restructuredtext' -__version__ = '$Id: mediaquery.py 1638 2009-01-13 20:39:33Z cthedot $' +__version__ = '$Id: mediaquery.py 1738 2009-05-02 13:03:28Z cthedot $' import cssutils import re @@ -21,8 +21,8 @@ class MediaQuery(cssutils.util.Base): media_query: [[only | not]? [ and ]*] | [ and ]* expression: ( [: ]? ) - media_type: all | aural | braille | handheld | print | - projection | screen | tty | tv | embossed + media_type: all | braille | handheld | print | + projection | speech | screen | tty | tv | embossed media_feature: width | min-width | max-width | height | min-height | max-height | device-width | min-device-width | max-device-width @@ -35,8 +35,8 @@ class MediaQuery(cssutils.util.Base): | scan | grid """ - MEDIA_TYPES = [u'all', u'aural', u'braille', u'embossed', u'handheld', - u'print', u'projection', u'screen', u'tty', u'tv'] + MEDIA_TYPES = [u'all', u'braille', u'embossed', u'handheld', + u'print', u'projection', u'screen', u'speech', u'tty', u'tv'] # From the HTML spec (see MediaQuery): # "[...] character that isn't a US ASCII letter [a-zA-Z] (Unicode diff --git a/src/cssutils/util.py b/src/cssutils/util.py index 81335910ac..6f8ba3bc5e 100644 --- a/src/cssutils/util.py +++ b/src/cssutils/util.py @@ -2,7 +2,7 @@ """ __all__ = [] __docformat__ = 'restructuredtext' -__version__ = '$Id: util.py 1654 2009-02-03 20:16:20Z cthedot $' +__version__ = '$Id: util.py 1743 2009-05-09 20:33:15Z cthedot $' from helper import normalize from itertools import ifilter @@ -307,7 +307,6 @@ class Base(_BaseClass): bracket == parant == 0) and typ in endtypes: # mediaqueryendonly with STRING break - if separateEnd: # TODO: use this method as generator, then this makes sense if resulttokens: From 1c56b03f95505f13da3612634271d0c729cadc95 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 13 May 2009 09:42:48 -0700 Subject: [PATCH 04/15] New recipes for HRT and RTS by Darko Miletic --- src/calibre/gui2/images/news/hrt.png | Bin 0 -> 606 bytes src/calibre/gui2/images/news/rts.png | Bin 0 -> 458 bytes src/calibre/web/feeds/recipes/__init__.py | 2 +- src/calibre/web/feeds/recipes/recipe_hrt.py | 66 ++++++++++++++++++++ src/calibre/web/feeds/recipes/recipe_rts.py | 60 ++++++++++++++++++ 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 src/calibre/gui2/images/news/hrt.png create mode 100644 src/calibre/gui2/images/news/rts.png create mode 100644 src/calibre/web/feeds/recipes/recipe_hrt.py create mode 100644 src/calibre/web/feeds/recipes/recipe_rts.py diff --git a/src/calibre/gui2/images/news/hrt.png b/src/calibre/gui2/images/news/hrt.png new file mode 100644 index 0000000000000000000000000000000000000000..828819e226825ea2bbdf022b0104aa8f19131d1f GIT binary patch literal 606 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b zK-vS0-A-oPfdtD69Mgd`SU*F|v9*U87#Q0mB1L(l)>N-JP5IrO9_# zR5obbi2T96Cg`_*kXoqM>@Ar&$*FUdHgbBt=I4mG5z(<>Q%Bp828l_3cYHs$amF1} zfwJ`v8o3YOnc4n%UiyPZRffNJ%dPL*)zs*IwOYi)5hk+3{q5?bXU~dj>&Ms5Vk=>&W=wZ1_8BJOo5%VVN*e<*I*K#v6d&0`ehpM+sb8Bc^scuR{moDkW@%QnNK$6-z*Z`*<(-V4|6)O=dfY$k0ul_}EiQe@W6j5(oMGV{O9 zd;VNpTE)}w=fg>dJ{`LIroPVXN7apMtv?M9KH^oLcqO)c|DOA040X9*87vr1RQ*U5dG+Mjn0LK>{@b?fO$hXV;(a!r`SyQHukhUqW9sWan{|9|TBUHprBiYR&wCx8 z)6at(RrofUEOKCQ$@#>$Y^vnPpp~0!{`Swfd0yxA5w~RC{r??a{S#_T@ZeY{^kCY2 zA-BU(waKz`vyb|*a4@d?$@W)SE@t}zrJ5}(&E`e`<43i`HKHWBC^fMpRW}7lFc=va zSn3*>>KYn{7+6{vTUdd(=2iv<`5bHNVH$GtQ!>*kF*Fz&SeY7InHoVf6ikfz57fZm M>FVdQ&MBb@02*iXy8r+H literal 0 HcmV?d00001 diff --git a/src/calibre/gui2/images/news/rts.png b/src/calibre/gui2/images/news/rts.png new file mode 100644 index 0000000000000000000000000000000000000000..278f45edd7af222703cb4d11cc6711967a97cc13 GIT binary patch literal 458 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b zK-vS0-A-oPfdtD69Mgd`SU*F|v9*VRoX?&vjv*GOk51m0d)Pq0Ri1h4_9kx8FAQ0C z)vYD+F1HrQS2T;Td2e?!<6t_bEfM7~cW)Eh0&eFck&9Hye_p*i@q4BGch3L3i-q6U zzKJh5H*-11uC(l2tDISB(_Xl$hlVODX>Px!cIZwCbCaRZM3J3s-?zUPkw5fH@%z2w z`sMTHKM=Uq9C3Y}%-N}nJz36K2462N>zXqskW;p^E|-g6=*udXmTkF$6Sqjp>fboR zIy-~mQJwtkw`PKWE6?ZucKmT?X%xrmgFjLuXE8aL$1{BY%(Z;(t}4c`NKV!Ro6c?TrIoRT6^LtYWnhrcv8EoTAvZrIGp!OsgP{RXwUvPdM8mHV^EjXe22WQ% Jmvv4FO#mNWrN#gN literal 0 HcmV?d00001 diff --git a/src/calibre/web/feeds/recipes/__init__.py b/src/calibre/web/feeds/recipes/__init__.py index 4d2adfb1c0..ae1ae24131 100644 --- a/src/calibre/web/feeds/recipes/__init__.py +++ b/src/calibre/web/feeds/recipes/__init__.py @@ -42,7 +42,7 @@ recipe_modules = ['recipe_' + r for r in ( 'moneynews', 'der_standard', 'diepresse', 'nzz_ger', 'hna', 'seattle_times', 'scott_hanselman', 'coding_horror', 'twitchfilms', 'stackoverflow', 'telepolis_artikel', 'zaobao', 'usnews', - 'straitstimes', 'index_hu', 'pcworld_hu', + 'straitstimes', 'index_hu', 'pcworld_hu', 'hrt', 'rts', )] import re, imp, inspect, time, os diff --git a/src/calibre/web/feeds/recipes/recipe_hrt.py b/src/calibre/web/feeds/recipes/recipe_hrt.py new file mode 100644 index 0000000000..d07b214e02 --- /dev/null +++ b/src/calibre/web/feeds/recipes/recipe_hrt.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2009, Darko Miletic ' + +''' +www.hrt.hr +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag + +class HRT(BasicNewsRecipe): + title = 'HRT: Vesti' + __author__ = 'Darko Miletic' + description = 'News from Croatia' + publisher = 'HRT' + category = 'news, politics, Croatia, HRT' + no_stylesheets = True + encoding = 'utf-8' + use_embedded_content = False + language = _("Croatian") + lang = 'hr-HR' + extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif}' + + html2lrf_options = [ + '--comment', description + , '--category', category + , '--publisher', publisher + ] + + html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"' + + + preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')] + + keep_only_tags = [dict(name='div', attrs={'class':'bigVijest'})] + + remove_tags = [dict(name=['object','link','embed'])] + + remove_tags_after = dict(name='div', attrs={'class':'nsAuthor'}) + + feeds = [ + (u'Vijesti' , u'http://www.hrt.hr/?id=316&type=100&rss=vijesti' ) + ,(u'Sport' , u'http://www.hrt.hr/?id=316&type=100&rss=sport' ) + ,(u'Zabava' , u'http://www.hrt.hr/?id=316&type=100&rss=zabava' ) + ,(u'Filmovi i serije' , u'http://www.hrt.hr/?id=316&type=100&rss=filmovi' ) + ,(u'Dokumentarni program', u'http://www.hrt.hr/?id=316&type=100&rss=dokumentarci') + ,(u'Glazba' , u'http://www.hrt.hr/?id=316&type=100&rss=glazba' ) + ,(u'Kultura' , u'http://www.hrt.hr/?id=316&type=100&rss=kultura' ) + ,(u'Mladi' , u'http://www.hrt.hr/?id=316&type=100&rss=mladi' ) + ,(u'Manjine' , u'http://www.hrt.hr/?id=316&type=100&rss=manjine' ) + ,(u'Radio' , u'http://www.hrt.hr/?id=316&type=100&rss=radio' ) + ] + + def preprocess_html(self, soup): + soup.html['xml:lang'] = self.lang + soup.html['lang'] = self.lang + mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)]) + mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=UTF-8")]) + soup.head.insert(0,mlang) + soup.head.insert(1,mcharset) + for item in soup.findAll(style=True): + del item['style'] + return self.adeify_images(soup) diff --git a/src/calibre/web/feeds/recipes/recipe_rts.py b/src/calibre/web/feeds/recipes/recipe_rts.py new file mode 100644 index 0000000000..57ee346d62 --- /dev/null +++ b/src/calibre/web/feeds/recipes/recipe_rts.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2009, Darko Miletic ' + +''' +www.rts.rs +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag + +class RTS(BasicNewsRecipe): + title = 'RTS: Vesti' + __author__ = 'Darko Miletic' + description = 'News from Serbia' + publisher = 'RTS' + category = 'news, politics, Serbia, RTS' + no_stylesheets = True + encoding = 'utf-8' + use_embedded_content = True + language = _("Serbian") + lang = 'sr-Latn-RS' + extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif}' + + html2lrf_options = [ + '--comment', description + , '--category', category + , '--publisher', publisher + ] + + html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"' + + + preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')] + + feeds = [ + (u'Vesti' , u'http://www.rts.rs/page/stories/sr/rss.html' ) + ,(u'Srbija' , u'http://www.rts.rs/page/stories/sr/rss/9/Srbija.html' ) + ,(u'Region' , u'http://www.rts.rs/page/stories/sr/rss/11/Region.html' ) + ,(u'Svet' , u'http://www.rts.rs/page/stories/sr/rss/10/Svet.html' ) + ,(u'Hronika' , u'http://www.rts.rs/page/stories/sr/rss/135/Hronika.html' ) + ,(u'Drustvo' , u'http://www.rts.rs/page/stories/sr/rss/125/Dru%C5%A1tvo.html') + ,(u'Ekonomija' , u'http://www.rts.rs/page/stories/sr/rss/13/Ekonomija.html' ) + ,(u'Nauka' , u'http://www.rts.rs/page/stories/sr/rss/14/Nauka.html' ) + ,(u'Kultura' , u'http://www.rts.rs/page/stories/sr/rss/16/Kultura.html' ) + ,(u'Zanimljivosti' , u'http://www.rts.rs/page/stories/sr/rss/15/Zanimljivosti.html') + ,(u'Sport' , u'http://www.rts.rs/page/sport/sr/rss.html' ) + ] + + def preprocess_html(self, soup): + soup.html['xml:lang'] = self.lang + soup.html['lang'] = self.lang + mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)]) + mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=UTF-8")]) + soup.head.insert(0,mlang) + soup.head.insert(1,mcharset) + return self.adeify_images(soup) + From 580497d0ee0773e7e7a135fadae76b3649bc38f6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 13 May 2009 10:08:17 -0700 Subject: [PATCH 05/15] IGN:Change in cssutils profiles API --- src/calibre/ebooks/oeb/stylizer.py | 3 ++- src/cssutils/errorhandler.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index a6803854e8..c4301322e8 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -17,7 +17,8 @@ from xml.dom import SyntaxErr as CSSSyntaxError import cssutils from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \ CSSValueList, cssproperties -from cssutils.profiles import profiles as cssprofiles +from cssutils.profiles import Profiles +cssprofiles = Profiles() from lxml import etree from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES diff --git a/src/cssutils/errorhandler.py b/src/cssutils/errorhandler.py index aecf3e5fb1..2d1814b6c8 100644 --- a/src/cssutils/errorhandler.py +++ b/src/cssutils/errorhandler.py @@ -27,7 +27,7 @@ class _ErrorHandler(object): """ handles all errors and log messages """ - def __init__(self, log, defaultloglevel=logging.INFO, + def __init__(self, log, defaultloglevel=logging.INFO, raiseExceptions=True): """ inits log if none given @@ -51,7 +51,7 @@ class _ErrorHandler(object): hdlr.setFormatter(formatter) self._log.addHandler(hdlr) self._log.setLevel(defaultloglevel) - + self.raiseExceptions = raiseExceptions def __getattr__(self, name): @@ -86,7 +86,7 @@ class _ErrorHandler(object): if error and self.raiseExceptions and not neverraise: if isinstance(error, urllib2.HTTPError) or isinstance(error, urllib2.URLError): raise - elif issubclass(error, xml.dom.DOMException): + elif issubclass(error, xml.dom.DOMException): error.line = line error.col = col # raise error(msg, line, col) From d03b6493455224ff15e6947a3bd0f01bb2849e51 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 13 May 2009 14:04:58 -0700 Subject: [PATCH 06/15] Add a copyright file to comply with the FSF guidelines --- copyright | 355 +++++++++++++++++++++++++++++ src/calibre/ebooks/oeb/stylizer.py | 3 +- 2 files changed, 356 insertions(+), 2 deletions(-) create mode 100644 copyright diff --git a/copyright b/copyright new file mode 100644 index 0000000000..d4aa70052d --- /dev/null +++ b/copyright @@ -0,0 +1,355 @@ +Format-Specification: http://wiki.debian.org/Proposals/CopyrightFormat?action=recall&rev=196 +Upstream-Name: calibre +Upstream-Maintainer: Kovid Goyal +Upstream-Source: http://calibre.kovidgoyal.net/downloads + +Files: * +Copyright: Copyright (C) 2008 Kovid Goyal +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/ebooks/BeautifulSoup.py +Copyright: Copyright (c) 2004-2007, Leonard Richardson +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/ebooks/chardet/* +Copyright: Copyright (C) 1998-2001 Netscape Communications Corporation +License: LGPL-2.1+ + The full text of the LGPL is distributed as in + /usr/share/common-licenses/LGPL-2.1 on Debian systems. + +Files: src/calibre/ebooks/hyphenate.py +Copyright: Copyright (C) 1990, 2004, 2005 Gerard D.C. Kuiken. +License: other + Copying and distribution of this file, with or without modification, + are permitted in any medium without royalty provided the copyright + notice and this notice are preserved. + +Files: /src/cherrypy/* +Copyright: Copyright (c) 2004-2007, CherryPy Team (team@cherrypy.org) +Copyright: Copyright (C) 2005, Tiago Cogumbreiro +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/odf/* +Copyright: Copyright (C) 2006-2008 Søren Roug, European Environment Agency +License: LGPL2.1+ + The full text of the LGPL is distributed as in + /usr/share/common-licenses/LGPL-2.1 on Debian systems. + +Files: src/odf/teletype.py +Files: src/odf/easyliststyle.py +Copyright: Copyright (C) 2008, J. David Eisenberg +License: GPL2+ + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-2 on Debian systems. + +Files: src/pyPdf/* +Copyright: Copyright (c) 2006, Mathieu Fenniak +Copyright: Copyright (c) 2007, Ashish Kulkarni +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/utils/genshi/* +Copyright: Copyright (C) 2006-2008 Edgewall Software +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/utils/lzx/* +Copyright: Copyright (C) 2002, Matthew T. Russotto +Copyright: Copyright (C) 2008, Marshall T. Vandegrift +Copyright: Copyright (C) 2006-2008, Alexander Chemeris +License: LGPL-2.1 + The full text of the LGPL is distributed as in + /usr/share/common-licenses/LGPL-2.1 on Debian systems. + +Files: src/calibre/utils/lzx/msstdint.h +Copyright: Copyright (C) 2006-2008, Alexander Chemeris +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/utils/pyparsing.py +Copyright: Copyright (c) 2003-2008, Paul T. McGuire +License: MIT + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Files: src/calibre/utils/PythonMagickWand.py +Copyright: (c) 2007 - Achim Domma - domma@procoders.net +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +Files: src/calibre/utils/msdes/d3des.h: +Files: src/calibre/utils/msdes/des.c: +Copyright: Copyright (C) 1988,1989,1990,1991,1992, Richard Outerbridge +License: Other + THIS SOFTWARE PLACED IN THE PUBLIC DOMAIN BY THE AUTHOUR + +Files: src/calibre/utils/msdes/msdesmodule.c +Copyright: Copyright (C) 2008, Marshall T. Vandegrift +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/utils/msdes/spr.h +Copyright: Copyright (C) 2002, Dan A. Jackson +License: GPL2+ + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-2 on Debian systems. + +Files: src/calibre/gui2/pictureflow/* +Copyright: (C) Copyright 2007 Trolltech ASA +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/ebooks/lit/* +Copyright: 2008, Marshall T. Vandegrift +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/ebooks/lrf/* +Copyright: 2008, Anatoly Shipitsin +Copyright: copyright 2002 Paul Henry Tremblay +Copyright: Copyright (C) 2008 B.Scott Wxby [bswxby] +Copyright: Copyright (C) 2007 David Chen SonyReaderDaveChenorg +Copyright: Copyright (c) 2007 Mike Higgins (Falstaff) +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/ebooks/BeautifulSoup.py +Copyright: Copyright (c) 2004-2007, Leonard Richardson +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/ebooks/rtf2xml/* +Copyright: copyright 2002 Paul Henry Tremblay +License: GPL + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL on Debian systems. + +Files: src/calibre/web/feeds/feedparser.py +Copyright: Copyright (c) 2002-2006, Mark Pilgrim +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/web/feeds/recipes/* +Copyright: 2008, Darko Miletic +Copyright: 2008, Mathieu Godlewski +Copyright: Copyright (C) 2008 B.Scott Wxby [bswxby] +Copyright: Copyright (C) 2007 David Chen SonyReaderDaveChenorg +Copyright: 2008, Derry FitzGerald +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/ebooks/metadata/* +Copyright: 2008, Ashish Kulkarni +Copyright: Copyright (C) 2006 Søren Roug, European Environment Agency +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/encutils/__init__.py +Copyright: 2005-2008: Christof Hoeke +License: LGPL-3+, CC-BY-3.0 + The full text of the LGPL is distributed as in + /usr/share/common-licenses/LGPL-3 on Debian systems. + +Files: src/calibre/translations/* +Copyright: Copyright (C) 2007, Kovid Goyal +Copyright: Copyright (C) 2008, Rosetta Contributors and Canonical Ltd. +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/gui2/viewer/jquery.js +Files: src/calibre/gui2/viewer/jquery_scrollTo.js +Files: src/calibre/library/static/date.js +Copyright: Copyright (C) 2008, John Resig (jquery.com) +Copyright: Copyright (C) 2007-2008, Ariel Flesler - aflesler@gmail.com | http://flesler.blogspot.com +Copyright: Copyright (C) 2006-2007, Coolite Inc. (http://www.coolite.com/) +License: MIT + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Files: src/calibre/ebooks/lrf/fonts/liberation/* +Copyright: Copyright (C) 2007, Red Hat, Inc. All rights reserved. +License: Other + Copyright (C) 2007, Red Hat, Inc. All rights reserved. + LIBERATION is a trademark of Red Hat, Inc. + + This agreement governs the use of the Software and any updates to the Software, + regardless of the delivery mechanism. Subject to the following terms, Red Hat, Inc. + ("Red Hat") grants to the user ("Client") a license to this work pursuant to + the GNU General Public License v.2 with the exceptions set forth below and such + other terms as our set forth in this End User License Agreement. + + 1. The Software and License Exception. LIBERATION font software (the "Software") + consists of TrueType-OpenType formatted font software for rendering LIBERATION + typefaces in sans serif, serif, and monospaced character styles. You are licensed + to use, modify, copy, and distribute the Software pursuant to the GNU General + Public License v.2 with the following exceptions: + + (a) As a special exception, if you create a document which uses this font, and + embed this font or unaltered portions of this font into the document, this + font does not by itself cause the resulting document to be covered by the + GNU General Public License. This exception does not however invalidate any + other reasons why the document might be covered by the GNU General Public + License. If you modify this font, you may extend this exception to your + version of the font, but you are not obligated to do so. If you do not + wish to do so, delete this exception statement from your version. + + (b) As a further exception, any distribution of the object code of the Software + in a physical product must provide you the right to access and modify the + source code for the Software and to reinstall that modified version of the + Software in object code form on the same physical product on which you + received it. + + 2. Intellectual Property Rights. The Software and each of its components, including + the source code, documentation, appearance, structure and organization are owned + by Red Hat and others and are protected under copyright and other laws. Title to + the Software and any component, or to any copy, modification, or merged portion + shall remain with the aforementioned, subject to the applicable license. + The "LIBERATION" trademark is a trademark of Red Hat, Inc. in the U.S. and other + countries. This agreement does not permit Client to distribute modified versions + of the Software using Red Hat's trademarks. If Client makes a redistribution of + a modified version of the Software, then Client must modify the files names to + remove any reference to the Red Hat trademarks and must not use the Red Hat + trademarks in any way to reference or promote the modified Software. + + 3. Limited Warranty. To the maximum extent permitted under applicable law, the + Software is provided and licensed "as is" without warranty of any kind, + expressed or implied, including the implied warranties of merchantability, + non-infringement or fitness for a particular purpose. Red Hat does not warrant + that the functions contained in the Software will meet Client's requirements or + that the operation of the Software will be entirely error free or appear precisely + as described in the accompanying documentation. + + 4. Limitation of Remedies and Liability. To the maximum extent permitted by applicable + law, Red Hat or any Red Hat authorized dealer will not be liable to Client for any + incidental or consequential damages, including lost profits or lost savings arising + out of the use or inability to use the Software, even if Red Hat or such dealer has + been advised of the possibility of such damages. + + 5. General. If any provision of this agreement is held to be unenforceable, that shall + not affect the enforceability of the remaining provisions. This agreement shall be + governed by the laws of the State of North Carolina and of the United States, without + regard to any conflict of laws provisions, except that the United Nations Convention + on the International Sale of Goods shall not apply. + + +Files: installer/cx_Freeze/* +Copyright: Copyright © 2007-2008, Colt Engineering, Edmonton, Alberta, Canada. +Copyright: Copyright © 2001-2006, Computronix (Canada) Ltd., Edmonton, Alberta, Canada. +License: other + All rights reserved. + + NOTE: this license is derived from the Python Software Foundation License + which can be found at http://www.python.org/psf/license + + License for cx_Freeze 4.0.1 + --------------------------- + + 1. This LICENSE AGREEMENT is between the copyright holders and the Individual + or Organization ("Licensee") accessing and otherwise using cx_Freeze + software in source or binary form and its associated documentation. + + 2. Subject to the terms and conditions of this License Agreement, the + copyright holders hereby grant Licensee a nonexclusive, royalty-free, + world-wide license to reproduce, analyze, test, perform and/or display + publicly, prepare derivative works, distribute, and otherwise use cx_Freeze + alone or in any derivative version, provided, however, that this License + Agreement and this notice of copyright are retained in cx_Freeze alone or in + any derivative version prepared by Licensee. + + 3. In the event Licensee prepares a derivative work that is based on or + incorporates cx_Freeze or any part thereof, and wants to make the derivative + work available to others as provided herein, then Licensee hereby agrees to + include in any such work a brief summary of the changes made to cx_Freeze. + + 4. The copyright holders are making cx_Freeze available to Licensee on an + "AS IS" basis. THE COPYRIGHT HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, + EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, THE COPYRIGHT + HOLDERS MAKE NO AND DISCLAIM ANY REPRESENTATION OR WARRANTY OF + MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF + CX_FREEZE WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + + 5. THE COPYRIGHT HOLDERS SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF + CX_FREEZE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS + A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING CX_FREEZE, OR ANY + DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + + 6. This License Agreement will automatically terminate upon a material breach + of its terms and conditions. + + 7. Nothing in this License Agreement shall be deemed to create any relationship + of agency, partnership, or joint venture between the copyright holders and + Licensee. This License Agreement does not grant permission to use + copyright holder's trademarks or trade name in a trademark sense to endorse + or promote products or services of Licensee, or any third party. + + 8. By copying, installing or otherwise using cx_Freeze, Licensee agrees to be + bound by the terms and conditions of this License Agreement. + + Computronix® is a registered trademark of Computronix (Canada) Ltd. + diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index c4301322e8..6eca033bb1 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -17,8 +17,7 @@ from xml.dom import SyntaxErr as CSSSyntaxError import cssutils from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \ CSSValueList, cssproperties -from cssutils.profiles import Profiles -cssprofiles = Profiles() +from cssutils import profile as cssprofiles from lxml import etree from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES From c3ba827f5735a83c813f354aa3977b51fb28c4b1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 13 May 2009 18:37:51 -0700 Subject: [PATCH 07/15] New job system works in linux (needs more testing especially on non-linux systems) --- src/calibre/__init__.py | 9 +- src/calibre/devices/bebook/driver.py | 6 +- src/calibre/ebooks/comic/input.py | 59 +- src/calibre/ebooks/conversion/plumber.py | 1 + src/calibre/ebooks/lit/output.py | 14 +- src/calibre/ebooks/oeb/base.py | 13 +- .../ebooks/oeb/transforms/structure.py | 8 +- src/calibre/ebooks/pdf/pdftohtml.py | 12 +- src/calibre/gui2/convert/__init__.py | 2 +- src/calibre/gui2/convert/single.py | 2 + src/calibre/gui2/device.py | 38 +- src/calibre/gui2/dialogs/jobs.py | 66 -- src/calibre/gui2/dialogs/jobs.ui | 43 +- src/calibre/gui2/dialogs/user_profiles.py | 2 +- src/calibre/gui2/jobs.py | 257 +++++ src/calibre/gui2/jobs2.py | 203 ---- src/calibre/gui2/main.py | 71 +- src/calibre/gui2/status.py | 77 +- src/calibre/gui2/widgets.py | 46 +- src/calibre/linux.py | 2 +- src/calibre/parallel.py | 980 ------------------ src/calibre/utils/ipc/job.py | 137 +++ src/calibre/utils/ipc/launch.py | 3 +- src/calibre/utils/ipc/server.py | 236 +++++ src/calibre/utils/ipc/worker.py | 20 +- 25 files changed, 891 insertions(+), 1416 deletions(-) delete mode 100644 src/calibre/gui2/dialogs/jobs.py create mode 100644 src/calibre/gui2/jobs.py delete mode 100644 src/calibre/gui2/jobs2.py delete mode 100644 src/calibre/parallel.py create mode 100644 src/calibre/utils/ipc/job.py diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 280bf02cae..a08f0417ee 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -2,7 +2,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, re, logging, time, subprocess, mimetypes, \ +import sys, os, re, logging, time, mimetypes, \ __builtin__, warnings, multiprocessing __builtin__.__dict__['dynamic_property'] = lambda(func): func(None) from htmlentitydefs import name2codepoint @@ -91,11 +91,16 @@ def prints(*args, **kwargs): file = kwargs.get('file', sys.stdout) sep = kwargs.get('sep', ' ') end = kwargs.get('end', '\n') + enc = preferred_encoding + if 'CALIBRE_WORKER' in os.environ: + enc = 'utf-8' for i, arg in enumerate(args): if isinstance(arg, unicode): - arg = arg.encode(preferred_encoding) + arg = arg.encode(enc) if not isinstance(arg, str): arg = str(arg) + if not isinstance(arg, unicode): + arg = arg.decode(preferred_encoding, 'replace').encode(enc) file.write(arg) if i != len(args)-1: file.write(sep) diff --git a/src/calibre/devices/bebook/driver.py b/src/calibre/devices/bebook/driver.py index 0980554387..d3e887eb74 100644 --- a/src/calibre/devices/bebook/driver.py +++ b/src/calibre/devices/bebook/driver.py @@ -18,9 +18,9 @@ class BEBOOK(USBMS): VENDOR_ID = [0x0525] PRODUCT_ID = [0x8803, 0x6803] - BCD = [0x312] + BCD = [0x312] - VENDOR_NAME = 'LINUX' + VENDOR_NAME = 'LINUX' WINDOWS_MAIN_MEM = 'FILE-STOR_GADGET' WINDOWS_CARD_MEM = 'FILE-STOR_GADGET' @@ -51,7 +51,7 @@ class BEBOOK_MINI(BEBOOK): VENDOR_ID = [0x0492] PRODUCT_ID = [0x8813] - BCD = [0x319] + BCD = [0x319] OSX_MAIN_MEM = 'BeBook Mini Internal Memory' OSX_CARD_MEM = 'BeBook Mini Storage Card' diff --git a/src/calibre/ebooks/comic/input.py b/src/calibre/ebooks/comic/input.py index b228182ef4..bf2aac1162 100755 --- a/src/calibre/ebooks/comic/input.py +++ b/src/calibre/ebooks/comic/input.py @@ -7,12 +7,14 @@ __docformat__ = 'restructuredtext en' Based on ideas from comiclrf created by FangornUK. ''' -import os, shutil, traceback, textwrap +import os, shutil, traceback, textwrap, time +from Queue import Empty from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from calibre import extract, CurrentDir +from calibre import extract, CurrentDir, prints from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.parallel import Server, ParallelJob +from calibre.utils.ipc.server import Server +from calibre.utils.ipc.job import ParallelJob def extract_comic(path_to_comic_file): ''' @@ -47,8 +49,8 @@ def find_pages(dir, sort_on_mtime=False, verbose=False): pages.sort(cmp=comparator) if verbose: - print 'Found comic pages...' - print '\t'+'\n\t'.join([os.path.basename(p) for p in pages]) + prints('Found comic pages...') + prints('\t'+'\n\t'.join([os.path.basename(p) for p in pages])) return pages class PageProcessor(list): @@ -181,7 +183,7 @@ class PageProcessor(list): p.DestroyPixelWand(pw) p.DestroyMagickWand(wand) -def render_pages(tasks, dest, opts, notification=None): +def render_pages(tasks, dest, opts, notification=lambda x, y: x): ''' Entry point for the job server. ''' @@ -197,30 +199,23 @@ def render_pages(tasks, dest, opts, notification=None): msg = _('Failed %s')%path if opts.verbose: msg += '\n' + traceback.format_exc() - if notification is not None: - notification(0.5, msg) + prints(msg) + notification(0.5, msg) return pages, failures -class JobManager(object): - ''' - Simple job manager responsible for keeping track of overall progress. - ''' +class Progress(object): def __init__(self, total, update): self.total = total self.update = update self.done = 0 - self.add_job = lambda j: j - self.output = lambda j: j - self.start_work = lambda j: j - self.job_done = lambda j: j - def status_update(self, job): + def __call__(self, percent, msg=''): self.done += 1 #msg = msg%os.path.basename(job.args[0]) - self.update(float(self.done)/self.total, job.msg) + self.update(float(self.done)/self.total, msg) def process_pages(pages, opts, update, tdir): ''' @@ -229,22 +224,38 @@ def process_pages(pages, opts, update, tdir): from calibre.utils.PythonMagickWand import ImageMagick ImageMagick - job_manager = JobManager(len(pages), update) + progress = Progress(len(pages), update) server = Server() jobs = [] + tasks = [(p, os.path.join(tdir, os.path.basename(p))) for p in pages] tasks = server.split(pages) for task in tasks: - jobs.append(ParallelJob('render_pages', lambda s:s, job_manager=job_manager, + jobs.append(ParallelJob('render_pages', '', progress, args=[task, tdir, opts])) server.add_job(jobs[-1]) - server.wait() - server.killall() + while True: + time.sleep(1) + running = False + for job in jobs: + while True: + try: + x = job.notifications.get_nowait() + progress(*x) + except Empty: + break + job.update() + if not job.is_finished: + running = True + if not running: + break server.close() ans, failures = [], [] for job in jobs: - if job.result is None: - raise Exception(_('Failed to process comic: %s\n\n%s')%(job.exception, job.traceback)) + if job.failed: + raw_input() + raise Exception(_('Failed to process comic: \n\n%s')% + job.log_file.read()) pages, failures_ = job.result ans += pages failures += failures_ diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index ee26c3001c..7387cf158e 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -668,6 +668,7 @@ OptionRecommendation(name='list_recipes', self.output_plugin.convert(self.oeb, self.output, self.input_plugin, self.opts, self.log) self.ui_reporter(1.) + self.log(self.output_fmt.upper(), 'output written to', self.output) def create_oebbook(log, path_or_stream, opts, input_plugin, reader=None): ''' diff --git a/src/calibre/ebooks/lit/output.py b/src/calibre/ebooks/lit/output.py index 42be1ecac7..c72d8eae96 100644 --- a/src/calibre/ebooks/lit/output.py +++ b/src/calibre/ebooks/lit/output.py @@ -7,7 +7,8 @@ __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from calibre.customize.conversion import OutputFormatPlugin +from calibre.customize.conversion import OutputFormatPlugin, \ + OptionRecommendation class LITOutput(OutputFormatPlugin): @@ -15,12 +16,23 @@ class LITOutput(OutputFormatPlugin): author = 'Marshall T. Vandegrift' file_type = 'lit' + recommendations = set([ + ('dont_split_on_page_breaks', False, OptionRecommendation.HIGH), + ]) + def convert(self, oeb, output_path, input_plugin, opts, log): self.log, self.opts, self.oeb = log, opts, oeb from calibre.ebooks.oeb.transforms.manglecase import CaseMangler from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer from calibre.ebooks.oeb.transforms.htmltoc import HTMLTOCAdder from calibre.ebooks.lit.writer import LitWriter + from calibre.ebooks.oeb.transforms.split import Split + split = Split(not self.opts.dont_split_on_page_breaks, + max_flow_size=0 + ) + split(self.oeb, self.opts) + + tocadder = HTMLTOCAdder() tocadder(oeb, opts) mangler = CaseMangler() diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index bbac34f0b1..bdf78f96e4 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -96,8 +96,12 @@ def iterlinks(root): for el in root.iter(): attribs = el.attrib + try: + tag = el.tag + except UnicodeDecodeError: + continue - if el.tag == XHTML('object'): + if tag == XHTML('object'): codebase = None ## tags have attributes that are relative to ## codebase @@ -122,7 +126,7 @@ def iterlinks(root): yield (el, attr, attribs[attr], 0) - if el.tag == XHTML('style') and el.text: + if tag == XHTML('style') and el.text: for match in _css_url_re.finditer(el.text): yield (el, None, match.group(1), match.start(1)) for match in _css_import_re.finditer(el.text): @@ -801,6 +805,11 @@ class Manifest(object): self.oeb.logger.warn( 'File %r missing element' % self.href) etree.SubElement(data, XHTML('body')) + + # Remove microsoft office markup + r = [x for x in data.iterdescendants(etree.Element) if 'microsoft-com' in x.tag] + for x in r: + x.tag = XHTML('span') return data def _parse_css(self, data): diff --git a/src/calibre/ebooks/oeb/transforms/structure.py b/src/calibre/ebooks/oeb/transforms/structure.py index 8ec3c7737a..5e3322a57a 100644 --- a/src/calibre/ebooks/oeb/transforms/structure.py +++ b/src/calibre/ebooks/oeb/transforms/structure.py @@ -12,7 +12,13 @@ from lxml import etree from urlparse import urlparse from calibre.ebooks.oeb.base import XPNSMAP, TOC, XHTML -XPath = lambda x: etree.XPath(x, namespaces=XPNSMAP) +from calibre.ebooks import ConversionError +def XPath(x): + try: + return etree.XPath(x, namespaces=XPNSMAP) + except etree.XPathSyntaxError: + raise ConversionError( + 'The syntax of the XPath expression %s is invalid.' % repr(x)) class DetectStructure(object): diff --git a/src/calibre/ebooks/pdf/pdftohtml.py b/src/calibre/ebooks/pdf/pdftohtml.py index e03d7d0647..2c0daf05ca 100644 --- a/src/calibre/ebooks/pdf/pdftohtml.py +++ b/src/calibre/ebooks/pdf/pdftohtml.py @@ -6,7 +6,7 @@ __copyright__ = '2008, Kovid Goyal , ' \ '2009, John Schember ' __docformat__ = 'restructuredtext en' -import errno, os, re, sys, subprocess +import errno, os, sys, subprocess from functools import partial from calibre.ebooks import ConversionError, DRMError @@ -33,7 +33,7 @@ def pdftohtml(pdf_path): if isinstance(pdf_path, unicode): pdf_path = pdf_path.encode(sys.getfilesystemencoding()) if not os.access(pdf_path, os.R_OK): - raise ConversionError, 'Cannot read from ' + pdf_path + raise ConversionError('Cannot read from ' + pdf_path) with TemporaryDirectory('_pdftohtml') as tdir: index = os.path.join(tdir, 'index.html') @@ -47,7 +47,7 @@ def pdftohtml(pdf_path): p = popen(cmd, stderr=subprocess.PIPE) except OSError, err: if err.errno == 2: - raise ConversionError(_('Could not find pdftohtml, check it is in your PATH'), True) + raise ConversionError(_('Could not find pdftohtml, check it is in your PATH')) else: raise @@ -63,13 +63,13 @@ def pdftohtml(pdf_path): if ret != 0: err = p.stderr.read() - raise ConversionError, err + raise ConversionError(err) if not os.path.exists(index) or os.stat(index).st_size < 100: raise DRMError() - + with open(index, 'rb') as i: raw = i.read() if not '\n' + raw diff --git a/src/calibre/gui2/convert/__init__.py b/src/calibre/gui2/convert/__init__.py index 67b6e47aa9..d7dde4c190 100644 --- a/src/calibre/gui2/convert/__init__.py +++ b/src/calibre/gui2/convert/__init__.py @@ -176,7 +176,7 @@ class Widget(QWidget): elif isinstance(g, QCheckBox): return bool(g.isChecked()) elif isinstance(g, XPathEdit): - return g.xpath + return g.xpath if g.xpath else None else: raise Exception('Can\'t get value from %s'%type(g)) diff --git a/src/calibre/gui2/convert/single.py b/src/calibre/gui2/convert/single.py index c0da36e5dd..531448b1f3 100644 --- a/src/calibre/gui2/convert/single.py +++ b/src/calibre/gui2/convert/single.py @@ -189,6 +189,8 @@ class Config(ResizableDialog, Ui_Dialog): def setup_input_output_formats(self, db, book_id, preferred_input_format, preferred_output_format): + if preferred_output_format: + preferred_output_format = preferred_output_format.lower() available_formats = db.formats(book_id, index_is_id=True) if not available_formats: available_formats = '' diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index b3dcf8a21c..7f2513361a 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1,7 +1,7 @@ from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -import os, traceback, Queue, time, socket +import os, traceback, Queue, time, socket, cStringIO from threading import Thread, RLock from itertools import repeat from functools import partial @@ -15,7 +15,7 @@ from calibre.customize.ui import available_input_formats, available_output_forma from calibre.devices.interface import DevicePlugin from calibre.constants import iswindows from calibre.gui2.dialogs.choose_format import ChooseFormatDialog -from calibre.parallel import Job +from calibre.utils.ipc.job import BaseJob from calibre.devices.scanner import DeviceScanner from calibre.gui2 import config, error_dialog, Dispatcher, dynamic, \ pixmap_to_data, warning_dialog, \ @@ -27,22 +27,46 @@ from calibre.devices.errors import FreeSpaceError from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \ config as email_config -class DeviceJob(Job): +class DeviceJob(BaseJob): - def __init__(self, func, *args, **kwargs): - Job.__init__(self, *args, **kwargs) + def __init__(self, func, done, job_manager, args=[], kwargs={}, + description=''): + BaseJob.__init__(self, description, done=done) self.func = func + self.args, self.kwargs = args, kwargs + self.job_manager = job_manager + self.job_manager.add_job(self) + self.details = _('No details available.') + + def start_work(self): + self.start_time = time.time() + self.job_manager.changed_queue.put(self) + + def job_done(self): + self.duration = time.time() - self.start_time() + self.job_manager.changed_queue.put(self) + self.job_manager.job_done(self) + + def report_progress(self, percent, msg=''): + self.notifications.put((percent, msg)) + self.job_manager.changed_queue.put(self) def run(self): self.start_work() try: self.result = self.func(*self.args, **self.kwargs) except (Exception, SystemExit), err: + self.failed = True + self.details = unicode(err) + '\n\n' + \ + traceback.format_exc() self.exception = err - self.traceback = traceback.format_exc() finally: self.job_done() + @property + def log_file(self): + return cStringIO.StringIO(self.details.encode('utf-8')) + class DeviceManager(Thread): @@ -113,7 +137,7 @@ class DeviceManager(Thread): job = self.next() if job is not None: self.current_job = job - self.device.set_progress_reporter(job.update_status) + self.device.set_progress_reporter(job.report_progress) self.current_job.run() self.current_job = None else: diff --git a/src/calibre/gui2/dialogs/jobs.py b/src/calibre/gui2/dialogs/jobs.py deleted file mode 100644 index c6907c6b60..0000000000 --- a/src/calibre/gui2/dialogs/jobs.py +++ /dev/null @@ -1,66 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal ' -'''Display active jobs''' - -from PyQt4.QtCore import Qt, QObject, SIGNAL, QSize, QString, QTimer -from PyQt4.QtGui import QDialog, QAbstractItemDelegate, QStyleOptionProgressBarV2, \ - QApplication, QStyle - -from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog -from calibre import __appname__ - -class ProgressBarDelegate(QAbstractItemDelegate): - - def sizeHint(self, option, index): - return QSize(120, 30) - - def paint(self, painter, option, index): - opts = QStyleOptionProgressBarV2() - opts.rect = option.rect - opts.minimum = 1 - opts.maximum = 100 - opts.textVisible = True - percent, ok = index.model().data(index, Qt.DisplayRole).toInt() - if not ok: - percent = 0 - opts.progress = percent - opts.text = QString(_('Unavailable') if percent == 0 else '%d%%'%percent) - QApplication.style().drawControl(QStyle.CE_ProgressBar, opts, painter) - -class JobsDialog(QDialog, Ui_JobsDialog): - def __init__(self, window, model): - QDialog.__init__(self, window) - Ui_JobsDialog.__init__(self) - self.setupUi(self) - self.jobs_view.setModel(model) - self.model = model - self.setWindowModality(Qt.NonModal) - self.setWindowTitle(__appname__ + _(' - Jobs')) - QObject.connect(self.jobs_view.model(), SIGNAL('modelReset()'), - self.jobs_view.resizeColumnsToContents) - QObject.connect(self.kill_button, SIGNAL('clicked()'), - self.kill_job) - QObject.connect(self, SIGNAL('kill_job(int, PyQt_PyObject)'), - self.jobs_view.model().kill_job) - self.pb_delegate = ProgressBarDelegate(self) - self.jobs_view.setItemDelegateForColumn(2, self.pb_delegate) - - self.running_time_timer = QTimer(self) - self.connect(self.running_time_timer, SIGNAL('timeout()'), self.update_running_time) - self.running_time_timer.start(1000) - - def update_running_time(self, *args): - try: - self.model.running_time_updated() - except: # Raises random exceptions on OS X - pass - - def kill_job(self): - for index in self.jobs_view.selectedIndexes(): - row = index.row() - self.model.kill_job(row, self) - return - - def closeEvent(self, e): - self.jobs_view.write_settings() - e.accept() diff --git a/src/calibre/gui2/dialogs/jobs.ui b/src/calibre/gui2/dialogs/jobs.ui index a14be17f78..3716c9fbb9 100644 --- a/src/calibre/gui2/dialogs/jobs.ui +++ b/src/calibre/gui2/dialogs/jobs.ui @@ -1,7 +1,8 @@ - + + JobsDialog - - + + 0 0 @@ -9,31 +10,32 @@ 542 - + Active Jobs - - :/images/jobs.svg + + + :/images/jobs.svg:/images/jobs.svg - + - - + + Qt::NoContextMenu - + QAbstractItemView::NoEditTriggers - + true - + QAbstractItemView::SingleSelection - + QAbstractItemView::SelectRows - + 32 32 @@ -42,12 +44,19 @@ - - + + &Stop selected job + + + + Show job &details + + + @@ -58,7 +67,7 @@ - + diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index cab28c66eb..c46b4ba392 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -86,7 +86,7 @@ class UserProfiles(ResizableDialog, Ui_Dialog): self.source_code.setPlainText('') else: self.source_code.setPlainText(src) - #self.highlighter = PythonHighlighter(self.source_code.document()) + self.highlighter = PythonHighlighter(self.source_code.document()) self.stacks.setCurrentIndex(1) self.toggle_mode_button.setText(_('Switch to Basic mode')) diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py new file mode 100644 index 0000000000..6be6188ab9 --- /dev/null +++ b/src/calibre/gui2/jobs.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + +''' +Job management. +''' + +from Queue import Empty, Queue + +from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \ + QTimer, SIGNAL, QIcon, QDialog, QAbstractItemDelegate, QApplication, \ + QSize, QStyleOptionProgressBarV2, QString, QStyle + +from calibre.utils.ipc.server import Server +from calibre.utils.ipc.job import ParallelJob +from calibre.gui2 import Dispatcher, error_dialog, NONE +from calibre.gui2.device import DeviceJob +from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog +from calibre import __appname__ + +class JobManager(QAbstractTableModel): + + def __init__(self): + QAbstractTableModel.__init__(self) + self.wait_icon = QVariant(QIcon(':/images/jobs.svg')) + self.running_icon = QVariant(QIcon(':/images/exec.svg')) + self.error_icon = QVariant(QIcon(':/images/dialog_error.svg')) + self.done_icon = QVariant(QIcon(':/images/ok.svg')) + + self.jobs = [] + self.add_job = Dispatcher(self._add_job) + self.job_done = Dispatcher(self._job_done) + self.server = Server(self.job_done) + self.changed_queue = Queue() + + self.timer = QTimer(self) + self.connect(self.timer, SIGNAL('timeout()'), self.update, + Qt.QueuedConnection) + self.timer.start(1000) + + def columnCount(self, parent=QModelIndex()): + return 4 + + def rowCount(self, parent=QModelIndex()): + return len(self.jobs) + + def headerData(self, section, orientation, role): + if role != Qt.DisplayRole: + return NONE + if orientation == Qt.Horizontal: + if section == 0: text = _('Job') + elif section == 1: text = _('Status') + elif section == 2: text = _('Progress') + elif section == 3: text = _('Running time') + return QVariant(text) + else: + return QVariant(section+1) + + def data(self, index, role): + try: + if role not in (Qt.DisplayRole, Qt.DecorationRole): + return NONE + row, col = index.row(), index.column() + job = self.jobs[row] + + if role == Qt.DisplayRole: + if col == 0: + desc = job.description + if not desc: + desc = _('Unknown job') + return QVariant(desc) + if col == 1: + return QVariant(job.status_text) + if col == 2: + return QVariant(job.percent) + if col == 3: + rtime = job.running_time + if rtime is None: + return NONE + return QVariant('%dm %ds'%(int(rtime)//60, int(rtime)%60)) + if role == Qt.DecorationRole and col == 0: + state = job.run_state + if state == job.WAITING: + return self.wait_icon + if state == job.RUNNING: + return self.running_icon + if job.killed or job.failed: + return self.error_icon + return self.done_icon + except: + import traceback + traceback.print_exc() + return NONE + + def update(self): + try: + self._update() + except BaseException: + pass + + def _update(self): + # Update running time + rows = set([]) + for i, j in enumerate(self.jobs): + if j.run_state == j.RUNNING: + idx = self.index(i, 3) + self.emit(SIGNAL('dataChanged(QModelIndex,QModelIndex)'), + idx, idx) + + # Update parallel jobs + jobs = set([]) + while True: + try: + jobs.add(self.server.changed_jobs_queue.get_nowait()) + except Empty: + break + while True: + try: + jobs.add(self.changed_queue.get_nowait()) + except Empty: + break + + if jobs: + needs_reset = False + for job in jobs: + orig_state = job.run_state + job.update() + if orig_state != job.run_state: + needs_reset = True + if needs_reset: + self.jobs.sort() + self.reset() + else: + for job in jobs: + idx = self.jobs.index(job) + self.emit(SIGNAL('dataChanged(QModelIndex,QModelIndex)'), + self.index(idx, 0), self.index(idx, 3)) + + + def _add_job(self, job): + self.emit(SIGNAL('layoutAboutToBeChanged()')) + self.jobs.append(job) + self.jobs.sort() + self.emit(SIGNAL('job_added(int)'), len(self.unfinished_jobs())) + self.emit(SIGNAL('layoutChanged()')) + + def done_jobs(self): + return [j for j in self.jobs if j.is_finished] + + def unfinished_jobs(self): + return [j for j in self.jobs if not j.is_finished] + + def row_to_job(self, row): + return self.jobs[row] + + def _job_done(self, job): + self.emit(SIGNAL('layoutAboutToBeChanged()')) + self.jobs.sort() + self.emit(SIGNAL('job_done(int)'), len(self.unfinished_jobs())) + self.emit(SIGNAL('layoutChanged()')) + + def has_device_jobs(self): + for job in self.jobs: + if job.is_running and isinstance(job, DeviceJob): + return True + return False + + def has_jobs(self): + for job in self.jobs: + if job.is_running: + return True + return False + + def run_job(self, done, name, args=[], kwargs={}, + description=''): + job = ParallelJob(name, description, done, args=args, kwargs=kwargs) + self.add_job(job) + self.server.add_job(job) + return job + + def launch_gui_app(self, name, args=[], kwargs={}, description=''): + job = ParallelJob(name, description, lambda x: x, + args=args, kwargs=kwargs) + self.server.run_job(job, gui=True, redirect_output=False) + + + def kill_job(self, row, view): + job = self.jobs[row] + if isinstance(job, DeviceJob): + return error_dialog(view, _('Cannot kill job'), + _('Cannot kill jobs that communicate with the device')).exec_() + if job.duration is not None: + return error_dialog(view, _('Cannot kill job'), + _('Job has already run')).exec_() + self.server.kill_job(job) + + def terminate_all_jobs(self): + self.server.killall() + + +class ProgressBarDelegate(QAbstractItemDelegate): + + def sizeHint(self, option, index): + return QSize(120, 30) + + def paint(self, painter, option, index): + opts = QStyleOptionProgressBarV2() + opts.rect = option.rect + opts.minimum = 1 + opts.maximum = 100 + opts.textVisible = True + percent, ok = index.model().data(index, Qt.DisplayRole).toInt() + if not ok: + percent = 0 + opts.progress = percent + opts.text = QString(_('Unavailable') if percent == 0 else '%d%%'%percent) + QApplication.style().drawControl(QStyle.CE_ProgressBar, opts, painter) + +class JobsDialog(QDialog, Ui_JobsDialog): + def __init__(self, window, model): + QDialog.__init__(self, window) + Ui_JobsDialog.__init__(self) + self.setupUi(self) + self.jobs_view.setModel(model) + self.model = model + self.setWindowModality(Qt.NonModal) + self.setWindowTitle(__appname__ + _(' - Jobs')) + self.connect(self.jobs_view.model(), SIGNAL('modelReset()'), + self.jobs_view.resizeColumnsToContents) + self.connect(self.kill_button, SIGNAL('clicked()'), + self.kill_job) + self.connect(self.details_button, SIGNAL('clicked()'), + self.show_details) + self.connect(self, SIGNAL('kill_job(int, PyQt_PyObject)'), + self.jobs_view.model().kill_job) + self.pb_delegate = ProgressBarDelegate(self) + self.jobs_view.setItemDelegateForColumn(2, self.pb_delegate) + + + def kill_job(self): + for index in self.jobs_view.selectedIndexes(): + row = index.row() + self.model.kill_job(row, self) + return + + def show_details(self): + for index in self.jobs_view.selectedIndexes(): + self.jobs_view.show_details(index) + return + + + + def closeEvent(self, e): + self.jobs_view.write_settings() + e.accept() diff --git a/src/calibre/gui2/jobs2.py b/src/calibre/gui2/jobs2.py deleted file mode 100644 index fc6ddb642e..0000000000 --- a/src/calibre/gui2/jobs2.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Job management. -''' -import time -from PyQt4.QtCore import QAbstractTableModel, QVariant, QModelIndex, Qt, SIGNAL -from PyQt4.QtGui import QIcon, QDialog - -from calibre.parallel import ParallelJob, Server -from calibre.gui2 import Dispatcher, error_dialog -from calibre.gui2.device import DeviceJob -from calibre.gui2.dialogs.job_view_ui import Ui_Dialog - -NONE = QVariant() - -class JobManager(QAbstractTableModel): - - def __init__(self): - QAbstractTableModel.__init__(self) - self.wait_icon = QVariant(QIcon(':/images/jobs.svg')) - self.running_icon = QVariant(QIcon(':/images/exec.svg')) - self.error_icon = QVariant(QIcon(':/images/dialog_error.svg')) - self.done_icon = QVariant(QIcon(':/images/ok.svg')) - - self.jobs = [] - self.server = Server() - self.add_job = Dispatcher(self._add_job) - self.status_update = Dispatcher(self._status_update) - self.start_work = Dispatcher(self._start_work) - self.job_done = Dispatcher(self._job_done) - - def columnCount(self, parent=QModelIndex()): - return 4 - - def rowCount(self, parent=QModelIndex()): - return len(self.jobs) - - def headerData(self, section, orientation, role): - if role != Qt.DisplayRole: - return NONE - if orientation == Qt.Horizontal: - if section == 0: text = _("Job") - elif section == 1: text = _("Status") - elif section == 2: text = _("Progress") - elif section == 3: text = _('Running time') - return QVariant(text) - else: - return QVariant(section+1) - - def data(self, index, role): - try: - if role not in (Qt.DisplayRole, Qt.DecorationRole): - return NONE - row, col = index.row(), index.column() - job = self.jobs[row] - - if role == Qt.DisplayRole: - if col == 0: - desc = job.description - if not desc: - desc = _('Unknown job') - return QVariant(desc) - if col == 1: - status = job.status() - if status == 'DONE': - st = _('Finished') - elif status == 'ERROR': - st = _('Error') - elif status == 'WAITING': - st = _('Waiting') - else: - st = _('Working') - return QVariant(st) - if col == 2: - pc = job.percent - if pc <=0: - percent = 0 - else: - percent = int(100*pc) - return QVariant(percent) - if col == 3: - if job.start_time is None: - return NONE - rtime = job.running_time if job.running_time is not None else \ - time.time() - job.start_time - return QVariant('%dm %ds'%(int(rtime)//60, int(rtime)%60)) - if role == Qt.DecorationRole and col == 0: - status = job.status() - if status == 'WAITING': - return self.wait_icon - if status == 'WORKING': - return self.running_icon - if status == 'ERROR': - return self.error_icon - if status == 'DONE': - return self.done_icon - except: - import traceback - traceback.print_exc() - return NONE - - def _add_job(self, job): - self.emit(SIGNAL('layoutAboutToBeChanged()')) - self.jobs.append(job) - self.jobs.sort() - self.emit(SIGNAL('job_added(int)'), self.rowCount()) - self.emit(SIGNAL('layoutChanged()')) - - def done_jobs(self): - return [j for j in self.jobs if j.status() in ['DONE', 'ERROR']] - - def row_to_job(self, row): - return self.jobs[row] - - def _start_work(self, job): - self.emit(SIGNAL('layoutAboutToBeChanged()')) - self.jobs.sort() - self.emit(SIGNAL('layoutChanged()')) - - def _job_done(self, job): - self.emit(SIGNAL('layoutAboutToBeChanged()')) - self.jobs.sort() - self.emit(SIGNAL('job_done(int)'), len(self.jobs) - len(self.done_jobs())) - self.emit(SIGNAL('layoutChanged()')) - - def _status_update(self, job): - try: - row = self.jobs.index(job) - except ValueError: # Job has been stopped - return - self.emit(SIGNAL('dataChanged(QModelIndex, QModelIndex)'), - self.index(row, 0), self.index(row, 3)) - - def running_time_updated(self, *args): - for job in self.jobs: - if not job.is_running: - continue - row = self.jobs.index(job) - self.emit(SIGNAL('dataChanged(QModelIndex, QModelIndex)'), - self.index(row, 3), self.index(row, 3)) - - def has_device_jobs(self): - for job in self.jobs: - if job.is_running and isinstance(job, DeviceJob): - return True - return False - - def has_jobs(self): - for job in self.jobs: - if job.is_running: - return True - return False - - def run_job(self, done, func, args=[], kwargs={}, - description=None): - job = ParallelJob(func, done, self, args=args, kwargs=kwargs, - description=description) - self.server.add_job(job) - return job - - - def output(self, job): - self.emit(SIGNAL('output_received()')) - - def kill_job(self, row, view): - job = self.jobs[row] - if isinstance(job, DeviceJob): - error_dialog(view, _('Cannot kill job'), - _('Cannot kill jobs that communicate with the device')).exec_() - return - if job.has_run: - error_dialog(view, _('Cannot kill job'), - _('Job has already run')).exec_() - return - if not job.is_running: - self.jobs.remove(job) - self.reset() - return - - - self.server.kill(job) - - def terminate_all_jobs(self): - pass - -class DetailView(QDialog, Ui_Dialog): - - def __init__(self, parent, job): - QDialog.__init__(self, parent) - self.setupUi(self) - self.setWindowTitle(job.description) - self.job = job - self.update() - - - def update(self): - self.log.setPlainText(self.job.console_text()) - vbar = self.log.verticalScrollBar() - vbar.setValue(vbar.maximum()) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index 47276d2519..d64591bcd7 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -13,7 +13,7 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \ from PyQt4.QtSvg import QSvgRenderer from calibre import __version__, __appname__, islinux, sanitize_file_name, \ - iswindows, isosx + iswindows, isosx, prints from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import prefs, dynamic from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \ @@ -32,10 +32,9 @@ from calibre.gui2.main_window import MainWindow, option_parser as _option_parser from calibre.gui2.main_ui import Ui_MainWindow from calibre.gui2.device import DeviceManager, DeviceMenu, DeviceGUI, Emailer from calibre.gui2.status import StatusBar -from calibre.gui2.jobs2 import JobManager +from calibre.gui2.jobs import JobManager, JobsDialog from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog -from calibre.gui2.dialogs.jobs import JobsDialog from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebook, \ fetch_scheduled_recipe from calibre.gui2.dialogs.config import ConfigDialog @@ -44,7 +43,6 @@ from calibre.gui2.dialogs.choose_format import ChooseFormatDialog from calibre.gui2.dialogs.book_info import BookInfo from calibre.ebooks import BOOK_EXTENSIONS from calibre.library.database2 import LibraryDatabase2, CoverCache -from calibre.parallel import JobKilled from calibre.gui2.dialogs.confirm_delete import confirm class SaveMenu(QMenu): @@ -626,9 +624,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): ''' Called once device information has been read. ''' - if job.exception is not None: - self.device_job_exception(job) - return + if job.failed: + return self.device_job_exception(job) info, cp, fs = job.result self.location_view.model().update_devices(cp, fs) self.device_info = _('Connected ')+info[0] @@ -641,7 +638,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): ''' Called once metadata has been read for all books on the device. ''' - if job.exception is not None: + if job.failed: if isinstance(job.exception, ExpatError): error_dialog(self, _('Device database corrupted'), _(''' @@ -823,8 +820,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): Called once deletion is done on the device ''' for view in (self.memory_view, self.card_a_view, self.card_b_view): - view.model().deletion_done(job, bool(job.exception)) - if job.exception is not None: + view.model().deletion_done(job, job.failed) + if job.failed: self.device_job_exception(job) return @@ -993,9 +990,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): progress.hide() def books_saved(self, job): - if job.exception is not None: - self.device_job_exception(job) - return + if job.failed: + return self.device_job_exception(job) ############################################################################ @@ -1013,9 +1009,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def scheduled_recipe_fetched(self, job): temp_files, fmt, recipe, callback = self.conversion_jobs.pop(job) pt = temp_files[0] - if job.exception is not None: - self.job_exception(job) - return + if job.failed: + return self.job_exception(job) id = self.library_view.model().add_news(pt.name, recipe) self.library_view.model().reset() sync = dynamic.get('news_to_be_synced', set([])) @@ -1098,9 +1093,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def book_auto_converted(self, job): temp_files, fmt, book_id, on_card = self.conversion_jobs.pop(job) try: - if job.exception is not None: - self.job_exception(job) - return + if job.failed: + return self.job_exception(job) data = open(temp_files[0].name, 'rb') self.library_view.model().db.add_format(book_id, fmt, data, index_is_id=True) data.close() @@ -1122,7 +1116,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def book_converted(self, job): temp_files, fmt, book_id = self.conversion_jobs.pop(job) try: - if job.exception is not None: + if job.failed: self.job_exception(job) return data = open(temp_files[-1].name, 'rb') @@ -1151,7 +1145,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self._view_file(fmt_path) def book_downloaded_for_viewing(self, job): - if job.exception: + if job.failed: self.device_job_exception(job) return self._view_file(job.result) @@ -1165,12 +1159,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): args.append('--raise-window') if name is not None: args.append(name) - self.job_manager.server.run_free_job(viewer, - kwdargs=dict(args=args)) + self.job_manager.launch_gui_app(viewer, + kwargs=dict(args=args)) else: QDesktopServices.openUrl(QUrl.fromLocalFile(name))#launch(name) - - time.sleep(5) # User feedback + time.sleep(2) # User feedback finally: self.unsetCursor() @@ -1395,7 +1388,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): ''' try: if 'Could not read 32 bytes on the control bus.' in \ - unicode(job.exception): + unicode(job.details): error_dialog(self, _('Error talking to device'), _('There was a temporary error talking to the ' 'device. Please unplug and reconnect the device ' @@ -1404,16 +1397,16 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): except: pass try: - print >>sys.stderr, job.console_text() + prints(job.details, file=sys.stderr) except: pass if not self.device_error_dialog.isVisible(): - self.device_error_dialog.set_message(job.gui_text()) + self.device_error_dialog.set_message(job.details) self.device_error_dialog.show() def job_exception(self, job): try: - if job.exception[0] == 'DRMError': + if 'calibre.ebooks.DRMError' in job.details: error_dialog(self, _('Conversion Error'), _('

Could not convert: %s

It is a ' 'DRMed book. You must first remove the ' @@ -1423,23 +1416,15 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): return except: pass - only_msg = getattr(job.exception, 'only_msg', False) + if job.killed: + return try: - print job.console_text() + prints(job.details, file=sys.stderr) except: pass - if only_msg: - try: - exc = unicode(job.exception) - except: - exc = repr(job.exception) - error_dialog(self, _('Conversion Error'), exc).exec_() - return - if isinstance(job.exception, JobKilled): - return error_dialog(self, _('Conversion Error'), - _('Failed to process')+': '+unicode(job.description), - det_msg=job.console_text()).exec_() + _('Failed')+': '+unicode(job.description), + det_msg=job.details).exec_() def initialize_database(self): @@ -1555,7 +1540,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def shutdown(self, write_settings=True): if write_settings: self.write_settings() - self.job_manager.terminate_all_jobs() + self.job_manager.server.close() self.device_manager.keep_going = False self.cover_cache.stop() self.hide() diff --git a/src/calibre/gui2/status.py b/src/calibre/gui2/status.py index 7b3dac4dc8..11b442fd17 100644 --- a/src/calibre/gui2/status.py +++ b/src/calibre/gui2/status.py @@ -10,10 +10,10 @@ from calibre.gui2 import qstring_to_unicode, config class BookInfoDisplay(QWidget): class BookCoverDisplay(QLabel): - + WIDTH = 81 HEIGHT = 108 - + def __init__(self, coverpath=':/images/book.svg'): QLabel.__init__(self) self.default_pixmap = QPixmap(coverpath).scaled(self.__class__.WIDTH, @@ -23,42 +23,42 @@ class BookInfoDisplay(QWidget): self.setScaledContents(True) self.setMaximumHeight(self.HEIGHT) self.setPixmap(self.default_pixmap) - - + + def setPixmap(self, pixmap): width, height = fit_image(pixmap.width(), pixmap.height(), self.WIDTH, self.HEIGHT)[1:] self.setMaximumHeight(height) self.setMaximumWidth(width) QLabel.setPixmap(self, pixmap) - + try: aspect_ratio = pixmap.width()/float(pixmap.height()) except ZeroDivisionError: aspect_ratio = 1 self.setMaximumWidth(int(aspect_ratio*self.HEIGHT)) - + def sizeHint(self): return QSize(self.__class__.WIDTH, self.__class__.HEIGHT) - - + + class BookDataDisplay(QLabel): def __init__(self): QLabel.__init__(self) self.setText('') self.setWordWrap(True) self.setSizePolicy(QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)) - + def mouseReleaseEvent(self, ev): self.emit(SIGNAL('mr(int)'), 1) - + WEIGHTS = collections.defaultdict(lambda : 100) WEIGHTS[_('Path')] = 0 WEIGHTS[_('Formats')] = 1 WEIGHTS[_('Comments')] = 2 WEIGHTS[_('Series')] = 3 WEIGHTS[_('Tags')] = 4 - + def __init__(self, clear_message): QWidget.__init__(self) self.setCursor(Qt.PointingHandCursor) @@ -74,16 +74,16 @@ class BookInfoDisplay(QWidget): self.data = {} self.setVisible(False) self._layout.setAlignment(self.cover_display, Qt.AlignTop|Qt.AlignLeft) - + def mouseReleaseEvent(self, ev): self.emit(SIGNAL('show_book_info()')) - + def show_data(self, data): if data.has_key('cover'): self.cover_display.setPixmap(QPixmap.fromImage(data.pop('cover'))) else: self.cover_display.setPixmap(self.cover_display.default_pixmap) - + rows = u'' self.book_data.setText('') self.data = data.copy() @@ -97,7 +97,7 @@ class BookInfoDisplay(QWidget): txt = txt.decode(preferred_encoding, 'replace') rows += u'

'%(key, txt) self.book_data.setText(u'
%s:%s
'+rows+u'
') - + self.clear_message() self.book_data.updateGeometry() self.updateGeometry() @@ -113,7 +113,7 @@ class MovieButton(QFrame): self.movie = movie self.layout().addWidget(self.movie_widget) self.jobs = QLabel(''+_('Jobs:')+' 0') - self.jobs.setAlignment(Qt.AlignHCenter|Qt.AlignBottom) + self.jobs.setAlignment(Qt.AlignHCenter|Qt.AlignBottom) self.layout().addWidget(self.jobs) self.layout().setAlignment(self.jobs, Qt.AlignHCenter) self.jobs.setMargin(0) @@ -125,8 +125,8 @@ class MovieButton(QFrame): movie.start() movie.setPaused(True) self.jobs_dialog.jobs_view.restore_column_widths() - - + + def mouseReleaseEvent(self, event): if self.jobs_dialog.isVisible(): self.jobs_dialog.jobs_view.write_settings() @@ -137,7 +137,7 @@ class MovieButton(QFrame): self.jobs_dialog.jobs_view.restore_column_widths() class CoverFlowButton(QToolButton): - + def __init__(self, parent=None): QToolButton.__init__(self, parent) self.setIconSize(QSize(80, 80)) @@ -149,17 +149,17 @@ class CoverFlowButton(QToolButton): self.connect(self, SIGNAL('toggled(bool)'), self.adjust_tooltip) self.adjust_tooltip(False) self.setCursor(Qt.PointingHandCursor) - + def adjust_tooltip(self, on): tt = _('Click to turn off Cover Browsing') if on else _('Click to browse books by their covers') self.setToolTip(tt) - + def disable(self, reason): self.setDisabled(True) self.setToolTip(_('

Browsing books by their covers is disabled.
Import of pictureflow module failed:
')+reason) - + class TagViewButton(QToolButton): - + def __init__(self, parent=None): QToolButton.__init__(self, parent) self.setIconSize(QSize(80, 80)) @@ -170,10 +170,10 @@ class TagViewButton(QToolButton): self.setCheckable(True) self.setChecked(False) self.setAutoRaise(True) - + class StatusBar(QStatusBar): - + def __init__(self, jobs_dialog, systray=None): QStatusBar.__init__(self) self.systray = systray @@ -192,11 +192,11 @@ class StatusBar(QStatusBar): self.addWidget(self.scroll_area, 100) self.setMinimumHeight(120) self.setMaximumHeight(120) - - + + def reset_info(self): self.book_info.show_data({}) - + def showMessage(self, msg, timeout=0): ret = QStatusBar.showMessage(self, msg, timeout) if self.systray is not None and not config['disable_tray_notification']: @@ -207,39 +207,38 @@ class StatusBar(QStatusBar): msg = msg.encode('utf-8') self.systray.showMessage('calibre', msg, self.systray.Information, 10000) return ret - + def jobs(self): src = qstring_to_unicode(self.movie_button.jobs.text()) return int(re.search(r'\d+', src).group()) - + def show_book_info(self): self.emit(SIGNAL('show_book_info()')) - + def job_added(self, nnum): jobs = self.movie_button.jobs src = qstring_to_unicode(jobs.text()) num = self.jobs() - nnum = num + 1 text = src.replace(str(num), str(nnum)) jobs.setText(text) if self.movie_button.movie.state() == QMovie.Paused: self.movie_button.movie.setPaused(False) - - def job_done(self, running): + + def job_done(self, nnum): jobs = self.movie_button.jobs src = qstring_to_unicode(jobs.text()) num = self.jobs() - text = src.replace(str(num), str(running)) + text = src.replace(str(num), str(nnum)) jobs.setText(text) - if running == 0: + if nnum == 0: self.no_more_jobs() - + def no_more_jobs(self): if self.movie_button.movie.state() == QMovie.Running: self.movie_button.movie.jumpToFrame(0) self.movie_button.movie.setPaused(True) QCoreApplication.instance().alert(self, 5000) - + if __name__ == '__main__': # Used to create the animated status icon from PyQt4.Qt import QApplication, QPainter, QSvgRenderer, QColor @@ -280,4 +279,4 @@ if __name__ == '__main__': os.remove(file) import sys create_mng(sys.argv[1]) - + diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index db36daa784..2e5c982791 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -4,16 +4,16 @@ __copyright__ = '2008, Kovid Goyal ' Miscellaneous widgets used in the GUI ''' import re, os, traceback -from PyQt4.QtGui import QListView, QIcon, QFont, QLabel, QListWidget, \ +from PyQt4.Qt import QListView, QIcon, QFont, QLabel, QListWidget, \ QListWidgetItem, QTextCharFormat, QApplication, \ - QSyntaxHighlighter, QCursor, QColor, QWidget, QDialog, \ - QPixmap, QMovie, QPalette -from PyQt4.QtCore import QAbstractListModel, QVariant, Qt, SIGNAL, \ + QSyntaxHighlighter, QCursor, QColor, QWidget, \ + QPixmap, QMovie, QPalette, QTimer, QDialog, \ + QAbstractListModel, QVariant, Qt, SIGNAL, \ QRegExp, QSettings, QSize, QModelIndex -from calibre.gui2.jobs2 import DetailView from calibre.gui2 import human_readable, NONE, TableView, \ qstring_to_unicode, error_dialog +from calibre.gui2.dialogs.job_view_ui import Ui_Dialog from calibre.gui2.filename_pattern_ui import Ui_Form from calibre import fit_image from calibre.utils.fontconfig import find_font_families @@ -249,6 +249,31 @@ class LocationView(QListView): if 0 <= row and row <= 3: self.model().location_changed(row) +class DetailView(QDialog, Ui_Dialog): + + def __init__(self, parent, job): + QDialog.__init__(self, parent) + self.setupUi(self) + self.setWindowTitle(job.description) + self.job = job + self.next_pos = 0 + self.update() + self.timer = QTimer(self) + self.connect(self.timer, SIGNAL('timeout()'), self.update) + self.timer.start(1000) + + + def update(self): + f = self.job.log_file + f.seek(self.next_pos) + more = f.read() + self.next_pos = f.tell() + if more: + self.log.appendPlainText(more.decode('utf-8', 'replace')) + vbar = self.log.verticalScrollBar() + vbar.setValue(vbar.maximum()) + + class JobsView(TableView): def __init__(self, parent): @@ -259,7 +284,6 @@ class JobsView(TableView): row = index.row() job = self.model().row_to_job(row) d = DetailView(self, job) - self.connect(self.model(), SIGNAL('output_received()'), d.update) d.exec_() @@ -539,12 +563,12 @@ class PythonHighlighter(QSyntaxHighlighter): return for regex, format in PythonHighlighter.Rules: - i = text.indexOf(regex) + i = regex.indexIn(text) while i >= 0: length = regex.matchedLength() self.setFormat(i, length, PythonHighlighter.Formats[format]) - i = text.indexOf(regex, i + length) + i = regex.indexIn(text, i + length) # Slow but good quality highlighting for comments. For more # speed, comment this out and add the following to __init__: @@ -569,12 +593,12 @@ class PythonHighlighter(QSyntaxHighlighter): self.setCurrentBlockState(NORMAL) - if text.indexOf(self.stringRe) != -1: + if self.stringRe.indexIn(text) != -1: return # This is fooled by triple quotes inside single quoted strings - for i, state in ((text.indexOf(self.tripleSingleRe), + for i, state in ((self.tripleSingleRe.indexIn(text), TRIPLESINGLE), - (text.indexOf(self.tripleDoubleRe), + (self.tripleDoubleRe.indexIn(text), TRIPLEDOUBLE)): if self.previousBlockState() == state: if i == -1: diff --git a/src/calibre/linux.py b/src/calibre/linux.py index 215236e83d..9582651ca0 100644 --- a/src/calibre/linux.py +++ b/src/calibre/linux.py @@ -29,7 +29,7 @@ entry_points = { 'calibre-debug = calibre.debug:main', 'calibredb = calibre.library.cli:main', 'calibre-fontconfig = calibre.utils.fontconfig:main', - 'calibre-parallel = calibre.parallel:main', + 'calibre-parallel = calibre.utils.ipc.worker:main', 'calibre-customize = calibre.customize.ui:main', 'calibre-complete = calibre.utils.complete:main', 'pdfmanipulate = calibre.ebooks.pdf.manipulate.cli:main', diff --git a/src/calibre/parallel.py b/src/calibre/parallel.py deleted file mode 100644 index 0ec7ed09cc..0000000000 --- a/src/calibre/parallel.py +++ /dev/null @@ -1,980 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Used to run jobs in parallel in separate processes. Features output streaming, -support for progress notification as well as job killing. The worker processes -are controlled via a simple protocol run over sockets. The control happens -mainly in two class, :class:`Server` and :class:`Overseer`. The worker is -encapsulated in the function :function:`worker`. Every worker process -has the environment variable :envvar:`CALIBRE_WORKER` defined. - -The worker control protocol has two modes of operation. In the first mode, the -worker process listens for commands from the controller process. The controller -process can either hand off a job to the worker or tell the worker to die. -Once a job is handed off to the worker, the protocol enters the second mode, where -the controller listens for messages from the worker. The worker can send progress updates -as well as console output (i.e. text that would normally have been written to stdout -or stderr by the job). Once the job completes (or raises an exception) the worker -returns the result (or exception) to the controller and the protocol reverts to the first mode. - -In the second mode, the controller can also send the worker STOP messages, in which case -the worker interrupts the job and dies. The sending of progress and console output messages -is buffered and asynchronous to prevent the job from being IO bound. -''' -import sys, os, gc, cPickle, traceback, cStringIO, time, signal, \ - subprocess, socket, collections, binascii, re, thread, tempfile, atexit -from select import select -from threading import RLock, Thread, Event -from math import ceil - -from calibre.ptempfile import PersistentTemporaryFile -from calibre import iswindows, detect_ncpus, isosx, preferred_encoding -from calibre.utils.config import prefs - -DEBUG = False - -#: A mapping from job names to functions that perform the jobs -PARALLEL_FUNCS = { - 'lrfviewer' : - ('calibre.gui2.lrf_renderer.main', 'main', {}, None), - - 'ebook-viewer' : - ('calibre.gui2.viewer.main', 'main', {}, None), - - 'render_pages' : - ('calibre.ebooks.comic.input', 'render_pages', {}, 'notification'), - - 'ebook-convert' : - ('calibre.ebooks.conversion.cli', 'main', {}, None), - - 'gui_convert' : - ('calibre.gui2.convert.gui_conversion', 'gui_convert', {}, 'notification'), -} - - -isfrozen = hasattr(sys, 'frozen') -isworker = False - -win32event = __import__('win32event') if iswindows else None -win32process = __import__('win32process') if iswindows else None -msvcrt = __import__('msvcrt') if iswindows else None - -SOCKET_TYPE = socket.AF_UNIX if not iswindows else socket.AF_INET - -class WorkerStatus(object): - ''' - A platform independent class to control child processes. Provides the - methods: - - .. method:: WorkerStatus.is_alive() - - Return True is the child process is alive (i.e. it hasn't exited and returned a return code). - - .. method:: WorkerStatus.returncode() - - Wait for the child process to exit and return its return code (blocks until child returns). - - .. method:: WorkerStatus.kill() - - Forcibly terminates child process using operating system specific semantics. - ''' - - def __init__(self, obj): - ''' - `obj`: On windows a process handle, on unix a subprocess.Popen object. - ''' - self.obj = obj - self.win32process = win32process # Needed if kill is called during shutdown of interpreter - self.os = os - self.signal = signal - ext = 'windows' if iswindows else 'unix' - for func in ('is_alive', 'returncode', 'kill'): - setattr(self, func, getattr(self, func+'_'+ext)) - - def is_alive_unix(self): - return self.obj.poll() == None - - def returncode_unix(self): - return self.obj.wait() - - def kill_unix(self): - os.kill(self.obj.pid, self.signal.SIGKILL) - - def is_alive_windows(self): - return win32event.WaitForSingleObject(self.obj, 0) != win32event.WAIT_OBJECT_0 - - def returncode_windows(self): - return win32process.GetExitCodeProcess(self.obj) - - def kill_windows(self, returncode=-1): - self.win32process.TerminateProcess(self.obj, returncode) - -class WorkerMother(object): - ''' - Platform independent object for launching child processes. All processes - have the environment variable :envvar:`CALIBRE_WORKER` set. - - ..method:: WorkerMother.spawn_free_spirit(arg) - - Launch a non monitored process with argument `arg`. - - ..method:: WorkerMother.spawn_worker(arg) - - Launch a monitored and controllable process with argument `arg`. - ''' - - def __init__(self): - ext = 'windows' if iswindows else 'osx' if isosx else 'linux' - self.os = os # Needed incase cleanup called when interpreter is shutting down - self.env = {} - if iswindows: - self.executable = os.path.join(os.path.dirname(sys.executable), - 'calibre-parallel.exe' if isfrozen else 'Scripts\\calibre-parallel.exe') - elif isosx: - self.executable = self.gui_executable = sys.executable - self.prefix = '' - if isfrozen: - fd = os.path.realpath(getattr(sys, 'frameworks_dir')) - contents = os.path.dirname(fd) - self.gui_executable = os.path.join(contents, 'MacOS', - os.path.basename(sys.executable)) - contents = os.path.join(contents, 'console.app', 'Contents') - exe = os.path.basename(sys.executable) - if 'python' not in exe: - exe = 'python' - self.executable = os.path.join(contents, 'MacOS', exe) - - resources = os.path.join(contents, 'Resources') - fd = os.path.join(contents, 'Frameworks') - sp = os.path.join(resources, 'lib', 'python'+sys.version[:3], 'site-packages.zip') - self.prefix += 'import sys; sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd - self.prefix += 'sys.path.insert(0, %s); '%repr(sp) - if fd not in os.environ['PATH']: - self.env['PATH'] = os.environ['PATH']+':'+fd - self.env['PYTHONHOME'] = resources - self.env['MAGICK_HOME'] = os.path.join(fd, 'ImageMagick') - self.env['DYLD_LIBRARY_PATH'] = os.path.join(fd, 'ImageMagick', 'lib') - else: - self.executable = os.path.join(getattr(sys, 'frozen_path'), 'calibre-parallel') \ - if isfrozen else 'calibre-parallel' - if isfrozen: - self.env['LD_LIBRARY_PATH'] = getattr(sys, 'frozen_path') + ':' + os.environ.get('LD_LIBRARY_PATH', '') - - self.spawn_worker_windows = lambda arg : self.spawn_free_spirit_windows(arg, type='worker') - self.spawn_worker_linux = lambda arg : self.spawn_free_spirit_linux(arg, type='worker') - self.spawn_worker_osx = lambda arg : self.spawn_free_spirit_osx(arg, type='worker') - - for func in ('spawn_free_spirit', 'spawn_worker'): - setattr(self, func, getattr(self, func+'_'+ext)) - - - def cleanup_child_windows(self, child, name=None, fd=None): - try: - child.kill() - except: - pass - try: - if fd is not None: - self.os.close(fd) - except: - pass - try: - if name is not None and os.path.exists(name): - self.os.unlink(name) - except: - pass - - def cleanup_child_linux(self, child): - try: - child.kill() - except: - pass - - def get_env(self): - env = dict(os.environ) - env['CALIBRE_WORKER'] = '1' - env['ORIGWD'] = os.path.abspath(os.getcwd()) - if hasattr(self, 'env'): - env.update(self.env) - return env - - def spawn_free_spirit_osx(self, arg, type='free_spirit'): - script = ('from calibre.parallel import main; ' - 'main(args=["calibre-parallel", %s]);')%repr(arg) - exe = self.gui_executable if type == 'free_spirit' else self.executable - cmdline = [exe, '-c', self.prefix+script] - child = WorkerStatus(subprocess.Popen(cmdline, env=self.get_env())) - atexit.register(self.cleanup_child_linux, child) - return child - - def spawn_free_spirit_linux(self, arg, type='free_spirit'): - cmdline = [self.executable, arg] - child = WorkerStatus(subprocess.Popen(cmdline, - env=self.get_env(), cwd=getattr(sys, 'frozen_path', None))) - atexit.register(self.cleanup_child_linux, child) - return child - - def spawn_free_spirit_windows(self, arg, type='free_spirit'): - priority = {'high':win32process.HIGH_PRIORITY_CLASS, 'normal':win32process.NORMAL_PRIORITY_CLASS, - 'low':win32process.IDLE_PRIORITY_CLASS}[prefs['worker_process_priority']] - fd, name = tempfile.mkstemp('.log', 'calibre_'+type+'_') - handle = msvcrt.get_osfhandle(fd) - si = win32process.STARTUPINFO() - si.hStdOutput = handle - si.hStdError = handle - cmdline = self.executable + ' ' + str(arg) - hProcess = \ - win32process.CreateProcess( - None, # Application Name - cmdline, # Command line - None, # processAttributes - None, # threadAttributes - 1, # bInheritHandles - win32process.CREATE_NO_WINDOW|priority, # Dont want ugly console popping up - self.get_env(), # New environment - None, # Current directory - si - )[0] - child = WorkerStatus(hProcess) - atexit.register(self.cleanup_child_windows, child, name, fd) - return child - - -mother = WorkerMother() - -_comm_lock = RLock() -def write(socket, msg, timeout=5): - ''' - Write a message on socket. If `msg` is unicode, it is encoded in utf-8. - Raises a `RuntimeError` if the socket is not ready for writing or the writing fails. - `msg` is broken into chunks of size 4096 and sent. The :function:`read` function - automatically re-assembles the chunks into whole message. - ''' - if isworker: - _comm_lock.acquire() - try: - if isinstance(msg, unicode): - msg = msg.encode('utf-8') - if DEBUG: - print >>sys.__stdout__, 'write(%s):'%('worker' if isworker else 'overseer'), repr(msg) - length = None - while len(msg) > 0: - if length is None: - length = len(msg) - chunk = ('%-12d'%length) + msg[:4096-12] - msg = msg[4096-12:] - else: - chunk, msg = msg[:4096], msg[4096:] - w = select([], [socket], [], timeout)[1] - if not w: - raise RuntimeError('Write to socket timed out') - if socket.sendall(chunk) is not None: - raise RuntimeError('Failed to write chunk to socket') - finally: - if isworker: - _comm_lock.release() - -def read(socket, timeout=5): - ''' - Read a message from `socket`. The message must have been sent with the :function:`write` - function. Raises a `RuntimeError` if the message is corrupted. Can return an - empty string. - ''' - if isworker: - _comm_lock.acquire() - try: - buf = cStringIO.StringIO() - length = None - while select([socket],[],[],timeout)[0]: - msg = socket.recv(4096) - if not msg: - break - if length is None: - try: - length, msg = int(msg[:12]), msg[12:] - except ValueError: - if DEBUG: - print >>sys.__stdout__, 'read(%s):'%('worker' if isworker else 'overseer'), 'no length in', msg - return '' - buf.write(msg) - if buf.tell() >= length: - break - if not length: - if DEBUG: - print >>sys.__stdout__, 'read(%s):'%('worker' if isworker else 'overseer'), 'nothing' - return '' - msg = buf.getvalue()[:length] - if len(msg) < length: - raise RuntimeError('Corrupted packet received') - if DEBUG: - print >>sys.__stdout__, 'read(%s):'%('worker' if isworker else 'overseer'), repr(msg) - return msg - finally: - if isworker: - _comm_lock.release() - -class RepeatingTimer(Thread): - ''' - Calls a specified function repeatedly at a specified interval. Runs in a - daemon thread (i.e. the interpreter can exit while it is still running). - Call :meth:`start()` to start it. - ''' - - def repeat(self): - while True: - self.event.wait(self.interval) - if self.event.isSet(): - break - self.action() - - def __init__(self, interval, func, name): - self.event = Event() - self.interval = interval - self.action = func - Thread.__init__(self, target=self.repeat, name=name) - self.setDaemon(True) - -class ControlError(Exception): - pass - -class Overseer(object): - ''' - Responsible for controlling worker processes. The main interface is the - methods, :meth:`initialize_job`, :meth:`control`. - ''' - - KILL_RESULT = 'Server: job killed by user|||#@#$%&*)*(*$#$%#$@&' - INTERVAL = 0.1 - - def __init__(self, server, port, timeout=5): - self.worker_status = mother.spawn_worker('127.0.0.1:'+str(port)) - self.socket = server.accept()[0] - # Needed if terminate called when interpreter is shutting down - self.os = os - self.signal = signal - self.on_probation = False - self.terminated = False - - self.working = False - self.timeout = timeout - self.last_job_time = time.time() - self._stop = False - if not select([self.socket], [], [], 120)[0]: - raise RuntimeError(_('Could not launch worker process.')) - ID = self.read().split(':') - if ID[0] != 'CALIBRE_WORKER': - raise RuntimeError('Impostor') - self.worker_pid = int(ID[1]) - self.write('OK') - if self.read() != 'WAITING': - raise RuntimeError('Worker sulking') - - def terminate(self): - 'Kill worker process.' - self.terminated = True - try: - if self.socket: - self.write('STOP:') - time.sleep(1) - self.socket.shutdown(socket.SHUT_RDWR) - except: - pass - if iswindows: - win32api = __import__('win32api') - try: - handle = win32api.OpenProcess(1, False, self.worker_pid) - win32api.TerminateProcess(handle, -1) - except: - pass - else: - try: - try: - self.os.kill(self.worker_pid, self.signal.SIGKILL) - time.sleep(0.5) - finally: - self.worker_status.kill() - except: - pass - - - def write(self, msg, timeout=None): - write(self.socket, msg, timeout=self.timeout if timeout is None else timeout) - - def read(self, timeout=None): - return read(self.socket, timeout=self.timeout if timeout is None else timeout) - - def __eq__(self, other): - return hasattr(other, 'process') and hasattr(other, 'worker_pid') and self.worker_pid == other.worker_pid - - def is_viable(self): - if self.terminated: - return False - return self.worker_status.is_alive() - - def select(self, timeout=0): - return select([self.socket], [self.socket], [self.socket], timeout) - - def initialize_job(self, job): - ''' - Sends `job` to worker process. Can raise `ControlError` if worker process - does not respond appropriately. In this case, this Overseer is useless - and should be discarded. - - `job`: An instance of :class:`Job`. - ''' - self.working = True - self.write('JOB:'+cPickle.dumps((job.func, job.args, job.kwargs), -1)) - msg = self.read() - if msg != 'OK': - raise ControlError('Failed to initialize job on worker %d:%s'%(self.worker_pid, msg)) - self.job = job - self.last_report = time.time() - job.start_work() - - def control(self): - ''' - Listens for messages from the worker process and dispatches them - appropriately. If the worker process dies unexpectedly, returns a result - of None with a ControlError indicating the worker died. - - Returns a :class:`Result` instance or None, if the worker is still working. - ''' - if select([self.socket],[],[],0)[0]: - msg = self.read() - if msg: - self.on_probation = False - self.last_report = time.time() - else: - if self.on_probation: - self.terminate() - self.job.result = None - self.job.exception = ControlError('Worker process died unexpectedly') - return - else: - self.on_probation = True - return - word, msg = msg.partition(':')[0], msg.partition(':')[-1] - if word == 'PING': - self.write('OK') - return - elif word == 'RESULT': - self.write('OK') - self.job.result = cPickle.loads(msg) - return True - elif word == 'OUTPUT': - self.write('OK') - try: - self.job.output(''.join(cPickle.loads(msg))) - except: - self.job.output('Bad output message: '+ repr(msg)) - elif word == 'PROGRESS': - self.write('OK') - percent = None - try: - percent, msg = cPickle.loads(msg)[-1] - except: - print 'Bad progress update:', repr(msg) - if percent is not None: - self.job.update_status(percent, msg) - elif word == 'ERROR': - self.write('OK') - exception, tb = cPickle.loads(msg) - self.job.output(u'%s\n%s'%(exception, tb)) - self.job.exception, self.job.traceback = exception, tb - return True - else: - self.terminate() - self.job.exception = ControlError('Worker sent invalid msg: %s'%repr(msg)) - return - if not self.worker_status.is_alive() or time.time() - self.last_report > 380: - self.terminate() - self.job.exception = ControlError('Worker process died unexpectedly') - return - -class JobKilled(Exception): - pass - -class Job(object): - - def __init__(self, job_done, job_manager=None, - args=[], kwargs={}, description=None): - self.args = args - self.kwargs = kwargs - self._job_done = job_done - self.job_manager = job_manager - self.is_running = False - self.has_run = False - self.percent = -1 - self.msg = None - self.description = description - self.start_time = None - self.running_time = None - - self.result = self.exception = self.traceback = self.log = None - - def __cmp__(self, other): - sstatus, ostatus = self.status(), other.status() - if sstatus == ostatus or (self.has_run and other.has_run): - if self.start_time == other.start_time: - return cmp(id(self), id(other)) - return cmp(self.start_time, other.start_time) - if sstatus == 'WORKING': - return -1 - if ostatus == 'WORKING': - return 1 - if sstatus == 'WAITING': - return -1 - if ostatus == 'WAITING': - return 1 - - - def job_done(self): - self.is_running, self.has_run = False, True - self.running_time = (time.time() - self.start_time) if \ - self.start_time is not None else 0 - if self.job_manager is not None: - self.job_manager.job_done(self) - self._job_done(self) - - def start_work(self): - self.is_running = True - self.has_run = False - self.start_time = time.time() - if self.job_manager is not None: - self.job_manager.start_work(self) - - def update_status(self, percent, msg=None): - self.percent = percent - self.msg = msg - if self.job_manager is not None: - try: - self.job_manager.status_update(self) - except: - traceback.print_exc() - - def status(self): - if self.is_running: - return 'WORKING' - if not self.has_run: - return 'WAITING' - if self.has_run: - if self.exception is None: - return 'DONE' - return 'ERROR' - - def console_text(self): - ans = [u'Job: '] - if self.description: - ans[0] += self.description - if self.exception is not None: - header = unicode(self.exception.__class__.__name__) if \ - hasattr(self.exception, '__class__') else u'Error' - header = u'**%s**'%header - header += u': ' - try: - header += unicode(self.exception) - except: - header += unicode(repr(self.exception)) - ans.append(header) - if self.traceback: - ans.append(u'**Traceback**:') - ans.extend(self.traceback.split('\n')) - - if self.log: - if isinstance(self.log, str): - self.log = unicode(self.log, 'utf-8', 'replace') - ans.append(self.log) - return (u'\n'.join(ans)).encode('utf-8') - - def gui_text(self): - ans = [u'Job: '] - if self.description: - if not isinstance(self.description, unicode): - self.description = self.description.decode('utf-8', 'replace') - ans[0] += u'**%s**'%self.description - if self.exception is not None: - header = unicode(self.exception.__class__.__name__) if \ - hasattr(self.exception, '__class__') else u'Error' - header = u'**%s**'%header - header += u': ' - try: - header += unicode(self.exception) - except: - header += unicode(repr(self.exception)) - ans.append(header) - if self.traceback: - ans.append(u'**Traceback**:') - ans.extend(self.traceback.split('\n')) - if self.log: - ans.append(u'**Log**:') - if isinstance(self.log, str): - self.log = unicode(self.log, 'utf-8', 'replace') - ans.extend(self.log.split('\n')) - - ans = [x.decode(preferred_encoding, 'replace') if isinstance(x, str) else x for x in ans] - - return u'
'.join(ans) - - -class ParallelJob(Job): - - def __init__(self, func, *args, **kwargs): - Job.__init__(self, *args, **kwargs) - self.func = func - self.done = self.job_done - - def output(self, msg): - if not self.log: - self.log = u'' - if not isinstance(msg, unicode): - msg = msg.decode('utf-8', 'replace') - if msg: - self.log += msg - if self.job_manager is not None: - self.job_manager.output(self) - - -def remove_ipc_socket(path): - os = __import__('os') - if os.path.exists(path): - os.unlink(path) - -class Server(Thread): - - KILL_RESULT = Overseer.KILL_RESULT - START_PORT = 10013 - PID = os.getpid() - - - def __init__(self, number_of_workers=detect_ncpus()): - Thread.__init__(self) - self.setDaemon(True) - self.server_socket = socket.socket(SOCKET_TYPE, socket.SOCK_STREAM) - self.port = tempfile.mktemp(prefix='calibre_server')+'_%d_'%self.PID if not iswindows else self.START_PORT - while True: - try: - address = ('localhost', self.port) if iswindows else self.port - self.server_socket.bind(address) - break - except socket.error: - self.port += (1 if iswindows else '1') - if not iswindows: - atexit.register(remove_ipc_socket, self.port) - self.server_socket.listen(5) - self.number_of_workers = number_of_workers - self.pool, self.jobs, self.working = [], collections.deque(), [] - atexit.register(self.killall) - atexit.register(self.close) - self.job_lock = RLock() - self.overseer_lock = RLock() - self.working_lock = RLock() - self.result_lock = RLock() - self.pool_lock = RLock() - self.start() - - def split(self, tasks): - ''' - Split a list into a list of sub lists, with the number of sub lists being - no more than the number of workers this server supports. Each sublist contains - two tuples of the form (i, x) where x is an element fro the original list - and i is the index of the element x in the original list. - ''' - ans, count, pos = [], 0, 0 - delta = int(ceil(len(tasks)/float(self.number_of_workers))) - while count < len(tasks): - section = [] - for t in tasks[pos:pos+delta]: - section.append((count, t)) - count += 1 - ans.append(section) - pos += delta - return ans - - - def close(self): - try: - self.server_socket.shutdown(socket.SHUT_RDWR) - except: - pass - - def add_job(self, job): - with self.job_lock: - self.jobs.append(job) - if job.job_manager is not None: - job.job_manager.add_job(job) - - def poll(self): - ''' - Return True if the server has either working or queued jobs - ''' - with self.job_lock: - with self.working_lock: - return len(self.jobs) + len(self.working) > 0 - - def wait(self, sleep=1): - ''' - Wait until job queue is empty - ''' - while self.poll(): - time.sleep(sleep) - - def run(self): - while True: - job = None - with self.job_lock: - if len(self.jobs) > 0 and len(self.working) < self.number_of_workers: - job = self.jobs.popleft() - with self.pool_lock: - o = None - while self.pool: - o = self.pool.pop() - try: - o.initialize_job(job) - break - except: - o.terminate() - if o is None: - o = Overseer(self.server_socket, self.port) - try: - o.initialize_job(job) - except Exception, err: - o.terminate() - job.exception = err - job.traceback = traceback.format_exc() - job.done() - o = None - if o and o.is_viable(): - with self.working_lock: - self.working.append(o) - - with self.working_lock: - done = [] - for o in self.working: - try: - if o.control() is not None or o.job.exception is not None: - o.job.done() - done.append(o) - except Exception, err: - o.job.exception = err - o.job.traceback = traceback.format_exc() - o.terminate() - o.job.done() - done.append(o) - for o in done: - self.working.remove(o) - if o and o.is_viable(): - with self.pool_lock: - self.pool.append(o) - - try: - time.sleep(1) - except: - return - - - def killall(self): - with self.pool_lock: - map(lambda x: x.terminate(), self.pool) - self.pool = [] - - - def kill(self, job): - with self.working_lock: - pop = None - for o in self.working: - if o.job == job or o == job: - try: - o.terminate() - except: pass - o.job.exception = JobKilled(_('Job stopped by user')) - try: - o.job.done() - except: pass - pop = o - break - if pop is not None: - self.working.remove(pop) - - def run_free_job(self, func, args=[], kwdargs={}): - pt = PersistentTemporaryFile('.pickle', '_IPC_') - pt.write(cPickle.dumps((func, args, kwdargs))) - pt.close() - mother.spawn_free_spirit(binascii.hexlify(pt.name)) - - -########################################################################################## -##################################### CLIENT CODE ##################################### -########################################################################################## - -class BufferedSender(object): - - def __init__(self, socket): - self.socket = socket - self.wbuf, self.pbuf = [], [] - self.wlock, self.plock = RLock(), RLock() - self.last_report = None - self.timer = RepeatingTimer(0.5, self.send, 'BufferedSender') - self.timer.start() - - - def write(self, msg): - if not isinstance(msg, basestring): - msg = unicode(msg) - with self.wlock: - self.wbuf.append(msg) - - def send(self): - if callable(select) and select([self.socket], [], [], 0)[0]: - msg = read(self.socket) - if msg == 'PING:': - write(self.socket, 'OK') - elif msg: - self.socket.shutdown(socket.SHUT_RDWR) - thread.interrupt_main() - time.sleep(1) - raise SystemExit - if not select([], [self.socket], [], 30)[1]: - print >>sys.__stderr__, 'Cannot pipe to overseer' - return - - reported = False - with self.wlock: - if self.wbuf: - msg = cPickle.dumps(self.wbuf, -1) - self.wbuf = [] - write(self.socket, 'OUTPUT:'+msg) - read(self.socket, 10) - reported = True - - with self.plock: - if self.pbuf: - msg = cPickle.dumps(self.pbuf, -1) - self.pbuf = [] - write(self.socket, 'PROGRESS:'+msg) - read(self.socket, 10) - reported = True - - if self.last_report is not None: - if reported: - self.last_report = time.time() - elif time.time() - self.last_report > 60: - write(self.socket, 'PING:') - read(self.socket, 10) - self.last_report = time.time() - - def notify(self, percent, msg=''): - with self.plock: - self.pbuf.append((percent, msg)) - - def flush(self): - pass - -def get_func(name): - module, func, kwdargs, notification = PARALLEL_FUNCS[name] - module = __import__(module, fromlist=[1]) - func = getattr(module, func) - return func, kwdargs, notification - -_atexit = collections.deque() -def myatexit(func, *args, **kwargs): - _atexit.append((func, args, kwargs)) - -def work(client_socket, func, args, kwdargs): - sys.stdout.last_report = time.time() - orig = atexit.register - atexit.register = myatexit - try: - func, kargs, notification = get_func(func) - if notification is not None and hasattr(sys.stdout, 'notify'): - kargs[notification] = sys.stdout.notify - kargs.update(kwdargs) - res = func(*args, **kargs) - if hasattr(sys.stdout, 'send'): - sys.stdout.send() - return res - finally: - atexit.register = orig - sys.stdout.last_report = None - while True: - try: - func, args, kwargs = _atexit.pop() - except IndexError: - break - try: - func(*args, **kwargs) - except (Exception, SystemExit): - continue - - time.sleep(5) # Give any in progress BufferedSend time to complete - - -def worker(host, port): - client_socket = socket.socket(SOCKET_TYPE, socket.SOCK_STREAM) - address = (host, port) if iswindows else port - client_socket.connect(address) - write(client_socket, 'CALIBRE_WORKER:%d'%os.getpid()) - msg = read(client_socket, timeout=10) - if msg != 'OK': - return 1 - write(client_socket, 'WAITING') - - sys.stdout = BufferedSender(client_socket) - sys.stderr = sys.stdout - - while True: - if not select([client_socket], [], [], 60)[0]: - time.sleep(1) - continue - msg = read(client_socket, timeout=60) - if msg.startswith('JOB:'): - func, args, kwdargs = cPickle.loads(msg[4:]) - write(client_socket, 'OK') - try: - result = work(client_socket, func, args, kwdargs) - write(client_socket, 'RESULT:'+ cPickle.dumps(result)) - except BaseException, err: - exception = (err.__class__.__name__, unicode(str(err), 'utf-8', 'replace')) - tb = unicode(traceback.format_exc(), 'utf-8', 'replace') - msg = 'ERROR:'+cPickle.dumps((exception, tb),-1) - write(client_socket, msg) - res = read(client_socket, 10) - if res != 'OK': - break - gc.collect() - elif msg == 'PING:': - write(client_socket, 'OK') - elif msg == 'STOP:': - client_socket.shutdown(socket.SHUT_RDWR) - return 0 - elif not msg: - time.sleep(1) - else: - print >>sys.__stderr__, 'Invalid protocols message', msg - return 1 - -def free_spirit(path): - func, args, kwdargs = cPickle.load(open(path, 'rb')) - try: - os.unlink(path) - except: - pass - func, kargs = get_func(func)[:2] - kargs.update(kwdargs) - func(*args, **kargs) - -def main(args=sys.argv): - global isworker - isworker = True - args = args[1].split(':') - if len(args) == 1: - free_spirit(binascii.unhexlify(re.sub(r'[^a-f0-9A-F]', '', args[0]))) - else: - worker(args[0].replace("'", ''), int(args[1]) if iswindows else args[1]) - return 0 - -if __name__ == '__main__': - sys.exit(main()) - diff --git a/src/calibre/utils/ipc/job.py b/src/calibre/utils/ipc/job.py new file mode 100644 index 0000000000..6a055706e3 --- /dev/null +++ b/src/calibre/utils/ipc/job.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +_count = 0 + +import time, cStringIO +from Queue import Queue, Empty + +class BaseJob(object): + + WAITING = 0 + RUNNING = 1 + FINISHED = 2 + + def __init__(self, description, done=lambda x: x): + global _count + _count += 1 + + self.id = _count + self.description = description + self.done = done + self.done2 = None + self.killed = False + self.failed = False + self.start_time = None + self.result = None + self.duration = None + self.log_path = None + self.notifications = Queue() + + self._run_state = self.WAITING + self.percent = 0 + self._message = None + self._status_text = _('Waiting...') + self._done_called = False + + def update(self): + if self.duration is not None: + self._run_state = self.FINISHED + self.percent = 1 + if self.killed: + self._status_text = _('Stopped') + else: + self._status_text = _('Error') if self.failed else _('Finished') + if not self._done_called: + self._done_called = True + try: + self.done(self) + except: + pass + try: + if callable(self.done2): + self.done2(self) + except: + pass + elif self.start_time is not None: + self._run_state = self.RUNNING + self._status_text = _('Working...') + + while True: + try: + self.percent, self._message = self.notifications.get_nowait() + self.percent *= 100. + except Empty: + break + + @property + def status_text(self): + if self._run_state == self.FINISHED or not self._message: + return self._status_text + return self._message + + @property + def run_state(self): + return self._run_state + + @property + def running_time(self): + if self.duration is not None: + return self.duration + if self.start_time is not None: + return time.time() - self.start_time + return None + + @property + def is_finished(self): + return self._run_state == self.FINISHED + + @property + def is_started(self): + return self._run_state != self.WAITING + + @property + def is_running(self): + return self.is_started and not self.is_finished + + def __cmp__(self, other): + if self.is_finished == other.is_finished: + if self.start_time is None: + if other.start_time is None: # Both waiting + return cmp(other.id, self.id) + else: + return 1 + else: + if other.start_time is None: + return -1 + else: # Both running + return cmp(other.start_time, self.start_time) + + else: + return 1 if self.is_finished else -1 + return 0 + + @property + def log_file(self): + if self.log_path: + return open(self.log_path, 'rb') + return cStringIO.StringIO(_('No details available.')) + + @property + def details(self): + return self.log_file.read().decode('utf-8') + + +class ParallelJob(BaseJob): + + def __init__(self, name, description, done, args=[], kwargs={}): + self.name, self.args, self.kwargs = name, args, kwargs + BaseJob.__init__(self, description, done) + + + diff --git a/src/calibre/utils/ipc/launch.py b/src/calibre/utils/ipc/launch.py index 6c0ba46885..14530d7fea 100644 --- a/src/calibre/utils/ipc/launch.py +++ b/src/calibre/utils/ipc/launch.py @@ -70,7 +70,7 @@ class Worker(object): @property def is_alive(self): - return hasattr(self, 'child') and self.child.poll() is not None + return hasattr(self, 'child') and self.child.poll() is None @property def returncode(self): @@ -144,6 +144,7 @@ class Worker(object): self.child = subprocess.Popen(cmd, **args) + self.log_path = ret return ret diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index 3d1a86922e..3dec90a644 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -6,5 +6,241 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os, cPickle, time, tempfile +from math import ceil +from threading import Thread, RLock +from Queue import Queue, Empty +from multiprocessing.connection import Listener +from multiprocessing import cpu_count +from collections import deque +from binascii import hexlify + +from calibre.utils.ipc.launch import Worker +from calibre.utils.ipc.worker import PARALLEL_FUNCS + +_counter = 0 + +class ConnectedWorker(Thread): + + def __init__(self, worker, conn, rfile): + Thread.__init__(self) + self.daemon = True + self.conn = conn + self.worker = worker + self.notifications = Queue() + self._returncode = 'dummy' + self.killed = False + self.log_path = worker.log_path + self.rfile = rfile + + def start_job(self, job): + notification = PARALLEL_FUNCS[job.name][-1] is not None + self.conn.send((job.name, job.args, job.kwargs)) + if notification: + self.start() + else: + self.conn.close() + self.job = job + + def run(self): + while True: + try: + x = self.conn.recv() + self.notifications.put(x) + except BaseException: + break + try: + self.conn.close() + except BaseException: + pass + + def kill(self): + self.killed = True + try: + self.worker.kill() + except BaseException: + pass + + @property + def is_alive(self): + return not self.killed and self.worker.is_alive + + @property + def returncode(self): + if self._returncode != 'dummy': + return self._returncode + r = self.worker.returncode + if self.killed and r is None: + self._returncode = 1 + return 1 + if r is not None: + self._returncode = r + return r + +class Server(Thread): + + def __init__(self, notify_on_job_done=lambda x: x, pool_size=None): + Thread.__init__(self) + self.daemon = True + global _counter + self.id = _counter+1 + _counter += 1 + + self.pool_size = cpu_count() if pool_size is None else pool_size + self.notify_on_job_done = notify_on_job_done + self.auth_key = os.urandom(32) + self.listener = Listener(authkey=self.auth_key, backlog=4) + self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() + self.kill_queue = Queue() + self.waiting_jobs, self.processing_jobs = deque(), deque() + self.pool, self.workers = deque(), deque() + self.launched_worker_count = 0 + self._worker_launch_lock = RLock() + + self.start() + + def launch_worker(self, gui=False, redirect_output=None): + with self._worker_launch_lock: + self.launched_worker_count += 1 + id = self.launched_worker_count + rfile = os.path.join(tempfile.gettempdir(), + 'calibre_ipc_result_%d_%d.pickle'%(self.id, id)) + + env = { + 'CALIBRE_WORKER_ADDRESS' : + hexlify(cPickle.dumps(self.listener.address, -1)), + 'CALIBRE_WORKER_KEY' : hexlify(self.auth_key), + 'CALIBRE_WORKER_RESULT' : hexlify(rfile), + } + w = Worker(env, gui=gui) + if redirect_output is None: + redirect_output = not gui + w(redirect_output=redirect_output) + conn = self.listener.accept() + if conn is None: + raise Exception('Failed to launch worker process') + return ConnectedWorker(w, conn, rfile) + + def add_job(self, job): + job.done2 = self.notify_on_job_done + self.add_jobs_queue.put(job) + + def run_job(self, job, gui=True, redirect_output=False): + w = self.launch_worker(gui=gui, redirect_output=redirect_output) + w.start_job(job) + + + def run(self): + while True: + try: + job = self.add_jobs_queue.get(True, 0.2) + if job is None: + break + self.waiting_jobs.append(job) + except Empty: + pass + + for worker in self.workers: + while True: + try: + n = worker.notifications.get_nowait() + worker.job.notifications.put(n) + self.changed_jobs_queue.put(job) + except Empty: + break + + for worker in [w for w in self.workers if not w.is_alive]: + self.workers.remove(worker) + job = worker.job + if worker.returncode != 0: + job.failed = True + job.returncode = worker.returncode + elif os.path.exists(worker.rfile): + job.result = cPickle.load(open(worker.rfile, 'rb')) + os.remove(worker.rfile) + job.duration = time.time() - job.start_time + self.changed_jobs_queue.put(job) + + if len(self.pool) + len(self.workers) < self.pool_size: + try: + self.pool.append(self.launch_worker()) + except: + break + + if len(self.pool) > 0 and len(self.waiting_jobs) > 0: + job = self.waiting_jobs.pop() + worker = self.pool.pop() + job.start_time = time.time() + worker.start_job(job) + self.workers.append(worker) + job.log_path = worker.log_path + self.changed_jobs_queue.put(job) + + while True: + try: + j = self.kill_queue.get_nowait() + self._kill_job(j) + except Empty: + break + + def kill_job(self, job): + self.kill_queue.put(job) + + def killall(self): + for job in self.workers: + self.kill_queue.put(job) + + def _kill_job(self, job): + if job.start_time is None: return + for worker in self.workers: + if job is worker.job: + worker.kill() + job.killed = True + break + + def split(self, tasks): + ''' + Split a list into a list of sub lists, with the number of sub lists being + no more than the number of workers this server supports. Each sublist contains + two tuples of the form (i, x) where x is an element from the original list + and i is the index of the element x in the original list. + ''' + ans, count, pos = [], 0, 0 + delta = int(ceil(len(tasks)/float(self.pool_size))) + while count < len(tasks): + section = [] + for t in tasks[pos:pos+delta]: + section.append((count, t)) + count += 1 + ans.append(section) + pos += delta + return ans + + + + def close(self): + try: + self.add_jobs_queue.put(None) + self.listener.close() + except: + pass + time.sleep(0.2) + for worker in self.workers: + try: + worker.kill() + except: + pass + for worker in self.pool: + try: + worker.kill() + except: + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py index 75b42c9a25..6c974568ac 100644 --- a/src/calibre/utils/ipc/worker.py +++ b/src/calibre/utils/ipc/worker.py @@ -6,11 +6,12 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, cPickle +import os, cPickle, sys from multiprocessing.connection import Client from threading import Thread -from queue import Queue +from Queue import Queue from contextlib import closing +from binascii import unhexlify PARALLEL_FUNCS = { 'lrfviewer' : @@ -29,8 +30,8 @@ PARALLEL_FUNCS = { class Progress(Thread): def __init__(self, conn): - self.daemon = True Thread.__init__(self) + self.daemon = True self.conn = conn self.queue = Queue() @@ -56,8 +57,9 @@ def get_func(name): return func, notification def main(): - address = cPickle.loads(os.environ['CALIBRE_WORKER_ADDRESS']) - key = os.environ['CALIBRE_WORKER_KEY'] + address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS'])) + key = unhexlify(os.environ['CALIBRE_WORKER_KEY']) + resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT']) with closing(Client(address, authkey=key)) as conn: name, args, kwargs = conn.recv() func, notification = get_func(name) @@ -66,13 +68,17 @@ def main(): kwargs[notification] = notifier notifier.start() - func(*args, **kwargs) + result = func(*args, **kwargs) + if result is not None: + cPickle.dump(result, open(resultf, 'wb'), -1) notifier.queue.put(None) + sys.stdout.flush() + sys.stderr.flush() return 0 if __name__ == '__main__': - raise SystemExit(main()) + sys.exit(main()) From 2e0ad5d1e082ef7339c4e744ed3156c208f84cf0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 13 May 2009 19:20:47 -0700 Subject: [PATCH 08/15] Allow recipes to specify overrides for conversion options --- src/calibre/manual/news_recipe.rst | 2 + src/calibre/web/feeds/input.py | 2 + src/calibre/web/feeds/news.py | 17 +-- .../web/feeds/recipes/recipe_barrons.py | 116 +++++++++--------- .../web/feeds/recipes/recipe_winsupersite.py | 5 +- 5 files changed, 74 insertions(+), 68 deletions(-) diff --git a/src/calibre/manual/news_recipe.rst b/src/calibre/manual/news_recipe.rst index 6872d8e532..6eb47a26a1 100644 --- a/src/calibre/manual/news_recipe.rst +++ b/src/calibre/manual/news_recipe.rst @@ -54,6 +54,8 @@ Customizing e-book download .. automember:: BasicNewsRecipe.timefmt +.. automember:: basicNewsRecipe.conversion_options + .. automember:: BasicNewsRecipe.feeds .. automember:: BasicNewsRecipe.no_stylesheets diff --git a/src/calibre/web/feeds/input.py b/src/calibre/web/feeds/input.py index ee003be0da..3052ffebed 100644 --- a/src/calibre/web/feeds/input.py +++ b/src/calibre/web/feeds/input.py @@ -57,6 +57,8 @@ class RecipeInput(InputFormatPlugin): ro = recipe(opts, log, self.report_progress) ro.download() + for key, val in recipe.conversion_options.items(): + setattr(opts, key, val) opts.output_profile.flow_size = 0 diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 3f6b9b9ae1..eca8eb0c93 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -156,13 +156,16 @@ class BasicNewsRecipe(Recipe): #: :attr:`BasicNewsRecipe.filter_regexps` should be defined. filter_regexps = [] - #: List of options to pass to html2lrf, to customize generation of LRF ebooks. - html2lrf_options = [] - - #: Options to pass to html2epub to customize generation of EPUB ebooks. - html2epub_options = '' - #: Options to pass to oeb2mobi to customize generation of MOBI ebooks. - oeb2mobi_options = '' + #: Recipe specific options to control the conversion of the downloaded + #: content into an e-book. These will override any user or plugin specified + #: values, so only use if absolutely necessary. For example: + #: conversion_options = { + #: 'base_font_size' : 16, + #: 'tags' : 'mytag1,mytag2', + #: 'title' : 'My Title', + #: 'linearize_tables' : True, + #: } + conversion_options = {} #: List of tags to be removed. Specified tags are removed from downloaded HTML. #: A tag is specified as a dictionary of the form:: diff --git a/src/calibre/web/feeds/recipes/recipe_barrons.py b/src/calibre/web/feeds/recipes/recipe_barrons.py index 164be20d3e..f9f501a9c3 100644 --- a/src/calibre/web/feeds/recipes/recipe_barrons.py +++ b/src/calibre/web/feeds/recipes/recipe_barrons.py @@ -1,76 +1,76 @@ ## -## web2lrf profile to download articles from Barrons.com -## can download subscriber-only content if username and +## web2lrf profile to download articles from Barrons.com +## can download subscriber-only content if username and ## password are supplied. ## -''' -''' - -import re - -from calibre.web.feeds.news import BasicNewsRecipe - -class Barrons(BasicNewsRecipe): - - title = 'Barron\'s' +''' +''' + +import re + +from calibre.web.feeds.news import BasicNewsRecipe + +class Barrons(BasicNewsRecipe): + + title = 'Barron\'s' max_articles_per_feed = 50 needs_subscription = True language = _('English') __author__ = 'Kovid Goyal' description = 'Weekly publication for investors from the publisher of the Wall Street Journal' - timefmt = ' [%a, %b %d, %Y]' - use_embedded_content = False + timefmt = ' [%a, %b %d, %Y]' + use_embedded_content = False no_stylesheets = False match_regexps = ['http://online.barrons.com/.*?html\?mod=.*?|file:.*'] - html2lrf_options = [('--ignore-tables'),('--base-font-size=10')] + conversion_options = {'linearize_tables': True} ##delay = 1 - - ## Don't grab articles more than 7 days old - oldest_article = 7 + + ## Don't grab articles more than 7 days old + oldest_article = 7 - preprocess_regexps = [(re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in - [ - ## Remove anything before the body of the article. - (r'