diff --git a/resources/images/news/hitro.png b/resources/images/news/hitro.png new file mode 100644 index 0000000000..75c08a1c25 Binary files /dev/null and b/resources/images/news/hitro.png differ diff --git a/resources/images/news/kamikaze.png b/resources/images/news/kamikaze.png new file mode 100644 index 0000000000..49ef2f50a1 Binary files /dev/null and b/resources/images/news/kamikaze.png differ diff --git a/resources/images/news/kompiutierra.png b/resources/images/news/kompiutierra.png new file mode 100644 index 0000000000..272e3d905f Binary files /dev/null and b/resources/images/news/kompiutierra.png differ diff --git a/resources/images/news/rbc_ru.png b/resources/images/news/rbc_ru.png new file mode 100644 index 0000000000..46c5d3fdce Binary files /dev/null and b/resources/images/news/rbc_ru.png differ diff --git a/resources/images/news/trombon.png b/resources/images/news/trombon.png new file mode 100644 index 0000000000..641b04f1b7 Binary files /dev/null and b/resources/images/news/trombon.png differ diff --git a/resources/images/news/wallstreetro.png b/resources/images/news/wallstreetro.png new file mode 100644 index 0000000000..d72bc70ca0 Binary files /dev/null and b/resources/images/news/wallstreetro.png differ diff --git a/resources/recipes/economist.recipe b/resources/recipes/economist.recipe index 17bf4c8c20..9447fe2193 100644 --- a/resources/recipes/economist.recipe +++ b/resources/recipes/economist.recipe @@ -24,7 +24,7 @@ class Economist(BasicNewsRecipe): cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg' remove_tags = [ dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), - dict(attrs={'class':['dblClkTrk', 'ec-article-info']}), + dict(attrs={'class':['dblClkTrk', 'ec-article-info', 'share_inline_header']}), {'class': lambda x: x and 'share-links-header' in x}, ] keep_only_tags = [dict(id='ec-article-body')] diff --git a/resources/recipes/economist_free.recipe b/resources/recipes/economist_free.recipe index f4a4efd932..d1766211d7 100644 --- a/resources/recipes/economist_free.recipe +++ b/resources/recipes/economist_free.recipe @@ -18,7 +18,8 @@ class Economist(BasicNewsRecipe): cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg' remove_tags = [ dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), - dict(attrs={'class':['dblClkTrk', 'ec-article-info']}), + dict(attrs={'class':['dblClkTrk', 'ec-article-info', + 'share_inline_header']}), {'class': lambda x: x and 'share-links-header' in x}, ] keep_only_tags = [dict(id='ec-article-body')] diff --git a/resources/recipes/el_pais_babelia.recipe b/resources/recipes/el_pais_babelia.recipe new file mode 100644 index 0000000000..31b983ec0b --- /dev/null +++ b/resources/recipes/el_pais_babelia.recipe @@ -0,0 +1,49 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class ElPaisBabelia(BasicNewsRecipe): + + title = 'El Pais Babelia' + __author__ = 'oneillpt' + description = 'El Pais Babelia' + INDEX = 'http://www.elpais.com/suple/babelia/' + language = 'es' + + remove_tags_before = dict(name='div', attrs={'class':'estructura_2col'}) + keep_tags = [dict(name='div', attrs={'class':'estructura_2col'})] + remove_tags = [dict(name='div', attrs={'class':'votos estirar'}), + dict(name='div', attrs={'id':'utilidades'}), + dict(name='div', attrs={'class':'info_relacionada'}), + dict(name='div', attrs={'class':'mod_apoyo'}), + dict(name='div', attrs={'class':'contorno_f'}), + dict(name='div', attrs={'class':'pestanias'}), + dict(name='div', attrs={'class':'otros_webs'}), + dict(name='div', attrs={'id':'pie'}) + ] + #no_stylesheets = True + remove_javascript = True + + def parse_index(self): + articles = [] + soup = self.index_to_soup(self.INDEX) + feeds = [] + for section in soup.findAll('div', attrs={'class':'contenedor_nuevo'}): + section_title = self.tag_to_string(section.find('h1')) + articles = [] + for post in section.findAll('a', href=True): + url = post['href'] + if url.startswith('/'): + url = 'http://www.elpais.es'+url + title = self.tag_to_string(post) + if str(post).find('class=') > 0: + klass = post['class'] + if klass != "": + self.log() + self.log('--> post: ', post) + self.log('--> url: ', url) + self.log('--> title: ', title) + self.log('--> class: ', klass) + articles.append({'title':title, 'url':url}) + if articles: + feeds.append((section_title, articles)) + return feeds + diff --git a/resources/recipes/evz.ro.recipe b/resources/recipes/evz.ro.recipe index bce151d1fc..841dc80429 100644 --- a/resources/recipes/evz.ro.recipe +++ b/resources/recipes/evz.ro.recipe @@ -1,52 +1,54 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + __license__ = 'GPL v3' -__copyright__ = '2010, Darko Miletic ' +__copyright__ = u'2011, Silviu Cotoar\u0103' ''' evz.ro ''' -import re from calibre.web.feeds.news import BasicNewsRecipe -class EVZ_Ro(BasicNewsRecipe): - title = 'evz.ro' - __author__ = 'Darko Miletic' - description = 'News from Romania' - publisher = 'evz.ro' - category = 'news, politics, Romania' - oldest_article = 2 - max_articles_per_feed = 200 - no_stylesheets = True - encoding = 'utf8' - use_embedded_content = False +class EvenimentulZilei(BasicNewsRecipe): + title = u'Evenimentul Zilei' + __author__ = u'Silviu Cotoar\u0103' + description = '' + publisher = u'Evenimentul Zilei' + oldest_article = 5 language = 'ro' - masthead_url = 'http://www.evz.ro/fileadmin/images/logo.gif' - extra_css = ' body{font-family: Georgia,Arial,Helvetica,sans-serif } .firstP{font-size: 1.125em} .author,.articleInfo{font-size: small} ' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Stiri' + encoding = 'utf-8' + cover_url = 'http://www.evz.ro/fileadmin/images/evzLogo.png' conversion_options = { - 'comment' : description - , 'tags' : category - , 'publisher' : publisher - , 'language' : language - } + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } - preprocess_regexps = [ - (re.compile(r'.*?', re.DOTALL|re.IGNORECASE),lambda match: '<head><title>') - ,(re.compile(r'.*?', re.DOTALL|re.IGNORECASE),lambda match: '') - ] + keep_only_tags = [ + dict(name='div', attrs={'class':'single'}) + , dict(name='img', attrs={'id':'placeholder'}) + , dict(name='a', attrs={'id':'holderlink'}) + ] - remove_tags = [ - dict(name=['form','embed','iframe','object','base','link','script','noscript']) - ,dict(attrs={'class':['section','statsInfo','email il']}) - ,dict(attrs={'id' :'gallery'}) - ] + remove_tags = [ + dict(name='p', attrs={'class':['articleInfo']}) + , dict(name='div', attrs={'id':['bannerAddoceansArticleJos']}) + , dict(name='div', attrs={'id':['bannerAddoceansArticle']}) + ] - remove_tags_after = dict(attrs={'class':'section'}) - keep_only_tags = [dict(attrs={'class':'single'})] - remove_attributes = ['height','width'] + remove_tags_after = [ + dict(name='div', attrs={'id':['bannerAddoceansArticleJos']}) + ] - feeds = [(u'Articles', u'http://www.evz.ro/rss.xml')] + feeds = [ + (u'Feeds', u'http://www.evz.ro/rss.xml') + ] def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - return soup + return self.adeify_images(soup) diff --git a/resources/recipes/hitro.recipe b/resources/recipes/hitro.recipe new file mode 100644 index 0000000000..3a85847c81 --- /dev/null +++ b/resources/recipes/hitro.recipe @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +hit.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Hit(BasicNewsRecipe): + title = u'HIT' + __author__ = u'Silviu Cotoar\u0103' + description = 'IT' + publisher = 'HIT' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Reviste,IT' + encoding = 'utf-8' + cover_url = 'http://www.hit.ro/lib/images/frontend/hit_logo.png' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='h1', attrs={'class':'art_titl'}) + , dict(name='div', attrs={'id':'continut_articol'}) + ] + + feeds = [ + (u'Feeds', u'http://www.hit.ro/rss') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/resources/recipes/kamikaze.recipe b/resources/recipes/kamikaze.recipe new file mode 100644 index 0000000000..1369cb6f85 --- /dev/null +++ b/resources/recipes/kamikaze.recipe @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +kamikazeonline.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Kamikaze(BasicNewsRecipe): + title = u'Kamikaze' + __author__ = u'Silviu Cotoar\u0103' + description = u'S\u0103pt\u0103m\u00e2nal sc\u0103pat de sub control' + publisher = 'Kamikaze' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Reviste' + encoding = 'utf-8' + cover_url = 'http://www.kamikazeonline.ro/wp-content/themes/kamikaze/images/kamikazeonline_header.gif' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='div', attrs={'id':'content'}) + ] + + remove_tags = [ + dict(name='div', attrs={'class':['connect_confirmation_cell connect_confirmation_cell_no_like']}) + , dict(name='h3', attrs={'id':['comments']}) + , dict(name='ul', attrs={'class':['addtoany_list']}) + , dict(name='p', attrs={'class':['postmetadata']}) + ] + + remove_tags_after = [ + dict(name='p', attrs={'class':['postmetadata']}) + ] + + feeds = [ + (u'Feeds', u'http://www.kamikazeonline.ro/feed/') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/resources/recipes/komchadluek.recipe b/resources/recipes/komchadluek.recipe new file mode 100644 index 0000000000..5f0d2f58a2 --- /dev/null +++ b/resources/recipes/komchadluek.recipe @@ -0,0 +1,46 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + +class KomChadLuek(BasicNewsRecipe): + + title= 'KomChadLuek' + description = 'Komchadluek News' + __author__ = 'ballsaii and Chotechai' + __license__ = 'GPL v3' + publisher= 'Nation Media Group' + category = 'news, Thai' + language = 'th' + + oldest_article = 1 + max_articles_per_feed = 100 + no_stylesheets= True + remove_javascript=True + + cover_url = 'http://www.komchadluek.net/images_layout2/komchadluek_headerlogo.png' + + keep_only_tags = [] + keep_only_tags.append(dict(name = 'h2')) + keep_only_tags.append(dict(name = 'div', attrs={'id':'news_detail_news'})) + + remove_tags_after=[dict(name='hr')] + + feeds =( +(u'\u0e01\u0e32\u0e23\u0e40\u0e21\u0e37\u0e2d\u0e07','http://www.komchadluek.net/rss/politic.xml'), +(u'\u0e15\u0e48\u0e32\u0e07\u0e1b\u0e23\u0e30\u0e40\u0e17\u0e28','http://www.komchadluek.net/rss/sport.xml'), +(u'\u0e40\u0e01\u0e29\u0e15\u0e23','http://www.komchadluek.net/rss/agriculture.xml'), +(u'\u0e15\u0e48\u0e32\u0e07\u0e1b\u0e23\u0e30\u0e40\u0e17\u0e28','http://www.komchadluek.net/rss/foreign.xml'), +(u'\u0e1a\u0e31\u0e19\u0e40\u0e17\u0e34\u0e07','http://www.komchadluek.net/rss/entertainment.xml'), +(u'\u0e1c\u0e39\u0e49\u0e2b\u0e0d\u0e34\u0e07-\u0e41\u0e1f\u0e0a\u0e31\u0e48\u0e19','http://www.komchadluek.net/rss/fashion.xml'), +(u'\u0e1e\u0e23\u0e30\u0e40\u0e04\u0e23\u0e37\u0e48\u0e2d\u0e07','http://www.komchadluek.net/rss/amulet.xml'), +(u'\u0e20\u0e39\u0e21\u0e34\u0e20\u0e32\u0e04-\u0e1b\u0e23\u0e30\u0e0a\u0e32\u0e04\u0e21\u0e17\u0e49\u0e2d\u0e07\u0e16\u0e34\u0e48\u0e19','http://www.komchadluek.net/rss/local.xml'), +(u'\u0e25\u0e38\u0e07\u0e41\u0e08\u0e48\u0e21','http://www.komchadluek.net/rss/unclecham.xml'), +(u'\u0e44\u0e25\u0e1f\u0e4c\u0e2a\u0e44\u0e15\u0e25\u0e4c','http://www.komchadluek.net/rss/lifestyle.xml'), +(u'\u0e40\u0e28\u0e23\u0e29\u0e10\u0e01\u0e34\u0e08-\u0e01\u0e32\u0e23\u0e15\u0e25\u0e32\u0e14','http://www.komchadluek.net/rss/economic.xml'), +(u'\u0e2d\u0e32\u0e2b\u0e32\u0e23','http://www.komchadluek.net/rss/food.xml'), +(u'\u0e04\u0e19\u0e23\u0e31\u0e01\u0e1a\u0e49\u0e32\u0e19-\u0e22\u0e32\u0e19\u0e22\u0e19\u0e15\u0e4c','http://www.komchadluek.net/rss/homecar.xml'), +(u'\u0e14\u0e39\u0e14\u0e27\u0e07-\u0e42\u0e2b\u0e23\u0e32\u0e28\u0e32\u0e2a\u0e15\u0e23\u0e4c','http://www.komchadluek.net/rss/horoscope.xml'), +(u'\u0e27\u0e34\u0e17\u0e22\u0e4c\u0e28\u0e32\u0e2a\u0e15\u0e23\u0e4c-\u0e44\u0e2d\u0e17\u0e35','http://www.komchadluek.net/rss/scienceit.xml'), +(u'\u0e28\u0e32\u0e2a\u0e19\u0e32 \u0e28\u0e34\u0e25\u0e1b\u0e30-\u0e27\u0e31\u0e12\u0e19\u0e18\u0e23\u0e23\u0e21 \u0e2a\u0e32\u0e18\u0e32\u0e23\u0e13\u0e2a\u0e38\u0e02','http://www.komchadluek.net/rss/artculture.xml'), +(u'\u0e01\u0e32\u0e23\u0e28\u0e36\u0e01\u0e29\u0e32', 'http://www.komchadluek.net/rss/education.xml'), +(u'\u0e1a\u0e17\u0e04\u0e27\u0e32\u0e21','http://www.komchadluek.net/rss/article.xml'), +(u'\u0e2d\u0e32\u0e0a\u0e0d\u0e32\u0e01\u0e23\u0e23\u0e21', 'http://www.komchadluek.net/rss/crime.xml') +) diff --git a/resources/recipes/kompiutierra.recipe b/resources/recipes/kompiutierra.recipe index 0d30afa3a7..a82db9aced 100644 --- a/resources/recipes/kompiutierra.recipe +++ b/resources/recipes/kompiutierra.recipe @@ -1,36 +1,37 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -__license__ = 'GPL v3' -__copyright__ = '2010, Vadim Dyadkin, dyadkin@gmail.com' -__author__ = 'Vadim Dyadkin' - -from calibre.web.feeds.news import BasicNewsRecipe - -class Computerra(BasicNewsRecipe): - title = u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u0440\u0430' - recursion = 50 - oldest_article = 100 - __author__ = 'Vadim Dyadkin' - max_articles_per_feed = 100 - use_embedded_content = False - simultaneous_downloads = 5 - language = 'ru' - description = u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u044b, \u043e\u043a\u043e\u043b\u043e\u043d\u0430\u0443\u0447\u043d\u044b\u0435 \u0438 \u043e\u043a\u043e\u043b\u043e\u0444\u0438\u043b\u043e\u0441\u043e\u0444\u0441\u043a\u0438\u0435 \u0441\u0442\u0430\u0442\u044c\u0438, \u0433\u0430\u0434\u0436\u0435\u0442\u044b.' - - keep_only_tags = [dict(name='div', attrs={'id': 'content'}),] - - - feeds = [(u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u0440\u0430', 'http://feeds.feedburner.com/ct_news/'),] - - remove_tags = [dict(name='div', attrs={'id': ['fin', 'idc-container', 'idc-noscript',]}), - dict(name='ul', attrs={'class': "related_post"}), - dict(name='p', attrs={'class': 'info'}), - dict(name='a', attrs={'rel': 'tag', 'class': 'twitter-share-button', 'type': 'button_count'}), - dict(name='h2', attrs={}),] - - extra_css = 'body { text-align: justify; }' - - def get_article_url(self, article): - return article.get('feedburner:origLink', article.get('guid')) - +#!/usr/bin/python +# -*- coding: utf-8 -*- + +__license__ = 'GPL v3' +__copyright__ = '2010, Vadim Dyadkin, dyadkin@gmail.com' +__author__ = 'Vadim Dyadkin' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Computerra(BasicNewsRecipe): + title = u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u0440\u0430' + oldest_article = 100 + __author__ = 'Vadim Dyadkin (edited by A. Chewi)' + max_articles_per_feed = 50 + use_embedded_content = False + remove_javascript = True + no_stylesheets = True + conversion_options = {'linearize_tables' : True} + simultaneous_downloads = 5 + language = 'ru' + description = u'Компьютерра: все новости про компьютеры, железо, новые технологии, информационные технологии' + + keep_only_tags = [dict(name='div', attrs={'id': 'content'}),] + + feeds = [(u'Компьютерра-Онлайн', 'http://feeds.feedburner.com/ct_news/'),] + + remove_tags = [ + dict(name='div', attrs={'id': ['fin', 'idc-container', 'idc-noscript',]}), + dict(name='ul', attrs={'class': "related_post"}), + dict(name='p', attrs={'class': 'info'}), + dict(name='a', attrs={'class': 'twitter-share-button'}), + dict(name='a', attrs={'type': 'button_count'}), + dict(name='h2', attrs={}) + ] + + def print_version(self, url): + return url + '?print=true' diff --git a/resources/recipes/ming_pao.recipe b/resources/recipes/ming_pao.recipe index bbdbbf7ace..4a405a59dd 100644 --- a/resources/recipes/ming_pao.recipe +++ b/resources/recipes/ming_pao.recipe @@ -1,7 +1,20 @@ __license__ = 'GPL v3' __copyright__ = '2010-2011, Eddie Lau' + +# Users of Kindle 3 (with limited system-level CJK support) +# please replace the following "True" with "False". +__MakePeriodical__ = True +# Turn it to True if your device supports display of CJK titles +__UseChineseTitle__ = False + + ''' Change Log: +2011/03/06: add new articles for finance section, also a new section "Columns" +2011/02/28: rearrange the sections + [Disabled until Kindle has better CJK support and can remember last (section,article) read in Sections & Articles + View] make it the same title if generating a periodical, so past issue will be automatically put into "Past Issues" + folder in Kindle 3 2011/02/20: skip duplicated links in finance section, put photos which may extend a whole page to the back of the articles clean up the indentation 2010/12/07: add entertainment section, use newspaper front page as ebook cover, suppress date display in section list @@ -19,55 +32,58 @@ import os, datetime, re from calibre.web.feeds.recipes import BasicNewsRecipe from contextlib import nested - from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.ebooks.metadata.toc import TOC from calibre.ebooks.metadata import MetaInformation class MPHKRecipe(BasicNewsRecipe): - IsCJKWellSupported = True # Set to False to avoid generating periodical in which CJK characters can't be displayed in section/article view - title = 'Ming Pao - Hong Kong' - oldest_article = 1 - max_articles_per_feed = 100 - __author__ = 'Eddie Lau' - description = ('Hong Kong Chinese Newspaper (http://news.mingpao.com). If' - 'you are using a Kindle with firmware < 3.1, customize the' - 'recipe') - publisher = 'MingPao' - category = 'Chinese, News, Hong Kong' - remove_javascript = True - use_embedded_content = False - no_stylesheets = True - language = 'zh' - encoding = 'Big5-HKSCS' - recursions = 0 - conversion_options = {'linearize_tables':True} - timefmt = '' - extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}' - masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif' - keep_only_tags = [dict(name='h1'), + title = 'Ming Pao - Hong Kong' + oldest_article = 1 + max_articles_per_feed = 100 + __author__ = 'Eddie Lau' + description = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)' + publisher = 'MingPao' + category = 'Chinese, News, Hong Kong' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + language = 'zh' + encoding = 'Big5-HKSCS' + recursions = 0 + conversion_options = {'linearize_tables':True} + timefmt = '' + extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}' + masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif' + keep_only_tags = [dict(name='h1'), dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page title - dict(attrs={'id':['newscontent']}), # entertainment page content + dict(name='font', attrs={'color':['AA0000']}), # for column articles title + dict(attrs={'id':['newscontent']}), # entertainment and column page content dict(attrs={'id':['newscontent01','newscontent02']}), dict(attrs={'class':['photo']}) ] - remove_tags = [dict(name='style'), - dict(attrs={'id':['newscontent135']})] # for the finance page - remove_attributes = ['width'] - preprocess_regexps = [ + remove_tags = [dict(name='style'), + dict(attrs={'id':['newscontent135']}), # for the finance page + dict(name='table')] # for content fetched from life.mingpao.com + remove_attributes = ['width'] + preprocess_regexps = [ (re.compile(r'
', re.DOTALL|re.IGNORECASE), lambda match: '

'), (re.compile(r'

', re.DOTALL|re.IGNORECASE), lambda match: ''), (re.compile(r'

', re.DOTALL|re.IGNORECASE), # for entertainment page - lambda match: '') + lambda match: ''), + # skip
after title in life.mingpao.com fetched article + (re.compile(r"

", re.DOTALL|re.IGNORECASE), + lambda match: "
"), + (re.compile(r"

", re.DOTALL|re.IGNORECASE), + lambda match: "") ] - def image_url_processor(cls, baseurl, url): - # trick: break the url at the first occurance of digit, add an additional - # '_' at the front - # not working, may need to move this to preprocess_html() method + def image_url_processor(cls, baseurl, url): + # trick: break the url at the first occurance of digit, add an additional + # '_' at the front + # not working, may need to move this to preprocess_html() method # minIdx = 10000 # i0 = url.find('0') # if i0 >= 0 and i0 < minIdx: @@ -99,253 +115,314 @@ class MPHKRecipe(BasicNewsRecipe): # i9 = url.find('9') # if i9 >= 0 and i9 < minIdx: # minIdx = i9 - return url + return url - def get_dtlocal(self): - dt_utc = datetime.datetime.utcnow() - # convert UTC to local hk time - at around HKT 6.00am, all news are available - dt_local = dt_utc - datetime.timedelta(-2.0/24) - return dt_local + def get_dtlocal(self): + dt_utc = datetime.datetime.utcnow() + # convert UTC to local hk time - at around HKT 6.00am, all news are available + dt_local = dt_utc - datetime.timedelta(-2.0/24) + return dt_local - def get_fetchdate(self): - return self.get_dtlocal().strftime("%Y%m%d") + def get_fetchdate(self): + return self.get_dtlocal().strftime("%Y%m%d") - def get_fetchformatteddate(self): - return self.get_dtlocal().strftime("%Y-%m-%d") + def get_fetchformatteddate(self): + return self.get_dtlocal().strftime("%Y-%m-%d") - def get_fetchday(self): - # convert UTC to local hk time - at around HKT 6.00am, all news are available - return self.get_dtlocal().strftime("%d") + def get_fetchday(self): + # convert UTC to local hk time - at around HKT 6.00am, all news are available + return self.get_dtlocal().strftime("%d") - def get_cover_url(self): - cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg' - br = BasicNewsRecipe.get_browser() - try: - br.open(cover) - except: - cover = None - return cover + def get_cover_url(self): + cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + cover = None + return cover - def parse_index(self): - feeds = [] - dateStr = self.get_fetchdate() - for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), - (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), - (u'\u793e\u8a55/\u7b46\u9663 Editorial', 'http://news.mingpao.com/' + dateStr + '/mrindex.htm'), - (u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), + def parse_index(self): + feeds = [] + dateStr = self.get_fetchdate() + + for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), + (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), + (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) + + # special- editorial + ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr') + if ed_articles: + feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles)) + + for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'), - (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm'), - ('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), - (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm'), - (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm'), - (u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'), + (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) + + # special - finance + #fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm') + fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea') + if fin_articles: + feeds.append((u'\u7d93\u6fdf Finance', fin_articles)) + + for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), + (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) + + # special - entertainment + ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm') + if ent_articles: + feeds.append((u'\u5f71\u8996 Film/TV', ent_articles)) + + for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'), (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]: - articles = self.parse_section(url) - if articles: - feeds.append((title, articles)) - # special - finance - fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm') - if fin_articles: - feeds.append((u'\u7d93\u6fdf Finance', fin_articles)) - # special - entertainment - ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm') - if ent_articles: - feeds.append((u'\u5f71\u8996 Film/TV', ent_articles)) - return feeds + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) - def parse_section(self, url): - dateStr = self.get_fetchdate() - soup = self.index_to_soup(url) - divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']}) - current_articles = [] - included_urls = [] - divs.reverse() - for i in divs: - a = i.find('a', href = True) - title = self.tag_to_string(a) - url = a.get('href', False) - url = 'http://news.mingpao.com/' + dateStr + '/' +url - if url not in included_urls and url.rfind('Redirect') == -1: - current_articles.append({'title': title, 'url': url, 'description':'', 'date':''}) - included_urls.append(url) - current_articles.reverse() - return current_articles - def parse_fin_section(self, url): - dateStr = self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href= True) - current_articles = [] - included_urls = [] - for i in a: - url = 'http://www.mpfinance.com/cfm/' + i.get('href', False) - if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1: - title = self.tag_to_string(i) - current_articles.append({'title': title, 'url': url, 'description':''}) - included_urls.append(url) - return current_articles + # special- columns + col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn') + if col_articles: + feeds.append((u'\u5c08\u6b04 Columns', col_articles)) - def parse_ent_section(self, url): - self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href=True) - a.reverse() - current_articles = [] - included_urls = [] - for i in a: - title = self.tag_to_string(i) - url = 'http://ol.mingpao.com/cfm/' + i.get('href', False) - if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1): - current_articles.append({'title': title, 'url': url, 'description': ''}) - included_urls.append(url) - current_articles.reverse() - return current_articles + return feeds - def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - for item in soup.findAll(style=True): - del item['width'] - for item in soup.findAll(stype=True): - del item['absmiddle'] - return soup + def parse_section(self, url): + dateStr = self.get_fetchdate() + soup = self.index_to_soup(url) + divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']}) + current_articles = [] + included_urls = [] + divs.reverse() + for i in divs: + a = i.find('a', href = True) + title = self.tag_to_string(a) + url = a.get('href', False) + url = 'http://news.mingpao.com/' + dateStr + '/' +url + if url not in included_urls and url.rfind('Redirect') == -1: + current_articles.append({'title': title, 'url': url, 'description':'', 'date':''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def create_opf(self, feeds, dir=None): - if dir is None: - dir = self.output_dir - if self.IsCJKWellSupported == True: - # use Chinese title - title = u'\u660e\u5831 (\u9999\u6e2f) ' + self.get_fetchformatteddate() - else: - # use English title - title = self.short_title() + ' ' + self.get_fetchformatteddate() - if True: # force date in title - # title += strftime(self.timefmt) - mi = MetaInformation(title, [self.publisher]) - mi.publisher = self.publisher - mi.author_sort = self.publisher - if self.IsCJKWellSupported == True: - mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() - else: - mi.publication_type = self.publication_type+':'+self.short_title() - #mi.timestamp = nowf() - mi.timestamp = self.get_dtlocal() - mi.comments = self.description - if not isinstance(mi.comments, unicode): - mi.comments = mi.comments.decode('utf-8', 'replace') - #mi.pubdate = nowf() - mi.pubdate = self.get_dtlocal() - opf_path = os.path.join(dir, 'index.opf') - ncx_path = os.path.join(dir, 'index.ncx') - opf = OPFCreator(dir, mi) - # Add mastheadImage entry to section - mp = getattr(self, 'masthead_path', None) - if mp is not None and os.access(mp, os.R_OK): - from calibre.ebooks.metadata.opf2 import Guide - ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu()) - ref.type = 'masthead' - ref.title = 'Masthead Image' - opf.guide.append(ref) + def parse_ed_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('nal') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))] - manifest.append(os.path.join(dir, 'index.html')) - manifest.append(os.path.join(dir, 'index.ncx')) + def parse_fin_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href= True) + current_articles = [] + included_urls = [] + for i in a: + #url = 'http://www.mpfinance.com/cfm/' + i.get('href', False) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + #if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1: + if url not in included_urls and (not url.rfind('txt') == -1) and (not url.rfind('nal') == -1): + title = self.tag_to_string(i) + current_articles.append({'title': title, 'url': url, 'description':''}) + included_urls.append(url) + return current_articles - # Get cover - cpath = getattr(self, 'cover_path', None) - if cpath is None: - pf = open(os.path.join(dir, 'cover.jpg'), 'wb') - if self.default_cover(pf): - cpath = pf.name - if cpath is not None and os.access(cpath, os.R_OK): - opf.cover = cpath - manifest.append(cpath) + def parse_ent_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://ol.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - # Get masthead - mpath = getattr(self, 'masthead_path', None) - if mpath is not None and os.access(mpath, os.R_OK): - manifest.append(mpath) + def parse_col_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('ncl') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - opf.create_manifest_from_files_in(manifest) - for mani in opf.manifest: - if mani.path.endswith('.ncx'): - mani.id = 'ncx' - if mani.path.endswith('mastheadImage.jpg'): - mani.id = 'masthead-image' - entries = ['index.html'] - toc = TOC(base_path=dir) - self.play_order_counter = 0 - self.play_order_map = {} + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll(style=True): + del item['width'] + for item in soup.findAll(stype=True): + del item['absmiddle'] + return soup - def feed_index(num, parent): - f = feeds[num] - for j, a in enumerate(f): - if getattr(a, 'downloaded', False): - adir = 'feed_%d/article_%d/'%(num, j) - auth = a.author - if not auth: - auth = None - desc = a.text_summary - if not desc: - desc = None - else: - desc = self.description_limiter(desc) - entries.append('%sindex.html'%adir) - po = self.play_order_map.get(entries[-1], None) - if po is None: - self.play_order_counter += 1 - po = self.play_order_counter - parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'), + def create_opf(self, feeds, dir=None): + if dir is None: + dir = self.output_dir + if __UseChineseTitle__ == True: + title = u'\u660e\u5831 (\u9999\u6e2f)' + else: + title = self.short_title() + # if not generating a periodical, force date to apply in title + if __MakePeriodical__ == False: + title = title + ' ' + self.get_fetchformatteddate() + if True: + mi = MetaInformation(title, [self.publisher]) + mi.publisher = self.publisher + mi.author_sort = self.publisher + if __MakePeriodical__ == True: + mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() + else: + mi.publication_type = self.publication_type+':'+self.short_title() + #mi.timestamp = nowf() + mi.timestamp = self.get_dtlocal() + mi.comments = self.description + if not isinstance(mi.comments, unicode): + mi.comments = mi.comments.decode('utf-8', 'replace') + #mi.pubdate = nowf() + mi.pubdate = self.get_dtlocal() + opf_path = os.path.join(dir, 'index.opf') + ncx_path = os.path.join(dir, 'index.ncx') + opf = OPFCreator(dir, mi) + # Add mastheadImage entry to section + mp = getattr(self, 'masthead_path', None) + if mp is not None and os.access(mp, os.R_OK): + from calibre.ebooks.metadata.opf2 import Guide + ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu()) + ref.type = 'masthead' + ref.title = 'Masthead Image' + opf.guide.append(ref) + + manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))] + manifest.append(os.path.join(dir, 'index.html')) + manifest.append(os.path.join(dir, 'index.ncx')) + + # Get cover + cpath = getattr(self, 'cover_path', None) + if cpath is None: + pf = open(os.path.join(dir, 'cover.jpg'), 'wb') + if self.default_cover(pf): + cpath = pf.name + if cpath is not None and os.access(cpath, os.R_OK): + opf.cover = cpath + manifest.append(cpath) + + # Get masthead + mpath = getattr(self, 'masthead_path', None) + if mpath is not None and os.access(mpath, os.R_OK): + manifest.append(mpath) + + opf.create_manifest_from_files_in(manifest) + for mani in opf.manifest: + if mani.path.endswith('.ncx'): + mani.id = 'ncx' + if mani.path.endswith('mastheadImage.jpg'): + mani.id = 'masthead-image' + entries = ['index.html'] + toc = TOC(base_path=dir) + self.play_order_counter = 0 + self.play_order_map = {} + + def feed_index(num, parent): + f = feeds[num] + for j, a in enumerate(f): + if getattr(a, 'downloaded', False): + adir = 'feed_%d/article_%d/'%(num, j) + auth = a.author + if not auth: + auth = None + desc = a.text_summary + if not desc: + desc = None + else: + desc = self.description_limiter(desc) + entries.append('%sindex.html'%adir) + po = self.play_order_map.get(entries[-1], None) + if po is None: + self.play_order_counter += 1 + po = self.play_order_counter + parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'), play_order=po, author=auth, description=desc) - last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep)) - for sp in a.sub_pages: - prefix = os.path.commonprefix([opf_path, sp]) - relp = sp[len(prefix):] - entries.append(relp.replace(os.sep, '/')) - last = sp + last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep)) + for sp in a.sub_pages: + prefix = os.path.commonprefix([opf_path, sp]) + relp = sp[len(prefix):] + entries.append(relp.replace(os.sep, '/')) + last = sp - if os.path.exists(last): - with open(last, 'rb') as fi: - src = fi.read().decode('utf-8') - soup = BeautifulSoup(src) - body = soup.find('body') - if body is not None: - prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last)))) - templ = self.navbar.generate(True, num, j, len(f), + if os.path.exists(last): + with open(last, 'rb') as fi: + src = fi.read().decode('utf-8') + soup = BeautifulSoup(src) + body = soup.find('body') + if body is not None: + prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last)))) + templ = self.navbar.generate(True, num, j, len(f), not self.has_single_feed, a.orig_url, self.publisher, prefix=prefix, center=self.center_navbar) - elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') - body.insert(len(body.contents), elem) - with open(last, 'wb') as fi: - fi.write(unicode(soup).encode('utf-8')) - if len(feeds) == 0: - raise Exception('All feeds are empty, aborting.') + elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') + body.insert(len(body.contents), elem) + with open(last, 'wb') as fi: + fi.write(unicode(soup).encode('utf-8')) + if len(feeds) == 0: + raise Exception('All feeds are empty, aborting.') - if len(feeds) > 1: - for i, f in enumerate(feeds): - entries.append('feed_%d/index.html'%i) - po = self.play_order_map.get(entries[-1], None) - if po is None: - self.play_order_counter += 1 - po = self.play_order_counter - auth = getattr(f, 'author', None) - if not auth: - auth = None - desc = getattr(f, 'description', None) - if not desc: - desc = None - feed_index(i, toc.add_item('feed_%d/index.html'%i, None, + if len(feeds) > 1: + for i, f in enumerate(feeds): + entries.append('feed_%d/index.html'%i) + po = self.play_order_map.get(entries[-1], None) + if po is None: + self.play_order_counter += 1 + po = self.play_order_counter + auth = getattr(f, 'author', None) + if not auth: + auth = None + desc = getattr(f, 'description', None) + if not desc: + desc = None + feed_index(i, toc.add_item('feed_%d/index.html'%i, None, f.title, play_order=po, description=desc, author=auth)) - else: - entries.append('feed_%d/index.html'%0) - feed_index(0, toc) + else: + entries.append('feed_%d/index.html'%0) + feed_index(0, toc) - for i, p in enumerate(entries): - entries[i] = os.path.join(dir, p.replace('/', os.sep)) - opf.create_spine(entries) - opf.set_toc(toc) + for i, p in enumerate(entries): + entries[i] = os.path.join(dir, p.replace('/', os.sep)) + opf.create_spine(entries) + opf.set_toc(toc) - with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): - opf.render(opf_file, ncx_file) + with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): + opf.render(opf_file, ncx_file) diff --git a/resources/recipes/nationalgeoro.recipe b/resources/recipes/nationalgeoro.recipe index a3c5727d38..8f989be74d 100644 --- a/resources/recipes/nationalgeoro.recipe +++ b/resources/recipes/nationalgeoro.recipe @@ -14,7 +14,7 @@ class NationalGeoRo(BasicNewsRecipe): __author__ = u'Silviu Cotoar\u0103' description = u'S\u0103 avem grij\u0103 de planet\u0103' publisher = 'National Geographic' - oldest_article = 5 + oldest_article = 35 language = 'ro' max_articles_per_feed = 100 no_stylesheets = True diff --git a/resources/recipes/nrc-nl-epub.recipe b/resources/recipes/nrc-nl-epub.recipe index da9b9195ce..2d190e4d0a 100644 --- a/resources/recipes/nrc-nl-epub.recipe +++ b/resources/recipes/nrc-nl-epub.recipe @@ -1,14 +1,14 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # -*- coding: utf-8 -*- -#Based on Lars Jacob's Taz Digiabo recipe +#Based on veezh's original recipe and Kovid Goyal's New York Times recipe __license__ = 'GPL v3' -__copyright__ = '2010, veezh' +__copyright__ = '2011, Snaab' ''' www.nrc.nl ''' -import os, urllib2, zipfile +import os, zipfile import time from calibre.web.feeds.news import BasicNewsRecipe from calibre.ptempfile import PersistentTemporaryFile @@ -17,41 +17,59 @@ from calibre.ptempfile import PersistentTemporaryFile class NRCHandelsblad(BasicNewsRecipe): title = u'NRC Handelsblad' - description = u'De EPUB-versie van NRC' + description = u'De ePaper-versie van NRC' language = 'nl' lang = 'nl-NL' + needs_subscription = True - __author__ = 'veezh' + __author__ = 'Snaab' conversion_options = { 'no_default_epub_cover' : True } + def get_browser(self): + br = BasicNewsRecipe.get_browser() + if self.username is not None and self.password is not None: + br.open('http://login.nrc.nl/login') + br.select_form(nr=0) + br['username'] = self.username + br['password'] = self.password + br.submit() + return br + def build_index(self): + today = time.strftime("%Y%m%d") + domain = "http://digitaleeditie.nrc.nl" url = domain + "/digitaleeditie/helekrant/epub/nrc_" + today + ".epub" -# print url + #print url try: - f = urllib2.urlopen(url) - except urllib2.HTTPError: + br = self.get_browser() + f = br.open(url) + except: self.report_progress(0,_('Kan niet inloggen om editie te downloaden')) raise ValueError('Krant van vandaag nog niet beschikbaar') + tmp = PersistentTemporaryFile(suffix='.epub') self.report_progress(0,_('downloading epub')) tmp.write(f.read()) - tmp.close() - - zfile = zipfile.ZipFile(tmp.name, 'r') - self.report_progress(0,_('extracting epub')) - - zfile.extractall(self.output_dir) + f.close() + br.close() + if zipfile.is_zipfile(tmp): + try: + zfile = zipfile.ZipFile(tmp.name, 'r') + zfile.extractall(self.output_dir) + self.report_progress(0,_('extracting epub')) + except zipfile.BadZipfile: + self.report_progress(0,_('BadZip error, continuing')) tmp.close() - index = os.path.join(self.output_dir, 'content.opf') + index = os.path.join(self.output_dir, 'metadata.opf') self.report_progress(1,_('epub downloaded and extracted')) diff --git a/resources/recipes/trombon.recipe b/resources/recipes/trombon.recipe new file mode 100644 index 0000000000..1a4e488a43 --- /dev/null +++ b/resources/recipes/trombon.recipe @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +trombon.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Trombon(BasicNewsRecipe): + title = u'Trombon' + __author__ = u'Silviu Cotoar\u0103' + description = u'Parodii si Pamflete' + publisher = u'Trombon' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Reviste,Fun' + encoding = 'utf-8' + cover_url = 'http://www.trombon.ro/i/trombon.gif' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='div', attrs={'class':'articol'}) + ] + + remove_tags = [ + dict(name='div', attrs={'class':['info_2']}) + , dict(name='iframe', attrs={'scrolling':['no']}) + ] + + remove_tags_after = [ + dict(name='div', attrs={'id':'article_vote'}) + ] + + feeds = [ + (u'Feeds', u'http://feeds.feedburner.com/trombon/ABWb?format=xml') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/resources/recipes/wallstreetro.recipe b/resources/recipes/wallstreetro.recipe new file mode 100644 index 0000000000..8a66aa3673 --- /dev/null +++ b/resources/recipes/wallstreetro.recipe @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +wall-street.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class WallStreetRo(BasicNewsRecipe): + title = u'Wall Street' + __author__ = u'Silviu Cotoar\u0103' + description = '' + publisher = 'Wall Street' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare' + encoding = 'utf-8' + cover_url = 'http://img.wall-street.ro/images/WS_new_logo.jpg' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='div', attrs={'class':'article_header'}) + , dict(name='div', attrs={'class':'article_text'}) + ] + + remove_tags = [ + dict(name='p', attrs={'class':['page_breadcrumbs']}) + , dict(name='div', attrs={'id':['article_user_toolbox']}) + , dict(name='p', attrs={'class':['comments_count_container']}) + , dict(name='div', attrs={'class':['article_left_column']}) + ] + + remove_tags_after = [ + dict(name='div', attrs={'class':'clearfloat'}) + ] + + feeds = [ + (u'Feeds', u'http://img.wall-street.ro/rssfeeds/wall-street.xml') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/resources/templates/fb2.xsl b/resources/templates/fb2.xsl index 77c03cdc74..273edd71ae 100644 --- a/resources/templates/fb2.xsl +++ b/resources/templates/fb2.xsl @@ -4,6 +4,7 @@ # # # # # copyright 2002 Paul Henry Tremblay # +# Copyright 2011 Kovid Goyal # # # This program is distributed in the hope that it will be useful, # # but WITHOUT ANY WARRANTY; without even the implied warranty of # @@ -19,21 +20,21 @@ ######################################################################### --> - - - - - - - - - - <xsl:value-of select="fb:description/fb:title-info/fb:book-title"/> - - + - - - -
- -
-
- - -
    - -
-
+ + + +
+ +
+
+ + +
    + +
+
- - - -
- - -

- -

- - - - - - - - - - - - + + + +
+ + +

+ +

+ + + + + + + + + + + + - -
- - - -
  • - - - , # - - - -
      - - - -
    -     - - - - - - - - - -
  • - - -
    -     -
    • - - - - - - - - - -
    -     + +
    + + + +
  • + + + , # + + + +
      + + + +
    +     + + + + + + + + + +
  • + + +
    +     +
    • + + +     + + + + + + +
    +     - + @@ -164,15 +165,15 @@ - - - - - + + + + + - - - + + + @@ -181,79 +182,79 @@ TOC_ - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - + - -
    -     - - - - - - - -
    - -
    -     - - + +
    +     + + + + + + + +
    + +
    +     + + paragraph - - - - + + + + - - - - - - - - - - - - - - - -
    -     +     + + + + + + + + + + + + + + +
    +     @@ -261,123 +262,140 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    Annotation

    - -     - - -
    - - - - - - -
    -     - - -
    - -
    -     - - -
    - - - - - - -
    -     - - -
    -
    -     - - - - -     -
    -     - -     -
    -     -     -     - - -
    - - - - - - -
    -     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    Annotation

    + +     + + + + +
    +     + + + + + + + + + + + + + +
    + + + + + + +
    +     + + +
    + +
    +     + + +
    + + + + + + +
    +     + + +
    +
    +     + + + + +     +
    +     + +     +
    +     +     +     + + +
    + + + + + + +
    +     - - - -
    -     - - - - - - - -
    -     - - -
    - - - - - - - - - - -
    -     + + + +
    +     + + + + + + + +
    +     + + +
    + + + + + + + + + + +
    +     diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 221f5911c6..fa9a8f2404 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -61,8 +61,9 @@ def osx_version(): if m: return int(m.group(1)), int(m.group(2)), int(m.group(3)) - _filename_sanitize = re.compile(r'[\xae\0\\|\?\*<":>\+/]') +_filename_sanitize_unicode = frozenset([u'\\', u'|', u'?', u'*', u'<', + u'"', u':', u'>', u'+', u'/'] + list(map(unichr, xrange(32)))) def sanitize_file_name(name, substitute='_', as_unicode=False): ''' @@ -83,8 +84,35 @@ def sanitize_file_name(name, substitute='_', as_unicode=False): one = one.decode(filesystem_encoding) one = one.replace('..', substitute) # Windows doesn't like path components that end with a period - if one.endswith('.'): + if one and one[-1] in ('.', ' '): one = one[:-1]+'_' + # Names starting with a period are hidden on Unix + if one.startswith('.'): + one = '_' + one[1:] + return one + +def sanitize_file_name_unicode(name, substitute='_'): + ''' + Sanitize the filename `name`. All invalid characters are replaced by `substitute`. + The set of invalid characters is the union of the invalid characters in Windows, + OS X and Linux. Also removes leading and trailing whitespace. + **WARNING:** This function also replaces path separators, so only pass file names + and not full paths to it. + ''' + if not isinstance(name, unicode): + return sanitize_file_name(name, substitute=substitute, as_unicode=True) + chars = [substitute if c in _filename_sanitize_unicode else c for c in + name] + one = u''.join(chars) + one = re.sub(r'\s', ' ', one).strip() + one = re.sub(r'^\.+$', '_', one) + one = one.replace('..', substitute) + # Windows doesn't like path components that end with a period or space + if one and one[-1] in ('.', ' '): + one = one[:-1]+'_' + # Names starting with a period are hidden on Unix + if one.startswith('.'): + one = '_' + one[1:] return one diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index 1918a36cc8..0d62a8f619 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -30,6 +30,7 @@ def strftime(epoch, zone=time.gmtime): def get_connected_device(): from calibre.customize.ui import device_plugins from calibre.devices.scanner import DeviceScanner + import uuid dev = None scanner = DeviceScanner() scanner.scan() @@ -47,7 +48,7 @@ def get_connected_device(): for d in connected_devices: try: - d.open() + d.open(str(uuid.uuid4())) except: continue else: diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 3724f02ca2..e1a806af0f 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -57,7 +57,7 @@ class ANDROID(USBMS): 0x413c : { 0xb007 : [0x0100, 0x0224]}, # LG - 0x1004 : { 0x61cc : [0x100] }, + 0x1004 : { 0x61cc : [0x100], 0x61ce : [0x100] }, # Archos 0x0e79 : { @@ -78,6 +78,9 @@ class ANDROID(USBMS): # Xperia 0x13d3 : { 0x3304 : [0x0001, 0x0002] }, + # CREEL?? Also Nextbook + 0x5e3 : { 0x726 : [0x222] }, + } EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books'] EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to ' diff --git a/src/calibre/devices/prs500/cli/main.py b/src/calibre/devices/prs500/cli/main.py index cd8395467b..8a73f3fa23 100755 --- a/src/calibre/devices/prs500/cli/main.py +++ b/src/calibre/devices/prs500/cli/main.py @@ -6,7 +6,7 @@ Provides a command-line and optional graphical interface to the SONY Reader PRS- For usage information run the script. """ -import StringIO, sys, time, os +import StringIO, sys, time, os, uuid from optparse import OptionParser from calibre import __version__, __appname__ @@ -213,7 +213,7 @@ def main(): for d in connected_devices: try: - d.open() + d.open(str(uuid.uuid4())) except: continue else: diff --git a/src/calibre/ebooks/chm/input.py b/src/calibre/ebooks/chm/input.py index 89efa2b4d1..f55a76d67e 100644 --- a/src/calibre/ebooks/chm/input.py +++ b/src/calibre/ebooks/chm/input.py @@ -22,7 +22,7 @@ class CHMInput(InputFormatPlugin): def _chmtohtml(self, output_dir, chm_path, no_images, log): from calibre.ebooks.chm.reader import CHMReader log.debug('Opening CHM file') - rdr = CHMReader(chm_path, log) + rdr = CHMReader(chm_path, log, self.opts) log.debug('Extracting CHM to %s' % output_dir) rdr.extract_content(output_dir) self._chm_reader = rdr @@ -32,13 +32,13 @@ class CHMInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): from calibre.ebooks.chm.metadata import get_metadata_from_reader from calibre.customize.ui import plugin_for_input_format + self.opts = options log.debug('Processing CHM...') with TemporaryDirectory('_chm2oeb') as tdir: html_input = plugin_for_input_format('html') for opt in html_input.options: setattr(options, opt.option.name, opt.recommended_value) - options.input_encoding = 'utf-8' no_images = False #options.no_images chm_name = stream.name #chm_data = stream.read() @@ -54,6 +54,7 @@ class CHMInput(InputFormatPlugin): odi = options.debug_pipeline options.debug_pipeline = None + options.input_encoding = 'utf-8' # try a custom conversion: #oeb = self._create_oebbook(mainpath, tdir, options, log, metadata) # try using html converter: diff --git a/src/calibre/ebooks/chm/reader.py b/src/calibre/ebooks/chm/reader.py index fe8b4fdbde..34d228ef3b 100644 --- a/src/calibre/ebooks/chm/reader.py +++ b/src/calibre/ebooks/chm/reader.py @@ -40,13 +40,14 @@ class CHMError(Exception): pass class CHMReader(CHMFile): - def __init__(self, input, log): + def __init__(self, input, log, opts): CHMFile.__init__(self) if isinstance(input, unicode): input = input.encode(filesystem_encoding) if not self.LoadCHM(input): raise CHMError("Unable to open CHM file '%s'"%(input,)) self.log = log + self.opts = opts self._sourcechm = input self._contents = None self._playorder = 0 @@ -151,6 +152,8 @@ class CHMReader(CHMFile): break def _reformat(self, data, htmlpath): + if self.opts.input_encoding: + data = data.decode(self.opts.input_encoding) try: data = xml_to_unicode(data, strip_encoding_pats=True)[0] soup = BeautifulSoup(data) diff --git a/src/calibre/ebooks/comic/input.py b/src/calibre/ebooks/comic/input.py index 7710d41fb3..56fa123249 100755 --- a/src/calibre/ebooks/comic/input.py +++ b/src/calibre/ebooks/comic/input.py @@ -131,9 +131,12 @@ class PageProcessor(list): # {{{ newsizey = int(newsizex / aspect) deltax = 0 deltay = (SCRHEIGHT - newsizey) / 2 - wand.size = (newsizex, newsizey) - wand.set_border_color(pw) - wand.add_border(pw, deltax, deltay) + if newsizex < 20000 and newsizey < 20000: + # Too large and resizing fails, so better + # to leave it as original size + wand.size = (newsizex, newsizey) + wand.set_border_color(pw) + wand.add_border(pw, deltax, deltay) elif self.opts.wide: # Keep aspect and Use device height as scaled image width so landscape mode is clean aspect = float(sizex) / float(sizey) @@ -152,11 +155,15 @@ class PageProcessor(list): # {{{ newsizey = int(newsizex / aspect) deltax = 0 deltay = (wscreeny - newsizey) / 2 - wand.size = (newsizex, newsizey) - wand.set_border_color(pw) - wand.add_border(pw, deltax, deltay) + if newsizex < 20000 and newsizey < 20000: + # Too large and resizing fails, so better + # to leave it as original size + wand.size = (newsizex, newsizey) + wand.set_border_color(pw) + wand.add_border(pw, deltax, deltay) else: - wand.size = (SCRWIDTH, SCRHEIGHT) + if SCRWIDTH < 20000 and SCRHEIGHT < 20000: + wand.size = (SCRWIDTH, SCRHEIGHT) if not self.opts.dont_sharpen: wand.sharpen(0.0, 1.0) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index a2599ab0b5..feb6ff4bb9 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -130,7 +130,7 @@ class Metadata(object): self.set_identifiers(val) elif field in STANDARD_METADATA_FIELDS: if val is None: - val = NULL_VALUES.get(field, None) + val = copy.copy(NULL_VALUES.get(field, None)) _data[field] = val elif field in _data['user_metadata'].iterkeys(): _data['user_metadata'][field]['#value#'] = val diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index efc8fe1463..0cd17387fe 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -8,11 +8,7 @@ from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2008, Marshall T. Vandegrift ' -import os -import itertools -import re -import logging -import copy +import os, itertools, re, logging, copy, unicodedata from weakref import WeakKeyDictionary from xml.dom import SyntaxErr as CSSSyntaxError import cssutils @@ -234,8 +230,18 @@ class Stylizer(object): for elem in matches: for x in elem.iter(): if x.text: - span = E.span(x.text[0]) - span.tail = x.text[1:] + punctuation_chars = [] + text = unicode(x.text) + while text: + if not unicodedata.category(text[0]).startswith('P'): + break + punctuation_chars.append(text[0]) + text = text[1:] + + special_text = u''.join(punctuation_chars) + \ + (text[0] if text else u'') + span = E.span(special_text) + span.tail = text[1:] x.text = None x.insert(0, span) self.style(span)._update_cssdict(cssdict) diff --git a/src/calibre/ebooks/pdf/writer.py b/src/calibre/ebooks/pdf/writer.py index b0884417f6..516509fdd7 100644 --- a/src/calibre/ebooks/pdf/writer.py +++ b/src/calibre/ebooks/pdf/writer.py @@ -46,7 +46,8 @@ def get_pdf_printer(opts, for_comic=False): printer = QPrinter(QPrinter.HighResolution) custom_size = get_custom_size(opts) - if opts.output_profile.short_name == 'default': + if opts.output_profile.short_name == 'default' or \ + opts.output_profile.width > 10000: if custom_size is None: printer.setPaperSize(paper_size(opts.paper_size)) else: diff --git a/src/calibre/ebooks/snb/snbfile.py b/src/calibre/ebooks/snb/snbfile.py index e42533f241..9a7d65e417 100644 --- a/src/calibre/ebooks/snb/snbfile.py +++ b/src/calibre/ebooks/snb/snbfile.py @@ -75,15 +75,20 @@ class SNBFile: for i in range(self.plainBlock): bzdc = bz2.BZ2Decompressor() if (i < self.plainBlock - 1): - bSize = self.blocks[self.binBlock + i + 1].Offset - self.blocks[self.binBlock + i].Offset; + bSize = self.blocks[self.binBlock + i + 1].Offset - self.blocks[self.binBlock + i].Offset else: - bSize = self.tailOffset - self.blocks[self.binBlock + i].Offset; - snbFile.seek(self.blocks[self.binBlock + i].Offset); + bSize = self.tailOffset - self.blocks[self.binBlock + i].Offset + snbFile.seek(self.blocks[self.binBlock + i].Offset) try: data = snbFile.read(bSize) - uncompressedData += bzdc.decompress(data) + if len(data) < 32768: + uncompressedData += bzdc.decompress(data) + else: + uncompressedData += data except Exception, e: print e + if len(uncompressedData) != self.plainStreamSizeUncompressed: + raise Exception() f.fileBody = uncompressedData[plainPos:plainPos+f.fileSize] plainPos += f.fileSize elif f.attr & 0x01000000 == 0x01000000: diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 0040acea28..cf67cd6cfa 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -204,15 +204,29 @@ class AddAction(InterfaceAction): to_device = self.gui.stack.currentIndex() != 0 self._add_books(paths, to_device) - def files_dropped_on_book(self, event, paths): + def remote_file_dropped_on_book(self, url, fname): + if self.gui.current_view() is not self.gui.library_view: + return + db = self.gui.library_view.model().db + current_idx = self.gui.library_view.currentIndex() + if not current_idx.isValid(): return + cid = db.id(current_idx.row()) + from calibre.gui2.dnd import DownloadDialog + d = DownloadDialog(url, fname, self.gui) + d.start_download() + if d.err is None: + self.files_dropped_on_book(None, [d.fpath], cid=cid) + + def files_dropped_on_book(self, event, paths, cid=None): accept = False if self.gui.current_view() is not self.gui.library_view: return db = self.gui.library_view.model().db cover_changed = False current_idx = self.gui.library_view.currentIndex() - if not current_idx.isValid(): return - cid = db.id(current_idx.row()) + if cid is None: + if not current_idx.isValid(): return + cid = db.id(current_idx.row()) if cid is None else cid for path in paths: ext = os.path.splitext(path)[1].lower() if ext: @@ -227,8 +241,9 @@ class AddAction(InterfaceAction): elif ext in BOOK_EXTENSIONS: db.add_format_with_hooks(cid, ext, path, index_is_id=True) accept = True - if accept: + if accept and event is not None: event.accept() + if current_idx.isValid(): self.gui.library_view.model().current_changed(current_idx, current_idx) if cover_changed: if self.gui.cover_flow: diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 63deccb2f0..a28759486e 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -5,7 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, collections, sys +import collections, sys from Queue import Queue from PyQt4.Qt import QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, \ @@ -14,7 +14,8 @@ from PyQt4.Qt import QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, \ from PyQt4.QtWebKit import QWebView from calibre import fit_image, prepare_string_for_xml -from calibre.gui2.widgets import IMAGE_EXTENSIONS +from calibre.gui2.dnd import dnd_has_image, dnd_get_image, dnd_get_files, \ + IMAGE_EXTENSIONS, dnd_has_extension from calibre.ebooks import BOOK_EXTENSIONS from calibre.constants import preferred_encoding from calibre.library.comments import comments_to_html @@ -165,11 +166,12 @@ class CoverView(QWidget): # {{{ def copy_to_clipboard(self): QApplication.instance().clipboard().setPixmap(self.pixmap) - def paste_from_clipboard(self): - cb = QApplication.instance().clipboard() - pmap = cb.pixmap() - if pmap.isNull() and cb.supportsSelection(): - pmap = cb.pixmap(cb.Selection) + def paste_from_clipboard(self, pmap=None): + if not isinstance(pmap, QPixmap): + cb = QApplication.instance().clipboard() + pmap = cb.pixmap() + if pmap.isNull() and cb.supportsSelection(): + pmap = cb.pixmap(cb.Selection) if not pmap.isNull(): self.pixmap = pmap self.do_layout() @@ -226,6 +228,7 @@ class BookInfo(QWebView): self._link_clicked = False self.setAttribute(Qt.WA_OpaquePaintEvent, False) palette = self.palette() + self.setAcceptDrops(False) palette.setBrush(QPalette.Base, Qt.transparent) self.page().setPalette(palette) @@ -388,36 +391,50 @@ class BookDetails(QWidget): # {{{ show_book_info = pyqtSignal() open_containing_folder = pyqtSignal(int) view_specific_format = pyqtSignal(int, object) - - # Drag 'n drop {{{ - DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS+BOOK_EXTENSIONS + remote_file_dropped = pyqtSignal(object, object) files_dropped = pyqtSignal(object, object) cover_changed = pyqtSignal(object, object) - # application/x-moz-file-promise-url - @classmethod - def paths_from_event(cls, event): - ''' - Accept a drop event and return a list of paths that can be read from - and represent files with extensions. - ''' - if event.mimeData().hasFormat('text/uri-list'): - urls = [unicode(u.toLocalFile()) for u in event.mimeData().urls()] - urls = [u for u in urls if os.path.splitext(u)[1] and os.access(u, os.R_OK)] - return [u for u in urls if os.path.splitext(u)[1][1:].lower() in cls.DROPABBLE_EXTENSIONS] + # Drag 'n drop {{{ + DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS+BOOK_EXTENSIONS def dragEnterEvent(self, event): - if int(event.possibleActions() & Qt.CopyAction) + \ - int(event.possibleActions() & Qt.MoveAction) == 0: - return - paths = self.paths_from_event(event) - if paths: + md = event.mimeData() + if dnd_has_extension(md, self.DROPABBLE_EXTENSIONS) or \ + dnd_has_image(md): event.acceptProposedAction() def dropEvent(self, event): - paths = self.paths_from_event(event) event.setDropAction(Qt.CopyAction) - self.files_dropped.emit(event, paths) + md = event.mimeData() + + x, y = dnd_get_image(md) + if x is not None: + # We have an image, set cover + event.accept() + if y is None: + # Local image + self.cover_view.paste_from_clipboard(x) + else: + self.remote_file_dropped.emit(x, y) + # We do not support setting cover *and* adding formats for + # a remote drop, anyway, so return + return + + # Now look for ebook files + urls, filenames = dnd_get_files(md, BOOK_EXTENSIONS) + if not urls: + # Nothing found + return + + if not filenames: + # Local files + self.files_dropped.emit(event, urls) + else: + # Remote files, use the first file + self.remote_file_dropped.emit(urls[0], filenames[0]) + event.accept() + def dragMoveEvent(self, event): event.acceptProposedAction() diff --git a/src/calibre/gui2/convert/xexp_edit.ui b/src/calibre/gui2/convert/xexp_edit.ui index 18b7c39b52..68c0c8c98e 100644 --- a/src/calibre/gui2/convert/xexp_edit.ui +++ b/src/calibre/gui2/convert/xexp_edit.ui @@ -43,6 +43,9 @@ 0 + + QComboBox::AdjustToMinimumContentsLengthWithIcon + 30 diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index fa7ba3c56d..beaca77a38 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -5,7 +5,6 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys from functools import partial from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \ @@ -85,7 +84,7 @@ class Int(Base): self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), QSpinBox(parent)] w = self.widgets[1] - w.setRange(-100, sys.maxint) + w.setRange(-100, 100000000) w.setSpecialValueText(_('Undefined')) w.setSingleStep(1) @@ -108,7 +107,7 @@ class Float(Int): self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), QDoubleSpinBox(parent)] w = self.widgets[1] - w.setRange(-100., float(sys.maxint)) + w.setRange(-100., float(100000000)) w.setDecimals(2) w.setSpecialValueText(_('Undefined')) w.setSingleStep(1) @@ -289,7 +288,7 @@ class Series(Base): self.widgets.append(QLabel('&'+self.col_metadata['name']+_(' index:'), parent)) w = QDoubleSpinBox(parent) - w.setRange(-100., float(sys.maxint)) + w.setRange(-100., float(100000000)) w.setDecimals(2) w.setSpecialValueText(_('Undefined')) w.setSingleStep(1) @@ -595,7 +594,7 @@ class BulkInt(BulkBase): def setup_ui(self, parent): self.make_widgets(parent, QSpinBox) - self.main_widget.setRange(-100, sys.maxint) + self.main_widget.setRange(-100, 100000000) self.main_widget.setSpecialValueText(_('Undefined')) self.main_widget.setSingleStep(1) @@ -617,7 +616,7 @@ class BulkFloat(BulkInt): def setup_ui(self, parent): self.make_widgets(parent, QDoubleSpinBox) - self.main_widget.setRange(-100., float(sys.maxint)) + self.main_widget.setRange(-100., float(100000000)) self.main_widget.setDecimals(2) self.main_widget.setSpecialValueText(_('Undefined')) self.main_widget.setSingleStep(1) @@ -795,6 +794,7 @@ class BulkEnumeration(BulkBase, Enumeration): return value def setup_ui(self, parent): + self.parent = parent self.make_widgets(parent, QComboBox) vals = self.col_metadata['display']['enum_values'] self.main_widget.blockSignals(True) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 298e541730..2cbecc134c 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1160,6 +1160,14 @@ class DeviceMixin(object): # {{{ ), bad) d.exec_() + def upload_dirtied_booklists(self): + ''' + Upload metadata to device. + ''' + plugboards = self.library_view.model().db.prefs.get('plugboards', {}) + self.device_manager.sync_booklists(Dispatcher(lambda x: x), + self.booklists(), plugboards) + def upload_booklists(self): ''' Upload metadata to device. diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index c1627d7e12..e270cd0a55 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -7,7 +7,7 @@ import re, os, inspect from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \ pyqtSignal, QDialogButtonBox, QInputDialog, QLineEdit, \ - QDate + QDate, QCompleter from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog from calibre.gui2.dialogs.tag_editor import TagEditor @@ -364,7 +364,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): (fm[f]['datatype'] in ['text', 'series', 'enumeration'] and fm[f].get('search_terms', None) and f not in ['formats', 'ondevice']) or - fm[f]['datatype'] in ['int', 'float', 'bool'] ): + (fm[f]['datatype'] in ['int', 'float', 'bool'] and + f not in ['id'])): self.all_fields.append(f) self.writable_fields.append(f) if fm[f]['datatype'] == 'composite': @@ -393,6 +394,14 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.book_1_text.setObjectName(name) self.testgrid.addWidget(w, i+offset, 2, 1, 1) + ident_types = sorted(self.db.get_all_identifier_types(), key=sort_key) + self.s_r_dst_ident.setCompleter(QCompleter(ident_types)) + try: + self.s_r_dst_ident.setPlaceholderText(_('Enter an identifier type')) + except: + pass + self.s_r_src_ident.addItems(ident_types) + self.main_heading = _( 'You can destroy your library using this feature. ' 'Changes are permanent. There is no undo function. ' @@ -449,6 +458,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.test_text.editTextChanged[str].connect(self.s_r_paint_results) self.comma_separated.stateChanged.connect(self.s_r_paint_results) self.case_sensitive.stateChanged.connect(self.s_r_paint_results) + self.s_r_src_ident.currentIndexChanged[int].connect(self.s_r_paint_results) + self.s_r_dst_ident.textChanged.connect(self.s_r_paint_results) self.s_r_template.lost_focus.connect(self.s_r_template_changed) self.central_widget.setCurrentIndex(0) @@ -471,6 +482,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.query_field.addItems(sorted([q for q in self.queries], key=sort_key)) self.query_field.currentIndexChanged[str].connect(self.s_r_query_change) self.query_field.setCurrentIndex(0) + self.search_field.setCurrentIndex(0) + self.s_r_search_field_changed(0) def s_r_sf_itemdata(self, idx): if idx is None: @@ -495,6 +508,13 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): val = mi.get(field, None) if isinstance(val, (int, float, bool)): val = str(val) + elif fm['is_csp']: + # convert the csp dict into a list + id_type = unicode(self.s_r_src_ident.currentText()) + if id_type: + val = [val.get(id_type, '')] + else: + val = [u'%s:%s'%(t[0], t[1]) for t in val.iteritems()] if val is None: val = [] if fm['is_multiple'] else [''] elif not fm['is_multiple']: @@ -512,12 +532,17 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.s_r_search_field_changed(self.search_field.currentIndex()) def s_r_search_field_changed(self, idx): - if self.search_mode.currentIndex() != 0 and idx == 1: # Template + self.s_r_template.setVisible(False) + self.template_label.setVisible(False) + self.s_r_src_ident_label.setVisible(False) + self.s_r_src_ident.setVisible(False) + if idx == 1: # Template self.s_r_template.setVisible(True) self.template_label.setVisible(True) - else: - self.s_r_template.setVisible(False) - self.template_label.setVisible(False) + elif self.s_r_sf_itemdata(idx) == 'identifiers': + self.s_r_src_ident_label.setVisible(True) + self.s_r_src_ident.setVisible(True) + for i in range(0, self.s_r_number_of_books): w = getattr(self, 'book_%d_text'%(i+1)) mi = self.db.get_metadata(self.ids[i], index_is_id=True) @@ -535,10 +560,15 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.s_r_paint_results(None) def s_r_destination_field_changed(self, idx): + self.s_r_dst_ident_label.setVisible(False) + self.s_r_dst_ident.setVisible(False) txt = self.s_r_df_itemdata(idx) if not txt: txt = self.s_r_sf_itemdata(None) if txt and txt in self.writable_fields: + if txt == 'identifiers': + self.s_r_dst_ident_label.setVisible(True) + self.s_r_dst_ident.setVisible(True) self.destination_field_fm = self.db.metadata_for_field(txt) self.s_r_paint_results(None) @@ -617,6 +647,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): dest = src dest_mode = self.replace_mode.currentIndex() + if self.destination_field_fm['is_csp']: + if not unicode(self.s_r_dst_ident.text()): + raise Exception(_('You must specify a destination identifier type')) + if self.destination_field_fm['is_multiple']: if self.comma_separated.isChecked(): if dest == 'authors': @@ -635,6 +669,13 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): if dest_mode != 0: dest_val = mi.get(dest, '') + if self.db.metadata_for_field(dest)['is_csp']: + dst_id_type = unicode(self.s_r_dst_ident.text()) + if dst_id_type: + dest_val = [dest_val.get(dst_id_type, '')] + else: + # convert the csp dict into a list + dest_val = [u'%s:%s'%(t[0], t[1]) for t in dest_val.iteritems()] if dest_val is None: dest_val = [] elif not isinstance(dest_val, list): @@ -717,6 +758,17 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): 'Book title %s not processed')%mi.title, show=True) return + # convert the colon-separated pair strings back into a dict, which + # is what set_identifiers wants + if dfm['is_csp']: + dst_id_type = unicode(self.s_r_dst_ident.text()) + if dst_id_type: + v = ''.join(val) + ids = mi.get(dest) + ids[dst_id_type] = v + val = ids + else: + val = dict([(t.split(':')) for t in val]) else: val = self.s_r_replace_mode_separator().join(val) if dest == 'title' and len(val) == 0: diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 1654ff8261..59a68d6514 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -732,6 +732,29 @@ Future conversion of these books will use the default settings. + + + + Identifier type: + + + s_r_src_ident + + + + + + + + 100 + 0 + + + + Choose which identifier type to operate upon + + + @@ -910,7 +933,30 @@ not multiple and the destination field is multiple - + + + + Identifier type: + + + s_r_dst_ident + + + + + + + + 100 + 0 + + + + Choose which identifier type to operate upon + + + + @@ -996,7 +1042,7 @@ not multiple and the destination field is multiple - + QFrame::NoFrame @@ -1120,6 +1166,7 @@ not multiple and the destination field is multiple remove_button search_field search_mode + s_r_src_ident s_r_template search_for case_sensitive @@ -1128,6 +1175,7 @@ not multiple and the destination field is multiple destination_field replace_mode comma_separated + s_r_dst_ident results_count starting_from multiple_separator diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index f2388d2981..5453a90766 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal ' import time, os from PyQt4.Qt import SIGNAL, QUrl, QAbstractListModel, Qt, \ - QVariant + QVariant, QFont from calibre.web.feeds.recipes import compile_recipe, custom_recipes from calibre.web.feeds.news import AutomaticNewsRecipe @@ -83,6 +83,9 @@ class UserProfiles(ResizableDialog, Ui_Dialog): self._model = self.model = CustomRecipeModel(recipe_model) self.available_profiles.setModel(self._model) self.available_profiles.currentChanged = self.current_changed + f = QFont() + f.setStyleHint(f.Monospace) + self.source_code.setFont(f) self.connect(self.remove_feed_button, SIGNAL('clicked(bool)'), self.added_feeds.remove_selected_items) diff --git a/src/calibre/gui2/dialogs/user_profiles.ui b/src/calibre/gui2/dialogs/user_profiles.ui index 97e3d37db2..7631c74768 100644 --- a/src/calibre/gui2/dialogs/user_profiles.ui +++ b/src/calibre/gui2/dialogs/user_profiles.ui @@ -410,11 +410,6 @@ p, li { white-space: pre-wrap; } 0 - - - DejaVu Sans Mono - - QTextEdit::NoWrap diff --git a/src/calibre/gui2/dnd.py b/src/calibre/gui2/dnd.py new file mode 100644 index 0000000000..928de72578 --- /dev/null +++ b/src/calibre/gui2/dnd.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import posixpath, os, urllib, re +from urlparse import urlparse, urlunparse +from threading import Thread +from Queue import Queue, Empty + +from PyQt4.Qt import QPixmap, Qt, QDialog, QLabel, QVBoxLayout, \ + QDialogButtonBox, QProgressBar, QTimer + +from calibre.constants import DEBUG, iswindows +from calibre.ptempfile import PersistentTemporaryFile +from calibre import browser, as_unicode, prints +from calibre.gui2 import error_dialog + +IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'gif', 'png', 'bmp'] + +class Worker(Thread): # {{{ + + def __init__(self, url, fpath, rq): + Thread.__init__(self) + self.url, self.fpath = url, fpath + self.daemon = True + self.rq = rq + self.err = self.tb = None + + def run(self): + try: + br = browser() + br.retrieve(self.url, self.fpath, self.callback) + except Exception, e: + self.err = as_unicode(e) + import traceback + self.tb = traceback.format_exc() + + def callback(self, a, b, c): + self.rq.put((a, b, c)) +# }}} + +class DownloadDialog(QDialog): # {{{ + + def __init__(self, url, fname, parent): + QDialog.__init__(self, parent) + self.setWindowTitle(_('Download %s')%fname) + self.l = QVBoxLayout(self) + self.purl = urlparse(url) + self.msg = QLabel(_('Downloading %s from %s')%(fname, + self.purl.netloc)) + self.msg.setWordWrap(True) + self.l.addWidget(self.msg) + self.pb = QProgressBar(self) + self.pb.setMinimum(0) + self.pb.setMaximum(0) + self.l.addWidget(self.pb) + self.bb = QDialogButtonBox(QDialogButtonBox.Cancel, Qt.Horizontal, self) + self.l.addWidget(self.bb) + self.bb.rejected.connect(self.reject) + sz = self.sizeHint() + self.resize(max(sz.width(), 400), sz.height()) + + fpath = PersistentTemporaryFile(os.path.splitext(fname)[1]) + fpath.close() + self.fpath = fpath.name + + self.worker = Worker(url, self.fpath, Queue()) + self.rejected = False + + def reject(self): + self.rejected = True + QDialog.reject(self) + + def start_download(self): + self.worker.start() + QTimer.singleShot(50, self.update) + self.exec_() + if self.worker.err is not None: + error_dialog(self.parent(), _('Download failed'), + _('Failed to download from %r with error: %s')%( + self.worker.url, self.worker.err), + det_msg=self.worker.tb, show=True) + + def update(self): + if self.rejected: + return + + try: + progress = self.worker.rq.get_nowait() + except Empty: + pass + else: + self.update_pb(progress) + + if not self.worker.is_alive(): + return self.accept() + QTimer.singleShot(50, self.update) + + def update_pb(self, progress): + transferred, block_size, total = progress + if total == -1: + self.pb.setMaximum(0) + self.pb.setMinimum(0) + self.pb.setValue(0) + else: + so_far = transferred * block_size + self.pb.setMaximum(max(total, so_far)) + self.pb.setValue(so_far) + + @property + def err(self): + return self.worker.err + +# }}} + +def dnd_has_image(md): + return md.hasImage() + +def data_as_string(f, md): + raw = bytes(md.data(f)) + if '/x-moz' in f: + try: + raw = raw.decode('utf-16') + except: + pass + return raw + +def dnd_has_extension(md, extensions): + if DEBUG: + prints('Debugging DND event') + for f in md.formats(): + f = unicode(f) + prints(f, repr(data_as_string(f, md))[:300], '\n') + print () + if has_firefox_ext(md, extensions): + return True + if md.hasUrls(): + urls = [unicode(u.toString()) for u in + md.urls()] + purls = [urlparse(u) for u in urls] + if DEBUG: + prints('URLS:', urls) + prints('Paths:', [u2p(x) for x in purls]) + + exts = frozenset([posixpath.splitext(u.path)[1][1:].lower() for u in + purls]) + return bool(exts.intersection(frozenset(extensions))) + return False + +def u2p(url): + path = url.path + if iswindows: + if path.startswith('/'): + path = path[1:] + ans = path.replace('/', os.sep) + if os.path.exists(ans): + return ans + # Try unquoting the URL + return urllib.unquote(ans) + +def dnd_get_image(md, image_exts=IMAGE_EXTENSIONS): + ''' + Get the image in the QMimeData object md. + + :return: None, None if no image is found + QPixmap, None if an image is found, the pixmap is guaranteed not + null + url, filename if a URL that points to an image is found + ''' + if dnd_has_image(md): + for x in md.formats(): + x = unicode(x) + if x.startswith('image/'): + cdata = bytes(md.data(x)) + pmap = QPixmap() + pmap.loadFromData(cdata) + if not pmap.isNull(): + return pmap, None + break + + # No image, look for a URL pointing to an image + if md.hasUrls(): + urls = [unicode(u.toString()) for u in + md.urls()] + purls = [urlparse(u) for u in urls] + # First look for a local file + images = [u2p(x) for x in purls if x.scheme in ('', 'file') and + posixpath.splitext(urllib.unquote(x.path))[1][1:].lower() in + image_exts] + images = [x for x in images if os.path.exists(x)] + p = QPixmap() + for path in images: + try: + with open(path, 'rb') as f: + p.loadFromData(f.read()) + except: + continue + if not p.isNull(): + return p, None + + # No local images, look for remote ones + + # First, see if this is from Firefox + rurl, fname = get_firefox_rurl(md, image_exts) + + if rurl and fname: + return rurl, fname + # Look through all remaining URLs + remote_urls = [x for x in purls if x.scheme in ('http', 'https', + 'ftp') and posixpath.splitext(x.path)[1][1:].lower() in image_exts] + if remote_urls: + rurl = remote_urls[0] + fname = posixpath.basename(urllib.unquote(rurl.path)) + return urlunparse(rurl), fname + + return None, None + +def dnd_get_files(md, exts): + ''' + Get the file in the QMimeData object md with an extension that is one of + the extensions in exts. + + :return: None, None if no file is found + [paths], None if a local file is found + [urls], [filenames] if URLs that point to a files are found + ''' + # Look for a URL pointing to a file + if md.hasUrls(): + urls = [unicode(u.toString()) for u in + md.urls()] + purls = [urlparse(u) for u in urls] + # First look for a local file + local_files = [u2p(x) for x in purls if x.scheme in ('', 'file') and + posixpath.splitext(urllib.unquote(x.path))[1][1:].lower() in + exts] + local_files = [x for x in local_files if os.path.exists(x)] + if local_files: + return local_files, None + + # No local files, look for remote ones + + # First, see if this is from Firefox + rurl, fname = get_firefox_rurl(md, exts) + if rurl and fname: + return [rurl], [fname] + + # Look through all remaining URLs + remote_urls = [x for x in purls if x.scheme in ('http', 'https', + 'ftp') and posixpath.splitext(x.path)[1][1:].lower() in exts] + if remote_urls: + filenames = [posixpath.basename(urllib.unquote(rurl.path)) for rurl in + remote_urls] + return [urlunparse(x) for x in remote_urls], filenames + + return None, None + +def _get_firefox_pair(md, exts, url, fname): + url = bytes(md.data(url)).decode('utf-16') + fname = bytes(md.data(fname)).decode('utf-16') + while url.endswith('\x00'): + url = url[:-1] + while fname.endswith('\x00'): + fname = fname[:-1] + if not url or not fname: + return None, None + ext = posixpath.splitext(fname)[1][1:].lower() + # Weird firefox bug on linux + ext = {'jpe':'jpg', 'epu':'epub', 'mob':'mobi'}.get(ext, ext) + fname = os.path.splitext(fname)[0] + '.' + ext + if DEBUG: + prints('Firefox file promise:', url, fname) + if ext not in exts: + fname = url = None + return url, fname + + +def get_firefox_rurl(md, exts): + formats = frozenset([unicode(x) for x in md.formats()]) + url = fname = None + if 'application/x-moz-file-promise-url' in formats and \ + 'application/x-moz-file-promise-dest-filename' in formats: + try: + url, fname = _get_firefox_pair(md, exts, + 'application/x-moz-file-promise-url', + 'application/x-moz-file-promise-dest-filename') + except: + if DEBUG: + import traceback + traceback.print_exc() + if url is None and 'text/x-moz-url-data' in formats and \ + 'text/x-moz-url-desc' in formats: + try: + url, fname = _get_firefox_pair(md, exts, + 'text/x-moz-url-data', 'text/x-moz-url-desc') + except: + if DEBUG: + import traceback + traceback.print_exc() + + if url is None and '_NETSCAPE_URL' in formats: + try: + raw = bytes(md.data('_NETSCAPE_URL')) + raw = raw.decode('utf-8') + lines = raw.splitlines() + if len(lines) > 1 and re.match(r'[a-z]+://', lines[1]) is None: + url, fname = lines[:2] + ext = posixpath.splitext(fname)[1][1:].lower() + if ext not in exts: + fname = url = None + except: + if DEBUG: + import traceback + traceback.print_exc() + if DEBUG: + prints('Firefox rurl:', url, fname) + return url, fname + +def has_firefox_ext(md, exts): + return bool(get_firefox_rurl(md, exts)[0]) + diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index e8c2712c83..80f1f1c2cf 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -44,13 +44,13 @@ class LibraryViewMixin(object): # {{{ for view in (self.library_view, self.memory_view, self.card_a_view, self.card_b_view): getattr(view, func)(*args) - self.memory_view.connect_dirtied_signal(self.upload_booklists) + self.memory_view.connect_dirtied_signal(self.upload_dirtied_booklists) self.memory_view.connect_upload_collections_signal( func=self.upload_collections, oncard=None) - self.card_a_view.connect_dirtied_signal(self.upload_booklists) + self.card_a_view.connect_dirtied_signal(self.upload_dirtied_booklists) self.card_a_view.connect_upload_collections_signal( func=self.upload_collections, oncard='carda') - self.card_b_view.connect_dirtied_signal(self.upload_booklists) + self.card_b_view.connect_dirtied_signal(self.upload_dirtied_booklists) self.card_b_view.connect_upload_collections_signal( func=self.upload_collections, oncard='cardb') self.book_on_device(None, reset=True) @@ -264,6 +264,9 @@ class LayoutMixin(object): # {{{ self.book_details.files_dropped.connect(self.iactions['Add Books'].files_dropped_on_book) self.book_details.cover_changed.connect(self.bd_cover_changed, type=Qt.QueuedConnection) + self.book_details.remote_file_dropped.connect( + self.iactions['Add Books'].remote_file_dropped_on_book, + type=Qt.QueuedConnection) self.book_details.open_containing_folder.connect(self.iactions['View'].view_folder_for_id) self.book_details.view_specific_format.connect(self.iactions['View'].view_format_by_id) diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py index 87da6818eb..3a090f8102 100644 --- a/src/calibre/gui2/library/delegates.py +++ b/src/calibre/gui2/library/delegates.py @@ -5,7 +5,6 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys from math import cos, sin, pi from PyQt4.Qt import QColor, Qt, QModelIndex, QSize, \ @@ -245,13 +244,13 @@ class CcTextDelegate(QStyledItemDelegate): # {{{ typ = m.custom_columns[col]['datatype'] if typ == 'int': editor = QSpinBox(parent) - editor.setRange(-100, sys.maxint) + editor.setRange(-100, 100000000) editor.setSpecialValueText(_('Undefined')) editor.setSingleStep(1) elif typ == 'float': editor = QDoubleSpinBox(parent) editor.setSpecialValueText(_('Undefined')) - editor.setRange(-100., float(sys.maxint)) + editor.setRange(-100., 100000000) editor.setDecimals(2) else: editor = MultiCompleteLineEdit(parent) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index b782cc7c72..33d12e8ab9 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -268,6 +268,15 @@ class BooksModel(QAbstractTableModel): # {{{ return None return self.get_current_highlighted_id() + def highlight_ids(self, ids_to_highlight): + self.ids_to_highlight = ids_to_highlight + self.ids_to_highlight_set = set(self.ids_to_highlight) + if self.ids_to_highlight: + self.current_highlighted_idx = 0 + else: + self.current_highlighted_idx = None + self.reset() + def search(self, text, reset=True): try: if self.highlight_only: diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index 5a4c34a5cd..5b8501ebb5 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -436,17 +436,18 @@ class SavedSearchBoxMixin(object): # {{{ b = getattr(self, x+'_search_button') b.setStatusTip(b.toolTip()) - def saved_searches_changed(self): + def saved_searches_changed(self, set_restriction=None): p = sorted(saved_searches().names(), key=sort_key) - t = unicode(self.search_restriction.currentText()) + if set_restriction is None: + set_restriction = unicode(self.search_restriction.currentText()) # rebuild the restrictions combobox using current saved searches self.search_restriction.clear() self.search_restriction.addItem('') self.tags_view.recount() for s in p: self.search_restriction.addItem(s) - if t: # redo the search restriction if there was one - self.apply_named_search_restriction(t) + if set_restriction: # redo the search restriction if there was one + self.apply_named_search_restriction(set_restriction) def do_saved_search_edit(self, search): d = SavedSearchEditor(self, search) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 586715afd0..c4871880a4 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -246,7 +246,7 @@ class TagsView(QTreeView): # {{{ self.add_subcategory.emit(key) return if action == 'search_category': - self.tags_marked.emit(key + ':' + search_state) + self._toggle(index, set_to=search_state) return if action == 'delete_user_category': self.delete_user_category.emit(key) @@ -320,6 +320,9 @@ class TagsView(QTreeView): # {{{ self.context_menu.addAction(_('Edit sort for %s')%tag.name, partial(self.context_menu_handler, action='edit_author_sort', index=tag.id)) + + # is_editable is also overloaded to mean 'can be added + # to a user category' m = self.context_menu.addMenu(self.user_category_icon, _('Add %s to user category')%tag.name) nt = self.model().category_node_tree @@ -345,7 +348,7 @@ class TagsView(QTreeView): # {{{ partial(self.context_menu_handler, action='delete_item_from_user_category', key = key, index = tag_item)) - # Add the search for value items + # Add the search for value items. All leaf nodes are searchable self.context_menu.addAction(self.search_icon, _('Search for %s')%tag.name, partial(self.context_menu_handler, action='search', @@ -373,7 +376,6 @@ class TagsView(QTreeView): # {{{ action='delete_user_category', key=key)) self.context_menu.addSeparator() # Hide/Show/Restore categories - #if not key.startswith('@') or key.find('.') < 0: self.context_menu.addAction(_('Hide category %s') % category, partial(self.context_menu_handler, action='hide', category=key)) @@ -384,16 +386,21 @@ class TagsView(QTreeView): # {{{ m.addAction(self.db.field_metadata[col]['name'], partial(self.context_menu_handler, action='show', category=col)) - # search by category - if key != 'search': + # search by category. Some categories are not searchable, such + # as search and news + if item.tag.is_searchable: self.context_menu.addAction(self.search_icon, _('Search for books in category %s')%category, - partial(self.context_menu_handler, action='search_category', - key=key, search_state='true')) + partial(self.context_menu_handler, + action='search_category', + index=self._model.createIndex(item.row(), 0, item), + search_state=TAG_SEARCH_STATES['mark_plus'])) self.context_menu.addAction(self.search_icon, _('Search for books not in category %s')%category, - partial(self.context_menu_handler, action='search_category', - key=key, search_state='false')) + partial(self.context_menu_handler, + action='search_category', + index=self._model.createIndex(item.row(), 0, item), + search_state=TAG_SEARCH_STATES['mark_minus'])) # Offer specific editors for tags/series/publishers/saved searches self.context_menu.addSeparator() if key in ['tags', 'publisher', 'series'] or \ @@ -559,8 +566,10 @@ class TagTreeItem(object): # {{{ self.bold_font = QVariant(self.bold_font) self.category_key = category_key self.temporary = temporary - self.tag = Tag(data) - self.tag.is_hierarchical = category_key.startswith('@') + self.tag = Tag(data, category=category_key, + is_editable=category_key not in ['news', 'search', 'identifiers'], + is_searchable=category_key not in ['news', 'search']) + elif self.type == self.TAG: self.icon_state_map[0] = QVariant(data.icon) self.tag = data @@ -660,14 +669,12 @@ class TagTreeItem(object): # {{{ ''' set_to: None => advance the state, otherwise a value from TAG_SEARCH_STATES ''' - basic_search_ok = self.tag.is_editable or \ - self.tag.category == 'formats' or self.tag.category == 'rating' if set_to is None: while True: self.tag.state = (self.tag.state + 1)%5 if self.tag.state == TAG_SEARCH_STATES['mark_plus'] or \ self.tag.state == TAG_SEARCH_STATES['mark_minus']: - if basic_search_ok: + if self.tag.is_searchable: break elif self.tag.state == TAG_SEARCH_STATES['mark_plusplus'] or\ self.tag.state == TAG_SEARCH_STATES['mark_minusminus']: @@ -766,6 +773,7 @@ class TagsModel(QAbstractItemModel): # {{{ self.category_nodes.append(node) node.can_be_edited = (not is_gst) and (i == (len(path_parts)-1)) node.is_gst = is_gst + node.tag.is_hierarchical = not is_gst if not is_gst: tree_root[p] = {} tree_root = tree_root[p] @@ -1240,9 +1248,6 @@ class TagsModel(QAbstractItemModel): # {{{ n.id_set |= tag.id_set category_child_map[tag.name, tag.category] = n self.endInsertRows() - tag.is_editable = key != 'formats' and (key == 'news' or \ - self.db.field_metadata[tag.category]['datatype'] in \ - ['text', 'series', 'enumeration']) else: for i,comp in enumerate(components): if i == 0: @@ -1258,12 +1263,13 @@ class TagsModel(QAbstractItemModel): # {{{ if i < len(components)-1: t = copy.copy(tag) t.original_name = '.'.join(components[:i+1]) + # This 'manufactured' intermediate node can + # be searched, but cannot be edited. t.is_editable = False else: t = tag if not in_uc: t.original_name = t.name - t.is_editable = True t.is_hierarchical = True t.name = comp self.beginInsertRows(category_index, 999999, 1) @@ -1340,7 +1346,8 @@ class TagsModel(QAbstractItemModel): # {{{ for c in sorted(user_cats.keys(), key=sort_key): if icu_lower(c).startswith(ckey_lower): if len(c) == len(ckey): - if nkey_lower in user_cat_keys_lower: + if strcmp(ckey, nkey) != 0 and \ + nkey_lower in user_cat_keys_lower: error_dialog(self.tags_view, _('Rename user category'), _('The name %s is already used')%nkey, show=True) return False @@ -1348,7 +1355,8 @@ class TagsModel(QAbstractItemModel): # {{{ del user_cats[ckey] elif c[len(ckey)] == '.': rest = c[len(ckey):] - if icu_lower(nkey + rest) in user_cat_keys_lower: + if strcmp(ckey, nkey) != 0 and \ + icu_lower(nkey + rest) in user_cat_keys_lower: error_dialog(self.tags_view, _('Rename user category'), _('The name %s is already used')%(nkey+rest), show=True) return False diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index de0f83a5b2..964616ab48 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -17,17 +17,19 @@ from calibre.gui2.viewer.bookmarkmanager import BookmarkManager from calibre.gui2.widgets import ProgressIndicator from calibre.gui2.main_window import MainWindow from calibre.gui2 import Application, ORG_NAME, APP_UID, choose_files, \ - info_dialog, error_dialog, open_url, available_height, gprefs + info_dialog, error_dialog, open_url, available_height from calibre.ebooks.oeb.iterator import EbookIterator from calibre.ebooks import DRMError from calibre.constants import islinux, isfreebsd, isosx, filesystem_encoding -from calibre.utils.config import Config, StringConfig, dynamic +from calibre.utils.config import Config, StringConfig, JSONConfig from calibre.gui2.search_box import SearchBox2 from calibre.ebooks.metadata import MetaInformation from calibre.customize.ui import available_input_formats from calibre.gui2.viewer.dictionary import Lookup from calibre import as_unicode, force_unicode, isbytestring +vprefs = JSONConfig('viewer') + class TOCItem(QStandardItem): def __init__(self, toc): @@ -303,7 +305,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer): m = self.open_history_menu m.clear() count = 0 - for path in gprefs.get('viewer_open_history', []): + for path in vprefs.get('viewer_open_history', []): if count > 9: break if os.path.exists(path): @@ -315,17 +317,17 @@ class EbookViewer(MainWindow, Ui_EbookViewer): return MainWindow.closeEvent(self, e) def save_state(self): - state = str(self.saveState(self.STATE_VERSION)) - dynamic['viewer_toolbar_state'] = state - dynamic.set('viewer_window_geometry', self.saveGeometry()) + state = bytearray(self.saveState(self.STATE_VERSION)) + vprefs['viewer_toolbar_state'] = state + vprefs.set('viewer_window_geometry', bytearray(self.saveGeometry())) if self.current_book_has_toc: - dynamic.set('viewer_toc_isvisible', bool(self.toc.isVisible())) + vprefs.set('viewer_toc_isvisible', bool(self.toc.isVisible())) if self.toc.isVisible(): - dynamic.set('viewer_splitter_state', + vprefs.set('viewer_splitter_state', bytearray(self.splitter.saveState())) def restore_state(self): - state = dynamic.get('viewer_toolbar_state', None) + state = vprefs.get('viewer_toolbar_state', None) if state is not None: try: state = QByteArray(state) @@ -676,13 +678,13 @@ class EbookViewer(MainWindow, Ui_EbookViewer): self.action_table_of_contents.setChecked(False) if isbytestring(pathtoebook): pathtoebook = force_unicode(pathtoebook, filesystem_encoding) - vh = gprefs.get('viewer_open_history', []) + vh = vprefs.get('viewer_open_history', []) try: vh.remove(pathtoebook) except: pass vh.insert(0, pathtoebook) - gprefs.set('viewer_open_history', vh[:50]) + vprefs.set('viewer_open_history', vh[:50]) self.build_recent_menu() self.action_table_of_contents.setDisabled(not self.iterator.toc) @@ -739,13 +741,13 @@ class EbookViewer(MainWindow, Ui_EbookViewer): c = config().parse() self.splitter.setSizes([1, 300]) if c.remember_window_size: - wg = dynamic.get('viewer_window_geometry', None) + wg = vprefs.get('viewer_window_geometry', None) if wg is not None: self.restoreGeometry(wg) - ss = dynamic.get('viewer_splitter_state', None) + ss = vprefs.get('viewer_splitter_state', None) if ss is not None: self.splitter.restoreState(ss) - self.show_toc_on_open = dynamic.get('viewer_toc_isvisible', False) + self.show_toc_on_open = vprefs.get('viewer_toc_isvisible', False) av = available_height() - 30 if self.height() > av: self.resize(self.width(), av) diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index aa9d6c8b9f..8ebf9c2c21 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -3,7 +3,7 @@ __copyright__ = '2008, Kovid Goyal ' ''' Miscellaneous widgets used in the GUI ''' -import re, os, traceback +import re, traceback from PyQt4.Qt import QIcon, QFont, QLabel, QListWidget, QAction, \ QListWidgetItem, QTextCharFormat, QApplication, \ @@ -22,6 +22,8 @@ from calibre.ebooks import BOOK_EXTENSIONS from calibre.ebooks.metadata.meta import metadata_from_filename from calibre.utils.config import prefs, XMLConfig, tweaks from calibre.gui2.progress_indicator import ProgressIndicator as _ProgressIndicator +from calibre.gui2.dnd import dnd_has_image, dnd_get_image, dnd_get_files, \ + IMAGE_EXTENSIONS, dnd_has_extension, DownloadDialog history = XMLConfig('history') @@ -141,36 +143,35 @@ class FilenamePattern(QWidget, Ui_Form): return pat -IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'gif', 'png', 'bmp'] - class FormatList(QListWidget): DROPABBLE_EXTENSIONS = BOOK_EXTENSIONS formats_dropped = pyqtSignal(object, object) delete_format = pyqtSignal() - @classmethod - def paths_from_event(cls, event): - ''' - Accept a drop event and return a list of paths that can be read from - and represent files with extensions. - ''' - if event.mimeData().hasFormat('text/uri-list'): - urls = [unicode(u.toLocalFile()) for u in event.mimeData().urls()] - urls = [u for u in urls if os.path.splitext(u)[1] and os.access(u, os.R_OK)] - return [u for u in urls if os.path.splitext(u)[1][1:].lower() in cls.DROPABBLE_EXTENSIONS] - def dragEnterEvent(self, event): - if int(event.possibleActions() & Qt.CopyAction) + \ - int(event.possibleActions() & Qt.MoveAction) == 0: - return - paths = self.paths_from_event(event) - if paths: + md = event.mimeData() + if dnd_has_extension(md, self.DROPABBLE_EXTENSIONS): event.acceptProposedAction() def dropEvent(self, event): - paths = self.paths_from_event(event) event.setDropAction(Qt.CopyAction) - self.formats_dropped.emit(event, paths) + md = event.mimeData() + # Now look for ebook files + urls, filenames = dnd_get_files(md, self.DROPABBLE_EXTENSIONS) + if not urls: + # Nothing found + return + + if not filenames: + # Local files + self.formats_dropped.emit(event, urls) + else: + # Remote files, use the first file + d = DownloadDialog(urls[0], filenames[0], self) + d.start_download() + if d.err is None: + self.formats_dropped.emit(event, [d.fpath]) + def dragMoveEvent(self, event): event.acceptProposedAction() @@ -183,7 +184,7 @@ class FormatList(QListWidget): class ImageDropMixin(object): # {{{ ''' - Adds support for dropping images onto widgets and a contect menu for + Adds support for dropping images onto widgets and a context menu for copy/pasting images. ''' DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS @@ -191,39 +192,36 @@ class ImageDropMixin(object): # {{{ def __init__(self): self.setAcceptDrops(True) - @classmethod - def paths_from_event(cls, event): - ''' - Accept a drop event and return a list of paths that can be read from - and represent files with extensions. - ''' - if event.mimeData().hasFormat('text/uri-list'): - urls = [unicode(u.toLocalFile()) for u in event.mimeData().urls()] - urls = [u for u in urls if os.path.splitext(u)[1] and os.access(u, os.R_OK)] - return [u for u in urls if os.path.splitext(u)[1][1:].lower() in cls.DROPABBLE_EXTENSIONS] - def dragEnterEvent(self, event): - if int(event.possibleActions() & Qt.CopyAction) + \ - int(event.possibleActions() & Qt.MoveAction) == 0: - return - paths = self.paths_from_event(event) - if paths: + md = event.mimeData() + if dnd_has_extension(md, self.DROPABBLE_EXTENSIONS) or \ + dnd_has_image(md): event.acceptProposedAction() def dropEvent(self, event): - paths = self.paths_from_event(event) event.setDropAction(Qt.CopyAction) - for path in paths: - pmap = QPixmap() - pmap.load(path) - if not pmap.isNull(): - self.handle_image_drop(path, pmap) - event.accept() - break + md = event.mimeData() - def handle_image_drop(self, path, pmap): + x, y = dnd_get_image(md) + if x is not None: + # We have an image, set cover + event.accept() + if y is None: + # Local image + self.handle_image_drop(x) + else: + # Remote files, use the first file + d = DownloadDialog(x, y, self) + d.start_download() + if d.err is None: + pmap = QPixmap() + pmap.loadFromData(open(d.fpath, 'rb').read()) + if not pmap.isNull(): + self.handle_image_drop(pmap) + + def handle_image_drop(self, pmap): self.set_pixmap(pmap) - self.cover_changed.emit(open(path, 'rb').read()) + self.cover_changed.emit(pixmap_to_data(pmap)) def dragMoveEvent(self, event): event.acceptProposedAction() diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 823ef77bc5..21a2622f33 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -7,7 +7,7 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import re, itertools, time, traceback -from itertools import repeat +from itertools import repeat, izip, imap from datetime import timedelta from threading import Thread @@ -194,6 +194,7 @@ class ResultCache(SearchQueryParser): # {{{ self.first_sort = True self.search_restriction = '' self.search_restriction_book_count = 0 + self.marked_ids_dict = {} self.field_metadata = field_metadata self.all_search_locations = field_metadata.get_search_terms() SearchQueryParser.__init__(self, self.all_search_locations, optimize=True) @@ -775,6 +776,36 @@ class ResultCache(SearchQueryParser): # {{{ def get_search_restriction_book_count(self): return self.search_restriction_book_count + def set_marked_ids(self, id_dict): + ''' + ids in id_dict are "marked". They can be searched for by + using the search term ``marked:true``. Pass in an empty dictionary or + set to clear marked ids. + + :param id_dict: Either a dictionary mapping ids to values or a set + of ids. In the latter case, the value is set to 'true' for all ids. If + a mapping is provided, then the search can be used to search for + particular values: ``marked:value`` + ''' + if not hasattr(id_dict, 'items'): + # Simple list. Make it a dict of string 'true' + self.marked_ids_dict = dict.fromkeys(id_dict, u'true') + else: + # Ensure that all the items in the dict are text + self.marked_ids_dict = dict(izip(id_dict.iterkeys(), imap(unicode, + id_dict.itervalues()))) + + # Set the values in the cache + marked_col = self.FIELD_MAP['marked'] + for r in self.iterall(): + r[marked_col] = None + + for id_, val in self.marked_ids_dict.iteritems(): + try: + self._data[id_][marked_col] = val + except: + pass + # }}} def remove(self, id): @@ -824,6 +855,7 @@ class ResultCache(SearchQueryParser): # {{{ self._data[id] = CacheRow(db, self.composites, db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]) self._data[id].append(db.book_on_device_string(id)) + self._data[id].append(self.marked_ids_dict.get(id, None)) except IndexError: return None try: @@ -840,6 +872,7 @@ class ResultCache(SearchQueryParser): # {{{ self._data[id] = CacheRow(db, self.composites, db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]) self._data[id].append(db.book_on_device_string(id)) + self._data[id].append(self.marked_ids_dict.get(id, None)) self._map[0:0] = ids self._map_filtered[0:0] = ids @@ -864,6 +897,15 @@ class ResultCache(SearchQueryParser): # {{{ for item in self._data: if item is not None: item.append(db.book_on_device_string(item[0])) + item.append(None) + + marked_col = self.FIELD_MAP['marked'] + for id_,val in self.marked_ids_dict.iteritems(): + try: + self._data[id_][marked_col] = val + except: + pass + self._map = [i[0] for i in self._data if i is not None] if field is not None: self.sort(field, ascending) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 38b70fc2bf..b506e7e82d 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -47,13 +47,15 @@ copyfile = os.link if hasattr(os, 'link') else shutil.copyfile class Tag(object): def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None, - tooltip=None, icon=None, category=None, id_set=None): + tooltip=None, icon=None, category=None, id_set=None, + is_editable = True, is_searchable=True): self.name = self.original_name = name self.id = id self.count = count self.state = state self.is_hierarchical = False - self.is_editable = True + self.is_editable = is_editable + self.is_searchable = is_searchable self.id_set = id_set self.avg_rating = avg/2.0 if avg is not None else 0 self.sort = sort @@ -372,6 +374,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.FIELD_MAP['ondevice'] = base = base+1 self.field_metadata.set_field_record_index('ondevice', base, prefer_custom=False) + self.FIELD_MAP['marked'] = base = base+1 + self.field_metadata.set_field_record_index('marked', base, prefer_custom=False) script = ''' DROP VIEW IF EXISTS meta2; @@ -419,6 +423,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.row = self.data.row self.has_id = self.data.has_id self.count = self.data.count + self.set_marked_ids = self.data.set_marked_ids for prop in ( 'author_sort', 'authors', 'comment', 'comments', @@ -1439,10 +1444,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): reverse=True items.sort(key=kf, reverse=reverse) + is_editable = category not in ['news', 'rating'] categories[category] = [tag_class(formatter(r.n), count=r.c, id=r.id, avg=avgr(r), sort=r.s, icon=icon, tooltip=tooltip, category=category, - id_set=r.id_set) + id_set=r.id_set, is_editable=is_editable) for r in items] #print 'end phase "tags list":', time.clock() - last, 'seconds' @@ -1479,7 +1485,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): all=False) if count > 0: categories['formats'].append(Tag(fmt, count=count, icon=icon, - category='formats')) + category='formats', is_editable=False)) if sort == 'popularity': categories['formats'].sort(key=lambda x: x.count, reverse=True) @@ -1507,7 +1513,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): all=False) if count > 0: categories['identifiers'].append(Tag(ident, count=count, icon=icon, - category='identifiers')) + category='identifiers', + is_editable=False)) if sort == 'popularity': categories['identifiers'].sort(key=lambda x: x.count, reverse=True) @@ -1566,7 +1573,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon = icon_map['search'] for srch in saved_searches().names(): items.append(Tag(srch, tooltip=saved_searches().lookup(srch), - sort=srch, icon=icon, category='search')) + sort=srch, icon=icon, category='search', + is_editable=False)) if len(items): if icon_map is not None: icon_map['search'] = icon_map['search'] @@ -2546,6 +2554,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return ans + def get_all_identifier_types(self): + idents = self.conn.get('SELECT DISTINCT type FROM identifiers') + return [ident[0] for ident in idents] + def _clean_identifier(self, typ, val): typ = icu_lower(typ).strip().replace(':', '').replace(',', '') val = val.strip().replace(',', '|').replace(':', '|') diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index ff38af6890..b8180f9f39 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -273,6 +273,16 @@ class FieldMetadata(dict): 'is_custom':False, 'is_category':False, 'is_csp': False}), + ('marked', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name': None, + 'search_terms':['marked'], + 'is_custom':False, + 'is_category':False, + 'is_csp': False}), ('series_index',{'table':None, 'column':None, 'datatype':'float', diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index de586048b7..96c42e6e0e 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -12,13 +12,13 @@ from calibre.constants import DEBUG from calibre.utils.config import Config, StringConfig, tweaks from calibre.utils.formatter import TemplateFormatter from calibre.utils.filenames import shorten_components_to, supports_long_names, \ - ascii_filename, sanitize_file_name + ascii_filename from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ebooks.metadata.meta import set_metadata -from calibre.constants import preferred_encoding, filesystem_encoding +from calibre.constants import preferred_encoding from calibre.ebooks.metadata import fmt_sidx from calibre.ebooks.metadata import title_sort -from calibre import strftime, prints +from calibre import strftime, prints, sanitize_file_name_unicode plugboard_any_device_value = 'any device' plugboard_any_format_value = 'any format' @@ -197,12 +197,10 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, format_args[key] = '' components = SafeFormat().safe_format(template, format_args, 'G_C-EXCEPTION!', mi) - components = [x.strip() for x in components.split('/') if x.strip()] + components = [x.strip() for x in components.split('/')] components = [sanitize_func(x) for x in components if x] if not components: components = [str(id)] - components = [x.encode(filesystem_encoding, 'replace') if isinstance(x, - unicode) else x for x in components] if to_lowercase: components = [x.lower() for x in components] if replace_whitespace: @@ -247,7 +245,7 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, return True, id_, mi.title components = get_components(opts.template, mi, id_, opts.timefmt, length, - ascii_filename if opts.asciiize else sanitize_file_name, + ascii_filename if opts.asciiize else sanitize_file_name_unicode, to_lowercase=opts.to_lowercase, replace_whitespace=opts.replace_whitespace) base_path = os.path.join(root, *components) @@ -329,8 +327,6 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, def _sanitize_args(root, opts): if opts is None: opts = config().parse() - if isinstance(root, unicode): - root = root.encode(filesystem_encoding) root = os.path.abspath(root) opts.template = preprocess_template(opts.template) diff --git a/src/calibre/startup.py b/src/calibre/startup.py index 41b20f3946..c883c43e8a 100644 --- a/src/calibre/startup.py +++ b/src/calibre/startup.py @@ -72,47 +72,6 @@ if not _run_once: pass ################################################################################ - # Improve builtin path functions to handle unicode sensibly - - _abspath = os.path.abspath - def my_abspath(path, encoding=sys.getfilesystemencoding()): - ''' - Work around for buggy os.path.abspath. This function accepts either byte strings, - in which it calls os.path.abspath, or unicode string, in which case it first converts - to byte strings using `encoding`, calls abspath and then decodes back to unicode. - ''' - to_unicode = False - if encoding is None: - encoding = preferred_encoding - if isinstance(path, unicode): - path = path.encode(encoding) - to_unicode = True - res = _abspath(path) - if to_unicode: - res = res.decode(encoding) - return res - - os.path.abspath = my_abspath - - _join = os.path.join - def my_join(a, *p): - encoding=sys.getfilesystemencoding() - if not encoding: - encoding = preferred_encoding - p = [a] + list(p) - _unicode = False - for i in p: - if isinstance(i, unicode): - _unicode = True - break - p = [i.encode(encoding) if isinstance(i, unicode) else i for i in p] - - res = _join(*p) - if _unicode: - res = res.decode(encoding) - return res - - os.path.join = my_join def local_open(name, mode='r', bufsize=-1): ''' diff --git a/src/calibre/trac/bzr_commit_plugin.py b/src/calibre/trac/bzr_commit_plugin.py index 2f91804315..325bac7a79 100644 --- a/src/calibre/trac/bzr_commit_plugin.py +++ b/src/calibre/trac/bzr_commit_plugin.py @@ -19,7 +19,7 @@ in the working tree you want to use it with:: trac_reponame_password = ''' -import os, re, xmlrpclib +import os, re, xmlrpclib, subprocess from bzrlib.builtins import cmd_commit as _cmd_commit, tree_files from bzrlib import branch import bzrlib @@ -115,5 +115,7 @@ class cmd_commit(_cmd_commit): server.ticket.update(int(bug), msg, {'status':'closed', 'resolution':'fixed'}, True) + subprocess.Popen('/home/kovid/work/kde/mail.py -f --delay 10'.split()) + bzrlib.commands.register_command(cmd_commit) diff --git a/src/calibre/utils/magick/__init__.py b/src/calibre/utils/magick/__init__.py index 834a798de5..6be5580d17 100644 --- a/src/calibre/utils/magick/__init__.py +++ b/src/calibre/utils/magick/__init__.py @@ -95,6 +95,26 @@ class DrawingWand(_magick.DrawingWand): # {{{ self.font_size_ = float(val) return property(fget=fget, fset=fset, doc=_magick.DrawingWand.font_size_.__doc__) + @dynamic_property + def stroke_color(self): + def fget(self): + return self.stroke_color_.color + def fset(self, val): + col = PixelWand() + col.color = unicode(val) + self.stroke_color_ = col + return property(fget=fget, fset=fset, doc=_magick.DrawingWand.font_size_.__doc__) + + @dynamic_property + def fill_color(self): + def fget(self): + return self.fill_color_.color + def fset(self, val): + col = PixelWand() + col.color = unicode(val) + self.fill_color_ = col + return property(fget=fget, fset=fset, doc=_magick.DrawingWand.font_size_.__doc__) + # }}} class Image(_magick.Image): # {{{ diff --git a/src/calibre/utils/magick/magick.c b/src/calibre/utils/magick/magick.c index 869b77c736..84c5f3a2ed 100644 --- a/src/calibre/utils/magick/magick.c +++ b/src/calibre/utils/magick/magick.c @@ -263,6 +263,78 @@ magick_DrawingWand_fontsize_setter(magick_DrawingWand *self, PyObject *val, void // }}} +// DrawingWand.stroke_color {{{ +static PyObject * +magick_DrawingWand_stroke_color_getter(magick_DrawingWand *self, void *closure) { + NULL_CHECK(NULL) + magick_PixelWand *pw; + PixelWand *wand = NewPixelWand(); + + if (wand == NULL) return PyErr_NoMemory(); + DrawGetStrokeColor(self->wand, wand); + + pw = (magick_PixelWand*) magick_PixelWandType.tp_alloc(&magick_PixelWandType, 0); + if (pw == NULL) return PyErr_NoMemory(); + pw->wand = wand; + return Py_BuildValue("O", (PyObject *)pw); +} + +static int +magick_DrawingWand_stroke_color_setter(magick_DrawingWand *self, PyObject *val, void *closure) { + NULL_CHECK(-1) + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete DrawingWand stroke color"); + return -1; + } + + magick_PixelWand *pw; + + pw = (magick_PixelWand*)val; + if (!IsPixelWand(pw->wand)) { PyErr_SetString(PyExc_TypeError, "Invalid PixelWand"); return -1; } + + DrawSetStrokeColor(self->wand, pw->wand); + + return 0; +} + +// }}} + +// DrawingWand.fill_color {{{ +static PyObject * +magick_DrawingWand_fill_color_getter(magick_DrawingWand *self, void *closure) { + NULL_CHECK(NULL) + magick_PixelWand *pw; + PixelWand *wand = NewPixelWand(); + + if (wand == NULL) return PyErr_NoMemory(); + DrawGetFillColor(self->wand, wand); + + pw = (magick_PixelWand*) magick_PixelWandType.tp_alloc(&magick_PixelWandType, 0); + if (pw == NULL) return PyErr_NoMemory(); + pw->wand = wand; + return Py_BuildValue("O", (PyObject *)pw); +} + +static int +magick_DrawingWand_fill_color_setter(magick_DrawingWand *self, PyObject *val, void *closure) { + NULL_CHECK(-1) + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete DrawingWand fill color"); + return -1; + } + + magick_PixelWand *pw; + + pw = (magick_PixelWand*)val; + if (!IsPixelWand(pw->wand)) { PyErr_SetString(PyExc_TypeError, "Invalid PixelWand"); return -1; } + + DrawSetFillColor(self->wand, pw->wand); + + return 0; +} + +// }}} + // DrawingWand.text_antialias {{{ static PyObject * magick_DrawingWand_textantialias_getter(magick_DrawingWand *self, void *closure) { @@ -336,6 +408,16 @@ static PyGetSetDef magick_DrawingWand_getsetters[] = { (char *)"DrawingWand fontsize", NULL}, + {(char *)"stroke_color_", + (getter)magick_DrawingWand_stroke_color_getter, (setter)magick_DrawingWand_stroke_color_setter, + (char *)"DrawingWand stroke color", + NULL}, + + {(char *)"fill_color_", + (getter)magick_DrawingWand_fill_color_getter, (setter)magick_DrawingWand_fill_color_setter, + (char *)"DrawingWand fill color", + NULL}, + {(char *)"text_antialias", (getter)magick_DrawingWand_textantialias_getter, (setter)magick_DrawingWand_textantialias_setter, (char *)"DrawingWand text antialias", diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index 67f19e40e4..f2e22c8f5e 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -193,8 +193,8 @@ class RecursiveFetcher(object): data = None self.log.debug('Fetching', url) delta = time.time() - self.last_fetch_at - if delta < self.delay: - time.sleep(delta) + if delta < self.delay: + time.sleep(self.delay - delta) if isinstance(url, unicode): url = url.encode('utf-8') # Not sure is this is really needed as I think mechanize