From db385fac26a541ac57f6d42c93d4d80fff47e183 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 7 May 2011 15:50:07 +0100 Subject: [PATCH 01/35] First cut at beam ebooks. Also fix format names in foyles and waterstones to match other stores. --- src/calibre/customize/builtins.py | 12 ++- .../gui2/store/beam_ebooks_de_plugin.py | 89 +++++++++++++++++++ src/calibre/gui2/store/foyles_uk_plugin.py | 2 +- .../gui2/store/waterstones_uk_plugin.py | 2 +- 4 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 src/calibre/gui2/store/beam_ebooks_de_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 36bcbdbfe2..ff9861c537 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1171,13 +1171,19 @@ class StoreFoylesUKStore(StoreBase): description = _('Foyles of London, online') actual_plugin = 'calibre.gui2.store.foyles_uk_plugin:FoylesUKStore' -class AmazonDEKindleStore(StoreBase): +class StoreAmazonDEKindleStore(StoreBase): name = 'Amazon DE Kindle' description = _('Kindle eBooks') actual_plugin = 'calibre.gui2.store.amazon_de_plugin:AmazonDEKindleStore' -plugins += [StoreAmazonKindleStore, AmazonDEKindleStore, StoreAmazonUKKindleStore, - StoreBaenWebScriptionStore, StoreBNStore, +class StoreBeamEBooksDEStore(StoreBase): + name = 'Beam EBooks DE' + description = _('Kindle eBooks') + actual_plugin = 'calibre.gui2.store.beam_ebooks_de_plugin:BeamEBooksDEStore' + +plugins += [StoreAmazonKindleStore, StoreAmazonDEKindleStore, + StoreAmazonUKKindleStore, + StoreBaenWebScriptionStore, StoreBNStore, StoreBeamEBooksDEStore, StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreEHarlequinStoretore, StoreFeedbooksStore, StoreFoylesUKStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, diff --git a/src/calibre/gui2/store/beam_ebooks_de_plugin.py b/src/calibre/gui2/store/beam_ebooks_de_plugin.py new file mode 100644 index 0000000000..18cff7a411 --- /dev/null +++ b/src/calibre/gui2/store/beam_ebooks_de_plugin.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + url = 'http://www.beam-ebooks.de' + url_details = 'http://www.beam-ebooks.de{0}' + + if external or self.config.get('open_external', False): + if detail_item: + url = url_details.format(detail_item) + open_url(QUrl(url)) + else: + detail_url = None + if detail_item: + detail_url = url_details.format(detail_item) + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.beam-ebooks.de/suchergebnis.php?Type=&sw=' + urllib2.quote(query) + print(url) + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + print(doc) + for data in doc.xpath('//table[tr/td/div[@class="stil2"]]'): + print('here1') + if counter <= 0: + break + + id = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/@href')).strip() + print('here', id) + if not id: + continue + cover_url = ''.join(data.xpath('./tr/td[1]/a/img/@src')) + if cover_url: + cover_url = 'http://www.beam-ebooks.de' + cover_url + title = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/b/text()')) + author = ' '.join(data.xpath('./tr/td/div[@class="stil2"]/child::b/text()|./tr/td/div[@class="stil2"]/child::strong/text()')) + price = ''.join(data.xpath('./tr/td[3]/text()')) + print(data.xpath('./tr/td[3]/a/img/@alt')) + pdf = data.xpath('boolean(./tr/td[3]/a/img[contains(@alt, "PDF")]/@alt)') + epub = data.xpath('boolean(./tr/td[3]/a/img[contains(@alt, "ePub")]/@alt)') + mobi = data.xpath('boolean(./tr/td[3]/a/img[contains(@alt, "Mobipocket")]/@alt)') + print(id, cover_url, title, author, price, pdf, epub, mobi) + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.drm = SearchResult.DRM_UNLOCKED + s.detail_item = id + formats = [] + if epub: + formats.append('ePub') + if pdf: + formats.append('PDF') + if mobi: + formats.append('MOBI') + s.formats = ', '.join(formats) + + yield s diff --git a/src/calibre/gui2/store/foyles_uk_plugin.py b/src/calibre/gui2/store/foyles_uk_plugin.py index ca35fb6bb2..1a997cd671 100644 --- a/src/calibre/gui2/store/foyles_uk_plugin.py +++ b/src/calibre/gui2/store/foyles_uk_plugin.py @@ -73,6 +73,6 @@ class FoylesUKStore(BasicStoreConfig, StorePlugin): s.price = price s.detail_item = id s.drm = SearchResult.DRM_LOCKED - s.formats = 'EPUB' + s.formats = 'ePub' yield s diff --git a/src/calibre/gui2/store/waterstones_uk_plugin.py b/src/calibre/gui2/store/waterstones_uk_plugin.py index d422165c47..a5065128ba 100644 --- a/src/calibre/gui2/store/waterstones_uk_plugin.py +++ b/src/calibre/gui2/store/waterstones_uk_plugin.py @@ -76,7 +76,7 @@ class WaterstonesUKStore(BasicStoreConfig, StorePlugin): s.detail_item = id formats = [] if epub: - formats.append('EPUB') + formats.append('ePub') if pdf: formats.append('PDF') s.formats = ', '.join(formats) From e219f7214dca2b84b054077506867b2096298572 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 8 May 2011 13:36:08 +0100 Subject: [PATCH 02/35] Add epubbuy. Continue a bit with beam. --- src/calibre/customize/builtins.py | 11 ++- .../gui2/store/beam_ebooks_de_plugin.py | 5 +- src/calibre/gui2/store/epubbuy_de_plugin.py | 73 +++++++++++++++++++ 3 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 src/calibre/gui2/store/epubbuy_de_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 7f1705e8ff..5faa2de232 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1126,7 +1126,7 @@ class StoreEbookscomStore(StoreBase): description = _('The digital bookstore.') actual_plugin = 'calibre.gui2.store.ebooks_com_plugin:EbookscomStore' -class StoreEHarlequinStoretore(StoreBase): +class StoreEHarlequinStore(StoreBase): name = 'eHarlequin' description = _('entertain, enrich, inspire.') actual_plugin = 'calibre.gui2.store.eharlequin_plugin:EHarlequinStore' @@ -1183,14 +1183,19 @@ class StoreAmazonDEKindleStore(StoreBase): class StoreBeamEBooksDEStore(StoreBase): name = 'Beam EBooks DE' - description = _('Kindle eBooks') + description = _('der eBook Shop') actual_plugin = 'calibre.gui2.store.beam_ebooks_de_plugin:BeamEBooksDEStore' +class StoreEPubBuyDEStore(StoreBase): + name = 'EPUBBuy DE' + description = _('EPUBReaders eBook Shop') + actual_plugin = 'calibre.gui2.store.epubbuy_de_plugin:EPubBuyDEStore' + plugins += [StoreAmazonKindleStore, StoreAmazonDEKindleStore, StoreAmazonUKKindleStore, StoreBaenWebScriptionStore, StoreBNStore, StoreBeamEBooksDEStore, StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, - StoreEHarlequinStoretore, StoreFeedbooksStore, + StoreEPubBuyDEStore, StoreEHarlequinStore, StoreFeedbooksStore, StoreFoylesUKStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore, StoreWaterstonesUKStore] diff --git a/src/calibre/gui2/store/beam_ebooks_de_plugin.py b/src/calibre/gui2/store/beam_ebooks_de_plugin.py index 18cff7a411..d5a9d8f12c 100644 --- a/src/calibre/gui2/store/beam_ebooks_de_plugin.py +++ b/src/calibre/gui2/store/beam_ebooks_de_plugin.py @@ -23,8 +23,8 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): - url = 'http://www.beam-ebooks.de' - url_details = 'http://www.beam-ebooks.de{0}' + url = 'http://www.affiliwelt.net/klick.php?log=no&prid=908&pid=2&bannerid=10072&url=http://www.beam-ebooks.de' + url_details = 'http://www.affiliwelt.net/klick.php?log=no&prid=908&pid=2&bannerid=10072&url=http://www.beam-ebooks.de{0}' if external or self.config.get('open_external', False): if detail_item: @@ -34,6 +34,7 @@ class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): detail_url = None if detail_item: detail_url = url_details.format(detail_item) + print(detail_url) d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(self.config.get('tags', '')) diff --git a/src/calibre/gui2/store/epubbuy_de_plugin.py b/src/calibre/gui2/store/epubbuy_de_plugin.py new file mode 100644 index 0000000000..6b92e101f8 --- /dev/null +++ b/src/calibre/gui2/store/epubbuy_de_plugin.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class EPubBuyDEStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + url = 'http://www.epubbuy.com/' + url_details = '{0}' + + if external or self.config.get('open_external', False): + if detail_item: + url = url_details.format(detail_item) + open_url(QUrl(url)) + else: + detail_url = None + if detail_item: + detail_url = url_details.format(detail_item) + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.epubbuy.com/search.php?search_query=' + urllib2.quote(query) + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//li[contains(@class, "ajax_block_product")]'): + if counter <= 0: + break + + id = ''.join(data.xpath('./div[@class="center_block"]/a[@class="product_img_link"]/@href')).strip() + if not id: + continue + cover_url = ''.join(data.xpath('./div[@class="center_block"]/a[@class="product_img_link"]/img/@src')) + if cover_url: + cover_url = 'http://www.epubbuy.com' + cover_url + title = ''.join(data.xpath('./div[@class="center_block"]/a[@class="product_img_link"]/@title')) + author = ''.join(data.xpath('./div[@class="center_block"]/a[2]/text()')) + price = ''.join(data.xpath('.//span[@class="price"]/text()')) + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.drm = SearchResult.DRM_UNLOCKED + s.detail_item = id + s.formats = 'ePub' + + yield s From a6261f3202d7caa46636f14f4d3437a28949b07b Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 8 May 2011 14:00:52 -0400 Subject: [PATCH 03/35] Borders plugin. --- src/calibre/customize/builtins.py | 8 +- src/calibre/gui2/store/borders_plugin.py | 94 ++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 src/calibre/gui2/store/borders_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index c1da8391e0..08049e48cf 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1116,6 +1116,11 @@ class StoreBeWriteStore(StoreBase): description = _('Publishers of fine books.') actual_plugin = 'calibre.gui2.store.bewrite_plugin:BeWriteStore' +class StoreBordersStore(StoreBase): + name = 'Borders' + description = _('Buy Books, Used Books, Music, DVDs & Blu-ray Online.') + actual_plugin = 'calibre.gui2.store.borders_plugin:BordersStore' + class StoreDieselEbooksStore(StoreBase): name = 'Diesel eBooks' description = _('World Famous eBook Store.') @@ -1183,7 +1188,8 @@ class AmazonDEKindleStore(StoreBase): plugins += [StoreAmazonKindleStore, AmazonDEKindleStore, StoreAmazonUKKindleStore, StoreBaenWebScriptionStore, StoreBNStore, - StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, + StoreBeWriteStore, StoreBordersStore, + StoreDieselEbooksStore, StoreEbookscomStore, StoreEHarlequinStoretore, StoreFeedbooksStore, StoreFoylesUKStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore, diff --git a/src/calibre/gui2/store/borders_plugin.py b/src/calibre/gui2/store/borders_plugin.py new file mode 100644 index 0000000000..a610c2eaef --- /dev/null +++ b/src/calibre/gui2/store/borders_plugin.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import re +import urllib +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class BordersStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + #m_url = 'http://www.dpbolvw.net/' + #h_click = 'click-4879827-10762497' + #d_click = 'click-4879827-10772898' + # Use Kovid's affiliate id 30% of the time. + #if random.randint(1, 10) in (1, 2, 3): + # h_click = 'click-4913808-10762497' + # d_click = 'click-4913808-10772898' + + #url = m_url + h_click + url = 'http://www.borders.com/online/store/Landing?nav=5185+700152' + detail_url = None + if detail_item: + detail_url = 'http://www.borders.com/online/store/TitleDetail?sku=%s' % detail_item + #detail_url = m_url + d_click + detail_item + + if external or self.config.get('open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.borders.com/online/store/SearchResults?type=44&simple=1&keyword=' + urllib.quote_plus(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//table[@class="browseResultsTable"]//tr'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//td[1]/a[1]/@href')) + if not id: + continue + id = re.search('(?<=sku=)\d+', id) + if not id: + continue + id = id.group() + + cover_url = ''.join(data.xpath('.//img[@class="jtip prod-item"]/@src')) + + title = ''.join(data.xpath('.//td[@align="left"][1]//a[1]//text()')) + author = ''.join(data.xpath('.//td[@align="left"][1]//a[2]/text()')) + + price = ''.join(data.xpath('.//div[@class="sale_price"]//text()')) + price = re.search('\$[\d.]+', price) + if not price: + continue + price = price.group() + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = id.strip() + #s.detail_item = '?url=http://www.kobobooks.com/' + id.strip() + s.drm = SearchResult.DRM_LOCKED + s.formats = 'EPUB' + + yield s From dd56bb7b6d9e32f811ee1a40eea630376f567ef8 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 9 May 2011 18:13:59 -0400 Subject: [PATCH 04/35] Remove Borders store as they do not want to be included. --- src/calibre/customize/builtins.py | 7 +- src/calibre/gui2/store/borders_plugin.py | 94 ------------------------ 2 files changed, 1 insertion(+), 100 deletions(-) delete mode 100644 src/calibre/gui2/store/borders_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 08049e48cf..1ca51c2f81 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1116,11 +1116,6 @@ class StoreBeWriteStore(StoreBase): description = _('Publishers of fine books.') actual_plugin = 'calibre.gui2.store.bewrite_plugin:BeWriteStore' -class StoreBordersStore(StoreBase): - name = 'Borders' - description = _('Buy Books, Used Books, Music, DVDs & Blu-ray Online.') - actual_plugin = 'calibre.gui2.store.borders_plugin:BordersStore' - class StoreDieselEbooksStore(StoreBase): name = 'Diesel eBooks' description = _('World Famous eBook Store.') @@ -1188,7 +1183,7 @@ class AmazonDEKindleStore(StoreBase): plugins += [StoreAmazonKindleStore, AmazonDEKindleStore, StoreAmazonUKKindleStore, StoreBaenWebScriptionStore, StoreBNStore, - StoreBeWriteStore, StoreBordersStore, + StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreEHarlequinStoretore, StoreFeedbooksStore, StoreFoylesUKStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, diff --git a/src/calibre/gui2/store/borders_plugin.py b/src/calibre/gui2/store/borders_plugin.py deleted file mode 100644 index a610c2eaef..0000000000 --- a/src/calibre/gui2/store/borders_plugin.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import (unicode_literals, division, absolute_import, print_function) - -__license__ = 'GPL 3' -__copyright__ = '2011, John Schember ' -__docformat__ = 'restructuredtext en' - -import random -import re -import urllib -from contextlib import closing - -from lxml import html - -from PyQt4.Qt import QUrl - -from calibre import browser, url_slash_cleaner -from calibre.gui2 import open_url -from calibre.gui2.store import StorePlugin -from calibre.gui2.store.basic_config import BasicStoreConfig -from calibre.gui2.store.search_result import SearchResult -from calibre.gui2.store.web_store_dialog import WebStoreDialog - -class BordersStore(BasicStoreConfig, StorePlugin): - - def open(self, parent=None, detail_item=None, external=False): - #m_url = 'http://www.dpbolvw.net/' - #h_click = 'click-4879827-10762497' - #d_click = 'click-4879827-10772898' - # Use Kovid's affiliate id 30% of the time. - #if random.randint(1, 10) in (1, 2, 3): - # h_click = 'click-4913808-10762497' - # d_click = 'click-4913808-10772898' - - #url = m_url + h_click - url = 'http://www.borders.com/online/store/Landing?nav=5185+700152' - detail_url = None - if detail_item: - detail_url = 'http://www.borders.com/online/store/TitleDetail?sku=%s' % detail_item - #detail_url = m_url + d_click + detail_item - - if external or self.config.get('open_external', False): - open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) - else: - d = WebStoreDialog(self.gui, url, parent, detail_url) - d.setWindowTitle(self.name) - d.set_tags(self.config.get('tags', '')) - d.exec_() - - def search(self, query, max_results=10, timeout=60): - url = 'http://www.borders.com/online/store/SearchResults?type=44&simple=1&keyword=' + urllib.quote_plus(query) - - br = browser() - - counter = max_results - with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) - for data in doc.xpath('//table[@class="browseResultsTable"]//tr'): - if counter <= 0: - break - - id = ''.join(data.xpath('.//td[1]/a[1]/@href')) - if not id: - continue - id = re.search('(?<=sku=)\d+', id) - if not id: - continue - id = id.group() - - cover_url = ''.join(data.xpath('.//img[@class="jtip prod-item"]/@src')) - - title = ''.join(data.xpath('.//td[@align="left"][1]//a[1]//text()')) - author = ''.join(data.xpath('.//td[@align="left"][1]//a[2]/text()')) - - price = ''.join(data.xpath('.//div[@class="sale_price"]//text()')) - price = re.search('\$[\d.]+', price) - if not price: - continue - price = price.group() - - counter -= 1 - - s = SearchResult() - s.cover_url = cover_url - s.title = title.strip() - s.author = author.strip() - s.price = price.strip() - s.detail_item = id.strip() - #s.detail_item = '?url=http://www.kobobooks.com/' + id.strip() - s.drm = SearchResult.DRM_LOCKED - s.formats = 'EPUB' - - yield s From fb493ff8de2efb4cb0fbe8fbca0776074398e16e Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 9 May 2011 18:44:00 -0400 Subject: [PATCH 05/35] Store: List of stores that do not want to be included. Weightless Books plugin. --- src/calibre/customize/builtins.py | 27 ++++--- src/calibre/gui2/store/declined.txt | 5 ++ .../gui2/store/weightless_books_plugin.py | 76 +++++++++++++++++++ 3 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 src/calibre/gui2/store/declined.txt create mode 100644 src/calibre/gui2/store/weightless_books_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 1ca51c2f81..2c516f22c0 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1096,6 +1096,11 @@ class StoreAmazonKindleStore(StoreBase): description = _('Kindle books from Amazon') actual_plugin = 'calibre.gui2.store.amazon_plugin:AmazonKindleStore' +class StoreAmazonDEKindleStore(StoreBase): + name = 'Amazon DE Kindle' + description = _('Kindle eBooks') + actual_plugin = 'calibre.gui2.store.amazon_de_plugin:AmazonDEKindleStore' + class StoreAmazonUKKindleStore(StoreBase): name = 'Amazon UK Kindle' description = _('Kindle books from Amazon.uk') @@ -1136,6 +1141,11 @@ class StoreFeedbooksStore(StoreBase): description = _('Read anywhere.') actual_plugin = 'calibre.gui2.store.feedbooks_plugin:FeedbooksStore' +class StoreFoylesUKStore(StoreBase): + name = 'Foyles UK' + description = _('Foyles of London, online') + actual_plugin = 'calibre.gui2.store.foyles_uk_plugin:FoylesUKStore' + class StoreGutenbergStore(StoreBase): name = 'Project Gutenberg' description = _('The first producer of free ebooks.') @@ -1171,23 +1181,18 @@ class StoreWaterstonesUKStore(StoreBase): description = _('Feel every word') actual_plugin = 'calibre.gui2.store.waterstones_uk_plugin:WaterstonesUKStore' -class StoreFoylesUKStore(StoreBase): - name = 'Foyles UK' - description = _('Foyles of London, online') - actual_plugin = 'calibre.gui2.store.foyles_uk_plugin:FoylesUKStore' +class StoreWeightlessBooksStore(StoreBase): + name = 'Weightless Books' + description = '(e)Books That Don\'t Weigh You Down' + actual_plugin = 'calibre.gui2.store.weightless_books_plugin:WeightlessBooksStore' -class AmazonDEKindleStore(StoreBase): - name = 'Amazon DE Kindle' - description = _('Kindle eBooks') - actual_plugin = 'calibre.gui2.store.amazon_de_plugin:AmazonDEKindleStore' - -plugins += [StoreAmazonKindleStore, AmazonDEKindleStore, StoreAmazonUKKindleStore, +plugins += [StoreAmazonKindleStore, StoreAmazonDEKindleStore, StoreAmazonUKKindleStore, StoreBaenWebScriptionStore, StoreBNStore, StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreEHarlequinStoretore, StoreFeedbooksStore, StoreFoylesUKStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore, - StoreWaterstonesUKStore] + StoreWaterstonesUKStore, StoreWeightlessBooksStore] # }}} diff --git a/src/calibre/gui2/store/declined.txt b/src/calibre/gui2/store/declined.txt new file mode 100644 index 0000000000..2b0e5caed2 --- /dev/null +++ b/src/calibre/gui2/store/declined.txt @@ -0,0 +1,5 @@ +This is a list of stores that objected, declined +or asked not to be included in the store integration. + +* Borders (http://www.borders.com/) +* WH Smith (http://www.whsmith.co.uk/) diff --git a/src/calibre/gui2/store/weightless_books_plugin.py b/src/calibre/gui2/store/weightless_books_plugin.py new file mode 100644 index 0000000000..3fa1c76851 --- /dev/null +++ b/src/calibre/gui2/store/weightless_books_plugin.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class WeightlessBooksStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + url = 'http://weightlessbooks.com/' + + if external or self.config.get('open_external', False): + open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://weightlessbooks.com/?s=' + urllib.quote_plus(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//li[@id="product"]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//div[@class="cover"]/a/@href')) + if not id: + continue + + cover_url = ''.join(data.xpath('.//div[@class="cover"]/a/img/@src')) + + price = ''.join(data.xpath('.//div[@class="buy_buttons"]/b[1]/text()')) + if not price: + continue + + formats = ', '.join(data.xpath('.//select[@class="eStore_variation"]//option//text()')) + formats = formats.upper() + + title = ''.join(data.xpath('.//h3/a/text()')) + author = ''.join(data.xpath('.//h3//text()')) + author = author.replace(title, '') + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = id.strip() + s.drm = SearchResult.DRM_UNLOCKED + s.formats = formats + + yield s From e1b0cfdacfdf772eaa9cf700191b431bc5d1f5df Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 9 May 2011 18:49:13 -0400 Subject: [PATCH 06/35] Store: Make mention of declined.txt in comments. --- src/calibre/gui2/store/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index 214ede3372..35fa440b28 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -43,6 +43,8 @@ class StorePlugin(object): # {{{ The easiest way to handle affiliate money payouts is to randomly select between the author's affiliate id and calibre's affiliate id so that 70% of the time the author's id is used. + + See declined.txt for a list of stores that do not want to be included. ''' def __init__(self, gui, name): From 4a20c220f13fe4922ed83a566cb4b03b27146406 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 9 May 2011 18:51:05 -0400 Subject: [PATCH 07/35] ... --- src/calibre/gui2/store/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index 35fa440b28..d58ccbda84 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' class StorePlugin(object): # {{{ ''' A plugin representing an online ebook repository (store). The store can - be a comercial store that sells ebooks or a source of free downloadable + be a commercial store that sells ebooks or a source of free downloadable ebooks. Note that this class is the base class for these plugins, however, to From f4878b5c7fa19fe85805263dce45a7f8d4b4a18e Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 9 May 2011 19:05:01 -0400 Subject: [PATCH 08/35] Store: Always sort stores in case-insentive alphabetical order. --- src/calibre/gui2/actions/store.py | 2 +- src/calibre/gui2/store/search/search.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index 90f2cac3cd..1989250bc8 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -27,7 +27,7 @@ class StoreAction(InterfaceAction): self.store_menu.clear() self.store_menu.addAction(_('Search'), self.search) self.store_menu.addSeparator() - for n, p in self.gui.istores.items(): + for n, p in sorted(self.gui.istores.items(), key=lambda x: x[0].lower()): self.store_menu.addAction(n, partial(self.open_store, p)) self.qaction.setMenu(self.store_menu) diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index 07d4afca54..62e4e97f11 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -47,7 +47,7 @@ class SearchDialog(QDialog, Ui_Dialog): # per search basis. stores_group_layout = QVBoxLayout() self.stores_group.setLayout(stores_group_layout) - for x in self.store_plugins: + for x in sorted(self.store_plugins.keys(), key=lambda x: x.lower()): cbox = QCheckBox(x) cbox.setChecked(True) stores_group_layout.addWidget(cbox) From 6ebdda7aa54af32316cb962f7b815a692eef7f75 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 10 May 2011 12:58:54 +0100 Subject: [PATCH 09/35] Beam store, with working links --- src/calibre/gui2/store/beam_ebooks_de_plugin.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/store/beam_ebooks_de_plugin.py b/src/calibre/gui2/store/beam_ebooks_de_plugin.py index d5a9d8f12c..385bbc5456 100644 --- a/src/calibre/gui2/store/beam_ebooks_de_plugin.py +++ b/src/calibre/gui2/store/beam_ebooks_de_plugin.py @@ -23,8 +23,8 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): - url = 'http://www.affiliwelt.net/klick.php?log=no&prid=908&pid=2&bannerid=10072&url=http://www.beam-ebooks.de' - url_details = 'http://www.affiliwelt.net/klick.php?log=no&prid=908&pid=2&bannerid=10072&url=http://www.beam-ebooks.de{0}' + url = 'http://klick.affiliwelt.net/klick.php?bannerid=10072&pid=32307&prid=908' + url_details = 'http://klick.affiliwelt.net/klick.php?bannerid=10730&pid=32307&prid=908&prodid={0}' if external or self.config.get('open_external', False): if detail_item: @@ -34,7 +34,6 @@ class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): detail_url = None if detail_item: detail_url = url_details.format(detail_item) - print(detail_url) d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(self.config.get('tags', '')) @@ -42,33 +41,28 @@ class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): def search(self, query, max_results=10, timeout=60): url = 'http://www.beam-ebooks.de/suchergebnis.php?Type=&sw=' + urllib2.quote(query) - print(url) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: doc = html.fromstring(f.read()) - print(doc) for data in doc.xpath('//table[tr/td/div[@class="stil2"]]'): - print('here1') if counter <= 0: break id = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/@href')).strip() - print('here', id) if not id: continue + id = id[7:] cover_url = ''.join(data.xpath('./tr/td[1]/a/img/@src')) if cover_url: cover_url = 'http://www.beam-ebooks.de' + cover_url title = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/b/text()')) author = ' '.join(data.xpath('./tr/td/div[@class="stil2"]/child::b/text()|./tr/td/div[@class="stil2"]/child::strong/text()')) price = ''.join(data.xpath('./tr/td[3]/text()')) - print(data.xpath('./tr/td[3]/a/img/@alt')) pdf = data.xpath('boolean(./tr/td[3]/a/img[contains(@alt, "PDF")]/@alt)') epub = data.xpath('boolean(./tr/td[3]/a/img[contains(@alt, "ePub")]/@alt)') mobi = data.xpath('boolean(./tr/td[3]/a/img[contains(@alt, "Mobipocket")]/@alt)') - print(id, cover_url, title, author, price, pdf, epub, mobi) counter -= 1 s = SearchResult() From c10349c5dd2b518cc7b6c8ab26dd102c28902a47 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 10 May 2011 15:18:16 +0100 Subject: [PATCH 10/35] Final versions of Beam and EPUBBuy. --- .../gui2/store/beam_ebooks_de_plugin.py | 19 ++++++++++++++----- src/calibre/gui2/store/epubbuy_de_plugin.py | 17 ++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/calibre/gui2/store/beam_ebooks_de_plugin.py b/src/calibre/gui2/store/beam_ebooks_de_plugin.py index 385bbc5456..ee73607c57 100644 --- a/src/calibre/gui2/store/beam_ebooks_de_plugin.py +++ b/src/calibre/gui2/store/beam_ebooks_de_plugin.py @@ -24,7 +24,8 @@ class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): url = 'http://klick.affiliwelt.net/klick.php?bannerid=10072&pid=32307&prid=908' - url_details = 'http://klick.affiliwelt.net/klick.php?bannerid=10730&pid=32307&prid=908&prodid={0}' + url_details = ('http://klick.affiliwelt.net/klick.php?' + 'bannerid=10730&pid=32307&prid=908&prodid={0}') if external or self.config.get('open_external', False): if detail_item: @@ -54,15 +55,23 @@ class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): if not id: continue id = id[7:] + print(id) cover_url = ''.join(data.xpath('./tr/td[1]/a/img/@src')) if cover_url: cover_url = 'http://www.beam-ebooks.de' + cover_url title = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/b/text()')) - author = ' '.join(data.xpath('./tr/td/div[@class="stil2"]/child::b/text()|./tr/td/div[@class="stil2"]/child::strong/text()')) + author = ' '.join(data.xpath('./tr/td/div[@class="stil2"]/' + 'child::b/text()' + '|' + './tr/td/div[@class="stil2"]/' + 'child::strong/text()')) price = ''.join(data.xpath('./tr/td[3]/text()')) - pdf = data.xpath('boolean(./tr/td[3]/a/img[contains(@alt, "PDF")]/@alt)') - epub = data.xpath('boolean(./tr/td[3]/a/img[contains(@alt, "ePub")]/@alt)') - mobi = data.xpath('boolean(./tr/td[3]/a/img[contains(@alt, "Mobipocket")]/@alt)') + pdf = data.xpath( + 'boolean(./tr/td[3]/a/img[contains(@alt, "PDF")]/@alt)') + epub = data.xpath( + 'boolean(./tr/td[3]/a/img[contains(@alt, "ePub")]/@alt)') + mobi = data.xpath( + 'boolean(./tr/td[3]/a/img[contains(@alt, "Mobipocket")]/@alt)') counter -= 1 s = SearchResult() diff --git a/src/calibre/gui2/store/epubbuy_de_plugin.py b/src/calibre/gui2/store/epubbuy_de_plugin.py index 6b92e101f8..242ef76793 100644 --- a/src/calibre/gui2/store/epubbuy_de_plugin.py +++ b/src/calibre/gui2/store/epubbuy_de_plugin.py @@ -23,8 +23,9 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog class EPubBuyDEStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): - url = 'http://www.epubbuy.com/' - url_details = '{0}' + url = 'http://klick.affiliwelt.net/klick.php?bannerid=47653&pid=32307&prid=2627' + url_details = ('http://klick.affiliwelt.net/klick.php?bannerid=47653' + '&pid=32307&prid=2627&prodid={0}') if external or self.config.get('open_external', False): if detail_item: @@ -50,13 +51,19 @@ class EPubBuyDEStore(BasicStoreConfig, StorePlugin): if counter <= 0: break - id = ''.join(data.xpath('./div[@class="center_block"]/a[@class="product_img_link"]/@href')).strip() + id = ''.join(data.xpath('./div[@class="center_block"]' + '/p[contains(text(), "artnr:")]/text()')).strip() if not id: continue - cover_url = ''.join(data.xpath('./div[@class="center_block"]/a[@class="product_img_link"]/img/@src')) + id = id[6:].strip() + if not id: + continue + cover_url = ''.join(data.xpath('./div[@class="center_block"]' + '/a[@class="product_img_link"]/img/@src')) if cover_url: cover_url = 'http://www.epubbuy.com' + cover_url - title = ''.join(data.xpath('./div[@class="center_block"]/a[@class="product_img_link"]/@title')) + title = ''.join(data.xpath('./div[@class="center_block"]' + '/a[@class="product_img_link"]/@title')) author = ''.join(data.xpath('./div[@class="center_block"]/a[2]/text()')) price = ''.join(data.xpath('.//span[@class="price"]/text()')) counter -= 1 From d9e0e361408aa64eecbcf683428d06c8ab10cac3 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 10 May 2011 16:18:13 +0100 Subject: [PATCH 11/35] Remove a print statement --- src/calibre/gui2/store/beam_ebooks_de_plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/gui2/store/beam_ebooks_de_plugin.py b/src/calibre/gui2/store/beam_ebooks_de_plugin.py index ee73607c57..b589a8c310 100644 --- a/src/calibre/gui2/store/beam_ebooks_de_plugin.py +++ b/src/calibre/gui2/store/beam_ebooks_de_plugin.py @@ -55,7 +55,6 @@ class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): if not id: continue id = id[7:] - print(id) cover_url = ''.join(data.xpath('./tr/td[1]/a/img/@src')) if cover_url: cover_url = 'http://www.beam-ebooks.de' + cover_url From 009090c99eee1fa83f028ac7dc8a21fbe8c62703 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 10 May 2011 18:23:18 -0400 Subject: [PATCH 12/35] Store: Add Wizards Tower Books plugin. --- src/calibre/customize/builtins.py | 7 +- .../gui2/store/wizards_tower_books_plugin.py | 88 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 src/calibre/gui2/store/wizards_tower_books_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 2c516f22c0..579403f6ba 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1186,6 +1186,11 @@ class StoreWeightlessBooksStore(StoreBase): description = '(e)Books That Don\'t Weigh You Down' actual_plugin = 'calibre.gui2.store.weightless_books_plugin:WeightlessBooksStore' +class StoreWizardsTowerBooksStore(StoreBase): + name = 'Wizards Tower Books' + description = 'Wizard\'s Tower Press' + actual_plugin = 'calibre.gui2.store.wizards_tower_books_plugin:WizardsTowerBooksStore' + plugins += [StoreAmazonKindleStore, StoreAmazonDEKindleStore, StoreAmazonUKKindleStore, StoreBaenWebScriptionStore, StoreBNStore, StoreBeWriteStore, @@ -1193,6 +1198,6 @@ plugins += [StoreAmazonKindleStore, StoreAmazonDEKindleStore, StoreAmazonUKKindl StoreEHarlequinStoretore, StoreFeedbooksStore, StoreFoylesUKStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore, - StoreWaterstonesUKStore, StoreWeightlessBooksStore] + StoreWaterstonesUKStore, StoreWeightlessBooksStore, StoreWizardsTowerBooksStore] # }}} diff --git a/src/calibre/gui2/store/wizards_tower_books_plugin.py b/src/calibre/gui2/store/wizards_tower_books_plugin.py new file mode 100644 index 0000000000..56bb00ff7e --- /dev/null +++ b/src/calibre/gui2/store/wizards_tower_books_plugin.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class WizardsTowerBooksStore(BasicStoreConfig, StorePlugin): + + url = 'http://www.wizardstowerbooks.com/' + + def open(self, parent=None, detail_item=None, external=False): + if detail_item: + detail_item = self.url + detail_item + + if external or self.config.get('open_external', False): + open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url))) + else: + d = WebStoreDialog(self.gui, self.url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.wizardstowerbooks.com/search.html?for=' + urllib.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//table[@class="gridp"]//td'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//span[@class="prti"]/a/@href')) + id = id.strip() + if not id: + continue + + cover_url = ''.join(data.xpath('.//div[@class="prim"]/a/img/@src')) + cover_url = url_slash_cleaner(self.url + cover_url.strip()) + + price = ''.join(data.xpath('.//font[@class="selling_price"]//text()')) + price = price.strip() + if not price: + continue + + title = ''.join(data.xpath('.//span[@class="prti"]/a/b/text()')) + author = ''.join(data.xpath('.//p[@class="last"]/text()')) + a, b, author = author.partition(' by ') + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = id.strip() + s.drm = SearchResult.DRM_UNLOCKED + + yield s + + def get_details(self, search_result, timeout): + br = browser() + with closing(br.open(url_slash_cleaner(self.url + search_result.detail_item), timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + + formats = ', '.join(idata.xpath('//select[@id="N1_"]//option//text()')) + search_result.formats = formats.upper() + + return True From 1073c627d63e07b766fbc02f982782664a513908 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 May 2011 22:45:59 -0600 Subject: [PATCH 13/35] Fix #780849 (Spelling error in Welcome to calibre) --- src/calibre/gui2/wizard/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/wizard/__init__.py b/src/calibre/gui2/wizard/__init__.py index a32347dc72..6b1a793fc8 100644 --- a/src/calibre/gui2/wizard/__init__.py +++ b/src/calibre/gui2/wizard/__init__.py @@ -435,7 +435,7 @@ class DevicePage(QWizardPage, DeviceUI): self.registerField("device", self.device_view) def initializePage(self): - self.label.setText(_('Choose you e-book device. If your device is' + self.label.setText(_('Choose your e-book device. If your device is' ' not in the list, choose a "%s" device.')%Device.manufacturer) self.man_model = ManufacturerModel() self.manufacturer_view.setModel(self.man_model) From 31e2bc3818b888503b14c025230c7468e3dcad78 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 09:53:28 -0600 Subject: [PATCH 14/35] Fix #781135 (ParseException when pressing "Next" or "Previous" in plugin search) --- src/calibre/gui2/preferences/plugins.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index 8888a64e84..4f88e5aa1d 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -75,6 +75,8 @@ class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{ def find(self, query): query = query.strip() + if not query: + return QModelIndex() matches = self.parse(query) if not matches: return QModelIndex() @@ -87,6 +89,8 @@ class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{ def find_next(self, idx, query, backwards=False): query = query.strip() + if not query: + return idx matches = self.parse(query) if not matches: return idx From 231df586c08b8d155a7996f6b000db65f7b438c3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 14:56:10 -0600 Subject: [PATCH 15/35] ODT Input: Handle inline special styles defined on tags. Fixes #780250 (Italic text not converting from ODT source) --- src/odf/odf2xhtml.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/odf/odf2xhtml.py b/src/odf/odf2xhtml.py index a04aa48bf7..0ae89b1663 100644 --- a/src/odf/odf2xhtml.py +++ b/src/odf/odf2xhtml.py @@ -1415,18 +1415,34 @@ ol, ul { padding-left: 2em; } self.writedata() c = attrs.get( (TEXTNS,'style-name'), None) htmlattrs = {} + # Changed by Kovid to handle inline apecial styles defined on tags. + # Apparently LibreOffice does this. + special = 'span' if c: c = c.replace(".","_") special = special_styles.get("S-"+c) - if special is None and self.generate_css: - htmlattrs['class'] = "S-%s" % c - self.opentag('span', htmlattrs) + if special is None: + special = 'span' + if self.generate_css: + htmlattrs['class'] = "S-%s" % c + + self.opentag(special, htmlattrs) self.purgedata() def e_text_span(self, tag, attrs): """ End the """ self.writedata() - self.closetag('span', False) + c = attrs.get( (TEXTNS,'style-name'), None) + # Changed by Kovid to handle inline apecial styles defined on tags. + # Apparently LibreOffice does this. + special = 'span' + if c: + c = c.replace(".","_") + special = special_styles.get("S-"+c) + if special is None: + special = 'span' + + self.closetag(special, False) self.purgedata() def s_text_tab(self, tag, attrs): From fa0c8702b0d6d51ae7d96f1c14e43c27beb9f7c7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 16:33:23 -0600 Subject: [PATCH 16/35] ... --- recipes/divahair.recipe | 53 ++++++++++++++++++++++++++++++++++ recipes/icons/divahair.png | Bin 0 -> 675 bytes recipes/icons/mayra.png | Bin 0 -> 620 bytes recipes/icons/moldovaazi.png | Bin 0 -> 243 bytes recipes/icons/newsmoldova.png | Bin 0 -> 837 bytes recipes/mayra.recipe | 51 ++++++++++++++++++++++++++++++++ recipes/moldovaazi.recipe | 50 ++++++++++++++++++++++++++++++++ recipes/newsmoldova.recipe | 50 ++++++++++++++++++++++++++++++++ 8 files changed, 204 insertions(+) create mode 100644 recipes/divahair.recipe create mode 100644 recipes/icons/divahair.png create mode 100644 recipes/icons/mayra.png create mode 100644 recipes/icons/moldovaazi.png create mode 100644 recipes/icons/newsmoldova.png create mode 100644 recipes/mayra.recipe create mode 100644 recipes/moldovaazi.recipe create mode 100644 recipes/newsmoldova.recipe diff --git a/recipes/divahair.recipe b/recipes/divahair.recipe new file mode 100644 index 0000000000..978ac19808 --- /dev/null +++ b/recipes/divahair.recipe @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +divahair.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class DivaHair(BasicNewsRecipe): + title = u'Diva Hair' + language = 'ro' + __author__ = u'Silviu Cotoar\u0103' + description = u'Coafuri, frizuri, tunsori ..' + publisher = u'Diva Hair' + category = u'Ziare,Stiri,Coafuri,Femei' + oldest_article = 5 + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + encoding = 'utf-8' + remove_javascript = True + cover_url = 'http://www.divahair.ro/imgs/logo.jpg' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='td', attrs={'class':'spatiuart'}) + , dict(name='div', attrs={'class':'spatiuart'}) + ] + + + remove_tags = [ + dict(name='div', attrs={'class':'categorie'}) + , dict(name='div', attrs={'class':'gri gri2 detaliiart'}) + , dict(name='div', attrs={'class':'articol_box_bottom'}) + ] + + remove_tags_after = [ + dict(name='div', attrs={'class':'articol_box_bottom'}) + ] + + feeds = [ (u'\u0218tiri', u'http://www.divahair.ro/feed') ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/recipes/icons/divahair.png b/recipes/icons/divahair.png new file mode 100644 index 0000000000000000000000000000000000000000..4cb2964687b0acf7fc5d93332ca5963bb1d96a63 GIT binary patch literal 675 zcmV;U0$lxxP)jf%BN7SpBJEufALZZ58cfd7ZMb2yJMDZK6 zM5*`@t`{VWwuMAuqDiB<_wCS=ru5g%!$ZHyVdT;~+ zXl^&cZbt~gLy@wn3HviY&dyF;!5|I6Aa=hWt-1M87(m**J{%n#I9~;E1w+`|+R$2B z5Kh;lFo1Lf0yw+7afL$I12528Tak^82>Cb+Af4SWafL!S+S{?YpCX?$A*B2-7(lv$ zA-?_m#mCuMrjto(C={@`xQJmG7={5rxm;#xX$hqiySuyW?Ckv8psR~7$s}*1QLd_0 z#>dA2n3|d*m&-9fKaWz1csx$ES_NQiY>aq3{#Ri4`LJv@qW%4hj*g=1I{ADauh&aB z97fl5EXyL5N-;P%$nx?s(P)%Hp@32fSp+iHQjY1_rn)7O5N`BQx17kw^r9xw$#|`uaFLJgm=z5X{WXFfuYiI-N$7lCRU# zoSmHT)fGxxTU4`IsMYEn+e4tby`_43N@;VG;_3?Jg9EHm=}z~KKv@>GT#oB>n&RpT zSKHg%oS&mi^WNX63zSk=l?uOgozmJG#q~8Rx{j(;>esk0e*pJl2G9Ddg zkKy*DcOS*DZoy&@d|KO>tUb-!fe4*7xRmEB>!Q?lw*0vS5Osp0EVNOb;cSQV}sH!*s zwp1o^G}FA&9%YOnc69{TOH+N61Hk#cgNb(RDQ=lKhOTQP)E~b@t;JE*=H%BlQL+DJ z5*a>>E^i_~SBMY_C`i75f2fB~Q!|8Ni0ADi8tvpqex4IU$C;V_!az1dskFxM>66eI z!7i0?Xrche5C#ph*?!XL6xC{tLZL_~!rM3RNvBg(DjO76iXaxOLc|ckbwDgm-+q$O z4&uo;$y5SVSzIhIIGAO9y-fc=hFYygDwV`8t)qn0@4cNFfj(aLY z?BW?+fa>a+8<_IPn1; oXT0iAyAf3;VL#XQd-Z>|sWM#kdXX$=fEF@%y85}Sb4q9e0J&XVnE(I) literal 0 HcmV?d00001 diff --git a/recipes/icons/newsmoldova.png b/recipes/icons/newsmoldova.png new file mode 100644 index 0000000000000000000000000000000000000000..a4faaa46357560501f42c45788b34e86896ff51e GIT binary patch literal 837 zcmV-L1G@Z)P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!~g&e!~vBn4jTXf0@_JLK~y+Tm66#` zQ$ZBQ@8998&n9Z3PyPoc#>A)*g#=A7CPp6=7YY~!0!ly-1VSmB>_MSn4TYAarIc1$ z*3#bY_jWI|R4k>He$EVmg!tr5CU?#^=lsrhW;p!E_vB*_#x(0^D!A@R9S#mt7+>6g z(;I=LNzUS-Sre?DAo>*syr><-^s*V9<4ZV5rda%?U&(T~%Qg%AoFhjFj41=sQ_ z@pW((&#L-ak)%^oB)0$lE^FXVvf!L zXySu+tqgFdK#Ki?V+YjYsCpeE3v0}QZqtc#*I(nt%Z7gey~Gx~gCZF&W|y*rtYR51 zWR+mc>0Fki$N4cria5}c)ebh zM>VU}icly7PNht_((*TQJi`!Qe&fe$CZ6;3!&yC4!V0GZAYg=H6YYLNlj zmi_2m38IFa?~;!78ZTOjCL^n6UWCHB3$A)zU|lFgz8;1_h@(R3IR#9ZcTl2mu`yb7 zK1i2*XeZ4YwLWx^KSzVAS%KNARC&;0piX>P^2E`!=*8zbfpw*Ez1quM(J^v*%qX5D zm)K1#PV-Up7=vga1#~28Eq~OGS7g Date: Wed, 11 May 2011 16:34:59 -0600 Subject: [PATCH 17/35] Fix #780728 (RTF to mobi conversion fails) --- src/calibre/ebooks/rtf/input.py | 2 +- src/calibre/ebooks/rtf2xml/process_tokens.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/rtf/input.py b/src/calibre/ebooks/rtf/input.py index 23c16f473d..f08aa76605 100644 --- a/src/calibre/ebooks/rtf/input.py +++ b/src/calibre/ebooks/rtf/input.py @@ -86,7 +86,7 @@ class RTFInput(InputFormatPlugin): run_lev = 4 self.log('Running RTFParser in debug mode') except: - pass + self.log.warn('Impossible to run RTFParser in debug mode') parser = ParseRtf( in_file = stream, out_file = ofile, diff --git a/src/calibre/ebooks/rtf2xml/process_tokens.py b/src/calibre/ebooks/rtf2xml/process_tokens.py index 65162d0d37..11aab48588 100755 --- a/src/calibre/ebooks/rtf2xml/process_tokens.py +++ b/src/calibre/ebooks/rtf2xml/process_tokens.py @@ -197,8 +197,8 @@ class ProcessTokens: # character info => ci 'b' : ('ci', 'bold______', self.bool_st_func), 'blue' : ('ci', 'blue______', self.color_func), - 'caps' : ('ci', 'caps______', self.bool_st_func), - 'cf' : ('ci', 'font-color', self.default_func), + 'caps' : ('ci', 'caps______', self.bool_st_func), + 'cf' : ('ci', 'font-color', self.colorz_func), 'chftn' : ('ci', 'footnot-mk', self.bool_st_func), 'dn' : ('ci', 'font-down_', self.divide_by_2), 'embo' : ('ci', 'emboss____', self.bool_st_func), @@ -624,6 +624,11 @@ class ProcessTokens: num = 'true' return 'cw<%s<%s Date: Wed, 11 May 2011 17:24:20 -0600 Subject: [PATCH 18/35] Fix #780804 (not found gdbm on Windows in 0.8.0 and Japanese) --- setup/resources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/resources.py b/setup/resources.py index 84ff136371..1501e28017 100644 --- a/setup/resources.py +++ b/setup/resources.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' import os, cPickle, re, anydbm, shutil, marshal, zipfile, glob from zlib import compress -from setup import Command, basenames, __appname__ +from setup import Command, basenames, __appname__, iswindows def get_opts_from_parser(parser): def do_opt(opt): @@ -128,7 +128,7 @@ class Resources(Command): if not os.path.exists(base): os.makedirs(base) - if self.newer(dest, src): + if self.newer(dest, src) or iswindows: self.info('\tGenerating Kanwadict') for line in open(src, "r"): From 1a38be575d68278caff8647446255f92e3eabf79 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 17:35:27 -0600 Subject: [PATCH 19/35] ... --- setup/resources.py | 4 ++-- src/odf/odf2xhtml.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/resources.py b/setup/resources.py index 1501e28017..afa8829988 100644 --- a/setup/resources.py +++ b/setup/resources.py @@ -139,7 +139,7 @@ class Resources(Command): dest = self.j(self.RESOURCES, 'localization', 'pykakasi','itaijidict2.pickle') - if self.newer(dest, src): + if self.newer(dest, src) or iswindows: self.info('\tGenerating Itaijidict') self.mkitaiji(src, dest) @@ -147,7 +147,7 @@ class Resources(Command): dest = self.j(self.RESOURCES, 'localization', 'pykakasi','kanadict2.pickle') - if self.newer(dest, src): + if self.newer(dest, src) or iswindows: self.info('\tGenerating kanadict') self.mkkanadict(src, dest) diff --git a/src/odf/odf2xhtml.py b/src/odf/odf2xhtml.py index 0ae89b1663..b1dbebb775 100644 --- a/src/odf/odf2xhtml.py +++ b/src/odf/odf2xhtml.py @@ -1415,7 +1415,7 @@ ol, ul { padding-left: 2em; } self.writedata() c = attrs.get( (TEXTNS,'style-name'), None) htmlattrs = {} - # Changed by Kovid to handle inline apecial styles defined on tags. + # Changed by Kovid to handle inline special styles defined on tags. # Apparently LibreOffice does this. special = 'span' if c: @@ -1433,7 +1433,7 @@ ol, ul { padding-left: 2em; } """ End the """ self.writedata() c = attrs.get( (TEXTNS,'style-name'), None) - # Changed by Kovid to handle inline apecial styles defined on tags. + # Changed by Kovid to handle inline special styles defined on tags. # Apparently LibreOffice does this. special = 'span' if c: From 48166bd87764c26ccb6401482048679d558a3d93 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 18:08:00 -0600 Subject: [PATCH 20/35] Ensure kakasi databases are always rebuilt on windows --- setup/commands.py | 5 +- setup/installer/windows/__init__.py | 1 + setup/installer/windows/wix-template.xml | 3 - setup/resources.py | 209 ++++++++++++----------- 4 files changed, 114 insertions(+), 104 deletions(-) diff --git a/setup/commands.py b/setup/commands.py index 7e22ff14f3..febc684c08 100644 --- a/setup/commands.py +++ b/setup/commands.py @@ -11,7 +11,7 @@ __all__ = [ 'build', 'build_pdf2xml', 'server', 'gui', 'develop', 'install', - 'resources', + 'kakasi', 'resources', 'check', 'sdist', 'manual', 'tag_release', @@ -49,8 +49,9 @@ gui = GUI() from setup.check import Check check = Check() -from setup.resources import Resources +from setup.resources import Resources, Kakasi resources = Resources() +kakasi = Kakasi() from setup.publish import Manual, TagRelease, Stage1, Stage2, \ Stage3, Stage4, Publish diff --git a/setup/installer/windows/__init__.py b/setup/installer/windows/__init__.py index b51eccc832..59042ac56c 100644 --- a/setup/installer/windows/__init__.py +++ b/setup/installer/windows/__init__.py @@ -32,6 +32,7 @@ class Win32(VMInstaller): FREEZE_TEMPLATE = 'python -OO setup.py {freeze_command} --no-ice' INSTALLER_EXT = 'msi' SHUTDOWN_CMD = ['shutdown.exe', '-s', '-f', '-t', '0'] + BUILD_BUILD = ['python setup.py kakasi',] + VMInstaller.BUILD_BUILD def download_installer(self): installer = self.installer() diff --git a/setup/installer/windows/wix-template.xml b/setup/installer/windows/wix-template.xml index b5d2f4b292..5de08e155f 100644 --- a/setup/installer/windows/wix-template.xml +++ b/setup/installer/windows/wix-template.xml @@ -11,9 +11,6 @@ SummaryCodepage='1252' /> - - Date: Wed, 11 May 2011 20:10:00 -0400 Subject: [PATCH 21/35] Store: Search selected book's title and author. Put stores into sub menu. --- src/calibre/gui2/actions/store.py | 41 +++++++++++++++++++++++-- src/calibre/gui2/library/models.py | 3 ++ src/calibre/gui2/store/search/search.py | 14 +++++++-- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index 1989250bc8..db505cf590 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -26,16 +26,51 @@ class StoreAction(InterfaceAction): def load_menu(self): self.store_menu.clear() self.store_menu.addAction(_('Search'), self.search) + self.store_menu.addAction(_('Search Author'), self.search_author) + self.store_menu.addAction(_('Search Title'), self.search_title) self.store_menu.addSeparator() + self.store_list_menu = self.store_menu.addMenu(_('Stores')) for n, p in sorted(self.gui.istores.items(), key=lambda x: x[0].lower()): - self.store_menu.addAction(n, partial(self.open_store, p)) + self.store_list_menu.addAction(n, partial(self.open_store, p)) self.qaction.setMenu(self.store_menu) - def search(self): + def search(self, query=''): self.show_disclaimer() from calibre.gui2.store.search.search import SearchDialog - sd = SearchDialog(self.gui.istores, self.gui) + sd = SearchDialog(self.gui.istores, self.gui, query) sd.exec_() + + def search_author(self): + rows = self.gui.current_view().selectionModel().selectedRows() + if not rows or len(rows) == 0: + return + row = rows[0].row() + + author = '' + if self.gui.current_view() is self.gui.library_view: + author = self.gui.library_view.model().authors(row) + else: + mi = self.gui.current_view().model().get_book_display_info(row) + author = ' & '.join(mi.authors) + + query = 'author:"%s"' % author + self.search(query) + + def search_title(self): + rows = self.gui.current_view().selectionModel().selectedRows() + if not rows or len(rows) == 0: + return + row = rows[0].row() + + title = '' + if self.gui.current_view() is self.gui.library_view: + title = self.gui.library_view.model().title(row) + else: + mi = self.gui.current_view().model().get_book_display_info(row) + title = mi.title + + query = 'title:"%s"' % title + self.search(query) def open_store(self, store_plugin): self.show_disclaimer() diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index dd5082c27f..fc1117167d 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -506,6 +506,9 @@ class BooksModel(QAbstractTableModel): # {{{ def id(self, row): return self.db.id(getattr(row, 'row', lambda:row)()) + def authors(self, row_number): + return self.db.authors(row_number) + def title(self, row_number): return self.db.title(row_number) diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index 62e4e97f11..f7e8c88cd9 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -23,8 +23,8 @@ TIMEOUT = 75 # seconds class SearchDialog(QDialog, Ui_Dialog): - def __init__(self, istores, *args): - QDialog.__init__(self, *args) + def __init__(self, istores, parent=None, query=''): + QDialog.__init__(self, parent) self.setupUi(self) self.config = JSONConfig('store/search') @@ -54,6 +54,9 @@ class SearchDialog(QDialog, Ui_Dialog): setattr(self, 'store_check_' + x, cbox) stores_group_layout.addStretch() + # Set the search query + self.search_edit.setText(query) + # Create and add the progress indicator self.pi = ProgressIndicator(self, 24) self.top_layout.addWidget(self.pi) @@ -93,7 +96,7 @@ class SearchDialog(QDialog, Ui_Dialog): # Store / Formats self.results_view.setColumnWidth(4, int(total*.25)) - def do_search(self, checked=False): + def do_search(self): # Stop all running threads. self.checker.stop() self.search_pool.abort() @@ -252,4 +255,9 @@ class SearchDialog(QDialog, Ui_Dialog): self.search_pool.abort() self.cache_pool.abort() self.save_state() + + def exec_(self): + if unicode(self.search_edit.text()).strip(): + self.do_search() + return QDialog.exec_(self) From abf31ec8262905669bb4b3e0323c3430be00c813 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 18:20:44 -0600 Subject: [PATCH 22/35] Use Qt 4.7.3 for windows build --- setup/installer/windows/freeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index f666427598..7fb60968e7 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -14,7 +14,7 @@ from setup.build_environment import msvc, MT, RC from setup.installer.windows.wix import WixMixIn OPENSSL_DIR = r'Q:\openssl' -QT_DIR = 'Q:\\Qt\\4.7.2' +QT_DIR = 'Q:\\Qt\\4.7.3' QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns'] LIBUSB_DIR = 'C:\\libusb' LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll' From 6259914a9c402f5a8f25ebdc40737a55c30c6375 Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 11 May 2011 20:41:07 -0400 Subject: [PATCH 23/35] Store: Search by author and title menu option. --- src/calibre/gui2/actions/store.py | 53 ++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index db505cf590..c629b2c52a 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -10,6 +10,7 @@ from functools import partial from PyQt4.Qt import QMenu +from calibre.gui2 import error_dialog from calibre.gui2.actions import InterfaceAction from calibre.gui2.dialogs.confirm_delete import confirm @@ -25,9 +26,10 @@ class StoreAction(InterfaceAction): def load_menu(self): self.store_menu.clear() - self.store_menu.addAction(_('Search'), self.search) - self.store_menu.addAction(_('Search Author'), self.search_author) - self.store_menu.addAction(_('Search Title'), self.search_title) + self.store_menu.addAction(_('Search for ebooks'), self.search) + self.store_menu.addAction(_('Search by this author'), self.search_author) + self.store_menu.addAction(_('Search by this title'), self.search_title) + self.store_menu.addAction(_('Search by this author and title'), self.search_author_title) self.store_menu.addSeparator() self.store_list_menu = self.store_menu.addMenu(_('Stores')) for n, p in sorted(self.gui.istores.items(), key=lambda x: x[0].lower()): @@ -40,12 +42,13 @@ class StoreAction(InterfaceAction): sd = SearchDialog(self.gui.istores, self.gui, query) sd.exec_() - def search_author(self): + def _get_selected_row(self): rows = self.gui.current_view().selectionModel().selectedRows() if not rows or len(rows) == 0: - return - row = rows[0].row() - + return None + return rows[0].row() + + def _get_author(self, row): author = '' if self.gui.current_view() is self.gui.library_view: author = self.gui.library_view.model().authors(row) @@ -53,23 +56,43 @@ class StoreAction(InterfaceAction): mi = self.gui.current_view().model().get_book_display_info(row) author = ' & '.join(mi.authors) - query = 'author:"%s"' % author + return author + + def search_author(self): + row = self._get_selected_row() + if row == None: + error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True) + return + + query = 'author:"%s"' % self._get_author(row) self.search(query) - def search_title(self): - rows = self.gui.current_view().selectionModel().selectedRows() - if not rows or len(rows) == 0: - return - row = rows[0].row() - + def _get_title(self, row): title = '' if self.gui.current_view() is self.gui.library_view: title = self.gui.library_view.model().title(row) else: mi = self.gui.current_view().model().get_book_display_info(row) title = mi.title + + return title + + def search_title(self): + row = self._get_selected_row() + if row == None: + error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True) + return + + query = 'title:"%s"' % self._get_title(row) + self.search(query) - query = 'title:"%s"' % title + def search_author_title(self): + row = self._get_selected_row() + if row == None: + error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True) + return + + query = 'author:"%s" title:"%s"' % (self._get_author(row), self._get_title(row)) self.search(query) def open_store(self, store_plugin): From 0fa971937519ab39801831d2135f69c7128a070f Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 11 May 2011 21:02:15 -0400 Subject: [PATCH 24/35] Store: Fix B&N plugin. Fix cleaning of query. --- src/calibre/gui2/store/bn_plugin.py | 4 ++-- src/calibre/gui2/store/search/download_thread.py | 13 +++++++++---- src/calibre/gui2/store/search/search.py | 9 ++++++--- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index f26a60c89d..62826e825d 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' import random import re -import urllib2 +import urllib from contextlib import closing from lxml import html @@ -48,7 +48,7 @@ class BNStore(BasicStoreConfig, StorePlugin): def search(self, query, max_results=10, timeout=60): url = 'http://productsearch.barnesandnoble.com/search/results.aspx?STORE=EBOOK&SZE=%s&WRD=' % max_results - url += urllib2.quote(query) + url += urllib.quote_plus(query) br = browser() diff --git a/src/calibre/gui2/store/search/download_thread.py b/src/calibre/gui2/store/search/download_thread.py index 6dd59cc5a7..97279d7773 100644 --- a/src/calibre/gui2/store/search/download_thread.py +++ b/src/calibre/gui2/store/search/download_thread.py @@ -12,6 +12,7 @@ from threading import Thread from Queue import Queue from calibre import browser +from calibre.constants import DEBUG from calibre.utils.magick.draw import thumbnail class GenericDownloadThreadPool(object): @@ -119,7 +120,8 @@ class SearchThread(Thread): self.results.put((res, store_plugin)) self.tasks.task_done() except: - traceback.print_exc() + if DEBUG: + traceback.print_exc() class CoverThreadPool(GenericDownloadThreadPool): @@ -157,7 +159,8 @@ class CoverThread(Thread): callback() self.tasks.task_done() except: - continue + if DEBUG: + traceback.print_exc() class DetailsThreadPool(GenericDownloadThreadPool): @@ -191,7 +194,8 @@ class DetailsThread(Thread): callback(result) self.tasks.task_done() except: - continue + if DEBUG: + traceback.print_exc() class CacheUpdateThreadPool(GenericDownloadThreadPool): @@ -221,4 +225,5 @@ class CacheUpdateThread(Thread): store_plugin, timeout = self.tasks.get() store_plugin.update_cache(timeout=timeout, suppress_progress=True) except: - traceback.print_exc() + if DEBUG: + traceback.print_exc() diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index f7e8c88cd9..eea1a692de 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -139,14 +139,17 @@ class SearchDialog(QDialog, Ui_Dialog): query = query.replace('>', '') query = query.replace('<', '') # Remove the prefix. - for loc in ( 'all', 'author', 'authors', 'title'): - query = re.sub(r'%s:"?(?P[^\s"]+)"?' % loc, '\g', query) + for loc in ('all', 'author', 'authors', 'title'): + query = re.sub(r'%s:"(?P[^\s"]+)"' % loc, '\g', query) + query = query.replace('%s:' % loc, '') # Remove the prefix and search text. for loc in ('cover', 'drm', 'format', 'formats', 'price', 'store'): query = re.sub(r'%s:"[^"]"' % loc, '', query) query = re.sub(r'%s:[^\s]*' % loc, '', query) # Remove logic. - query = re.sub(r'(^|\s)(and|not|or)(\s|$)', ' ', query) + query = re.sub(r'(^|\s)(and|not|or|a|the|is|of)(\s|$)', ' ', query) + # Remove " + query = query.replace('"', '') # Remove excess whitespace. query = re.sub(r'\s{2,}', ' ', query) query = query.strip() From 8c6c5c668fd6d95a6a035d72b7eb0fe4413641bc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 22:28:24 -0600 Subject: [PATCH 25/35] ... --- recipes/arcamax.recipe | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/recipes/arcamax.recipe b/recipes/arcamax.recipe index db4d753cef..d1c1c6766d 100644 --- a/recipes/arcamax.recipe +++ b/recipes/arcamax.recipe @@ -93,7 +93,7 @@ class Arcamax(BasicNewsRecipe): for page in pages: page_soup = self.index_to_soup(url) if page_soup: - title = page_soup.find(name='div', attrs={'class':'comics-header'}).h1.contents[0] + title = self.tag_to_string(page_soup.find(name='div', attrs={'class':'comics-header'}).h1.contents[0]) page_url = url # orig prev_page_url = 'http://www.arcamax.com' + page_soup.find('a', attrs={'class':'prev'}, text='Previous').parent['href'] prev_page_url = 'http://www.arcamax.com' + page_soup.find('span', text='Previous').parent.parent['href'] @@ -127,4 +127,3 @@ class Arcamax(BasicNewsRecipe): p{font-family:Arial,Helvetica,sans-serif;font-size:small;} body{font-family:Helvetica,Arial,sans-serif;font-size:small;} ''' - From da90da98e3dcf73c855ceac5edc9092608dafed3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 09:07:29 -0600 Subject: [PATCH 26/35] Handle people with legacy databases that have custom columns with non ascii search names. Fixes #781490 (KeyError happens whenever I click on a book in the main screen (list)) --- src/calibre/ebooks/metadata/book/base.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 7927517b22..ceb6751238 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -112,10 +112,15 @@ class Metadata(object): Be careful with numeric fields since this will return True for zero as well as None. + + Also returns True if the field does not exist. ''' - null_val = NULL_VALUES.get(field, None) - val = getattr(self, field, None) - return not val or val == null_val + try: + null_val = NULL_VALUES.get(field, None) + val = getattr(self, field, None) + return not val or val == null_val + except: + return True def __getattribute__(self, field): _data = object.__getattribute__(self, '_data') From c7c9ade376c0461e4bac6bb3fd531437e6d14955 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 09:40:52 -0600 Subject: [PATCH 27/35] Driver for Dell Streak on windows --- src/calibre/devices/android/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index ca84271778..db473a755e 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -109,7 +109,7 @@ class ANDROID(USBMS): 'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H', 'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD', '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2', - 'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB'] + 'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD'] From 16c92f3d23728b76c1cf4240cfe9aa02f60f9918 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 10:10:14 -0600 Subject: [PATCH 28/35] Updated Ming Pao and Various Taiwanese news sources by Eddie Lau --- recipes/china_times.recipe | 42 +++ recipes/liberty_times.recipe | 44 +++ recipes/ming_pao.recipe | 656 +++++++++++++++++++---------------- recipes/united_daily.recipe | 67 ++++ 4 files changed, 505 insertions(+), 304 deletions(-) create mode 100644 recipes/china_times.recipe create mode 100644 recipes/liberty_times.recipe create mode 100644 recipes/united_daily.recipe diff --git a/recipes/china_times.recipe b/recipes/china_times.recipe new file mode 100644 index 0000000000..8c1493d71f --- /dev/null +++ b/recipes/china_times.recipe @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +__license__ = 'GPL v3' +# dug from http://www.mobileread.com/forums/showthread.php?p=1012294 + +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1277443634(BasicNewsRecipe): + title = u'中時電子報' + oldest_article = 1 + max_articles_per_feed = 100 + + feeds = [(u'焦點', u'http://rss.chinatimes.com/rss/focus-u.rss'), + (u'政治', u'http://rss.chinatimes.com/rss/Politic-u.rss'), + (u'社會', u'http://rss.chinatimes.com/rss/social-u.rss'), + (u'國際', u'http://rss.chinatimes.com/rss/international-u.rss'), + (u'兩岸', u'http://rss.chinatimes.com/rss/mainland-u.rss'), + (u'地方', u'http://rss.chinatimes.com/rss/local-u.rss'), + (u'言論', u'http://rss.chinatimes.com/rss/comment-u.rss'), + (u'科技', u'http://rss.chinatimes.com/rss/technology-u.rss'), + (u'運動', u'http://rss.chinatimes.com/rss/sport-u.rss'), + (u'藝文', u'http://rss.chinatimes.com/rss/philology-u.rss'), + #(u'旺報', u'http://rss.chinatimes.com/rss/want-u.rss'), + #(u'財經', u'http://rss.chinatimes.com/rss/finance-u.rss'), # broken links + #(u'股市', u'http://rss.chinatimes.com/rss/stock-u.rss') # broken links + ] + + __author__ = 'einstuerzende, updated by Eddie Lau' + __version__ = '1.0' + language = 'zh' + publisher = 'China Times Group' + description = 'China Times (Taiwan)' + category = 'News, Chinese, Taiwan' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + encoding = 'big5' + conversion_options = {'linearize_tables':True} + masthead_url = 'http://www.fcuaa.org/gif/chinatimeslogo.gif' + cover_url = 'http://www.fcuaa.org/gif/chinatimeslogo.gif' + keep_only_tags = [dict(name='div', attrs={'class':['articlebox','articlebox clearfix']})] + remove_tags = [dict(name='div', attrs={'class':['focus-news']})] + diff --git a/recipes/liberty_times.recipe b/recipes/liberty_times.recipe new file mode 100644 index 0000000000..c3a9d106da --- /dev/null +++ b/recipes/liberty_times.recipe @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +__license__ = 'GPL v3' +# dug from http://www.mobileread.com/forums/showthread.php?p=1012294 + + +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1277443634(BasicNewsRecipe): + title = u'自由電子報' + oldest_article = 1 + max_articles_per_feed = 100 + + feeds = [(u'焦點新聞', u'http://www.libertytimes.com.tw/rss/fo.xml'), + (u'政治新聞', u'http://www.libertytimes.com.tw/rss/p.xml'), + (u'生活新聞', u'http://www.libertytimes.com.tw/rss/life.xml'), + (u'國際新聞', u'http://www.libertytimes.com.tw/rss/int.xml'), + (u'自由廣場', u'http://www.libertytimes.com.tw/rss/o.xml'), + (u'社會新聞', u'http://www.libertytimes.com.tw/rss/so.xml'), + (u'體育新聞', u'http://www.libertytimes.com.tw/rss/sp.xml'), + (u'財經焦點', u'http://www.libertytimes.com.tw/rss/e.xml'), + (u'證券理財', u'http://www.libertytimes.com.tw/rss/stock.xml'), + (u'影視焦點', u'http://www.libertytimes.com.tw/rss/show.xml'), + (u'北部新聞', u'http://www.libertytimes.com.tw/rss/north.xml'), + (u'中部新聞', u'http://www.libertytimes.com.tw/rss/center.xml'), + (u'南部新聞', u'http://www.libertytimes.com.tw/rss/south.xml'), + (u'大台北新聞', u'http://www.libertytimes.com.tw/rss/taipei.xml'), + (u'藝術文化', u'http://www.libertytimes.com.tw/rss/art.xml'), + ] + extra_css = '''span[class='insubject1'][id='newtitle'] {font-size:200%; font-weight:bold;}''' + __author__ = 'einstuerzende, updated by Eddie Lau' + __version__ = '1.1' + language = 'zh' + publisher = 'Liberty Times Group' + description = 'Liberty Times (Taiwan)' + category = 'News, Chinese, Taiwan' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + encoding = 'big5' + conversion_options = {'linearize_tables':True} + masthead_url = 'http://www.libertytimes.com.tw/2008/images/img_auto/005/logo_new.gif' + cover_url = 'http://www.libertytimes.com.tw/2008/images/img_auto/005/logo_new.gif' + keep_only_tags = [dict(name='td', attrs={'id':['newsContent']})] + diff --git a/recipes/ming_pao.recipe b/recipes/ming_pao.recipe index 4a405a59dd..08ee20cb15 100644 --- a/recipes/ming_pao.recipe +++ b/recipes/ming_pao.recipe @@ -1,15 +1,18 @@ +# -*- coding: utf-8 -*- __license__ = 'GPL v3' __copyright__ = '2010-2011, Eddie Lau' -# Users of Kindle 3 (with limited system-level CJK support) +# Users of Kindle 3 with limited system-level CJK support # please replace the following "True" with "False". __MakePeriodical__ = True -# Turn it to True if your device supports display of CJK titles +# Turn below to true if your device supports display of CJK titles __UseChineseTitle__ = False - +# Trun below to true if you wish to use life.mingpao.com as the main article source +__UseLife__ = True ''' Change Log: +2011/05/12: switch the main parse source to life.mingpao.com, which has more photos on the article pages 2011/03/06: add new articles for finance section, also a new section "Columns" 2011/02/28: rearrange the sections [Disabled until Kindle has better CJK support and can remember last (section,article) read in Sections & Articles @@ -32,41 +35,43 @@ import os, datetime, re from calibre.web.feeds.recipes import BasicNewsRecipe from contextlib import nested + from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.ebooks.metadata.toc import TOC from calibre.ebooks.metadata import MetaInformation class MPHKRecipe(BasicNewsRecipe): - title = 'Ming Pao - Hong Kong' - oldest_article = 1 - max_articles_per_feed = 100 - __author__ = 'Eddie Lau' - description = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)' - publisher = 'MingPao' - category = 'Chinese, News, Hong Kong' - remove_javascript = True - use_embedded_content = False - no_stylesheets = True - language = 'zh' - encoding = 'Big5-HKSCS' - recursions = 0 - conversion_options = {'linearize_tables':True} - timefmt = '' - extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}' - masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif' - keep_only_tags = [dict(name='h1'), + title = 'Ming Pao - Hong Kong' + oldest_article = 1 + max_articles_per_feed = 100 + __author__ = 'Eddie Lau' + description = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)' + publisher = 'MingPao' + category = 'Chinese, News, Hong Kong' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + language = 'zh' + encoding = 'Big5-HKSCS' + recursions = 0 + conversion_options = {'linearize_tables':True} + timefmt = '' + extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}' + masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif' + keep_only_tags = [dict(name='h1'), dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page title dict(name='font', attrs={'color':['AA0000']}), # for column articles title dict(attrs={'id':['newscontent']}), # entertainment and column page content dict(attrs={'id':['newscontent01','newscontent02']}), - dict(attrs={'class':['photo']}) + dict(attrs={'class':['photo']}), + dict(name='img', attrs={'width':['180'], 'alt':['按圖放大']}) # images for source from life.mingpao.com ] - remove_tags = [dict(name='style'), - dict(attrs={'id':['newscontent135']}), # for the finance page - dict(name='table')] # for content fetched from life.mingpao.com - remove_attributes = ['width'] - preprocess_regexps = [ + remove_tags = [dict(name='style'), + dict(attrs={'id':['newscontent135']}), # for the finance page from mpfinance.com + dict(name='table')] # for content fetched from life.mingpao.com + remove_attributes = ['width'] + preprocess_regexps = [ (re.compile(r'
', re.DOTALL|re.IGNORECASE), lambda match: '

'), (re.compile(r'

', re.DOTALL|re.IGNORECASE), @@ -80,10 +85,10 @@ class MPHKRecipe(BasicNewsRecipe): lambda match: "") ] - def image_url_processor(cls, baseurl, url): - # trick: break the url at the first occurance of digit, add an additional - # '_' at the front - # not working, may need to move this to preprocess_html() method + def image_url_processor(cls, baseurl, url): + # trick: break the url at the first occurance of digit, add an additional + # '_' at the front + # not working, may need to move this to preprocess_html() method # minIdx = 10000 # i0 = url.find('0') # if i0 >= 0 and i0 < minIdx: @@ -115,314 +120,357 @@ class MPHKRecipe(BasicNewsRecipe): # i9 = url.find('9') # if i9 >= 0 and i9 < minIdx: # minIdx = i9 - return url + return url - def get_dtlocal(self): - dt_utc = datetime.datetime.utcnow() - # convert UTC to local hk time - at around HKT 6.00am, all news are available - dt_local = dt_utc - datetime.timedelta(-2.0/24) - return dt_local + def get_dtlocal(self): + dt_utc = datetime.datetime.utcnow() + # convert UTC to local hk time - at around HKT 6.00am, all news are available + dt_local = dt_utc - datetime.timedelta(-2.0/24) + return dt_local - def get_fetchdate(self): - return self.get_dtlocal().strftime("%Y%m%d") + def get_fetchdate(self): + return self.get_dtlocal().strftime("%Y%m%d") - def get_fetchformatteddate(self): - return self.get_dtlocal().strftime("%Y-%m-%d") + def get_fetchformatteddate(self): + return self.get_dtlocal().strftime("%Y-%m-%d") - def get_fetchday(self): - # convert UTC to local hk time - at around HKT 6.00am, all news are available - return self.get_dtlocal().strftime("%d") + def get_fetchday(self): + # dt_utc = datetime.datetime.utcnow() + # convert UTC to local hk time - at around HKT 6.00am, all news are available + # dt_local = dt_utc - datetime.timedelta(-2.0/24) + return self.get_dtlocal().strftime("%d") - def get_cover_url(self): - cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg' - br = BasicNewsRecipe.get_browser() - try: - br.open(cover) - except: - cover = None - return cover + def get_cover_url(self): + cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + cover = None + return cover - def parse_index(self): - feeds = [] - dateStr = self.get_fetchdate() + def parse_index(self): + feeds = [] + dateStr = self.get_fetchdate() - for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), - (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), - (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]: - articles = self.parse_section(url) - if articles: - feeds.append((title, articles)) + if __UseLife__: + for title, url, keystr in [(u'\u8981\u805e Headline', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalga', 'nal'), + (u'\u6e2f\u805e Local', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgb', 'nal'), + (u'\u6559\u80b2 Education', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgf', 'nal'), + (u'\u793e\u8a55/\u7b46\u9663 Editorial', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr', 'nal'), + (u'\u8ad6\u58c7 Forum', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalfa', 'nal'), + (u'\u4e2d\u570b China', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalca', 'nal'), + (u'\u570b\u969b World', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalta', 'nal'), + (u'\u7d93\u6fdf Finance', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea', 'nal'), + (u'\u9ad4\u80b2 Sport', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalsp', 'nal'), + (u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal'), + (u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl')]: + articles = self.parse_section2(url, keystr) + if articles: + feeds.append((title, articles)) - # special- editorial - ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr') - if ed_articles: - feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles)) + for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'), + (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) + else: + for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), + (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), + (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) - for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), - (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'), - (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]: - articles = self.parse_section(url) - if articles: - feeds.append((title, articles)) + # special- editorial + ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr') + if ed_articles: + feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles)) - # special - finance - #fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm') - fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea') - if fin_articles: - feeds.append((u'\u7d93\u6fdf Finance', fin_articles)) + for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), + (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'), + (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) - for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), - (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]: - articles = self.parse_section(url) - if articles: - feeds.append((title, articles)) + # special - finance + #fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm') + fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea') + if fin_articles: + feeds.append((u'\u7d93\u6fdf Finance', fin_articles)) - # special - entertainment - ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm') - if ent_articles: - feeds.append((u'\u5f71\u8996 Film/TV', ent_articles)) + for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), + (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) - for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'), - (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]: - articles = self.parse_section(url) - if articles: - feeds.append((title, articles)) + # special - entertainment + ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm') + if ent_articles: + feeds.append((u'\u5f71\u8996 Film/TV', ent_articles)) + + for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'), + (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) - # special- columns - col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn') - if col_articles: - feeds.append((u'\u5c08\u6b04 Columns', col_articles)) + # special- columns + col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn') + if col_articles: + feeds.append((u'\u5c08\u6b04 Columns', col_articles)) - return feeds + return feeds - def parse_section(self, url): - dateStr = self.get_fetchdate() - soup = self.index_to_soup(url) - divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']}) - current_articles = [] - included_urls = [] - divs.reverse() - for i in divs: - a = i.find('a', href = True) - title = self.tag_to_string(a) - url = a.get('href', False) - url = 'http://news.mingpao.com/' + dateStr + '/' +url - if url not in included_urls and url.rfind('Redirect') == -1: - current_articles.append({'title': title, 'url': url, 'description':'', 'date':''}) - included_urls.append(url) - current_articles.reverse() - return current_articles + # parse from news.mingpao.com + def parse_section(self, url): + dateStr = self.get_fetchdate() + soup = self.index_to_soup(url) + divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']}) + current_articles = [] + included_urls = [] + divs.reverse() + for i in divs: + a = i.find('a', href = True) + title = self.tag_to_string(a) + url = a.get('href', False) + url = 'http://news.mingpao.com/' + dateStr + '/' +url + if url not in included_urls and url.rfind('Redirect') == -1: + current_articles.append({'title': title, 'url': url, 'description':'', 'date':''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def parse_ed_section(self, url): - self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href=True) - a.reverse() - current_articles = [] - included_urls = [] - for i in a: - title = self.tag_to_string(i) - url = 'http://life.mingpao.com/cfm/' + i.get('href', False) - if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('nal') == -1): - current_articles.append({'title': title, 'url': url, 'description': ''}) - included_urls.append(url) - current_articles.reverse() - return current_articles + # parse from life.mingpao.com + def parse_section2(self, url, keystr): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind(keystr) == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def parse_fin_section(self, url): - self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href= True) - current_articles = [] - included_urls = [] - for i in a: - #url = 'http://www.mpfinance.com/cfm/' + i.get('href', False) - url = 'http://life.mingpao.com/cfm/' + i.get('href', False) - #if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1: - if url not in included_urls and (not url.rfind('txt') == -1) and (not url.rfind('nal') == -1): - title = self.tag_to_string(i) - current_articles.append({'title': title, 'url': url, 'description':''}) - included_urls.append(url) - return current_articles + def parse_ed_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('nal') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def parse_ent_section(self, url): - self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href=True) - a.reverse() - current_articles = [] - included_urls = [] - for i in a: - title = self.tag_to_string(i) - url = 'http://ol.mingpao.com/cfm/' + i.get('href', False) - if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1): - current_articles.append({'title': title, 'url': url, 'description': ''}) - included_urls.append(url) - current_articles.reverse() - return current_articles + def parse_fin_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href= True) + current_articles = [] + included_urls = [] + for i in a: + #url = 'http://www.mpfinance.com/cfm/' + i.get('href', False) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + #if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1: + if url not in included_urls and (not url.rfind('txt') == -1) and (not url.rfind('nal') == -1): + title = self.tag_to_string(i) + current_articles.append({'title': title, 'url': url, 'description':''}) + included_urls.append(url) + return current_articles - def parse_col_section(self, url): - self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href=True) - a.reverse() - current_articles = [] - included_urls = [] - for i in a: - title = self.tag_to_string(i) - url = 'http://life.mingpao.com/cfm/' + i.get('href', False) - if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('ncl') == -1): - current_articles.append({'title': title, 'url': url, 'description': ''}) - included_urls.append(url) - current_articles.reverse() - return current_articles + def parse_ent_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://ol.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - for item in soup.findAll(style=True): - del item['width'] - for item in soup.findAll(stype=True): - del item['absmiddle'] - return soup + def parse_col_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('ncl') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def create_opf(self, feeds, dir=None): - if dir is None: - dir = self.output_dir - if __UseChineseTitle__ == True: - title = u'\u660e\u5831 (\u9999\u6e2f)' - else: - title = self.short_title() - # if not generating a periodical, force date to apply in title - if __MakePeriodical__ == False: - title = title + ' ' + self.get_fetchformatteddate() - if True: - mi = MetaInformation(title, [self.publisher]) - mi.publisher = self.publisher - mi.author_sort = self.publisher - if __MakePeriodical__ == True: - mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() - else: - mi.publication_type = self.publication_type+':'+self.short_title() - #mi.timestamp = nowf() - mi.timestamp = self.get_dtlocal() - mi.comments = self.description - if not isinstance(mi.comments, unicode): - mi.comments = mi.comments.decode('utf-8', 'replace') - #mi.pubdate = nowf() - mi.pubdate = self.get_dtlocal() - opf_path = os.path.join(dir, 'index.opf') - ncx_path = os.path.join(dir, 'index.ncx') - opf = OPFCreator(dir, mi) - # Add mastheadImage entry to section - mp = getattr(self, 'masthead_path', None) - if mp is not None and os.access(mp, os.R_OK): - from calibre.ebooks.metadata.opf2 import Guide - ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu()) - ref.type = 'masthead' - ref.title = 'Masthead Image' - opf.guide.append(ref) + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll(style=True): + del item['width'] + for item in soup.findAll(stype=True): + del item['absmiddle'] + return soup - manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))] - manifest.append(os.path.join(dir, 'index.html')) - manifest.append(os.path.join(dir, 'index.ncx')) + def create_opf(self, feeds, dir=None): + if dir is None: + dir = self.output_dir + if __UseChineseTitle__ == True: + title = u'\u660e\u5831 (\u9999\u6e2f)' + else: + title = self.short_title() + # if not generating a periodical, force date to apply in title + if __MakePeriodical__ == False: + title = title + ' ' + self.get_fetchformatteddate() + if True: + mi = MetaInformation(title, [self.publisher]) + mi.publisher = self.publisher + mi.author_sort = self.publisher + if __MakePeriodical__ == True: + mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() + else: + mi.publication_type = self.publication_type+':'+self.short_title() + #mi.timestamp = nowf() + mi.timestamp = self.get_dtlocal() + mi.comments = self.description + if not isinstance(mi.comments, unicode): + mi.comments = mi.comments.decode('utf-8', 'replace') + #mi.pubdate = nowf() + mi.pubdate = self.get_dtlocal() + opf_path = os.path.join(dir, 'index.opf') + ncx_path = os.path.join(dir, 'index.ncx') + opf = OPFCreator(dir, mi) + # Add mastheadImage entry to section + mp = getattr(self, 'masthead_path', None) + if mp is not None and os.access(mp, os.R_OK): + from calibre.ebooks.metadata.opf2 import Guide + ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu()) + ref.type = 'masthead' + ref.title = 'Masthead Image' + opf.guide.append(ref) - # Get cover - cpath = getattr(self, 'cover_path', None) - if cpath is None: - pf = open(os.path.join(dir, 'cover.jpg'), 'wb') - if self.default_cover(pf): - cpath = pf.name - if cpath is not None and os.access(cpath, os.R_OK): - opf.cover = cpath - manifest.append(cpath) + manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))] + manifest.append(os.path.join(dir, 'index.html')) + manifest.append(os.path.join(dir, 'index.ncx')) - # Get masthead - mpath = getattr(self, 'masthead_path', None) - if mpath is not None and os.access(mpath, os.R_OK): - manifest.append(mpath) + # Get cover + cpath = getattr(self, 'cover_path', None) + if cpath is None: + pf = open(os.path.join(dir, 'cover.jpg'), 'wb') + if self.default_cover(pf): + cpath = pf.name + if cpath is not None and os.access(cpath, os.R_OK): + opf.cover = cpath + manifest.append(cpath) - opf.create_manifest_from_files_in(manifest) - for mani in opf.manifest: - if mani.path.endswith('.ncx'): - mani.id = 'ncx' - if mani.path.endswith('mastheadImage.jpg'): - mani.id = 'masthead-image' - entries = ['index.html'] - toc = TOC(base_path=dir) - self.play_order_counter = 0 - self.play_order_map = {} + # Get masthead + mpath = getattr(self, 'masthead_path', None) + if mpath is not None and os.access(mpath, os.R_OK): + manifest.append(mpath) - def feed_index(num, parent): - f = feeds[num] - for j, a in enumerate(f): - if getattr(a, 'downloaded', False): - adir = 'feed_%d/article_%d/'%(num, j) - auth = a.author - if not auth: - auth = None - desc = a.text_summary - if not desc: - desc = None - else: - desc = self.description_limiter(desc) - entries.append('%sindex.html'%adir) - po = self.play_order_map.get(entries[-1], None) - if po is None: - self.play_order_counter += 1 - po = self.play_order_counter - parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'), + opf.create_manifest_from_files_in(manifest) + for mani in opf.manifest: + if mani.path.endswith('.ncx'): + mani.id = 'ncx' + if mani.path.endswith('mastheadImage.jpg'): + mani.id = 'masthead-image' + entries = ['index.html'] + toc = TOC(base_path=dir) + self.play_order_counter = 0 + self.play_order_map = {} + + def feed_index(num, parent): + f = feeds[num] + for j, a in enumerate(f): + if getattr(a, 'downloaded', False): + adir = 'feed_%d/article_%d/'%(num, j) + auth = a.author + if not auth: + auth = None + desc = a.text_summary + if not desc: + desc = None + else: + desc = self.description_limiter(desc) + entries.append('%sindex.html'%adir) + po = self.play_order_map.get(entries[-1], None) + if po is None: + self.play_order_counter += 1 + po = self.play_order_counter + parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'), play_order=po, author=auth, description=desc) - last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep)) - for sp in a.sub_pages: - prefix = os.path.commonprefix([opf_path, sp]) - relp = sp[len(prefix):] - entries.append(relp.replace(os.sep, '/')) - last = sp + last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep)) + for sp in a.sub_pages: + prefix = os.path.commonprefix([opf_path, sp]) + relp = sp[len(prefix):] + entries.append(relp.replace(os.sep, '/')) + last = sp - if os.path.exists(last): - with open(last, 'rb') as fi: - src = fi.read().decode('utf-8') - soup = BeautifulSoup(src) - body = soup.find('body') - if body is not None: - prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last)))) - templ = self.navbar.generate(True, num, j, len(f), + if os.path.exists(last): + with open(last, 'rb') as fi: + src = fi.read().decode('utf-8') + soup = BeautifulSoup(src) + body = soup.find('body') + if body is not None: + prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last)))) + templ = self.navbar.generate(True, num, j, len(f), not self.has_single_feed, a.orig_url, self.publisher, prefix=prefix, center=self.center_navbar) - elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') - body.insert(len(body.contents), elem) - with open(last, 'wb') as fi: - fi.write(unicode(soup).encode('utf-8')) - if len(feeds) == 0: - raise Exception('All feeds are empty, aborting.') + elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') + body.insert(len(body.contents), elem) + with open(last, 'wb') as fi: + fi.write(unicode(soup).encode('utf-8')) + if len(feeds) == 0: + raise Exception('All feeds are empty, aborting.') - if len(feeds) > 1: - for i, f in enumerate(feeds): - entries.append('feed_%d/index.html'%i) - po = self.play_order_map.get(entries[-1], None) - if po is None: - self.play_order_counter += 1 - po = self.play_order_counter - auth = getattr(f, 'author', None) - if not auth: - auth = None - desc = getattr(f, 'description', None) - if not desc: - desc = None - feed_index(i, toc.add_item('feed_%d/index.html'%i, None, + if len(feeds) > 1: + for i, f in enumerate(feeds): + entries.append('feed_%d/index.html'%i) + po = self.play_order_map.get(entries[-1], None) + if po is None: + self.play_order_counter += 1 + po = self.play_order_counter + auth = getattr(f, 'author', None) + if not auth: + auth = None + desc = getattr(f, 'description', None) + if not desc: + desc = None + feed_index(i, toc.add_item('feed_%d/index.html'%i, None, f.title, play_order=po, description=desc, author=auth)) - else: - entries.append('feed_%d/index.html'%0) - feed_index(0, toc) + else: + entries.append('feed_%d/index.html'%0) + feed_index(0, toc) - for i, p in enumerate(entries): - entries[i] = os.path.join(dir, p.replace('/', os.sep)) - opf.create_spine(entries) - opf.set_toc(toc) + for i, p in enumerate(entries): + entries[i] = os.path.join(dir, p.replace('/', os.sep)) + opf.create_spine(entries) + opf.set_toc(toc) + + with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): + opf.render(opf_file, ncx_file) - with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): - opf.render(opf_file, ncx_file) diff --git a/recipes/united_daily.recipe b/recipes/united_daily.recipe new file mode 100644 index 0000000000..6954a7e725 --- /dev/null +++ b/recipes/united_daily.recipe @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +__license__ = 'GPL v3' + +from calibre.web.feeds.news import BasicNewsRecipe + +class UnitedDaily(BasicNewsRecipe): + title = u'聯合新聞網' + oldest_article = 1 + max_articles_per_feed = 100 + + feeds = [(u'焦點', u'http://udn.com/udnrss/focus.xml'), + (u'政治', u'http://udn.com/udnrss/politics.xml'), + (u'社會', u'http://udn.com/udnrss/social.xml'), + (u'生活', u'http://udn.com/udnrss/life.xml'), + (u'綜合', u'http://udn.com/udnrss/education.xml'), + (u'意見評論', u'http://udn.com/udnrss/opinion.xml'), + (u'大台北', u'http://udn.com/udnrss/local_taipei.xml'), + (u'桃竹苗', u'http://udn.com/udnrss/local_tyhcml.xml'), + (u'中彰投', u'http://udn.com/udnrss/local_tcchnt.xml'), + (u'雲嘉南', u'http://udn.com/udnrss/local_ylcytn.xml'), + (u'高屏離島', u'http://udn.com/udnrss/local_ksptisland.xml'), + (u'基宜花東', u'http://udn.com/udnrss/local_klilhltt.xml'), + (u'台灣百寶鄉', u'http://udn.com/udnrss/local_oddlyenough.xml'), + (u'兩岸要聞', u'http://udn.com/udnrss/mainland.xml'), + (u'國際焦點', u'http://udn.com/udnrss/international.xml'), + (u'台商經貿', u'http://udn.com/udnrss/financechina.xml'), + (u'國際財經', u'http://udn.com/udnrss/financeworld.xml'), + (u'財經焦點', u'http://udn.com/udnrss/financesfocus.xml'), + (u'股市要聞', u'http://udn.com/udnrss/stock.xml'), + (u'股市快訊', u'http://udn.com/udnrss/stklatest.xml'), + (u'稅務法務', u'http://udn.com/udnrss/tax.xml'), + (u'房市情報', u'http://udn.com/udnrss/houses.xml'), + (u'棒球', u'http://udn.com/udnrss/baseball.xml'), + (u'籃球', u'http://udn.com/udnrss/basketball.xml'), + (u'體壇動態', u'http://udn.com/udnrss/sportsfocus.xml'), + (u'熱門星聞', u'http://udn.com/udnrss/starsfocus.xml'), + (u'廣電港陸', u'http://udn.com/udnrss/tv.xml'), + (u'海外星球', u'http://udn.com/udnrss/starswestern.xml'), + (u'日韓星情', u'http://udn.com/udnrss/starsjk.xml'), + (u'電影世界', u'http://udn.com/udnrss/movie.xml'), + (u'流行音樂', u'http://udn.com/udnrss/music.xml'), + (u'觀點專題', u'http://udn.com/udnrss/starssubject.xml'), + (u'食樂指南', u'http://udn.com/udnrss/food.xml'), + (u'折扣好康', u'http://udn.com/udnrss/shopping.xml'), + (u'醫藥新聞', u'http://udn.com/udnrss/health.xml'), + (u'家婦繽紛', u'http://udn.com/udnrss/benfen.xml'), + (u'談星論命', u'http://udn.com/udnrss/astrology.xml'), + (u'文化副刊', u'http://udn.com/udnrss/reading.xml'), + ] + + extra_css = '''div[id='story_title'] {font-size:200%; font-weight:bold;}''' + + __author__ = 'Eddie Lau' + __version__ = '1.0' + language = 'zh' + publisher = 'United Daily News Group' + description = 'United Daily (Taiwan)' + category = 'News, Chinese, Taiwan' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + encoding = 'big5' + conversion_options = {'linearize_tables':True} + masthead_url = 'http://udn.com/NEWS/2004/images/logo_udn.gif' + cover_url = 'http://udn.com/NEWS/2004/images/logo_udn.gif' + keep_only_tags = [dict(name='div', attrs={'id':['story_title','story_author', 'story']})] + remove_tags = [dict(name='div', attrs={'id':['mvouter']})] From a204568f2ee507292716ff42bc3c03bd0fa85755 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 10:38:07 -0600 Subject: [PATCH 29/35] Fix #779994 (Metadata download API priority only partially considered) --- .../ebooks/metadata/sources/identify.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index 31998dfcb2..b084f86294 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -372,6 +372,18 @@ def identify(log, abort, # {{{ longest, lp = -1, '' for plugin, presults in results.iteritems(): presults.sort(key=plugin.identify_results_keygen(**sort_kwargs)) + + # Throw away lower priority results from the same source that have exactly the same + # title and authors as a higher priority result + filter_results = set() + filtered_results = [] + for r in presults: + key = (r.title, tuple(r.authors)) + if key not in filter_results: + filtered_results.append(r) + filter_results.add(key) + presults = filtered_results + plog = logs[plugin].getvalue().strip() log('\n'+'*'*30, plugin.name, '*'*30) log('Request extra headers:', plugin.browser.addheaders) @@ -479,7 +491,7 @@ if __name__ == '__main__': # tests {{{ ( {'title':'Magykal Papers', 'authors':['Sage']}, - [title_test('The Magykal Papers', exact=True)], + [title_test('Septimus Heap: The Magykal Papers', exact=True)], ), @@ -506,12 +518,6 @@ if __name__ == '__main__': # tests {{{ exact=True), authors_test(['Dan Brown'])] ), - ( # No ISBN - {'title':'Justine', 'authors':['Durrel']}, - [title_test('Justine', exact=True), - authors_test(['Lawrence Durrel'])] - ), - ( # A newer book {'identifiers':{'isbn': '9780316044981'}}, [title_test('The Heroes', exact=True), From 0f7272d1b4fafcfa1be45a78d02017a0ab3ca789 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 10:58:54 -0600 Subject: [PATCH 30/35] Fix #781759 (Identifiers lost by 0.8 metadata download) --- src/calibre/gui2/actions/edit_metadata.py | 4 ++++ src/calibre/gui2/metadata/single.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 4ab4950179..ac475cb027 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -478,6 +478,10 @@ class EditMetadataAction(InterfaceAction): try: set_title = not mi.is_null('title') set_authors = not mi.is_null('authors') + idents = db.get_identifiers(i, index_is_id=True) + if mi.identifiers: + idents.update(mi.identifiers) + mi.identifiers = idents db.set_metadata(i, mi, commit=False, set_title=set_title, set_authors=set_authors, notify=False) self.applied_ids.append(i) diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index 5c4e241bba..099831ccba 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -336,7 +336,9 @@ class MetadataSingleDialogBase(ResizableDialog): if not mi.is_null('tags'): self.tags.current_val = mi.tags if not mi.is_null('identifiers'): - self.identifiers.current_val = mi.identifiers + current = self.identifiers.current_val + current.update(mi.identifiers) + self.identifiers.current_val = current if not mi.is_null('pubdate'): self.pubdate.current_val = mi.pubdate if not mi.is_null('series') and mi.series.strip(): From 28dfc420d758cef69b9a4ea048406152b20636bb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 11:18:03 -0600 Subject: [PATCH 31/35] Fix #778208 (Fetch news from Readers Digest) --- recipes/readers_digest.recipe | 150 ++-------------------------------- 1 file changed, 9 insertions(+), 141 deletions(-) diff --git a/recipes/readers_digest.recipe b/recipes/readers_digest.recipe index 3689ca4c53..caf5cf081d 100644 --- a/recipes/readers_digest.recipe +++ b/recipes/readers_digest.recipe @@ -3,7 +3,6 @@ __license__ = 'GPL v3' ''' ''' from calibre.web.feeds.recipes import BasicNewsRecipe -from calibre.web.feeds import Feed class ReadersDigest(BasicNewsRecipe): @@ -38,151 +37,20 @@ class ReadersDigest(BasicNewsRecipe): ''' - remove_tags = [ - dict(name='h4', attrs={'class':'close'}), - dict(name='div', attrs={'class':'fromLine'}), - dict(name='img', attrs={'class':'colorTag'}), - dict(name='div', attrs={'id':'sponsorArticleHeader'}), - dict(name='div', attrs={'class':'horizontalAd'}), - dict(name='div', attrs={'id':'imageCounterLeft'}), - dict(name='div', attrs={'id':'commentsPrint'}) - ] - - feeds = [ - ('New in RD', 'http://feeds.rd.com/ReadersDigest'), - ('Jokes', 'http://feeds.rd.com/ReadersDigestJokes'), - ('Cartoons', 'http://feeds.rd.com/ReadersDigestCartoons'), - ('Blogs','http://feeds.rd.com/ReadersDigestBlogs') + ('Food', 'http://www.rd.com/food/feed'), + ('Health', 'http://www.rd.com/health/feed'), + ('Home', 'http://www.rd.com/home/feed'), + ('Family', 'http://www.rd.com/family/feed'), + ('Money', 'http://www.rd.com/money/feed'), + ('Travel', 'http://www.rd.com/travel/feed'), ] cover_url = 'http://www.rd.com/images/logo-main-rd.gif' - - -#------------------------------------------------------------------------------------------------- - - def print_version(self, url): - - # Get the identity number of the current article and append it to the root print URL - - if url.find('/article') > 0: - ident = url[url.find('/article')+8:url.find('.html?')-4] - url = 'http://www.rd.com/content/printContent.do?contentId=' + ident - - elif url.find('/post') > 0: - - # in this case, have to get the page itself to derive the Print page. - soup = self.index_to_soup(url) - newsoup = soup.find('ul',attrs={'class':'printBlock'}) - url = 'http://www.rd.com' + newsoup('a')[0]['href'] - url = url[0:url.find('&Keep')] - - return url - -#------------------------------------------------------------------------------------------------- - - def parse_index(self): - - pages = [ - ('Your America','http://www.rd.com/your-america-inspiring-people-and-stories', 'channelLeftContainer',{'class':'moreLeft'}), - # useless recipes ('Living Healthy','http://www.rd.com/living-healthy', 'channelLeftContainer',{'class':'moreLeft'}), - ('Advice and Know-How','http://www.rd.com/advice-and-know-how', 'channelLeftContainer',{'class':'moreLeft'}) - + keep_only_tags = dict(id='main-content') + remove_tags = [ + {'class':['post-categories']}, ] - feeds = [] - - for page in pages: - section, url, divider, attrList = page - newArticles = self.page_parse(url, divider, attrList) - feeds.append((section,newArticles)) - - # after the pages of the site have been processed, parse several RSS feeds for additional sections - newfeeds = Feed() - newfeeds = self.parse_rss() - - - # The utility code in parse_rss returns a Feed object. Convert each feed/article combination into a form suitable - # for this module (parse_index). - - for feed in newfeeds: - newArticles = [] - for article in feed.articles: - newArt = { - 'title' : article.title, - 'url' : article.url, - 'date' : article.date, - 'description' : article.text_summary - } - newArticles.append(newArt) - - - # New and Blogs should be the first two feeds. - if feed.title == 'New in RD': - feeds.insert(0,(feed.title,newArticles)) - elif feed.title == 'Blogs': - feeds.insert(1,(feed.title,newArticles)) - else: - feeds.append((feed.title,newArticles)) - - - return feeds - -#------------------------------------------------------------------------------------------------- - - def page_parse(self, mainurl, divider, attrList): - - articles = [] - mainsoup = self.index_to_soup(mainurl) - for item in mainsoup.findAll(attrs=attrList): - newArticle = { - 'title' : item('img')[0]['alt'], - 'url' : 'http://www.rd.com'+item('a')[0]['href'], - 'date' : '', - 'description' : '' - } - articles.append(newArticle) - - - - return articles - - - -#------------------------------------------------------------------------------------------------- - - def parse_rss (self): - - # Do the "official" parse_feeds first - feeds = BasicNewsRecipe.parse_feeds(self) - - - # Loop thru the articles in all feeds to find articles with "recipe" in it - recipeArticles = [] - for curfeed in feeds: - delList = [] - for a,curarticle in enumerate(curfeed.articles): - if curarticle.title.upper().find('RECIPE') >= 0: - recipeArticles.append(curarticle) - delList.append(curarticle) - if len(delList)>0: - for d in delList: - index = curfeed.articles.index(d) - curfeed.articles[index:index+1] = [] - - # If there are any recipes found, create a new Feed object and append. - if len(recipeArticles) > 0: - pfeed = Feed() - pfeed.title = 'Recipes' - pfeed.descrition = 'Recipe Feed (Virtual)' - pfeed.image_url = None - pfeed.oldest_article = 30 - pfeed.id_counter = len(recipeArticles) - # Create a new Feed, add the recipe articles, and then append - # to "official" list of feeds - pfeed.articles = recipeArticles[:] - feeds.append(pfeed) - - return feeds From 751890a83f5fef83968c0de39313980c7be3d7e7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 13:15:52 -0600 Subject: [PATCH 32/35] ... --- src/calibre/ebooks/metadata/sources/identify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index b084f86294..0cc070c3c6 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -382,7 +382,7 @@ def identify(log, abort, # {{{ if key not in filter_results: filtered_results.append(r) filter_results.add(key) - presults = filtered_results + results[plugin] = presults = filtered_results plog = logs[plugin].getvalue().strip() log('\n'+'*'*30, plugin.name, '*'*30) From e19edba3efe5fa257591ed0fe1fbfb286317257d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 14:31:21 -0600 Subject: [PATCH 33/35] EPUB Input: Ignore missing cover file when converting, instead of erroring out. Fixes #781848 ([Errno 2] No such file or directory while converting) --- src/calibre/ebooks/epub/input.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/epub/input.py b/src/calibre/ebooks/epub/input.py index 917c5ad8ae..ac1d61ce59 100644 --- a/src/calibre/ebooks/epub/input.py +++ b/src/calibre/ebooks/epub/input.py @@ -103,10 +103,11 @@ class EPUBInput(InputFormatPlugin): t.set('href', guide_cover) t.set('title', 'Title Page') from calibre.ebooks import render_html_svg_workaround - renderer = render_html_svg_workaround(guide_cover, log) - if renderer is not None: - open('calibre_raster_cover.jpg', 'wb').write( - renderer) + if os.path.exists(guide_cover): + renderer = render_html_svg_workaround(guide_cover, log) + if renderer is not None: + open('calibre_raster_cover.jpg', 'wb').write( + renderer) def find_opf(self): def attr(n, attr): From 953c8e939558ed380ae0a817cd89303a6fc959f7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 15:05:55 -0600 Subject: [PATCH 34/35] Allow the use of condensed/expanded fonts as interface fonts --- src/calibre/gui2/__init__.py | 6 +++++- src/calibre/gui2/preferences/look_feel.py | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 1dfe1d8d14..28504f2a31 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -620,7 +620,11 @@ class Application(QApplication): self.original_font = QFont(QApplication.font()) fi = gprefs['font'] if fi is not None: - QApplication.setFont(QFont(*fi)) + font = QFont(*(fi[:4])) + s = gprefs.get('font_stretch', None) + if s is not None: + font.setStretch(s) + QApplication.setFont(font) def _send_file_open_events(self): with self._file_open_lock: diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index 620113cc3f..ee2d7a5428 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -161,7 +161,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): def initialize(self): ConfigWidgetBase.initialize(self) - self.current_font = self.initial_font = gprefs['font'] + font = gprefs['font'] + if font is not None: + font = list(font) + font.append(gprefs.get('font_stretch', QFont.Unstretched)) + self.current_font = self.initial_font = font self.update_font_display() self.display_model.initialize() @@ -178,7 +182,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): def build_font_obj(self): font_info = self.current_font if font_info is not None: - font = QFont(*font_info) + font = QFont(*(font_info[:4])) + font.setStretch(font_info[4]) else: font = qt_app.original_font return font @@ -215,15 +220,18 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if fd.exec_() == fd.Accepted: font = fd.selectedFont() fi = QFontInfo(font) - self.current_font = (unicode(fi.family()), fi.pointSize(), - fi.weight(), fi.italic()) + self.current_font = [unicode(fi.family()), fi.pointSize(), + fi.weight(), fi.italic(), font.stretch()] self.update_font_display() self.changed_signal.emit() def commit(self, *args): rr = ConfigWidgetBase.commit(self, *args) if self.current_font != self.initial_font: - gprefs['font'] = self.current_font + gprefs['font'] = (self.current_font[:4] if self.current_font else + None) + gprefs['font_stretch'] = (self.current_font[4] if self.current_font + is not None else QFont.Unstretched) QApplication.setFont(self.font_display.font()) rr = True self.display_model.commit() From af23efd3d6992b488803048f48efb1a1a1f7b908 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 15:09:41 -0600 Subject: [PATCH 35/35] Fix Strategy+Business --- recipes/strategy-business.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/strategy-business.recipe b/recipes/strategy-business.recipe index ab58965e98..a4697ecfcd 100644 --- a/recipes/strategy-business.recipe +++ b/recipes/strategy-business.recipe @@ -33,7 +33,7 @@ class StrategyBusinessRecipe(BasicNewsRecipe): elif c.name.endswith('_password'): br[c.name] = self.password raw = br.submit().read() - if '>Logout' not in raw: + if 'You have been logged in' not in raw: raise ValueError('Failed to login, check your username and password') return br