From 5507e72f45aa0dc642d0c966bfa7777d6f782b75 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 9 Oct 2011 15:29:36 +0200 Subject: [PATCH 01/15] Add Amazon.fr --- src/calibre/customize/builtins.py | 11 ++ .../gui2/store/stores/amazon_fr_plugin.py | 114 ++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 src/calibre/gui2/store/stores/amazon_fr_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index a2c0596e0b..79e5259c00 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1143,6 +1143,16 @@ class StoreAmazonDEKindleStore(StoreBase): formats = ['KINDLE'] affiliate = True +class StoreAmazonFRKindleStore(StoreBase): + name = 'Amazon FR Kindle' + author = 'Charles Haley' + description = u'Tous les ebooks Kindle' + actual_plugin = 'calibre.gui2.store.stores.amazon_fr_plugin:AmazonFRKindleStore' + + headquarters = 'DE' + formats = ['KINDLE'] + affiliate = True + class StoreAmazonUKKindleStore(StoreBase): name = 'Amazon UK Kindle' author = 'Charles Haley' @@ -1520,6 +1530,7 @@ plugins += [ StoreArchiveOrgStore, StoreAmazonKindleStore, StoreAmazonDEKindleStore, + StoreAmazonFRKindleStore, StoreAmazonUKKindleStore, StoreBaenWebScriptionStore, StoreBNStore, diff --git a/src/calibre/gui2/store/stores/amazon_fr_plugin.py b/src/calibre/gui2/store/stores/amazon_fr_plugin.py new file mode 100644 index 0000000000..a5b97751ca --- /dev/null +++ b/src/calibre/gui2/store/stores/amazon_fr_plugin.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.search_result import SearchResult + +class AmazonFRKindleStore(StorePlugin): + ''' + For comments on the implementation, please see amazon_plugin.py + ''' + + def open(self, parent=None, detail_item=None, external=False): + aff_id = {'tag': 'charhale-21'} + store_link = 'http://www.amazon.fr/livres-kindle/b?ie=UTF8&node=695398031&ref_=sa_menu_kbo1&_encoding=UTF8&tag=%(tag)s&linkCode=ur2&camp=1642&creative=19458' % aff_id + + if detail_item: + aff_id['asin'] = detail_item + store_link = 'http://www.amazon.fr/gp/redirect.html?ie=UTF8&location=http://www.amazon.fr/dp/%(asin)s&tag=%(tag)s&linkCode=ur2&camp=1634&creative=6738' % aff_id + open_url(QUrl(store_link)) + + def search(self, query, max_results=10, timeout=60): + search_url = 'http://www.amazon.fr/s/?url=search-alias%3Ddigital-text&field-keywords=' + url = search_url + urllib.quote_plus(query) + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + + data_xpath = '//div[contains(@class, "result") and contains(@class, "product")]' + format_xpath = './/span[@class="format"]/text()' + cover_xpath = './/img[@class="productImage"]/@src' + + for data in doc.xpath(data_xpath): + if counter <= 0: + break + + # Even though we are searching digital-text only Amazon will still + # put in results for non Kindle books (author pages). So we need + # to explicitly check if the item is a Kindle book and ignore it + # if it isn't. + format = ''.join(data.xpath(format_xpath)) + if 'kindle' not in format.lower(): + continue + + # We must have an asin otherwise we can't easily reference the + # book later. + asin = ''.join(data.xpath("@name")) + + cover_url = ''.join(data.xpath(cover_xpath)) + + title = ''.join(data.xpath('.//div[@class="title"]/a/text()')) + price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) + author = unicode(''.join(data.xpath('.//div[@class="title"]/span[@class="ptBrand"]/text()'))) + author = author.split('de ')[-1] + +# print (author, asin, cover_url, title, price) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url.strip() + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = asin.strip() + s.formats = 'Kindle' + s.DRM = SearchResult.DRM_UNKNOWN + yield s + + def get_details(self, search_result, timeout): + # We might already have been called. + if search_result.drm: + return + + url = 'http://amazon.fr/dp/' + drm_search_text = u'Simultaneous Device Usage' + drm_free_text = u'Unlimited' + + br = browser() + with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + if not search_result.author: + search_result.author = ''.join(idata.xpath('//div[@class="buying" and contains(., "Author")]/a/text()')) + is_kindle = idata.xpath('boolean(//div[@class="buying"]/h1/span/span[contains(text(), "Kindle Edition")])') + if is_kindle: + search_result.formats = 'Kindle' + if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' + + drm_search_text + '")])'): + if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' + + drm_free_text + '") and contains(b, "' + + drm_search_text + '")])'): + search_result.drm = SearchResult.DRM_UNLOCKED + else: + search_result.drm = SearchResult.DRM_UNKNOWN + else: + search_result.drm = SearchResult.DRM_LOCKED + return True + + From 7003ae2aa991052d226ba7b4da9e55d764e9438a Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 9 Oct 2011 10:54:19 -0400 Subject: [PATCH 02/15] Fix encoding issues when searching and displaying results for Amazon plugins. --- .../gui2/store/stores/amazon_de_plugin.py | 5 +-- .../gui2/store/stores/amazon_fr_plugin.py | 43 +++---------------- .../gui2/store/stores/amazon_plugin.py | 5 +-- .../gui2/store/stores/amazon_uk_plugin.py | 5 +-- 4 files changed, 11 insertions(+), 47 deletions(-) diff --git a/src/calibre/gui2/store/stores/amazon_de_plugin.py b/src/calibre/gui2/store/stores/amazon_de_plugin.py index 9f26e765e6..4948a48714 100644 --- a/src/calibre/gui2/store/stores/amazon_de_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_de_plugin.py @@ -6,7 +6,6 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import urllib from contextlib import closing from lxml import html @@ -37,12 +36,12 @@ class AmazonDEKindleStore(StorePlugin): def search(self, query, max_results=10, timeout=60): search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords=' - url = search_url + urllib.quote_plus(query) + url = search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+') br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) + doc = html.fromstring(f.read().decode('latin-1', 'replace')) # Amazon has two results pages. # 20110725: seems that is_shot is gone. diff --git a/src/calibre/gui2/store/stores/amazon_fr_plugin.py b/src/calibre/gui2/store/stores/amazon_fr_plugin.py index a5b97751ca..186ca8d4b4 100644 --- a/src/calibre/gui2/store/stores/amazon_fr_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_fr_plugin.py @@ -6,7 +6,6 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import urllib from contextlib import closing from lxml import html @@ -34,12 +33,12 @@ class AmazonFRKindleStore(StorePlugin): def search(self, query, max_results=10, timeout=60): search_url = 'http://www.amazon.fr/s/?url=search-alias%3Ddigital-text&field-keywords=' - url = search_url + urllib.quote_plus(query) + url = search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+') br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) + doc = html.fromstring(f.read().decode('latin-1', 'replace')) data_xpath = '//div[contains(@class, "result") and contains(@class, "product")]' format_xpath = './/span[@class="format"]/text()' @@ -66,9 +65,7 @@ class AmazonFRKindleStore(StorePlugin): title = ''.join(data.xpath('.//div[@class="title"]/a/text()')) price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) author = unicode(''.join(data.xpath('.//div[@class="title"]/span[@class="ptBrand"]/text()'))) - author = author.split('de ')[-1] - -# print (author, asin, cover_url, title, price) + author = author.split('et ')[-1] counter -= 1 @@ -79,36 +76,6 @@ class AmazonFRKindleStore(StorePlugin): s.price = price.strip() s.detail_item = asin.strip() s.formats = 'Kindle' - s.DRM = SearchResult.DRM_UNKNOWN + s.drm = SearchResult.DRM_UNKNOWN + yield s - - def get_details(self, search_result, timeout): - # We might already have been called. - if search_result.drm: - return - - url = 'http://amazon.fr/dp/' - drm_search_text = u'Simultaneous Device Usage' - drm_free_text = u'Unlimited' - - br = browser() - with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: - idata = html.fromstring(nf.read()) - if not search_result.author: - search_result.author = ''.join(idata.xpath('//div[@class="buying" and contains(., "Author")]/a/text()')) - is_kindle = idata.xpath('boolean(//div[@class="buying"]/h1/span/span[contains(text(), "Kindle Edition")])') - if is_kindle: - search_result.formats = 'Kindle' - if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' + - drm_search_text + '")])'): - if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' + - drm_free_text + '") and contains(b, "' + - drm_search_text + '")])'): - search_result.drm = SearchResult.DRM_UNLOCKED - else: - search_result.drm = SearchResult.DRM_UNKNOWN - else: - search_result.drm = SearchResult.DRM_LOCKED - return True - - diff --git a/src/calibre/gui2/store/stores/amazon_plugin.py b/src/calibre/gui2/store/stores/amazon_plugin.py index 693ef883fb..89a6278535 100644 --- a/src/calibre/gui2/store/stores/amazon_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_plugin.py @@ -8,7 +8,6 @@ __docformat__ = 'restructuredtext en' import random import re -import urllib from contextlib import closing from lxml import html @@ -122,12 +121,12 @@ class AmazonKindleStore(StorePlugin): open_url(QUrl(store_link)) def search(self, query, max_results=10, timeout=60): - url = self.search_url + urllib.quote_plus(query) + url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+') br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) + doc = html.fromstring(f.read().decode('latin-1', 'replace')) # Amazon has two results pages. is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])') diff --git a/src/calibre/gui2/store/stores/amazon_uk_plugin.py b/src/calibre/gui2/store/stores/amazon_uk_plugin.py index 86603f3fc3..3b2a4d05cc 100644 --- a/src/calibre/gui2/store/stores/amazon_uk_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_uk_plugin.py @@ -6,7 +6,6 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import urllib from contextlib import closing from lxml import html @@ -34,12 +33,12 @@ class AmazonUKKindleStore(StorePlugin): def search(self, query, max_results=10, timeout=60): search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords=' - url = search_url + urllib.quote_plus(query) + url = search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+') br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) + doc = html.fromstring(f.read().decode('latin-1', 'replace')) # Amazon has two results pages. # 20110725: seems that is_shot is gone. From 9e114a9e1f763c8160c652eb45c1f80e85a527b5 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 9 Oct 2011 17:39:25 +0200 Subject: [PATCH 03/15] Clean up/fix code to strip leading "by " in German and French. --- .../gui2/store/stores/amazon_de_plugin.py | 17 ++--------------- .../gui2/store/stores/amazon_fr_plugin.py | 5 +++-- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/calibre/gui2/store/stores/amazon_de_plugin.py b/src/calibre/gui2/store/stores/amazon_de_plugin.py index 4948a48714..ea92839268 100644 --- a/src/calibre/gui2/store/stores/amazon_de_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_de_plugin.py @@ -43,20 +43,9 @@ class AmazonDEKindleStore(StorePlugin): with closing(br.open(url, timeout=timeout)) as f: doc = html.fromstring(f.read().decode('latin-1', 'replace')) - # Amazon has two results pages. - # 20110725: seems that is_shot is gone. -# is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])') -# # Horizontal grid of books. -# if is_shot: -# data_xpath = '//div[contains(@class, "result")]' -# format_xpath = './/div[@class="productTitle"]/text()' -# cover_xpath = './/div[@class="productTitle"]//img/@src' -# # Vertical list of books. -# else: data_xpath = '//div[contains(@class, "result") and contains(@class, "product")]' format_xpath = './/span[@class="format"]/text()' cover_xpath = './/img[@class="productImage"]/@src' -# end is_shot else for data in doc.xpath(data_xpath): if counter <= 0: @@ -79,11 +68,9 @@ class AmazonDEKindleStore(StorePlugin): title = ''.join(data.xpath('.//div[@class="title"]/a/text()')) price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) -# if is_shot: -# author = format.split(' von ')[-1] -# else: author = ''.join(data.xpath('.//div[@class="title"]/span[@class="ptBrand"]/text()')) - author = author.split('von ')[-1] + if author.startswith('von '): + author = author[4:] counter -= 1 diff --git a/src/calibre/gui2/store/stores/amazon_fr_plugin.py b/src/calibre/gui2/store/stores/amazon_fr_plugin.py index 186ca8d4b4..ca36f1055b 100644 --- a/src/calibre/gui2/store/stores/amazon_fr_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_fr_plugin.py @@ -65,7 +65,8 @@ class AmazonFRKindleStore(StorePlugin): title = ''.join(data.xpath('.//div[@class="title"]/a/text()')) price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) author = unicode(''.join(data.xpath('.//div[@class="title"]/span[@class="ptBrand"]/text()'))) - author = author.split('et ')[-1] + if author.startswith('de '): + author = author[3:] counter -= 1 @@ -77,5 +78,5 @@ class AmazonFRKindleStore(StorePlugin): s.detail_item = asin.strip() s.formats = 'Kindle' s.drm = SearchResult.DRM_UNKNOWN - + yield s From 18ed5671c699b90dbee72dd7dd2a4e426ee84148 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 9 Oct 2011 17:48:40 +0200 Subject: [PATCH 04/15] Fix 'by' splitting in the UK plugin. --- .../gui2/store/stores/amazon_uk_plugin.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/calibre/gui2/store/stores/amazon_uk_plugin.py b/src/calibre/gui2/store/stores/amazon_uk_plugin.py index 3b2a4d05cc..ef15951d50 100644 --- a/src/calibre/gui2/store/stores/amazon_uk_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_uk_plugin.py @@ -40,20 +40,9 @@ class AmazonUKKindleStore(StorePlugin): with closing(br.open(url, timeout=timeout)) as f: doc = html.fromstring(f.read().decode('latin-1', 'replace')) - # Amazon has two results pages. - # 20110725: seems that is_shot is gone. -# is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])') -# # Horizontal grid of books. -# if is_shot: -# data_xpath = '//div[contains(@class, "result")]' -# format_xpath = './/div[@class="productTitle"]/text()' -# cover_xpath = './/div[@class="productTitle"]//img/@src' -# # Vertical list of books. -# else: data_xpath = '//div[contains(@class, "result") and contains(@class, "product")]' format_xpath = './/span[@class="format"]/text()' cover_xpath = './/img[@class="productImage"]/@src' -# end is_shot else for data in doc.xpath(data_xpath): if counter <= 0: @@ -76,11 +65,9 @@ class AmazonUKKindleStore(StorePlugin): title = ''.join(data.xpath('.//div[@class="title"]/a/text()')) price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) -# if is_shot: -# author = format.split(' von ')[-1] -# else: author = ''.join(data.xpath('.//div[@class="title"]/span[@class="ptBrand"]/text()')) - author = author.split('by ')[-1] + if author.startswith('by '): + author = author[3:] counter -= 1 From 64f0f0fd9d02e55ccf4e8da3338e956bc0315ae6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Oct 2011 15:02:14 +0530 Subject: [PATCH 05/15] ... --- src/calibre/library/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 29deaa680b..65752eb183 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -47,6 +47,9 @@ def get_parser(usage): def get_db(dbpath, options): if options.library_path is not None: dbpath = options.library_path + if dbpath is None: + raise ValueError('No saved library path, either run the GUI or use the' + ' --with-library option') dbpath = os.path.abspath(dbpath) return LibraryDatabase2(dbpath) From b25724a8eaa713797f543b2c3321ba3c382970a8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Oct 2011 16:34:57 +0530 Subject: [PATCH 06/15] Device drivers: Add a prepare_addable_books API method --- src/calibre/devices/interface.py | 6 ++++++ src/calibre/gui2/actions/add.py | 1 + 2 files changed, 7 insertions(+) diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 56c950bd16..ad21632a50 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -518,3 +518,9 @@ class BookList(list): ''' raise NotImplementedError() + def prepare_addable_books(self, paths): + ''' + Given a list of paths, returns another list of paths. These paths + point to addable versions of the books. + ''' + return paths diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 08385f4f3f..dc709f221e 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -397,6 +397,7 @@ class AddAction(InterfaceAction): d = error_dialog(self.gui, _('Add to library'), _('No book files found')) d.exec_() return + paths = self.gui.device_manager.device.prepare_addable_books(paths) from calibre.gui2.add import Adder self.__adder_func = partial(self._add_from_device_adder, on_card=None, model=view.model()) From f5ac39d932fa491f930f9aac73ae6117ade0d73e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Oct 2011 04:12:02 +0530 Subject: [PATCH 07/15] Defense News by DM. Fixes #871916 (New recipe for DefenseNews) --- recipes/defensenews.recipe | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 recipes/defensenews.recipe diff --git a/recipes/defensenews.recipe b/recipes/defensenews.recipe new file mode 100644 index 0000000000..8c0f9b0be7 --- /dev/null +++ b/recipes/defensenews.recipe @@ -0,0 +1,64 @@ +__license__ = 'GPL v3' +__copyright__ = '2011, Darko Miletic ' +''' +www.defensenews.com +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup + +class DefenseNews(BasicNewsRecipe): + title = 'Defense News' + __author__ = 'Darko Miletic' + description = 'Find late-breaking defense news from the leading defense news weekly' + publisher = 'Gannett Government Media Corporation' + category = 'defense news, defence news, defense, defence, defence budget, defence policy' + oldest_article = 31 + max_articles_per_feed = 200 + no_stylesheets = True + encoding = 'utf8' + use_embedded_content = False + language = 'en' + remove_empty_feeds = True + publication_type = 'newspaper' + masthead_url = 'http://www.defensenews.com/images/logo_defensenews2.jpg' + extra_css = """ + body{font-family: Arial,Helvetica,sans-serif } + img{margin-bottom: 0.4em; display:block} + .info{font-size: small; color: gray} + """ + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + } + + remove_tags = [ + dict(name=['meta','link']) + ,dict(attrs={'class':['toolbar','related','left','right']}) + ] + remove_tags_before = attrs={'class':'storyWrp'} + remove_tags_after = attrs={'class':'middle'} + + remove_attributes=['lang'] + + feeds = [ + (u'Europe' , u'http://www.defensenews.com/rss/eur/' ) + ,(u'Americas', u'http://www.defensenews.com/rss/ame/' ) + ,(u'Asia & Pacific rim', u'http://www.defensenews.com/rss/asi/' ) + ,(u'Middle east & Africa', u'http://www.defensenews.com/rss/mid/') + ,(u'Air', u'http://www.defensenews.com/rss/air/' ) + ,(u'Land', u'http://www.defensenews.com/rss/lan/' ) + ,(u'Naval', u'http://www.defensenews.com/rss/sea/' ) + ] + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll('img'): + if not item.has_key('alt'): + item['alt'] = 'image' + return soup From 831d301d802315c2834c720d12fbeadfc3773b90 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Oct 2011 04:18:38 +0530 Subject: [PATCH 08/15] ... --- src/calibre/gui2/preferences/server.ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/server.ui b/src/calibre/gui2/preferences/server.ui index b07a8cac34..be68c2448f 100644 --- a/src/calibre/gui2/preferences/server.ui +++ b/src/calibre/gui2/preferences/server.ui @@ -206,7 +206,7 @@ - Run server &automatically on startup + Run server &automatically when calibre starts From 983a3a76c5f6ae9b5add7544a8b18735bd517f79 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Oct 2011 04:30:57 +0530 Subject: [PATCH 09/15] Merco Press and Penguin news by Russell Phillips --- recipes/merco_press.recipe | 27 +++++++++++++++++++++++++++ recipes/penguin_news.recipe | 17 +++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 recipes/merco_press.recipe create mode 100644 recipes/penguin_news.recipe diff --git a/recipes/merco_press.recipe b/recipes/merco_press.recipe new file mode 100644 index 0000000000..efa2d6ec08 --- /dev/null +++ b/recipes/merco_press.recipe @@ -0,0 +1,27 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class MercoPress(BasicNewsRecipe): + title = u'Merco Press' + description = u"Read News, Stories and Insight Analysis from Latin America and Mercosur. Politics, Economy, Business and Investments in South America." + cover_url = 'http://en.mercopress.com/web/img/en/mercopress-logo.gif' + + __author__ = 'Russell Phillips' + language = 'en' + + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + + extra_css = 'img{padding-bottom:1ex; display:block; text-align: center;}' + remove_tags = [dict(name='a')] + + feeds = [('Antarctica', 'http://en.mercopress.com/rss/antarctica'), + ('Argentina', 'http://en.mercopress.com/rss/argentina'), + ('Brazil', 'http://en.mercopress.com/rss/brazil'), + ('Falkland Islands', 'http://en.mercopress.com/rss/falkland-islands'), + ('International News', 'http://en.mercopress.com/rss/international'), + ('Latin America', 'http://en.mercopress.com/rss/latin-america'), + ('Mercosur', 'http://en.mercopress.com/rss/mercosur'), + ('Paraguay', 'http://en.mercopress.com/rss/paraguay'), + ('United States', 'http://en.mercopress.com/rss/united-states'), + ('Uruguay://en.mercopress.com/rss/uruguay')] diff --git a/recipes/penguin_news.recipe b/recipes/penguin_news.recipe new file mode 100644 index 0000000000..6761623a55 --- /dev/null +++ b/recipes/penguin_news.recipe @@ -0,0 +1,17 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class MercoPress(BasicNewsRecipe): + title = u'Penguin News' + description = u"Penguin News: the Falkland Islands' only newspaper." + cover_url = 'http://www.penguin-news.com/templates/rt_syndicate_j15/images/logo/light/logo1.png' + language = 'en' + + __author__ = 'Russell Phillips' + + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + + extra_css = 'img{padding-bottom:1ex; display:block; text-align: center;}' + + feeds = [(u'Penguin News - Falkland Islands', u'http://www.penguin-news.com/index.php?format=feed&type=rss')] From a099448d6ac6398048db364cc176c6f3dbc3c8a9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Oct 2011 07:28:56 +0530 Subject: [PATCH 10/15] Changes to the build process to accomodate my current circumstances --- setup/build_environment.py | 5 ++++- setup/installer/__init__.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/setup/build_environment.py b/setup/build_environment.py index d4a7af833b..eb34763fb4 100644 --- a/setup/build_environment.py +++ b/setup/build_environment.py @@ -225,7 +225,10 @@ except: try: HOST=get_ip_address('wlan0') except: - HOST='192.168.1.2' + try: + HOST=get_ip_address('ppp0') + except: + HOST='192.168.1.2' PROJECT=os.path.basename(os.path.abspath('.')) diff --git a/setup/installer/__init__.py b/setup/installer/__init__.py index 79bb942cde..8374f93e38 100644 --- a/setup/installer/__init__.py +++ b/setup/installer/__init__.py @@ -20,17 +20,23 @@ for x in [ EXCLUDES.extend(['--exclude', x]) SAFE_EXCLUDES = ['"%s"'%x if '*' in x else x for x in EXCLUDES] +def get_rsync_pw(): + return open('/home/kovid/work/kde/conf/buildbot').read().partition( + ':')[-1].strip() + class Rsync(Command): description = 'Sync source tree from development machine' SYNC_CMD = ' '.join(BASE_RSYNC+SAFE_EXCLUDES+ - ['rsync://{host}/work/{project}', '..']) + ['rsync://buildbot@{host}/work/{project}', '..']) def run(self, opts): cmd = self.SYNC_CMD.format(host=HOST, project=PROJECT) + env = dict(os.environ) + env['RSYNC_PASSWORD'] = get_rsync_pw() self.info(cmd) - subprocess.check_call(cmd, shell=True) + subprocess.check_call(cmd, shell=True, env=env) class Push(Command): @@ -81,7 +87,8 @@ class VMInstaller(Command): def get_build_script(self): - ans = '\n'.join(self.BUILD_PREFIX)+'\n\n' + rs = ['export RSYNC_PASSWORD=%s'%get_rsync_pw()] + ans = '\n'.join(self.BUILD_PREFIX + rs)+'\n\n' ans += ' && \\\n'.join(self.BUILD_RSYNC) + ' && \\\n' ans += ' && \\\n'.join(self.BUILD_CLEAN) + ' && \\\n' ans += ' && \\\n'.join(self.BUILD_BUILD) + ' && \\\n' From 96fdd1799f39c83786289593ac194791f6779173 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Oct 2011 09:07:38 +0530 Subject: [PATCH 11/15] WoW Insider by Krittika Goyal --- recipes/wow.recipe | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 recipes/wow.recipe diff --git a/recipes/wow.recipe b/recipes/wow.recipe new file mode 100644 index 0000000000..9024f8eaf4 --- /dev/null +++ b/recipes/wow.recipe @@ -0,0 +1,17 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class WoW(BasicNewsRecipe): + title = u'WoW Insider' + language = 'en' + __author__ = 'Krittika Goyal' + oldest_article = 1 #days + max_articles_per_feed = 25 + use_embedded_content = False + + no_stylesheets = True + auto_cleanup = True + + feeds = [ +('WoW', + 'http://wow.joystiq.com/rss.xml') +] From f21132e16d3d7f5ea570611ef3bf776f03d6fec1 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 11 Oct 2011 09:39:38 +0200 Subject: [PATCH 12/15] Fix for the problem where setting the restriction to an empty current search clears the restriction box but does not clear the restriction. --- src/calibre/gui2/search_restriction_mixin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index ffebc9e131..1319f8d17d 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -37,6 +37,7 @@ class SearchRestrictionMixin(object): search = unicode(search) if not search: self.search_restriction.setCurrentIndex(0) + self._apply_search_restriction('') else: s = '*' + search if self.search_restriction.count() > 1: From 430f67c3f5a1156afa936ce12ea7f4dd1fd696dd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Oct 2011 17:26:51 +0530 Subject: [PATCH 13/15] ... --- src/calibre/devices/prst1/driver.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index ca8e2ae435..22ef567280 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -253,8 +253,11 @@ class PRST1(USBMS): # Get Metadata We Want lpath = book.lpath - author = newmi.authors[0] - title = newmi.title + try: + author = newmi.authors[0] + except: + author = _('Unknown') + title = newmi.title or _('Unknown') if lpath not in db_books: query = ''' @@ -405,34 +408,34 @@ class PRST1(USBMS): def upload_cover(self, path, filename, metadata, filepath): debug_print('PRS-T1: uploading cover') - + if filepath.startswith(self._main_prefix): prefix = self._main_prefix source_id = 0 else: prefix = self._card_a_prefix source_id = 1 - + metadata.lpath = filepath.partition(prefix)[2] dbpath = self.normalize_path(prefix + DBPATH) debug_print("SQLite DB Path: " + dbpath) - with closing(sqlite.connect(dbpath)) as connection: + with closing(sqlite.connect(dbpath)) as connection: cursor = connection.cursor() - + query = 'SELECT _id FROM books WHERE file_path = ?' t = (metadata.lpath,) cursor.execute(query, t) - + for i, row in enumerate(cursor): metadata.bookId = row[0] - + cursor.close() - + if metadata.bookId is not None: debug_print('PRS-T1: refreshing cover for book being sent') self.upload_book_cover(connection, metadata, source_id) - + debug_print('PRS-T1: done uploading cover') def upload_book_cover(self, connection, book, source_id): From 967ec83a7cbfa8991f9a060e07b7e3165cc48cc6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Oct 2011 17:44:25 +0530 Subject: [PATCH 14/15] ... --- recipes/guardian.recipe | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/recipes/guardian.recipe b/recipes/guardian.recipe index 05d6616ace..f063934b3d 100644 --- a/recipes/guardian.recipe +++ b/recipes/guardian.recipe @@ -119,10 +119,8 @@ class Guardian(BasicNewsRecipe): } def parse_index(self): - try: - feeds = [] - for title, href in self.find_sections(): - feeds.append((title, list(self.find_articles(href)))) - return feeds - except: - raise NotImplementedError + feeds = [] + for title, href in self.find_sections(): + feeds.append((title, list(self.find_articles(href)))) + return feeds + From bffa00773d14f5cffec23f685a5fdd1642614201 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Oct 2011 18:52:41 +0530 Subject: [PATCH 15/15] T1 driver ad detection of SD card on windows --- src/calibre/devices/prst1/driver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 22ef567280..44c93af4cc 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -47,6 +47,9 @@ class PRST1(USBMS): WINDOWS_MAIN_MEM = re.compile( r'(PRS-T1&)' ) + WINDOWS_CARD_A_MEM = re.compile( + r'(PRS-T1__SD&)' + ) MAIN_MEMORY_VOLUME_LABEL = 'SONY Reader Main Memory' STORAGE_CARD_VOLUME_LABEL = 'SONY Reader Storage Card'