From 02c48304e2b648de5bcbaf5a77b8d8f3a14aa2ce Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 26 Apr 2011 10:34:46 +0100 Subject: [PATCH 01/29] Add amazon_uk_plugin.py to repository --- src/calibre/customize/builtins.py | 8 +- src/calibre/gui2/store/amazon_uk_plugin.py | 197 +++++++++++++++++++++ 2 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 src/calibre/gui2/store/amazon_uk_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index c27fa2a57b..4f426d7490 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1109,6 +1109,11 @@ class StoreAmazonKindleStore(StoreBase): description = _('Kindle books from Amazon') actual_plugin = 'calibre.gui2.store.amazon_plugin:AmazonKindleStore' +class StoreAmazonUKKindleStore(StoreBase): + name = 'Amazon UK Kindle' + description = _('Kindle books from Amazon.uk') + actual_plugin = 'calibre.gui2.store.amazon_uk_plugin:AmazonUKKindleStore' + class StoreBaenWebScriptionStore(StoreBase): name = 'Baen WebScription' description = _('Ebooks for readers.') @@ -1174,7 +1179,8 @@ class StoreSmashwordsStore(StoreBase): description = _('Your ebook. Your way.') actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' -plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreBNStore, +plugins += [StoreAmazonKindleStore, StoreAmazonUKKindleStore, + StoreBaenWebScriptionStore, StoreBNStore, StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreEHarlequinStoretore, StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, diff --git a/src/calibre/gui2/store/amazon_uk_plugin.py b/src/calibre/gui2/store/amazon_uk_plugin.py new file mode 100644 index 0000000000..a68e4611f0 --- /dev/null +++ b/src/calibre/gui2/store/amazon_uk_plugin.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.search_result import SearchResult + +class AmazonKindleStore(StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + ''' + Amazon comes with a number of difficulties. + + QWebView has major issues with Amazon.com. The largest of + issues is it simply doesn't work on a number of pages. + + When connecting to a number parts of Amazon.com (Kindle library + for instance) QNetworkAccessManager fails to connect with a + NetworkError of 399 - ProtocolFailure. The strange thing is, + when I check QNetworkRequest.HttpStatusCodeAttribute when the + 399 error is returned the status code is 200 (Ok). However, once + the QNetworkAccessManager decides there was a NetworkError it + does not download the page from Amazon. So I can't even set the + HTML in the QWebView myself. + + There is http://bugreports.qt.nokia.com/browse/QTWEBKIT-259 an + open bug about the issue but it is not correct. We can set the + useragent (Arora does) to something else and the above issue + will persist. This http://developer.qt.nokia.com/forums/viewthread/793 + gives a bit more information about the issue but as of now (27/Feb/2011) + there is no solution or work around. + + We cannot change the The linkDelegationPolicy to allow us to avoid + QNetworkAccessManager because it only works links. Forms aren't + included so the same issue persists on any part of the site (login) + that use a form to load a new page. + + Using an aStore was evaluated but I've decided against using it. + There are three major issues with an aStore. Because checkout is + handled by sending the user to Amazon we can't put it in a QWebView. + If we're sending the user to Amazon sending them there directly is + nicer. Also, we cannot put the aStore in a QWebView and let it open the + redirection the users default browser because the cookies with the + shopping cart won't transfer. + + Another issue with the aStore is how it handles the referral. It only + counts the referral for the items in the shopping card / the item + that directed the user to Amazon. Kindle books do not use the shopping + cart and send the user directly to Amazon for the purchase. In this + instance we would only get referral credit for the one book that the + aStore directs to Amazon that the user buys. Any other purchases we + won't get credit for. + + The last issue with the aStore is performance. Even though it's an + Amazon site it's alow. So much slower than Amazon.com that it makes + me not want to browse books using it. The look and feel are lesser + issues. So is the fact that it almost seems like the purchase is + with calibre. This can cause some support issues because we can't + do much for issues with Amazon.com purchase hiccups. + + Another option that was evaluated was the Product Advertising API. + The reasons against this are complexity. It would take a lot of work + to basically re-create Amazon.com within calibre. The Product + Advertising API is also designed with being run on a server not + in an app. The signing keys would have to be made avaliable to ever + calibre user which means bad things could be done with our account. + + The Product Advertising API also assumes the same browser for easy + shopping cart transfer to Amazon. With QWebView not working and there + not being an easy way to transfer cookies between a QWebView and the + users default browser this won't work well. + + We could create our own website on the calibre server and create an + Amazon Product Advertising API store. However, this goes back to the + complexity argument. Why spend the time recreating Amazon.com + + The final and largest issue against using the Product Advertising API + is the Efficiency Guidelines: + + "Each account used to access the Product Advertising API will be allowed + an initial usage limit of 2,000 requests per hour. Each account will + receive an additional 500 requests per hour (up to a maximum of 25,000 + requests per hour) for every $1 of shipped item revenue driven per hour + in a trailing 30-day period. Usage thresholds are recalculated daily based + on revenue performance." + + With over two million users a limit of 2,000 request per hour could + render our store unusable for no other reason than Amazon rate + limiting our traffic. + + The best (I use the term lightly here) solution is to open Amazon.com + in the users default browser and set the affiliate id as part of the url. + ''' + aff_id = {'tag': 'josbl0e-cpb-20'} + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + aff_id['tag'] = 'calibrebs-20' + store_link = 'http://www.amazon.com/Kindle-eBooks/b/?ie=UTF&node=1286228011&ref_=%(tag)s&ref=%(tag)s&tag=%(tag)s&linkCode=ur2&camp=1789&creative=390957' % aff_id + if detail_item: + aff_id['asin'] = detail_item + store_link = 'http://www.amazon.com/dp/%(asin)s/?tag=%(tag)s' % aff_id + open_url(QUrl(store_link)) + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' + urllib2.quote(query) + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="productData"]'): + if counter <= 0: + break + + # Even though we are searching digital-text only Amazon will still + # put in results for non Kindle books (author pages). Se we need + # to explicitly check if the item is a Kindle book and ignore it + # if it isn't. + type = ''.join(data.xpath('//span[@class="format"]/text()')) + if 'kindle' not in type.lower(): + continue + + # We must have an asin otherwise we can't easily reference the + # book later. + asin_href = None + asin_a = data.xpath('div[@class="productTitle"]/a[1]') + if asin_a: + asin_href = asin_a[0].get('href', '') + m = re.search(r'/dp/(?P.+?)(/|$)', asin_href) + if m: + asin = m.group('asin') + else: + continue + else: + continue + + cover_url = '' + if asin_href: + cover_img = data.xpath('//div[@class="productImage"]/a[@href="%s"]/img/@src' % asin_href) + if cover_img: + cover_url = cover_img[0] + parts = cover_url.split('/') + bn = parts[-1] + f, _, ext = bn.rpartition('.') + if '_' in f: + bn = f.partition('_')[0]+'_SL160_.'+ext + parts[-1] = bn + cover_url = '/'.join(parts) + + title = ''.join(data.xpath('div[@class="productTitle"]/a/text()')) + author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) + author = author.split('by')[-1] + price = ''.join(data.xpath('div[@class="newPrice"]/span/text()')) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = asin.strip() + s.formats = 'Kindle' + + yield s + + def get_details(self, search_result, timeout): + url = 'http://amazon.com/dp/' + + br = browser() + with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "Simultaneous Device Usage")])'): + if idata.xpath('boolean(//div[@class="content"]//li[contains(., "Unlimited") and contains(b, "Simultaneous Device Usage")])'): + search_result.drm = SearchResult.DRM_UNLOCKED + else: + search_result.drm = SearchResult.DRM_UNKNOWN + else: + search_result.drm = SearchResult.DRM_LOCKED + return True + + From 5398c0fc64b68fb67cf30285cb746431aa0488e5 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 26 Apr 2011 11:17:50 +0100 Subject: [PATCH 02/29] First cut at amazon.uk store plugin. --- src/calibre/gui2/store/amazon_plugin.py | 6 +- src/calibre/gui2/store/amazon_uk_plugin.py | 188 ++------------------- 2 files changed, 15 insertions(+), 179 deletions(-) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index a68e4611f0..55fb613288 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -116,8 +116,9 @@ class AmazonKindleStore(StorePlugin): store_link = 'http://www.amazon.com/dp/%(asin)s/?tag=%(tag)s' % aff_id open_url(QUrl(store_link)) + search_url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' def search(self, query, max_results=10, timeout=60): - url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' + urllib2.quote(query) + url = self.search_url + urllib2.quote(query) br = browser() counter = max_results @@ -179,8 +180,9 @@ class AmazonKindleStore(StorePlugin): yield s + details_url = 'http://amazon.com/dp/' def get_details(self, search_result, timeout): - url = 'http://amazon.com/dp/' + url = self.details_url br = browser() with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: diff --git a/src/calibre/gui2/store/amazon_uk_plugin.py b/src/calibre/gui2/store/amazon_uk_plugin.py index a68e4611f0..e24ddb485e 100644 --- a/src/calibre/gui2/store/amazon_uk_plugin.py +++ b/src/calibre/gui2/store/amazon_uk_plugin.py @@ -6,192 +6,26 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import random -import re -import urllib2 -from contextlib import closing - -from lxml import html from PyQt4.Qt import QUrl -from calibre import browser from calibre.gui2 import open_url -from calibre.gui2.store import StorePlugin -from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.amazon_plugin import AmazonKindleStore -class AmazonKindleStore(StorePlugin): +class AmazonUKKindleStore(AmazonKindleStore): + + ''' + For comments on the implementation, please see amazon_plugin.py + ''' def open(self, parent=None, detail_item=None, external=False): - ''' - Amazon comes with a number of difficulties. - - QWebView has major issues with Amazon.com. The largest of - issues is it simply doesn't work on a number of pages. - - When connecting to a number parts of Amazon.com (Kindle library - for instance) QNetworkAccessManager fails to connect with a - NetworkError of 399 - ProtocolFailure. The strange thing is, - when I check QNetworkRequest.HttpStatusCodeAttribute when the - 399 error is returned the status code is 200 (Ok). However, once - the QNetworkAccessManager decides there was a NetworkError it - does not download the page from Amazon. So I can't even set the - HTML in the QWebView myself. - - There is http://bugreports.qt.nokia.com/browse/QTWEBKIT-259 an - open bug about the issue but it is not correct. We can set the - useragent (Arora does) to something else and the above issue - will persist. This http://developer.qt.nokia.com/forums/viewthread/793 - gives a bit more information about the issue but as of now (27/Feb/2011) - there is no solution or work around. - - We cannot change the The linkDelegationPolicy to allow us to avoid - QNetworkAccessManager because it only works links. Forms aren't - included so the same issue persists on any part of the site (login) - that use a form to load a new page. - - Using an aStore was evaluated but I've decided against using it. - There are three major issues with an aStore. Because checkout is - handled by sending the user to Amazon we can't put it in a QWebView. - If we're sending the user to Amazon sending them there directly is - nicer. Also, we cannot put the aStore in a QWebView and let it open the - redirection the users default browser because the cookies with the - shopping cart won't transfer. - - Another issue with the aStore is how it handles the referral. It only - counts the referral for the items in the shopping card / the item - that directed the user to Amazon. Kindle books do not use the shopping - cart and send the user directly to Amazon for the purchase. In this - instance we would only get referral credit for the one book that the - aStore directs to Amazon that the user buys. Any other purchases we - won't get credit for. - - The last issue with the aStore is performance. Even though it's an - Amazon site it's alow. So much slower than Amazon.com that it makes - me not want to browse books using it. The look and feel are lesser - issues. So is the fact that it almost seems like the purchase is - with calibre. This can cause some support issues because we can't - do much for issues with Amazon.com purchase hiccups. - - Another option that was evaluated was the Product Advertising API. - The reasons against this are complexity. It would take a lot of work - to basically re-create Amazon.com within calibre. The Product - Advertising API is also designed with being run on a server not - in an app. The signing keys would have to be made avaliable to ever - calibre user which means bad things could be done with our account. - - The Product Advertising API also assumes the same browser for easy - shopping cart transfer to Amazon. With QWebView not working and there - not being an easy way to transfer cookies between a QWebView and the - users default browser this won't work well. - - We could create our own website on the calibre server and create an - Amazon Product Advertising API store. However, this goes back to the - complexity argument. Why spend the time recreating Amazon.com - - The final and largest issue against using the Product Advertising API - is the Efficiency Guidelines: - - "Each account used to access the Product Advertising API will be allowed - an initial usage limit of 2,000 requests per hour. Each account will - receive an additional 500 requests per hour (up to a maximum of 25,000 - requests per hour) for every $1 of shipped item revenue driven per hour - in a trailing 30-day period. Usage thresholds are recalculated daily based - on revenue performance." - - With over two million users a limit of 2,000 request per hour could - render our store unusable for no other reason than Amazon rate - limiting our traffic. - - The best (I use the term lightly here) solution is to open Amazon.com - in the users default browser and set the affiliate id as part of the url. - ''' - aff_id = {'tag': 'josbl0e-cpb-20'} - # Use Kovid's affiliate id 30% of the time. - if random.randint(1, 10) in (1, 2, 3): - aff_id['tag'] = 'calibrebs-20' - store_link = 'http://www.amazon.com/Kindle-eBooks/b/?ie=UTF&node=1286228011&ref_=%(tag)s&ref=%(tag)s&tag=%(tag)s&linkCode=ur2&camp=1789&creative=390957' % aff_id + aff_id = {'tag': 'calcharles-21'} + store_link = 'http://www.amazon.co.uk/Kindle-eBooks/b/?ie=UTF&node=1286228011&ref_=%(tag)s&ref=%(tag)s&tag=%(tag)s&linkCode=ur2&camp=1789&creative=390957' % aff_id if detail_item: aff_id['asin'] = detail_item - store_link = 'http://www.amazon.com/dp/%(asin)s/?tag=%(tag)s' % aff_id + store_link = 'http://www.amazon.co.uk/dp/%(asin)s/?tag=%(tag)s' % aff_id open_url(QUrl(store_link)) - def search(self, query, max_results=10, timeout=60): - url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' + urllib2.quote(query) - br = browser() - - counter = max_results - with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) - for data in doc.xpath('//div[@class="productData"]'): - if counter <= 0: - break - - # Even though we are searching digital-text only Amazon will still - # put in results for non Kindle books (author pages). Se we need - # to explicitly check if the item is a Kindle book and ignore it - # if it isn't. - type = ''.join(data.xpath('//span[@class="format"]/text()')) - if 'kindle' not in type.lower(): - continue - - # We must have an asin otherwise we can't easily reference the - # book later. - asin_href = None - asin_a = data.xpath('div[@class="productTitle"]/a[1]') - if asin_a: - asin_href = asin_a[0].get('href', '') - m = re.search(r'/dp/(?P.+?)(/|$)', asin_href) - if m: - asin = m.group('asin') - else: - continue - else: - continue - - cover_url = '' - if asin_href: - cover_img = data.xpath('//div[@class="productImage"]/a[@href="%s"]/img/@src' % asin_href) - if cover_img: - cover_url = cover_img[0] - parts = cover_url.split('/') - bn = parts[-1] - f, _, ext = bn.rpartition('.') - if '_' in f: - bn = f.partition('_')[0]+'_SL160_.'+ext - parts[-1] = bn - cover_url = '/'.join(parts) - - title = ''.join(data.xpath('div[@class="productTitle"]/a/text()')) - author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) - author = author.split('by')[-1] - price = ''.join(data.xpath('div[@class="newPrice"]/span/text()')) - - counter -= 1 - - s = SearchResult() - s.cover_url = cover_url - s.title = title.strip() - s.author = author.strip() - s.price = price.strip() - s.detail_item = asin.strip() - s.formats = 'Kindle' - - yield s - - def get_details(self, search_result, timeout): - url = 'http://amazon.com/dp/' - - br = browser() - with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: - idata = html.fromstring(nf.read()) - if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "Simultaneous Device Usage")])'): - if idata.xpath('boolean(//div[@class="content"]//li[contains(., "Unlimited") and contains(b, "Simultaneous Device Usage")])'): - search_result.drm = SearchResult.DRM_UNLOCKED - else: - search_result.drm = SearchResult.DRM_UNKNOWN - else: - search_result.drm = SearchResult.DRM_LOCKED - return True - + search_url = 'http://www.amazon.co.uk/s/url=search-alias%3Ddigital-text&field-keywords=' + details_url = 'http://amazon.co.uk/dp/' From 335b933c2ba46d6d8e6518b47e8725be4232d039 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 26 Apr 2011 12:45:43 +0100 Subject: [PATCH 03/29] First iteration of waterstones store plugin --- src/calibre/customize/builtins.py | 8 +- .../gui2/store/waterstones_uk_plugin.py | 73 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 src/calibre/gui2/store/waterstones_uk_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 4f426d7490..1e885baf0c 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1179,11 +1179,17 @@ class StoreSmashwordsStore(StoreBase): description = _('Your ebook. Your way.') actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' +class StoreWaterstonesUKStore(StoreBase): + name = 'Waterstones UK' + description = _('Feel every word') + actual_plugin = 'calibre.gui2.store.waterstones_uk_plugin:WaterstonesUKStore' + plugins += [StoreAmazonKindleStore, StoreAmazonUKKindleStore, StoreBaenWebScriptionStore, StoreBNStore, StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreEHarlequinStoretore, StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, - StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore] + StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore, + StoreWaterstonesUKStore] # }}} diff --git a/src/calibre/gui2/store/waterstones_uk_plugin.py b/src/calibre/gui2/store/waterstones_uk_plugin.py new file mode 100644 index 0000000000..6c0f415891 --- /dev/null +++ b/src/calibre/gui2/store/waterstones_uk_plugin.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class WaterstonesUKStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + url = 'http://clkuk.tradedoubler.com/click?p=51196&a=1951604&g=19333484' + + if external or self.config.get('open_external', False): + if detail_item: + url = url + detail_item + open_url(QUrl(url_slash_cleaner(url))) + else: + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.waterstones.com/waterstonesweb/advancedSearch.do?buttonClicked=1&format=3757&bookkeywords=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[contains(@class, "results-pane")]'): + print('here') + if counter <= 0: + break + + cover_url = ''.join(data.xpath('.//div[@class="image"]/a/img/@src')) + + title = ''.join(data.xpath('./div/div/h2/a/text()')) + author = ', '.join(data.xpath('.//p[@class="byAuthor"]/a/text()')) + price = ''.join(data.xpath('.//p[@class="price"]/span[@class="priceStandard"]/text()')) + print(title, author, price) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.drm = SearchResult.DRM_LOCKED + s.formats = 'EPUB' + + yield s From 97300dee5b74c9135dc6a86758246b57f673ac2e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 26 Apr 2011 13:09:08 +0100 Subject: [PATCH 04/29] Fixes for waterstone store plugin --- src/calibre/gui2/store/waterstones_uk_plugin.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/store/waterstones_uk_plugin.py b/src/calibre/gui2/store/waterstones_uk_plugin.py index 6c0f415891..a066979f25 100644 --- a/src/calibre/gui2/store/waterstones_uk_plugin.py +++ b/src/calibre/gui2/store/waterstones_uk_plugin.py @@ -6,8 +6,6 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import random -import re import urllib2 from contextlib import closing @@ -15,7 +13,7 @@ from lxml import html from PyQt4.Qt import QUrl -from calibre import browser, url_slash_cleaner +from calibre import browser from calibre.gui2 import open_url from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig @@ -29,12 +27,12 @@ class WaterstonesUKStore(BasicStoreConfig, StorePlugin): if external or self.config.get('open_external', False): if detail_item: - url = url + detail_item - open_url(QUrl(url_slash_cleaner(url))) + url = detail_item + open_url(QUrl(url)) else: detail_url = None if detail_item: - detail_url = url + detail_item + detail_url = detail_item d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(self.config.get('tags', '')) @@ -49,12 +47,13 @@ class WaterstonesUKStore(BasicStoreConfig, StorePlugin): with closing(br.open(url, timeout=timeout)) as f: doc = html.fromstring(f.read()) for data in doc.xpath('//div[contains(@class, "results-pane")]'): - print('here') if counter <= 0: break + id = ''.join(data.xpath('./div/div/h2/a/@href')).strip() + if not id: + continue cover_url = ''.join(data.xpath('.//div[@class="image"]/a/img/@src')) - title = ''.join(data.xpath('./div/div/h2/a/text()')) author = ', '.join(data.xpath('.//p[@class="byAuthor"]/a/text()')) price = ''.join(data.xpath('.//p[@class="price"]/span[@class="priceStandard"]/text()')) @@ -68,6 +67,7 @@ class WaterstonesUKStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price s.drm = SearchResult.DRM_LOCKED + s.detail_item = id s.formats = 'EPUB' yield s From 996c9e77088c199bddf61a76b3749909a79996b6 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 26 Apr 2011 14:35:54 +0100 Subject: [PATCH 05/29] Add Foyles. Fix waterstones to use deep linking. --- src/calibre/customize/builtins.py | 10 ++- src/calibre/gui2/store/foyles_uk_plugin.py | 75 +++++++++++++++++++ .../gui2/store/waterstones_uk_plugin.py | 6 +- 3 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 src/calibre/gui2/store/foyles_uk_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 1e885baf0c..29b9954caa 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1184,12 +1184,16 @@ class StoreWaterstonesUKStore(StoreBase): description = _('Feel every word') actual_plugin = 'calibre.gui2.store.waterstones_uk_plugin:WaterstonesUKStore' -plugins += [StoreAmazonKindleStore, StoreAmazonUKKindleStore, - StoreBaenWebScriptionStore, StoreBNStore, +class StoreFoylesUKStore(StoreBase): + name = 'Foyles UK' + description = _('Foyles of London, online') + actual_plugin = 'calibre.gui2.store.foyles_uk_plugin:FoylesUKStore' + +plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreBNStore, StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreEHarlequinStoretore, StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore, - StoreWaterstonesUKStore] + StoreAmazonUKKindleStore, StoreFoylesUKStore, StoreWaterstonesUKStore] # }}} diff --git a/src/calibre/gui2/store/foyles_uk_plugin.py b/src/calibre/gui2/store/foyles_uk_plugin.py new file mode 100644 index 0000000000..134f710ef0 --- /dev/null +++ b/src/calibre/gui2/store/foyles_uk_plugin.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class FoylesUKStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + url = 'http://www.awin1.com/cread.php?awinmid=1414&awinaffid=120917&clickref=&p=' + url_redirect = 'http://www.foyles.co.uk' + + if external or self.config.get('open_external', False): + if detail_item: + url = url + url_redirect + detail_item + open_url(QUrl(url_slash_cleaner(url))) + else: + detail_url = None + if detail_item: + detail_url = url + url_redirect + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.foyles.co.uk/Public/Shop/Search.aspx?fFacetId=1015&searchBy=1&quick=true&term=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//table[contains(@id, "MainContent")]/tr/td/div[contains(@class, "Item")]'): + if counter <= 0: + break + id = ''.join(data.xpath('.//a[@class="Title"]/@href')).strip() + if not id: + continue + + cover_url = ''.join(data.xpath('.//div[@class="image"]/a/img/@src')) + + title = ''.join(data.xpath('.//a[@class="Title"]/text()')) + author = ', '.join(data.xpath('.//span[@class="Author"]/text()')) + price = ''.join(data.xpath('./ul/li[@class="Strong"]/text()')) + price = price[price.rfind(' '):] + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.detail_item = id + s.drm = SearchResult.DRM_LOCKED + s.formats = 'EPUB' + + yield s diff --git a/src/calibre/gui2/store/waterstones_uk_plugin.py b/src/calibre/gui2/store/waterstones_uk_plugin.py index a066979f25..198b855393 100644 --- a/src/calibre/gui2/store/waterstones_uk_plugin.py +++ b/src/calibre/gui2/store/waterstones_uk_plugin.py @@ -24,15 +24,17 @@ class WaterstonesUKStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): url = 'http://clkuk.tradedoubler.com/click?p=51196&a=1951604&g=19333484' + url_details = 'http://clkuk.tradedoubler.com/click?p(51196)a(1951604)g(16460516)url({0})' if external or self.config.get('open_external', False): if detail_item: - url = detail_item + url = url_details.format(detail_item) open_url(QUrl(url)) else: detail_url = None if detail_item: - detail_url = detail_item + detail_url = url_details.format(detail_item) + print(detail_url) d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(self.config.get('tags', '')) From 81bd4d1a6e1b2a63be9afc824941b1e29a8aa267 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 26 Apr 2011 14:47:20 +0100 Subject: [PATCH 06/29] Remove some print statements --- src/calibre/gui2/store/waterstones_uk_plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/calibre/gui2/store/waterstones_uk_plugin.py b/src/calibre/gui2/store/waterstones_uk_plugin.py index 198b855393..6870f4cf86 100644 --- a/src/calibre/gui2/store/waterstones_uk_plugin.py +++ b/src/calibre/gui2/store/waterstones_uk_plugin.py @@ -34,7 +34,6 @@ class WaterstonesUKStore(BasicStoreConfig, StorePlugin): detail_url = None if detail_item: detail_url = url_details.format(detail_item) - print(detail_url) d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(self.config.get('tags', '')) @@ -59,7 +58,6 @@ class WaterstonesUKStore(BasicStoreConfig, StorePlugin): title = ''.join(data.xpath('./div/div/h2/a/text()')) author = ', '.join(data.xpath('.//p[@class="byAuthor"]/a/text()')) price = ''.join(data.xpath('.//p[@class="price"]/span[@class="priceStandard"]/text()')) - print(title, author, price) counter -= 1 From 2811b452fbc1e9ba9991939b7746be193f877a9e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 26 Apr 2011 16:09:17 +0100 Subject: [PATCH 07/29] Fix links. --- src/calibre/gui2/store/amazon_uk_plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/amazon_uk_plugin.py b/src/calibre/gui2/store/amazon_uk_plugin.py index e24ddb485e..7e128dc62f 100644 --- a/src/calibre/gui2/store/amazon_uk_plugin.py +++ b/src/calibre/gui2/store/amazon_uk_plugin.py @@ -20,10 +20,11 @@ class AmazonUKKindleStore(AmazonKindleStore): def open(self, parent=None, detail_item=None, external=False): aff_id = {'tag': 'calcharles-21'} - store_link = 'http://www.amazon.co.uk/Kindle-eBooks/b/?ie=UTF&node=1286228011&ref_=%(tag)s&ref=%(tag)s&tag=%(tag)s&linkCode=ur2&camp=1789&creative=390957' % aff_id + store_link = 'http://www.amazon.co.uk/gp/redirect.html?ie=UTF8&location=http://www.amazon.co.uk/Kindle-eBooks/b?ie=UTF8&node=341689031&ref_=sa_menu_kbo2&tag=%(tag)s&linkCode=ur2&camp=1634&creative=19450' % aff_id + if detail_item: aff_id['asin'] = detail_item - store_link = 'http://www.amazon.co.uk/dp/%(asin)s/?tag=%(tag)s' % aff_id + store_link = 'http://www.amazon.co.uk/gp/redirect.html?ie=UTF8&location=http://www.amazon.co.uk/dp/%(asin)s&tag=%(tag)s&linkCode=ur2&camp=1634&creative=6738' % aff_id open_url(QUrl(store_link)) search_url = 'http://www.amazon.co.uk/s/url=search-alias%3Ddigital-text&field-keywords=' From 7169ebe7d94d12d71c1e217ae88fee9f9ae18ead Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 26 Apr 2011 18:05:14 +0100 Subject: [PATCH 08/29] amazon.de store plugin --- src/calibre/customize/builtins.py | 10 +++++-- src/calibre/gui2/store/amazon_de_plugin.py | 31 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 src/calibre/gui2/store/amazon_de_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 29b9954caa..8973548b9d 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1189,11 +1189,17 @@ class StoreFoylesUKStore(StoreBase): description = _('Foyles of London, online') actual_plugin = 'calibre.gui2.store.foyles_uk_plugin:FoylesUKStore' -plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreBNStore, +class AmazonDEKindleStore(StoreBase): + name = 'Amazon DE Kindle' + description = _('Kindle eBooks') + actual_plugin = 'calibre.gui2.store.amazon_de_plugin:AmazonDEKindleStore' + +plugins += [StoreAmazonKindleStore, AmazonDEKindleStore, StoreAmazonUKKindleStore, + StoreBaenWebScriptionStore, StoreBNStore, StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreEHarlequinStoretore, StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore, - StoreAmazonUKKindleStore, StoreFoylesUKStore, StoreWaterstonesUKStore] + StoreFoylesUKStore, StoreWaterstonesUKStore] # }}} diff --git a/src/calibre/gui2/store/amazon_de_plugin.py b/src/calibre/gui2/store/amazon_de_plugin.py new file mode 100644 index 0000000000..94583f8987 --- /dev/null +++ b/src/calibre/gui2/store/amazon_de_plugin.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + + +from PyQt4.Qt import QUrl + +from calibre.gui2 import open_url +from calibre.gui2.store.amazon_plugin import AmazonKindleStore + +class AmazonDEKindleStore(AmazonKindleStore): + + ''' + For comments on the implementation, please see amazon_plugin.py + ''' + + def open(self, parent=None, detail_item=None, external=False): + aff_id = {'tag': 'charhale0a-21'} + store_link = 'http://www.amazon.de/gp/redirect.html?ie=UTF8&location=http://www.amazon.de/&site-redirect=de&tag=%(tag)s&linkCode=ur2&camp=1638&creative=6742' % aff_id + if detail_item: + aff_id['asin'] = detail_item + store_link = 'http://www.amazon.de/gp/redirect.html?ie=UTF8&location=http://www.amazon.de/dp/%(asin)s&site-redirect=de&tag=%(tag)s&linkCode=ur2&camp=1638&creative=6742' % aff_id + open_url(QUrl(store_link)) + + search_url = 'http://www.amazon.de/s/url=search-alias%3Ddigital-text&field-keywords=' + details_url = 'http://amazon.de/dp/' + From 061fbd44c3423f2498edf165dae8ae504d61682f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 27 Apr 2011 10:21:15 +0100 Subject: [PATCH 09/29] 1) Fix cover problem with FoylesUKStore 2) Add PDF format and DRM verification to WaterstonesUKStore 3) Make store URL go to kindle store for amazon_de_plugin.py 4) Change amazon_plugin.py and amazon_de_plugin.py to recognize DRM words in German --- src/calibre/gui2/store/amazon_de_plugin.py | 13 +++++++++---- src/calibre/gui2/store/amazon_plugin.py | 9 +++++++-- src/calibre/gui2/store/foyles_uk_plugin.py | 5 ++++- src/calibre/gui2/store/waterstones_uk_plugin.py | 15 +++++++++++++-- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/store/amazon_de_plugin.py b/src/calibre/gui2/store/amazon_de_plugin.py index 94583f8987..2baa90360c 100644 --- a/src/calibre/gui2/store/amazon_de_plugin.py +++ b/src/calibre/gui2/store/amazon_de_plugin.py @@ -6,7 +6,6 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' - from PyQt4.Qt import QUrl from calibre.gui2 import open_url @@ -20,12 +19,18 @@ class AmazonDEKindleStore(AmazonKindleStore): def open(self, parent=None, detail_item=None, external=False): aff_id = {'tag': 'charhale0a-21'} - store_link = 'http://www.amazon.de/gp/redirect.html?ie=UTF8&location=http://www.amazon.de/&site-redirect=de&tag=%(tag)s&linkCode=ur2&camp=1638&creative=6742' % aff_id + store_link = ('http://www.amazon.de/gp/redirect.html?ie=UTF8&site-redirect=de' + '&tag=%(tag)s&linkCode=ur2&camp=1638&creative=19454' + '&location=http://www.amazon.de/ebooks-kindle/b?node=530886031') % aff_id if detail_item: aff_id['asin'] = detail_item - store_link = 'http://www.amazon.de/gp/redirect.html?ie=UTF8&location=http://www.amazon.de/dp/%(asin)s&site-redirect=de&tag=%(tag)s&linkCode=ur2&camp=1638&creative=6742' % aff_id + store_link = ('http://www.amazon.de/gp/redirect.html?ie=UTF8' + '&location=http://www.amazon.de/dp/%(asin)s&site-redirect=de' + '&tag=%(tag)s&linkCode=ur2&camp=1638&creative=6742') % aff_id open_url(QUrl(store_link)) search_url = 'http://www.amazon.de/s/url=search-alias%3Ddigital-text&field-keywords=' - details_url = 'http://amazon.de/dp/' + details_url = 'http://amazon.de/dp/' + drm_search_text = u'Gleichzeitige Verwendung von Geräten' + drm_free_text = u'Keine Einschränkung' \ No newline at end of file diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 55fb613288..179c6ffaef 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -181,14 +181,19 @@ class AmazonKindleStore(StorePlugin): yield s details_url = 'http://amazon.com/dp/' + drm_search_text = u'Simultaneous Device Usage' + drm_free_text = u'Unlimited' def get_details(self, search_result, timeout): url = self.details_url br = browser() with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: idata = html.fromstring(nf.read()) - if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "Simultaneous Device Usage")])'): - if idata.xpath('boolean(//div[@class="content"]//li[contains(., "Unlimited") and contains(b, "Simultaneous Device Usage")])'): + if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' + + self.drm_search_text + '")])'): + if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' + + self.drm_free_text + '") and contains(b, "' + + self.drm_search_text + '")])'): search_result.drm = SearchResult.DRM_UNLOCKED else: search_result.drm = SearchResult.DRM_UNKNOWN diff --git a/src/calibre/gui2/store/foyles_uk_plugin.py b/src/calibre/gui2/store/foyles_uk_plugin.py index 134f710ef0..b2ad3e5de1 100644 --- a/src/calibre/gui2/store/foyles_uk_plugin.py +++ b/src/calibre/gui2/store/foyles_uk_plugin.py @@ -54,7 +54,10 @@ class FoylesUKStore(BasicStoreConfig, StorePlugin): if not id: continue - cover_url = ''.join(data.xpath('.//div[@class="image"]/a/img/@src')) + cover_url = ''.join(data.xpath('.//a[@class="Jacket"]/img/@src')) + if cover_url: + cover_url = 'http://www.foyles.co.uk' + cover_url + print(cover_url) title = ''.join(data.xpath('.//a[@class="Title"]/text()')) author = ', '.join(data.xpath('.//span[@class="Author"]/text()')) diff --git a/src/calibre/gui2/store/waterstones_uk_plugin.py b/src/calibre/gui2/store/waterstones_uk_plugin.py index 6870f4cf86..d422165c47 100644 --- a/src/calibre/gui2/store/waterstones_uk_plugin.py +++ b/src/calibre/gui2/store/waterstones_uk_plugin.py @@ -58,6 +58,9 @@ class WaterstonesUKStore(BasicStoreConfig, StorePlugin): title = ''.join(data.xpath('./div/div/h2/a/text()')) author = ', '.join(data.xpath('.//p[@class="byAuthor"]/a/text()')) price = ''.join(data.xpath('.//p[@class="price"]/span[@class="priceStandard"]/text()')) + drm = data.xpath('boolean(.//td[@headers="productFormat" and contains(., "DRM")])') + pdf = data.xpath('boolean(.//td[@headers="productFormat" and contains(., "PDF")])') + epub = data.xpath('boolean(.//td[@headers="productFormat" and contains(., "EPUB")])') counter -= 1 @@ -66,8 +69,16 @@ class WaterstonesUKStore(BasicStoreConfig, StorePlugin): s.title = title.strip() s.author = author.strip() s.price = price - s.drm = SearchResult.DRM_LOCKED + if drm: + s.drm = SearchResult.DRM_LOCKED + else: + s.drm = SearchResult.DRM_UNKNOWN s.detail_item = id - s.formats = 'EPUB' + formats = [] + if epub: + formats.append('EPUB') + if pdf: + formats.append('PDF') + s.formats = ', '.join(formats) yield s From 86530031b1d254b0b470415b633288cb126cf192 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Apr 2011 09:04:25 -0600 Subject: [PATCH 10/29] Social Diva by Soilviu Cotoara --- recipes/icons/socialdiva.png | Bin 0 -> 1071 bytes recipes/socialdiva.recipe | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 recipes/icons/socialdiva.png create mode 100644 recipes/socialdiva.recipe diff --git a/recipes/icons/socialdiva.png b/recipes/icons/socialdiva.png new file mode 100644 index 0000000000000000000000000000000000000000..262a94432d8aa1ed6e0df34e05bd4967650d3add GIT binary patch literal 1071 zcmV+~1kn45P)+q_s{IChvy~Ddc!u^{VtPl=9_oE=Xsy^`QCAW zI=e8E@hAW{Cf)Ic9n>3;JW2z0$glVK#ntG{E>QAS80jQRxrM9GN-)-{n9LxEclyWv7 z2zk{I9gM0PQl1=duCNL~HjxBi?DViARmDiIBW|Ik5yug8m zc5w}Wxq0pIa5x6QUfbKMudNmyECTS#>(2r3#<>>&@XI9t+(ZHZC$PxIM6IT)>q+&I zn6y;3Q?&UE%v6NDv!+um>!{9uGw)vr3Bf1tzYW0gr-V4Zs+>m>23b;K%hhu)>@{2? zQ6Y{-RTcLZ3P39`;FtprCE5VUx?-Z_R{`iwNd04jqX7J|{u2P{^}7HB_k`I0HLv&D z6)$z757^FqU_=#`bXI)&(FD)}3^-{wTnBh9rJk)TA~oMN4FrMe{&c1TfNx#-!};+c z07g2x6{ufc@Ac~aa7Io!KQ#@&FV|)O`10~q0G6iz1X_Rr&l(s4%T=4@{GlNa1zPdL zid1{K_zwV6pDC9UAG`^`k*8At)ErgK-?xQ)qa>fn9MgZol}qXsE560wZA4Exyr}hn zced1_3|ciL&B=Z&$tU{lRLpK_ecV&EB&{P_S(H;2*MvA~XGd4hApkDC^Em+JYGa4h zddOrZ3p1px5fcTXEYD&Psr_52Q-`4`de3T0x+{E zh7WbAYU=eC(SYMv0~|3eJnjLr)&X@|4|u&S1V?(*@86pI7=UZn^_Tn2q#W|Q$s53S pe^B$KrG@2!&E>FV>oc0pz`utgI)|LX4L|?@002ovPDHLkV1hk5{W$;t literal 0 HcmV?d00001 diff --git a/recipes/socialdiva.recipe b/recipes/socialdiva.recipe new file mode 100644 index 0000000000..1befd03d4a --- /dev/null +++ b/recipes/socialdiva.recipe @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011' +''' +socialdiva.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class SocialDiva(BasicNewsRecipe): + title = u'Social Diva' + __author__ = u'Silviu Cotoara' + description = u'When in doubt, wear red' + publisher = 'Social Diva' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Reviste,Femei' + encoding = 'utf-8' + cover_url = 'http://www.socialdiva.ro/images/logo.png' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='div', attrs={'class':'col-alpha mt5 content_articol'}), + dict(name='div', attrs={'class':'mt5'}) + ] + + remove_tags = [ + dict(name='a', attrs={'class':['comments float-left scroll mt5']}), + dict(name='a', attrs={'class':['comments float-left scroll']}), + dict(name='div', attrs={'class':['rating-container relative float-left']}), + dict(name='div', attrs={'class':['float-right social_articol']}) + ] + + remove_tags_after = [ + dict(name='a', attrs={'class':['comments float-left scroll mt5']}) + ] + + feeds = [ + (u'Feeds', u'http://www.socialdiva.ro/rss.html') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) From 1a212506e2687ee5745659d0ba9e36a022f8b284 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Apr 2011 10:04:42 -0600 Subject: [PATCH 11/29] When adding quick start guide also add its cover --- resources/quick_start.epub | Bin 130557 -> 130640 bytes src/calibre/gui2/ui.py | 7 ++----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/resources/quick_start.epub b/resources/quick_start.epub index 2d590ebef268689c4d742c29ae161f719efd4f1c..3b289537a68d390883048ea0c099950fd46893c6 100644 GIT binary patch delta 1111 zcmezSoBhHc_6<6W^-lh??LK`@yK|0-f#EYR1A`!g41>R`Q$VnOa(-S(YF>$6enDDj z2qyzGx9tvJ5H79YW?*D_!OXw_CbmxZEuL*4(DwePs8npUK*I6(=D9PM8PEN4t+4nT z$Hrq_8x=fE&h3A%B;y>sJke?AjSX-1{oW9@Mc|$czvj<+*4Ak=qk`^#)nHWJ6tODB z(|dDi+LVmLp=U1p9Gqm=yv%&{YJ=xV=Y7;&D3nRAFCt*$-m&9+%ssBFJY$dbL-d##6HVqFA&G$@;JMw7Fyy>o4Q^j@zWVcxnK|y8 zGwgc*NV=FsD&Jjl`n1N>s&=)&>Bp+fuHD)grL1I>8j#@W+P32ISe@CpOMb_s=nKzO)Ocl1tYe7DoXi>Csl2syBRhMCYWbPs zxV66XmIoCyAA0lRh?-Ns{|$BCaPRt8?;2dMuPIfZn)Z6jS{?25Zxfu00<&}H1z)-J zEuTFhW?`xBk6YKmObblTMCoVz+ams2^rlVN*pzUE`aAJk_iHw8^h2axE9#FU*>$r}cZDtAUOdcMiw_HZD z67?z9L{rQRPVCV>Q4=As|Mj_7{ChV4iY*WOU;k#!x2x*MEwB6in!YicN#^BRO}SSw zS_&5XUT?hX@E8O4gaDP+Uf5Wr?MrQww$o?BY zk0<|?t9|5#ZerQS!szowQI1)Jfq{X80W*_G0L2*?7^mm{V3d*w4Pk|3545Z?z5NHH7Nhm_ zO+Of|7)z(W{lVzW*fHJkC!+^*>KexB6Sp&JOwV7#$Uc4bPev`~HR~Wk(|$1;GUslB z%Ey3s>$ih>`qTIS0xFE#1CHb|#vdSW$wMe_-9wKOn4 delta 1008 zcmccchyCww_6<6W^-Uo&?93P%udHWcVA#mZz#z!Lz~Jxd6cDVRoS#>cnpdKiUy!yn z{BHhi1A+SY@rgf6nLb~2P+D-wD)ZI7Woyn)G!%a%#>2XK+5f*iiJY%5iBA0}QgrA2 z;pd0hAD-QK;-dcw>9{VA+sj%7Il`93qzL9cy~kmYr6Q_%)|*kVuCbo&{*mxR*PF^` z88mlfDAi2ZbW7mVpFcUDw%nSXI#=-crkbFz>Yk{B^P>zZxST%Owp$&VfBMtMcieW~ z)?QZooHTSCE=}>e6`UVbmbCts)%7!)$`hOy-jGU~ZFbOkoly00ujLC&g0@YJc_wiG z(j0Zu1@Cp!`!lpNbuV5IljJ?AS^u0R(KI0PnulQSE2HqM3)UX_xc!sDg=w-2JQiHr zq4=@q%*qo{q3a}C{4D<3r_RZ4uBg)USHD_v#p}(#IM+wp#Ab(C-my4STGL+D_WoAq z!<}&nYu1)#t@)NWVa?wodxfU1t;B?zVTj7w1&S|=ERN`CfTfj-zwf|TxCb9_WoMbFeTV|Ur21~ zYOUuzUamfUZd$@qKYG6Xbn|-0t91ceb53Na%07Sbe#Xr5lTQ{ly-~Uzw(m*T>WlX5 z44Ipjz2uBPer%0U-=r5WSkKA-O1q!6d-=MnWxm(zvv#}b2EEfcZPd?xQ`hhJyh%UO z*6zBUFZL={`CEwcw?l4oR=Uk;?cEXFyMtT$<1(p_VrE6QS5JIBAL_J84~SK`yX&hFg!BL3h?rmQ9X%PMY! z#4rRs-YtJynfp|1L(r?`zq1XCEb}jI_|0%_USnxm!d=OTe70{@Y-Og*ug@P~Gv4sJ zKl!a>ZR-D<;ffdeoA;gBb-t}J;G)CYeTyH==U>X_@?%^5{de;Ze>`38!FFx}&;00* zSM>wDnOUYk_`xXE9Lcghl7&(4i?s-|2rzYXFknmbKnXBd(x}9UB?nBu^@CB1kzqRP zPev<7tLY9u8J!t@r?>oM^kCLn!#I86K}L<~-+wY1G8e9c@`8T>c}AO{y!AjHW6AV~ zzZj#LmG?je(|-e%cpe1v^rxTt&8WrPei$kz{|BhSd3xXeKuFG8!?)PCxjUF`7|+y6itjO)g0`28MIrf!<*N0B`Wj+yDRo diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 9a4e0ca70a..4885f7b2db 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -278,11 +278,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.library_view.model().count_changed_signal.connect( self.iactions['Choose Library'].count_changed) if not gprefs.get('quick_start_guide_added', False): - from calibre.ebooks.metadata import MetaInformation - mi = MetaInformation(_('Calibre Quick Start Guide'), ['John Schember']) - mi.author_sort = 'Schember, John' - mi.comments = "A guide to get you up and running with calibre" - mi.publisher = 'calibre' + from calibre.ebooks.metadata.meta import get_metadata + mi = get_metadata(open(P('quick_start.epub'), 'rb'), 'epub') self.library_view.model().add_books([P('quick_start.epub')], ['epub'], [mi]) gprefs['quick_start_guide_added'] = True From 007ad0e7c4bcd4038536c08e4ae36a74316aae1e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Apr 2011 10:34:10 -0600 Subject: [PATCH 12/29] Avoid running all the schema upgrades when creating a new database by merging them into the base SQL --- resources/metadata_sqlite.sql | 311 +++++++++++++++++++++++++------ src/calibre/library/database2.py | 7 +- src/calibre/utils/resources.py | 3 +- 3 files changed, 264 insertions(+), 57 deletions(-) diff --git a/resources/metadata_sqlite.sql b/resources/metadata_sqlite.sql index 2d95f735e2..9c4f666449 100644 --- a/resources/metadata_sqlite.sql +++ b/resources/metadata_sqlite.sql @@ -7,17 +7,30 @@ CREATE TABLE books ( id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL DEFAULT 'Unknown' COLLATE NOCASE, sort TEXT COLLATE NOCASE, timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - uri TEXT, - series_index INTEGER NOT NULL DEFAULT 1, + pubdate TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + series_index REAL NOT NULL DEFAULT 1.0, author_sort TEXT COLLATE NOCASE, isbn TEXT DEFAULT "" COLLATE NOCASE, - path TEXT NOT NULL DEFAULT "" - ); + lccn TEXT DEFAULT "" COLLATE NOCASE, + path TEXT NOT NULL DEFAULT "", + flags INTEGER NOT NULL DEFAULT 1 + , uuid TEXT, has_cover BOOL DEFAULT 0, last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00"); CREATE TABLE books_authors_link ( id INTEGER PRIMARY KEY, book INTEGER NOT NULL, author INTEGER NOT NULL, UNIQUE(book, author) ); +CREATE TABLE books_languages_link ( id INTEGER PRIMARY KEY, + book INTEGER NOT NULL, + lang_code INTEGER NOT NULL, + item_order INTEGER NOT NULL DEFAULT 0, + UNIQUE(book, lang_code) + ); +CREATE TABLE books_plugin_data(id INTEGER PRIMARY KEY, + book INTEGER NON NULL, + name TEXT NON NULL, + val TEXT NON NULL, + UNIQUE(book,name)); CREATE TABLE books_publishers_link ( id INTEGER PRIMARY KEY, book INTEGER NOT NULL, publisher INTEGER NOT NULL, @@ -49,11 +62,51 @@ CREATE TABLE conversion_options ( id INTEGER PRIMARY KEY, data BLOB NOT NULL, UNIQUE(format,book) ); +CREATE TABLE custom_columns ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT NOT NULL, + name TEXT NOT NULL, + datatype TEXT NOT NULL, + mark_for_delete BOOL DEFAULT 0 NOT NULL, + editable BOOL DEFAULT 1 NOT NULL, + display TEXT DEFAULT "{}" NOT NULL, + is_multiple BOOL DEFAULT 0 NOT NULL, + normalized BOOL NOT NULL, + UNIQUE(label) + ); +CREATE TABLE data ( id INTEGER PRIMARY KEY, + book INTEGER NON NULL, + format TEXT NON NULL COLLATE NOCASE, + uncompressed_size INTEGER NON NULL, + name TEXT NON NULL, + UNIQUE(book, format) +); CREATE TABLE feeds ( id INTEGER PRIMARY KEY, title TEXT NOT NULL, script TEXT NOT NULL, UNIQUE(title) ); +CREATE TABLE identifiers ( id INTEGER PRIMARY KEY, + book INTEGER NON NULL, + type TEXT NON NULL DEFAULT "isbn" COLLATE NOCASE, + val TEXT NON NULL COLLATE NOCASE, + UNIQUE(book, type) + ); +CREATE TABLE languages ( id INTEGER PRIMARY KEY, + lang_code TEXT NON NULL COLLATE NOCASE, + UNIQUE(lang_code) + ); +CREATE TABLE library_id ( id INTEGER PRIMARY KEY, + uuid TEXT NOT NULL, + UNIQUE(uuid) + ); +CREATE TABLE metadata_dirtied(id INTEGER PRIMARY KEY, + book INTEGER NOT NULL, + UNIQUE(book)); +CREATE TABLE preferences(id INTEGER PRIMARY KEY, + key TEXT NON NULL, + val TEXT NON NULL, + UNIQUE(key)); CREATE TABLE publishers ( id INTEGER PRIMARY KEY, name TEXT NOT NULL COLLATE NOCASE, sort TEXT COLLATE NOCASE, @@ -72,34 +125,143 @@ CREATE TABLE tags ( id INTEGER PRIMARY KEY, name TEXT NOT NULL COLLATE NOCASE, UNIQUE (name) ); -CREATE TABLE data ( id INTEGER PRIMARY KEY, - book INTEGER NON NULL, - format TEXT NON NULL COLLATE NOCASE, - uncompressed_size INTEGER NON NULL, - name TEXT NON NULL, - UNIQUE(book, format) -); - CREATE VIEW meta AS - SELECT id, title, - (SELECT concat(name) FROM authors WHERE authors.id IN (SELECT author from books_authors_link WHERE book=books.id)) authors, - (SELECT name FROM publishers WHERE publishers.id IN (SELECT publisher from books_publishers_link WHERE book=books.id)) publisher, - (SELECT rating FROM ratings WHERE ratings.id IN (SELECT rating from books_ratings_link WHERE book=books.id)) rating, - timestamp, - (SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size, - (SELECT concat(name) FROM tags WHERE tags.id IN (SELECT tag from books_tags_link WHERE book=books.id)) tags, - (SELECT text FROM comments WHERE book=books.id) comments, - (SELECT name FROM series WHERE series.id IN (SELECT series FROM books_series_link WHERE book=books.id)) series, - series_index, - sort, - author_sort, - (SELECT concat(format) FROM data WHERE data.book=books.id) formats, - isbn - FROM books; + SELECT id, title, + (SELECT sortconcat(bal.id, name) FROM books_authors_link AS bal JOIN authors ON(author = authors.id) WHERE book = books.id) authors, + (SELECT name FROM publishers WHERE publishers.id IN (SELECT publisher from books_publishers_link WHERE book=books.id)) publisher, + (SELECT rating FROM ratings WHERE ratings.id IN (SELECT rating from books_ratings_link WHERE book=books.id)) rating, + timestamp, + (SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size, + (SELECT concat(name) FROM tags WHERE tags.id IN (SELECT tag from books_tags_link WHERE book=books.id)) tags, + (SELECT text FROM comments WHERE book=books.id) comments, + (SELECT name FROM series WHERE series.id IN (SELECT series FROM books_series_link WHERE book=books.id)) series, + series_index, + sort, + author_sort, + (SELECT concat(format) FROM data WHERE data.book=books.id) formats, + isbn, + path, + lccn, + pubdate, + flags, + uuid + FROM books; +CREATE VIEW tag_browser_authors AS SELECT + id, + name, + (SELECT COUNT(id) FROM books_authors_link WHERE author=authors.id) count, + (SELECT AVG(ratings.rating) + FROM books_authors_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.author=authors.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0) avg_rating, + sort AS sort + FROM authors; +CREATE VIEW tag_browser_filtered_authors AS SELECT + id, + name, + (SELECT COUNT(books_authors_link.id) FROM books_authors_link WHERE + author=authors.id AND books_list_filter(book)) count, + (SELECT AVG(ratings.rating) + FROM books_authors_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.author=authors.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0 AND + books_list_filter(bl.book)) avg_rating, + sort AS sort + FROM authors; +CREATE VIEW tag_browser_filtered_publishers AS SELECT + id, + name, + (SELECT COUNT(books_publishers_link.id) FROM books_publishers_link WHERE + publisher=publishers.id AND books_list_filter(book)) count, + (SELECT AVG(ratings.rating) + FROM books_publishers_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.publisher=publishers.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0 AND + books_list_filter(bl.book)) avg_rating, + name AS sort + FROM publishers; +CREATE VIEW tag_browser_filtered_ratings AS SELECT + id, + rating, + (SELECT COUNT(books_ratings_link.id) FROM books_ratings_link WHERE + rating=ratings.id AND books_list_filter(book)) count, + (SELECT AVG(ratings.rating) + FROM books_ratings_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.rating=ratings.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0 AND + books_list_filter(bl.book)) avg_rating, + rating AS sort + FROM ratings; +CREATE VIEW tag_browser_filtered_series AS SELECT + id, + name, + (SELECT COUNT(books_series_link.id) FROM books_series_link WHERE + series=series.id AND books_list_filter(book)) count, + (SELECT AVG(ratings.rating) + FROM books_series_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.series=series.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0 AND + books_list_filter(bl.book)) avg_rating, + (title_sort(name)) AS sort + FROM series; +CREATE VIEW tag_browser_filtered_tags AS SELECT + id, + name, + (SELECT COUNT(books_tags_link.id) FROM books_tags_link WHERE + tag=tags.id AND books_list_filter(book)) count, + (SELECT AVG(ratings.rating) + FROM books_tags_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.tag=tags.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0 AND + books_list_filter(bl.book)) avg_rating, + name AS sort + FROM tags; +CREATE VIEW tag_browser_publishers AS SELECT + id, + name, + (SELECT COUNT(id) FROM books_publishers_link WHERE publisher=publishers.id) count, + (SELECT AVG(ratings.rating) + FROM books_publishers_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.publisher=publishers.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0) avg_rating, + name AS sort + FROM publishers; +CREATE VIEW tag_browser_ratings AS SELECT + id, + rating, + (SELECT COUNT(id) FROM books_ratings_link WHERE rating=ratings.id) count, + (SELECT AVG(ratings.rating) + FROM books_ratings_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.rating=ratings.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0) avg_rating, + rating AS sort + FROM ratings; +CREATE VIEW tag_browser_series AS SELECT + id, + name, + (SELECT COUNT(id) FROM books_series_link WHERE series=series.id) count, + (SELECT AVG(ratings.rating) + FROM books_series_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.series=series.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0) avg_rating, + (title_sort(name)) AS sort + FROM series; +CREATE VIEW tag_browser_tags AS SELECT + id, + name, + (SELECT COUNT(id) FROM books_tags_link WHERE tag=tags.id) count, + (SELECT AVG(ratings.rating) + FROM books_tags_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.tag=tags.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0) avg_rating, + name AS sort + FROM tags; CREATE INDEX authors_idx ON books (author_sort COLLATE NOCASE); CREATE INDEX books_authors_link_aidx ON books_authors_link (author); CREATE INDEX books_authors_link_bidx ON books_authors_link (book); CREATE INDEX books_idx ON books (sort COLLATE NOCASE); +CREATE INDEX books_languages_link_aidx ON books_languages_link (lang_code); +CREATE INDEX books_languages_link_bidx ON books_languages_link (book); CREATE INDEX books_publishers_link_aidx ON books_publishers_link (publisher); CREATE INDEX books_publishers_link_bidx ON books_publishers_link (book); CREATE INDEX books_ratings_link_aidx ON books_ratings_link (rating); @@ -111,32 +273,38 @@ CREATE INDEX books_tags_link_bidx ON books_tags_link (book); CREATE INDEX comments_idx ON comments (book); CREATE INDEX conversion_options_idx_a ON conversion_options (format COLLATE NOCASE); CREATE INDEX conversion_options_idx_b ON conversion_options (book); +CREATE INDEX custom_columns_idx ON custom_columns (label); CREATE INDEX data_idx ON data (book); +CREATE INDEX formats_idx ON data (format); +CREATE INDEX languages_idx ON languages (lang_code COLLATE NOCASE); CREATE INDEX publishers_idx ON publishers (name COLLATE NOCASE); -CREATE INDEX series_idx ON series (sort COLLATE NOCASE); +CREATE INDEX series_idx ON series (name COLLATE NOCASE); CREATE INDEX tags_idx ON tags (name COLLATE NOCASE); CREATE TRIGGER books_delete_trg - AFTER DELETE ON books - BEGIN - DELETE FROM books_authors_link WHERE book=OLD.id; - DELETE FROM books_publishers_link WHERE book=OLD.id; - DELETE FROM books_ratings_link WHERE book=OLD.id; - DELETE FROM books_series_link WHERE book=OLD.id; - DELETE FROM books_tags_link WHERE book=OLD.id; - DELETE FROM data WHERE book=OLD.id; - DELETE FROM comments WHERE book=OLD.id; - DELETE FROM conversion_options WHERE book=OLD.id; + AFTER DELETE ON books + BEGIN + DELETE FROM books_authors_link WHERE book=OLD.id; + DELETE FROM books_publishers_link WHERE book=OLD.id; + DELETE FROM books_ratings_link WHERE book=OLD.id; + DELETE FROM books_series_link WHERE book=OLD.id; + DELETE FROM books_tags_link WHERE book=OLD.id; + DELETE FROM books_languages_link WHERE book=OLD.id; + DELETE FROM data WHERE book=OLD.id; + DELETE FROM comments WHERE book=OLD.id; + DELETE FROM conversion_options WHERE book=OLD.id; + DELETE FROM books_plugin_data WHERE book=OLD.id; + DELETE FROM identifiers WHERE book=OLD.id; END; -CREATE TRIGGER books_insert_trg - AFTER INSERT ON books +CREATE TRIGGER books_insert_trg AFTER INSERT ON books BEGIN - UPDATE books SET sort=title_sort(NEW.title) WHERE id=NEW.id; + UPDATE books SET sort=title_sort(NEW.title),uuid=uuid4() WHERE id=NEW.id; END; CREATE TRIGGER books_update_trg - AFTER UPDATE ON books - BEGIN - UPDATE books SET sort=title_sort(NEW.title) WHERE id=NEW.id; - END; + AFTER UPDATE ON books + BEGIN + UPDATE books SET sort=title_sort(NEW.title) + WHERE id=NEW.id AND OLD.title <> NEW.title; + END; CREATE TRIGGER fkc_comments_insert BEFORE INSERT ON comments BEGIN @@ -169,23 +337,41 @@ CREATE TRIGGER fkc_data_update THEN RAISE(ABORT, 'Foreign key violation: book not in books') END; END; -CREATE TRIGGER fkc_delete_books_authors_link +CREATE TRIGGER fkc_delete_on_authors BEFORE DELETE ON authors BEGIN SELECT CASE - WHEN (SELECT COUNT(id) FROM books_authors_link WHERE book=OLD.book) > 0 - THEN RAISE(ABORT, 'Foreign key violation: author is still referenced') + WHEN (SELECT COUNT(id) FROM books_authors_link WHERE author=OLD.id) > 0 + THEN RAISE(ABORT, 'Foreign key violation: authors is still referenced') END; END; -CREATE TRIGGER fkc_delete_books_publishers_link +CREATE TRIGGER fkc_delete_on_languages + BEFORE DELETE ON languages + BEGIN + SELECT CASE + WHEN (SELECT COUNT(id) FROM books_languages_link WHERE lang_code=OLD.id) > 0 + THEN RAISE(ABORT, 'Foreign key violation: language is still referenced') + END; + END; +CREATE TRIGGER fkc_delete_on_languages_link + BEFORE INSERT ON books_languages_link + BEGIN + SELECT CASE + WHEN (SELECT id from books WHERE id=NEW.book) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: book not in books') + WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages') + END; + END; +CREATE TRIGGER fkc_delete_on_publishers BEFORE DELETE ON publishers BEGIN SELECT CASE - WHEN (SELECT COUNT(id) FROM books_publishers_link WHERE book=OLD.book) > 0 - THEN RAISE(ABORT, 'Foreign key violation: publisher is still referenced') + WHEN (SELECT COUNT(id) FROM books_publishers_link WHERE publisher=OLD.id) > 0 + THEN RAISE(ABORT, 'Foreign key violation: publishers is still referenced') END; END; -CREATE TRIGGER fkc_delete_books_series_link +CREATE TRIGGER fkc_delete_on_series BEFORE DELETE ON series BEGIN SELECT CASE @@ -193,12 +379,12 @@ CREATE TRIGGER fkc_delete_books_series_link THEN RAISE(ABORT, 'Foreign key violation: series is still referenced') END; END; -CREATE TRIGGER fkc_delete_books_tags_link +CREATE TRIGGER fkc_delete_on_tags BEFORE DELETE ON tags BEGIN SELECT CASE WHEN (SELECT COUNT(id) FROM books_tags_link WHERE tag=OLD.id) > 0 - THEN RAISE(ABORT, 'Foreign key violation: tag is still referenced') + THEN RAISE(ABORT, 'Foreign key violation: tags is still referenced') END; END; CREATE TRIGGER fkc_insert_books_authors_link @@ -267,6 +453,22 @@ CREATE TRIGGER fkc_update_books_authors_link_b THEN RAISE(ABORT, 'Foreign key violation: author not in authors') END; END; +CREATE TRIGGER fkc_update_books_languages_link_a + BEFORE UPDATE OF book ON books_languages_link + BEGIN + SELECT CASE + WHEN (SELECT id from books WHERE id=NEW.book) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: book not in books') + END; + END; +CREATE TRIGGER fkc_update_books_languages_link_b + BEFORE UPDATE OF lang_code ON books_languages_link + BEGIN + SELECT CASE + WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages') + END; + END; CREATE TRIGGER fkc_update_books_publishers_link_a BEFORE UPDATE OF book ON books_publishers_link BEGIN @@ -341,3 +543,4 @@ CREATE TRIGGER series_update_trg BEGIN UPDATE series SET sort=NEW.name WHERE id=NEW.id; END; +pragma user_version=20; diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 3702de45c5..9d58ae4456 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -464,9 +464,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.refresh_ondevice = None def initialize_database(self): - metadata_sqlite = open(P('metadata_sqlite.sql'), 'rb').read() + metadata_sqlite = P('metadata_sqlite.sql', data=True, + allow_user_override=False).decode('utf-8') self.conn.executescript(metadata_sqlite) - self.user_version = 1 + self.conn.commit() + if self.user_version == 0: + self.user_version = 1 def last_modified(self): ''' Return last modified time as a UTC datetime object''' diff --git a/src/calibre/utils/resources.py b/src/calibre/utils/resources.py index 97c14926e4..00777973bb 100644 --- a/src/calibre/utils/resources.py +++ b/src/calibre/utils/resources.py @@ -65,7 +65,8 @@ _resolver = PathResolver() def get_path(path, data=False, allow_user_override=True): fpath = _resolver(path, allow_user_override=allow_user_override) if data: - return open(fpath, 'rb').read() + with open(fpath, 'rb') as f: + return f.read() return fpath def get_image_path(path, data=False, allow_user_override=True): From a36e9c42431a845b9dd595003aba62fc4dcc27bc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Apr 2011 11:05:46 -0600 Subject: [PATCH 13/29] Fix adding html books from the top of a deep folder hierarchy very slow --- src/calibre/ebooks/html/input.py | 15 ++++++++------- src/calibre/ebooks/oeb/base.py | 15 ++++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index 079e990de3..b22f7d2791 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -309,9 +309,9 @@ class HTMLInput(InputFormatPlugin): def create_oebbook(self, htmlpath, basedir, opts, log, mi): from calibre.ebooks.conversion.plumber import create_oebbook - from calibre.ebooks.oeb.base import DirContainer, \ - rewrite_links, urlnormalize, urldefrag, BINARY_MIME, OEB_STYLES, \ - xpath + from calibre.ebooks.oeb.base import (DirContainer, + rewrite_links, urlnormalize, urldefrag, BINARY_MIME, OEB_STYLES, + xpath) from calibre import guess_type from calibre.ebooks.oeb.transforms.metadata import \ meta_info_to_oeb_metadata @@ -345,7 +345,8 @@ class HTMLInput(InputFormatPlugin): htmlfile_map = {} for f in filelist: path = f.path - oeb.container = DirContainer(os.path.dirname(path), log) + oeb.container = DirContainer(os.path.dirname(path), log, + ignore_opf=True) bname = os.path.basename(path) id, href = oeb.manifest.generate(id='html', href=ascii_filename(bname)) @@ -369,7 +370,7 @@ class HTMLInput(InputFormatPlugin): for f in filelist: path = f.path dpath = os.path.dirname(path) - oeb.container = DirContainer(dpath, log) + oeb.container = DirContainer(dpath, log, ignore_opf=True) item = oeb.manifest.hrefs[htmlfile_map[path]] rewrite_links(item.data, partial(self.resource_adder, base=dpath)) @@ -409,7 +410,7 @@ class HTMLInput(InputFormatPlugin): if not item.linear: continue toc.add(title, item.href) - oeb.container = DirContainer(os.getcwdu(), oeb.log) + oeb.container = DirContainer(os.getcwdu(), oeb.log, ignore_opf=True) return oeb def link_to_local_path(self, link_, base=None): @@ -456,7 +457,7 @@ class HTMLInput(InputFormatPlugin): href=bhref) self.oeb.log.debug('Added', link) self.oeb.container = self.DirContainer(os.path.dirname(link), - self.oeb.log) + self.oeb.log, ignore_opf=True) # Load into memory guessed = self.guess_type(href)[0] media_type = guessed or self.BINARY_MIME diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index f2c9696976..c07386e1fd 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -446,22 +446,23 @@ class NullContainer(object): class DirContainer(object): """Filesystem directory container.""" - def __init__(self, path, log): + def __init__(self, path, log, ignore_opf=False): self.log = log if isbytestring(path): path = path.decode(filesystem_encoding) + self.opfname = None ext = os.path.splitext(path)[1].lower() if ext == '.opf': self.opfname = os.path.basename(path) self.rootdir = os.path.dirname(path) return self.rootdir = path - for path in self.namelist(): - ext = os.path.splitext(path)[1].lower() - if ext == '.opf': - self.opfname = path - return - self.opfname = None + if not ignore_opf: + for path in self.namelist(): + ext = os.path.splitext(path)[1].lower() + if ext == '.opf': + self.opfname = path + return def read(self, path): if path is None: From 92de7e180790366c8f73f6639f5a596354c6b6db Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Apr 2011 11:12:53 -0600 Subject: [PATCH 14/29] De Volksrant (subscriber version) by Selcal --- recipes/volksrant_sub.recipe | 115 +++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 recipes/volksrant_sub.recipe diff --git a/recipes/volksrant_sub.recipe b/recipes/volksrant_sub.recipe new file mode 100644 index 0000000000..8a5f1543b5 --- /dev/null +++ b/recipes/volksrant_sub.recipe @@ -0,0 +1,115 @@ +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe + +class Volkskrant_full(BasicNewsRecipe): + # This recipe will download the Volkskrant newspaper, + # from the subscribers site. It requires a password. + # Known issues are: articles that are spread out over + # multiple pages will appear multiple times. Pages + # that contain only adverts will appear, but empty. + # The supplement 'Volkskrant Magazine' on saturday + # is currently not downloaded. + # You can set a manual date, to download an archived + # newspaper. Volkskrant stores over a month at the + # moment of writing. To do so I suggest you unmark + # the date on the line below, and insert it in the title. Then + # follow the instructions marked further below. + + title = 'De Volkskrant (subscription)' # [za, 13 nov 2010]' + __author__ = u'Selcal' + description = u"Volkskrant" + oldest_article = 30 + max_articles_per_feed = 100 + no_stylesheets = True + language = 'nl' + use_embedded_content = False + simultaneous_downloads = 1 + delay = 1 + needs_subscription = True + # Set RETRIEVEDATE to 'yyyymmdd' to load an older + # edition. Otherwise keep '%Y%m%d' + # When setting a manual date, unmark and add the date + # to the title above, and unmark the timefmt line to stop + # Calibre from adding today's date in addition. + + # timefmt = '' + RETRIEVEDATE = strftime('%Y%m%d') + INDEX_MAIN = 'http://www.volkskrant.nl/vk-online/VK/' + RETRIEVEDATE + '___/VKN01_001/#text' + INDEX_ARTICLE = 'http://www.volkskrant.nl/vk-online/VK/' + RETRIEVEDATE + '___/VKN01_001/' + LOGIN = 'http://www.volkskrant.nl/vk/user/loggedIn.do' + remove_tags = [dict(name='address')] + cover_url = 'http://www.volkskrant.nl/vk-online/VK/' + RETRIEVEDATE + '___/VKN01_001/page.jpg' + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + + if self.username is not None and self.password is not None: + br.open(self.LOGIN) + br.select_form(nr = 0) + br['username'] = self.username + br['password'] = self.password + br.submit() + return br + + def parse_index(self): + krant = [] + def strip_title(_title): + i = 0 + while ((_title[i] <> ":") and (i <= len(_title))): + i = i + 1 + return(_title[0:i]) + for temp in range (5): + try: + soup = self.index_to_soup(self.INDEX_MAIN) + break + except: + #print '(Retrying main index load)' + continue + mainsoup = soup.find('td', attrs={'id': 'select_page_top'}) + for option in mainsoup.findAll('option'): + articles = [] + _INDEX = 'http://www.volkskrant.nl/vk-online/VK/' + self.RETRIEVEDATE + '___/' + option['value'] + '/#text' + _INDEX_ARTICLE = 'http://www.volkskrant.nl/vk-online/VK/' + self.RETRIEVEDATE + '___/' + option['value'] + '/' + #print '' + #print '<------- Processing section: ' + _INDEX + ' ------------------------->' + for temp in range (5): + try: + soup = self.index_to_soup(_INDEX) + break + except: + #print '(Retrying index load)' + continue + for item in soup.findAll('area'): + art_nr = item['class'] + attrname = art_nr[0:12] + '_section' + option['value'][0:5] + '_' + art_nr[26:len(art_nr)] + #print '==> Found: ' + attrname; + index_title = soup.find('div', attrs={'class': attrname}) + get_title = index_title['title']; + _ARTICLE = _INDEX_ARTICLE + attrname + '.html#text' + title = get_title; + #print '--> Title: ' + title; + #print '--> URL: ' + _ARTICLE; + for temp in range (5): + try: + souparticle = self.index_to_soup(_ARTICLE); + break + except: + print '(Retrying URL load)' + continue + headerurl = souparticle.findAll('frame')[0]['src']; + #print '--> Read frame name for header: ' + headerurl; + url = _INDEX_ARTICLE + headerurl[0:len(headerurl)-12] + '_text.html'; + #print '--> Corrected URL: ' + url; + if (get_title <> ''): + title = strip_title(get_title) + date = strftime(' %B %Y') + if (title <> ''): + articles.append({ + 'title' :title + ,'date' :date + ,'url' :url + ,'description':'' + }) + krant.append( (option.string, articles)) + return krant + From 631afb7eac79b76b48fb410332b72679342a6c92 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Apr 2011 13:00:49 -0600 Subject: [PATCH 15/29] Improve The Marker --- recipes/the_marker.recipe | 66 ++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/recipes/the_marker.recipe b/recipes/the_marker.recipe index e5f1ffc761..12b2f5e2ff 100644 --- a/recipes/the_marker.recipe +++ b/recipes/the_marker.recipe @@ -3,7 +3,7 @@ from calibre.web.feeds.news import BasicNewsRecipe class AdvancedUserRecipe1283848012(BasicNewsRecipe): description = 'TheMarker Financial News in Hebrew' - __author__ = 'TonyTheBookworm, Marbs' + __author__ = 'Marbs' cover_url = 'http://static.ispot.co.il/wp-content/upload/2009/09/themarker.jpg' title = u'TheMarker' language = 'he' @@ -11,42 +11,38 @@ class AdvancedUserRecipe1283848012(BasicNewsRecipe): remove_javascript = True timefmt = '[%a, %d %b, %Y]' oldest_article = 1 - remove_tags = [dict(name='tr', attrs={'bgcolor':['#738A94']}) ] - max_articles_per_feed = 10 + keep_only_tags =dict(name='div', attrs={'id':'content'}) + remove_attributes = ['width','float','margin-left'] + no_stylesheets = True + remove_tags = [dict(name='div', attrs={'class':['social-nav article-social-nav','prsnlArticleEnvelope','cb']}) , + dict(name='a', attrs={'href':['/misc/mobile']}) , + dict(name='span', attrs={'class':['post-summ']}) ] + max_articles_per_feed = 100 extra_css='body{direction: rtl;} .article_description{direction: rtl; } a.article{direction: rtl; } .calibre_feed_description{direction: rtl; }' - feeds = [(u'Head Lines', u'http://www.themarker.com/tmc/content/xml/rss/hpfeed.xml'), - (u'TA Market', u'http://www.themarker.com/tmc/content/xml/rss/sections/marketfeed.xml'), - (u'Real Estate', u'http://www.themarker.com/tmc/content/xml/rss/sections/realEstaterfeed.xml'), - (u'Wall Street & Global', u'http://www.themarker.com/tmc/content/xml/rss/sections/wallsfeed.xml'), - (u'Law', u'http://www.themarker.com/tmc/content/xml/rss/sections/lawfeed.xml'), - (u'Media', u'http://www.themarker.com/tmc/content/xml/rss/sections/mediafeed.xml'), - (u'Consumer', u'http://www.themarker.com/tmc/content/xml/rss/sections/consumerfeed.xml'), - (u'Career', u'http://www.themarker.com/tmc/content/xml/rss/sections/careerfeed.xml'), - (u'Car', u'http://www.themarker.com/tmc/content/xml/rss/sections/carfeed.xml'), - (u'High Tech', u'http://www.themarker.com/tmc/content/xml/rss/sections/hightechfeed.xml'), - (u'Investor Guide', u'http://www.themarker.com/tmc/content/xml/rss/sections/investorGuidefeed.xml')] + feeds = [(u'Head Lines', u'http://www.themarker.com/cmlink/1.144'), + (u'TA Market', u'http://www.themarker.com/cmlink/1.243'), + (u'Real Estate', u'http://www.themarker.com/cmlink/1.605656'), + (u'Global', u'http://www.themarker.com/cmlink/1.605658'), + (u'Wall Street', u'http://www.themarker.com/cmlink/1.613713'), + (u'SmartPhone', u'http://www.themarker.com/cmlink/1.605661'), + (u'Law', u'http://www.themarker.com/cmlink/1.605664'), + (u'Media', u'http://www.themarker.com/cmlink/1.605660'), + (u'Consumer', u'http://www.themarker.com/cmlink/1.605662'), + (u'Career', u'http://www.themarker.com/cmlink/1.605665'), + (u'Car', u'http://www.themarker.com/cmlink/1.605663'), + (u'High Tech', u'http://www.themarker.com/cmlink/1.605659'), + (u'Small Business', u'http://www.themarker.com/cmlink/1.605666')] def print_version(self, url): - split1 = url.split("=") - weblinks = url + #split1 = url.split("/") + #print_url='http://www.themarker.com/misc/article-print-page/'+split1[-1] + txt=url - if weblinks is not None: - for link in weblinks: - #--------------------------------------------------------- - #here we need some help with some regexpressions - #we are trying to find it.themarker.com in a url - #----------------------------------------------------------- - re1='.*?' # Non-greedy match on filler - re2='(it\\.themarker\\.com)' # Fully Qualified Domain Name 1 - rg = re.compile(re1+re2,re.IGNORECASE|re.DOTALL) - m = rg.search(url) + re1='.*?' # Non-greedy match on filler + re2='(tv)' # Word 1 - - if m: - split2 = url.split("article/") - print_url = 'http://it.themarker.com/tmit/PrintArticle/' + split2[1] - - else: - print_url = 'http://www.themarker.com/ibo/misc/printFriendly.jhtml?ElementId=%2Fibo%2Frepositories%2Fstories%2Fm1_2000%2F' + split1[1]+'.xml' - - return print_url + rg = re.compile(re1+re2,re.IGNORECASE|re.DOTALL) + m = rg.search(txt) + if m: + #print 'bad link' + return 1 From 8887b7f0579e151765f2d2616b797d7ab48a8990 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Apr 2011 14:19:25 -0600 Subject: [PATCH 16/29] CHM Input: Store extracted files in the input/ sub dir for easy debugging when --debug-pipeline is specified --- src/calibre/ebooks/chm/input.py | 12 ++++++++---- src/calibre/ebooks/chm/reader.py | 9 ++++++--- src/calibre/ebooks/conversion/plumber.py | 3 ++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/calibre/ebooks/chm/input.py b/src/calibre/ebooks/chm/input.py index fce07c2359..b5074e8a72 100644 --- a/src/calibre/ebooks/chm/input.py +++ b/src/calibre/ebooks/chm/input.py @@ -19,12 +19,12 @@ class CHMInput(InputFormatPlugin): description = 'Convert CHM files to OEB' file_types = set(['chm']) - def _chmtohtml(self, output_dir, chm_path, no_images, log): + def _chmtohtml(self, output_dir, chm_path, no_images, log, debug_dump=False): from calibre.ebooks.chm.reader import CHMReader log.debug('Opening CHM file') rdr = CHMReader(chm_path, log, self.opts) log.debug('Extracting CHM to %s' % output_dir) - rdr.extract_content(output_dir) + rdr.extract_content(output_dir, debug_dump=debug_dump) self._chm_reader = rdr return rdr.hhc_path @@ -47,7 +47,12 @@ class CHMInput(InputFormatPlugin): stream.close() log.debug('tdir=%s' % tdir) log.debug('stream.name=%s' % stream.name) - mainname = self._chmtohtml(tdir, chm_name, no_images, log) + debug_dump = False + odi = options.debug_pipeline + if odi: + debug_dump = os.path.join(odi, 'input') + mainname = self._chmtohtml(tdir, chm_name, no_images, log, + debug_dump=debug_dump) mainpath = os.path.join(tdir, mainname) metadata = get_metadata_from_reader(self._chm_reader) @@ -56,7 +61,6 @@ class CHMInput(InputFormatPlugin): #from calibre import ipython #ipython() - odi = options.debug_pipeline options.debug_pipeline = None options.input_encoding = 'utf-8' # try a custom conversion: diff --git a/src/calibre/ebooks/chm/reader.py b/src/calibre/ebooks/chm/reader.py index 24814a34f9..5f23ad0241 100644 --- a/src/calibre/ebooks/chm/reader.py +++ b/src/calibre/ebooks/chm/reader.py @@ -97,7 +97,7 @@ class CHMReader(CHMFile): raise CHMError("'%s' is zero bytes in length!"%(path,)) return data - def ExtractFiles(self, output_dir=os.getcwdu()): + def ExtractFiles(self, output_dir=os.getcwdu(), debug_dump=False): html_files = set([]) for path in self.Contents(): lpath = os.path.join(output_dir, path) @@ -123,6 +123,9 @@ class CHMReader(CHMFile): self.log.warn('%r filename too long, skipping'%path) continue raise + if debug_dump: + import shutil + shutil.copytree(output_dir, os.path.join(debug_dump, 'debug_dump')) for lpath in html_files: with open(lpath, 'r+b') as f: data = f.read() @@ -249,8 +252,8 @@ class CHMReader(CHMFile): if not os.path.isdir(dir): os.makedirs(dir) - def extract_content(self, output_dir=os.getcwdu()): - self.ExtractFiles(output_dir=output_dir) + def extract_content(self, output_dir=os.getcwdu(), debug_dump=False): + self.ExtractFiles(output_dir=output_dir, debug_dump=debug_dump) diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 96ea3e5884..3eb59a21b9 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -854,7 +854,8 @@ OptionRecommendation(name='sr3_replace', if isinstance(ret, basestring): shutil.copytree(output_dir, out_dir) else: - os.makedirs(out_dir) + if not os.path.exists(out_dir): + os.makedirs(out_dir) self.dump_oeb(ret, out_dir) if self.input_fmt == 'recipe': zf = ZipFile(os.path.join(self.opts.debug_pipeline, From 1bfe6b06d77d3bb606f387dce411047cefe3d199 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 27 Apr 2011 21:49:28 +0100 Subject: [PATCH 17/29] Fix (I hope) template program_text retrieval when not running from source --- src/calibre/gui2/preferences/template_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/template_functions.py b/src/calibre/gui2/preferences/template_functions.py index 8ffd65b2b5..20724b7667 100644 --- a/src/calibre/gui2/preferences/template_functions.py +++ b/src/calibre/gui2/preferences/template_functions.py @@ -186,7 +186,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.argument_count.setValue(func.arg_count) self.documentation.setText(func.doc) if txt in self.builtins: - if hasattr(func, 'program_text'): + if hasattr(func, 'program_text') and func.program_text: self.program.setPlainText(func.program_text) elif txt in self.builtin_source_dict: self.program.setPlainText(self.builtin_source_dict[txt]) From 16cf2f86fa77e3ce3fa9a3b4c97c276e54a35613 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Apr 2011 14:53:50 -0600 Subject: [PATCH 18/29] ... --- src/calibre/gui2/preferences/template_functions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/preferences/template_functions.py b/src/calibre/gui2/preferences/template_functions.py index 8ffd65b2b5..23a2f08bbf 100644 --- a/src/calibre/gui2/preferences/template_functions.py +++ b/src/calibre/gui2/preferences/template_functions.py @@ -74,9 +74,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): def initialize(self): try: - with open(P('template-functions.json'), 'rb') as f: - self.builtin_source_dict = json.load(f, encoding='utf-8') + json.loads(P('template-functions.json', data=True, + allow_user_override=False).decode('utf-8')) except: + traceback.print_exc() self.builtin_source_dict = {} self.funcs = formatter_functions.get_functions() From fed64f9a172e38d68344827b65d2047d75400577 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Apr 2011 15:01:08 -0600 Subject: [PATCH 19/29] ... --- src/calibre/gui2/preferences/template_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/template_functions.py b/src/calibre/gui2/preferences/template_functions.py index 2c9547baf4..fcb4c87372 100644 --- a/src/calibre/gui2/preferences/template_functions.py +++ b/src/calibre/gui2/preferences/template_functions.py @@ -74,7 +74,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): def initialize(self): try: - json.loads(P('template-functions.json', data=True, + self.builtin_source_dict = json.loads(P('template-functions.json', data=True, allow_user_override=False).decode('utf-8')) except: traceback.print_exc() From 0a31b52a42a9f17a632ab44caae463e768e7bef3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Apr 2011 15:28:48 -0600 Subject: [PATCH 20/29] Add API docs for the Metadata object to the template tutorial --- src/calibre/ebooks/metadata/book/base.py | 36 ++++++++++++++++++++++-- src/calibre/manual/template_lang.rst | 16 +++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 3099de12e4..a031c4886d 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -68,7 +68,19 @@ composite_formatter = SafeFormat() class Metadata(object): ''' - A class representing all the metadata for a book. + A class representing all the metadata for a book. The various standard metadata + fields are available as attributes of this object. You can also stick + arbitrary attributes onto this object. + + Metadata from custom columns should be accessed via the get() method, + passing in the lookup name for the column, for example: "#mytags". + + Use the :meth:`is_null` method to test if a filed is null. + + This object also has functions to format fields into strings. + + The list of standard metadata fields grows with time is in + :data:`STANDARD_METADATA_FIELDS`. Please keep the method based API of this class to a minimum. Every method becomes a reserved field name. @@ -88,11 +100,19 @@ class Metadata(object): if title: self.title = title if authors: - #: List of strings or [] + # List of strings or [] self.author = list(authors) if authors else []# Needed for backward compatibility self.authors = list(authors) if authors else [] def is_null(self, field): + ''' + Return True if the value of filed is null in this object. + 'null' means it is unknown or evaluates to False. So a title of + _('Unknown') is null or a language of 'und' is null. + + Be careful with numeric fields since this will return True for zero as + well as None. + ''' null_val = NULL_VALUES.get(field, None) val = getattr(self, field, None) return not val or val == null_val @@ -547,13 +567,16 @@ class Metadata(object): return unicode(self.rating) def format_field(self, key, series_with_index=True): + ''' + Returns the tuple (display_name, formatted_value) + ''' name, val, ign, ign = self.format_field_extended(key, series_with_index) return (name, val) def format_field_extended(self, key, series_with_index=True): from calibre.ebooks.metadata import authors_to_string ''' - returns the tuple (field_name, formatted_value, original_value, + returns the tuple (display_name, formatted_value, original_value, field_metadata) ''' @@ -637,6 +660,10 @@ class Metadata(object): return (None, None, None, None) def __unicode__(self): + ''' + A string representation of this object, suitable for printing to + console + ''' from calibre.ebooks.metadata import authors_to_string ans = [] def fmt(x, y): @@ -680,6 +707,9 @@ class Metadata(object): return u'\n'.join(ans) def to_html(self): + ''' + A HTML representation of this object. + ''' from calibre.ebooks.metadata import authors_to_string ans = [(_('Title'), unicode(self.title))] ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))] diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst index a77f0d1697..b4bf7473e7 100644 --- a/src/calibre/manual/template_lang.rst +++ b/src/calibre/manual/template_lang.rst @@ -396,3 +396,19 @@ You might find the following tips useful. * In a plugboard, you can set a field to empty (or whatever is equivalent to empty) by using the special template ``{null}``. This template will always evaluate to an empty string. * The technique described above to show numbers even if they have a zero value works with the standard field series_index. +API of the Metadata objects +---------------------------- + +.. module:: calibre.ebooks.metadata.book.base + +.. autoclass:: Metadata + :members: + :member-order: bysource + +.. data:: STANDARD_METADATA_FIELDS + + The set of standard metadata fields. + +.. literalinclude:: ../ebooks/metadata/book/__init__.py + :lines: 7- + From 29b9dfa8bbf838936c969d7b36d6008cbb98c353 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 28 Apr 2011 05:32:43 +0100 Subject: [PATCH 21/29] Finish the 'rating in template' job. Make save_to_disk divide them by two. --- src/calibre/ebooks/metadata/book/base.py | 6 ++++-- src/calibre/library/save_to_disk.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index a031c4886d..5619ef7806 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -563,8 +563,10 @@ class Metadata(object): def format_tags(self): return u', '.join([unicode(t) for t in sorted(self.tags, key=sort_key)]) - def format_rating(self): - return unicode(self.rating) + def format_rating(self, v = None): + if v is None: + return unicode(self.rating/2) + return unicode(v/2) def format_field(self, key, series_with_index=True): ''' diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 709e3645a6..8eb2e8d788 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -211,6 +211,8 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, format_args[key] = strftime(timefmt, format_args[key].timetuple()) elif cm['datatype'] == 'bool': format_args[key] = _('yes') if format_args[key] else _('no') + elif cm['datatype'] == 'rating': + format_args[key] = mi.format_rating(format_args[key]) elif cm['datatype'] in ['int', 'float']: if format_args[key] != 0: format_args[key] = unicode(format_args[key]) From 22477dc1e4692211bc85e1e0ba4926e6262a22d1 Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 28 Apr 2011 07:40:07 -0400 Subject: [PATCH 22/29] Fix bug #772267: Underlined words (following quotes?) fail to become italics --- src/calibre/ebooks/conversion/utils.py | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index 1546644f95..7488df4609 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -156,17 +156,17 @@ class HeuristicProcessor(object): ] ITALICIZE_STYLE_PATS = [ - r'(?msu)(?<=[\s>])_(?P[^_]+)_', - r'(?msu)(?<=[\s>])/(?P[^/\*>]+)/', - r'(?msu)(?<=[\s>])~~(?P[^~]+)~~', - r'(?msu)(?<=[\s>])\*(?P[^\*]+)\*', - r'(?msu)(?<=[\s>])~(?P[^~]+)~', - r'(?msu)(?<=[\s>])_/(?P[^/_]+)/_', - r'(?msu)(?<=[\s>])_\*(?P[^\*_]+)\*_', - r'(?msu)(?<=[\s>])\*/(?P[^/\*]+)/\*', - r'(?msu)(?<=[\s>])_\*/(?P[^\*_]+)/\*_', - r'(?msu)(?<=[\s>])/:(?P[^:/]+):/', - r'(?msu)(?<=[\s>])\|:(?P[^:\|]+):\|', + ur'(?msu)(?<=[\s>"“\'‘])_(?P[^_]+)_', + ur'(?msu)(?<=[\s>"“\'‘])/(?P[^/\*>]+)/', + ur'(?msu)(?<=[\s>"“\'‘])~~(?P[^~]+)~~', + ur'(?msu)(?<=[\s>"“\'‘])\*(?P[^\*]+)\*', + ur'(?msu)(?<=[\s>"“\'‘])~(?P[^~]+)~', + ur'(?msu)(?<=[\s>"“\'‘])_/(?P[^/_]+)/_', + ur'(?msu)(?<=[\s>"“\'‘])_\*(?P[^\*_]+)\*_', + ur'(?msu)(?<=[\s>"“\'‘])\*/(?P[^/\*]+)/\*', + ur'(?msu)(?<=[\s>"“\'‘])_\*/(?P[^\*_]+)/\*_', + ur'(?msu)(?<=[\s>"“\'‘])/:(?P[^:/]+):/', + ur'(?msu)(?<=[\s>"“\'‘])\|:(?P[^:\|]+):\|', ] for word in ITALICIZE_WORDS: @@ -518,13 +518,13 @@ class HeuristicProcessor(object): if re.findall('(<|>)', replacement_break): if re.match('^\d+).*', '\g', replacement_break)) - replacement_break = re.sub('(?i)(width=\d+\%?|width:\s*\d+(\%|px|pt|em)?;?)', '', replacement_break) - divpercent = (100 - width) / 2 - hr_open = re.sub('45', str(divpercent), hr_open) - scene_break = hr_open+replacement_break+'' + width = int(re.sub('.*?width(:|=)(?P\d+).*', '\g', replacement_break)) + replacement_break = re.sub('(?i)(width=\d+\%?|width:\s*\d+(\%|px|pt|em)?;?)', '', replacement_break) + divpercent = (100 - width) / 2 + hr_open = re.sub('45', str(divpercent), hr_open) + scene_break = hr_open+replacement_break+'' else: - scene_break = hr_open+'
' + scene_break = hr_open+'
' elif re.match('^' else: @@ -584,10 +584,10 @@ class HeuristicProcessor(object): #print "styles for this line are: "+str(styles) split_styles = [] for style in styles: - #print "style is: "+str(style) - newstyle = style.split(':') - #print "newstyle is: "+str(newstyle) - split_styles.append(newstyle) + #print "style is: "+str(style) + newstyle = style.split(':') + #print "newstyle is: "+str(newstyle) + split_styles.append(newstyle) styles = split_styles for style, setting in styles: if style == 'text-align' and setting != 'left': From c2a7d26b5147bcbffcf423f995af23107acbe425 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 28 Apr 2011 08:13:16 -0600 Subject: [PATCH 23/29] ... --- src/calibre/gui2/threaded_jobs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py index eb8667a713..9e16f88a1a 100644 --- a/src/calibre/gui2/threaded_jobs.py +++ b/src/calibre/gui2/threaded_jobs.py @@ -100,7 +100,8 @@ class ThreadedJob(BaseJob): try: self.consolidate_log() except: - self.log.exception('Log consolidation failed') + if self.log is not None: + self.log.exception('Log consolidation failed') # No need to keep references to these around anymore self.func = self.args = self.kwargs = self.notifications = None From 213df8e3d7e7d91dd88db3523e0d36db112b72b8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 28 Apr 2011 08:48:54 -0600 Subject: [PATCH 24/29] Add command line option to shutdown running calibre --- src/calibre/gui2/main.py | 22 ++++++++++++++++------ src/calibre/gui2/ui.py | 7 +++++-- src/calibre/manual/creating_plugins.rst | 5 +++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index ee18d8e9ca..eadfa55549 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -40,6 +40,11 @@ path_to_ebook to the database. parser.add_option('--ignore-plugins', default=False, action='store_true', help=_('Ignore custom plugins, useful if you installed a plugin' ' that is preventing calibre from starting')) + parser.add_option('-s', '--shutdown-running-calibre', default=False, + action='store_true', + help=_('Cause a running calibre instance, if any, to be' + ' shutdown. Note that if there are running jobs, they ' + 'will be silently aborted, so use with care.')) return parser def init_qt(args): @@ -339,7 +344,7 @@ def cant_start(msg=_('If you are sure it is not running')+', ', raise SystemExit(1) -def communicate(args): +def communicate(opts, args): t = RC() t.start() time.sleep(3) @@ -348,9 +353,12 @@ def communicate(args): cant_start(what=_('try deleting the file')+': '+f) raise SystemExit(1) - if len(args) > 1: - args[1] = os.path.abspath(args[1]) - t.conn.send('launched:'+repr(args)) + if opts.shutdown_running_calibre: + t.conn.send('shutdown:') + else: + if len(args) > 1: + args[1] = os.path.abspath(args[1]) + t.conn.send('launched:'+repr(args)) t.conn.close() raise SystemExit(0) @@ -365,6 +373,8 @@ def main(args=sys.argv): from calibre.utils.lock import singleinstance from multiprocessing.connection import Listener si = singleinstance('calibre GUI') + if si and opts.shutdown_running_calibre: + return 0 if si: try: listener = Listener(address=ADDRESS) @@ -390,10 +400,10 @@ def main(args=sys.argv): else: # On windows only singleinstance can be trusted otherinstance = True if iswindows else False - if not otherinstance: + if not otherinstance and not opts.shutdown_running_calibre: return run_gui(opts, args, actions, listener, app, gui_debug=gui_debug) - communicate(args) + communicate(opts, args) return 0 diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 4885f7b2db..435b9ebe78 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -446,6 +446,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.library_view.model().refresh() self.library_view.model().research() self.tags_view.recount() + elif msg.startswith('shutdown:'): + self.quit(confirm_quit=False) else: print msg @@ -599,8 +601,9 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ dynamic.set('sort_history', self.library_view.model().sort_history) self.save_layout_state() - def quit(self, checked=True, restart=False, debug_on_restart=False): - if not self.confirm_quit(): + def quit(self, checked=True, restart=False, debug_on_restart=False, + confirm_quit=True): + if confirm_quit and not self.confirm_quit(): return try: self.shutdown() diff --git a/src/calibre/manual/creating_plugins.rst b/src/calibre/manual/creating_plugins.rst index 3b6b9611af..4a69cc8753 100644 --- a/src/calibre/manual/creating_plugins.rst +++ b/src/calibre/manual/creating_plugins.rst @@ -195,9 +195,10 @@ It can get tiresome to keep re-adding a plugin to calibre to test small changes. Once you've located the zip file of your plugin you can then directly update it with your changes instead of re-adding it each time. To do so from the command line, in the directory that contains your plugin source code, use:: - zip -R /path/to/plugin/zip/file.zip * + calibre -s; sleep 4s; zip -R /path/to/plugin/zip/file.zip *; calibre -This will update all changed files. It relies on the freely available zip command line tool. Note that you should quit calibre before running this command. +This will shutdown a running calibre. Wait for the shutdown to complete, then update your plugin files and relaunch calibre. +It relies on the freely available zip command line tool. More plugin examples ---------------------- From ec12901d8a6e7d62b44b3408c196e8ce33a110a5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 28 Apr 2011 10:11:00 -0600 Subject: [PATCH 25/29] ... --- resources/templates/book_details.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/resources/templates/book_details.css b/resources/templates/book_details.css index 5059a8f4c3..5fc2691850 100644 --- a/resources/templates/book_details.css +++ b/resources/templates/book_details.css @@ -2,6 +2,11 @@ a { text-decoration: none; color: blue } + +a:hover { + color: red +} + .comments { margin-top: 0; padding-top: 0; From 8182deb97f785c26fe10b7ff4425b182e6eb38a1 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 28 Apr 2011 18:50:34 +0100 Subject: [PATCH 26/29] Correctly parenthesize searches that are used to make search restrictions. --- src/calibre/library/caches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 543c6eab96..98fd3a9fbc 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -783,7 +783,7 @@ class ResultCache(SearchQueryParser): # {{{ else: q = query if search_restriction: - q = u'%s (%s)' % (search_restriction, query) + q = u'(%s) and (%s)' % (search_restriction, query) if not q: if set_restriction_count: self.search_restriction_book_count = len(self._map) From a8985831f255c0c42b90d645bdabf7cb517ed1a1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 28 Apr 2011 15:42:42 -0600 Subject: [PATCH 27/29] ... --- recipes/hbr_blogs.recipe | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recipes/hbr_blogs.recipe b/recipes/hbr_blogs.recipe index b15203fce1..2846ba79a3 100644 --- a/recipes/hbr_blogs.recipe +++ b/recipes/hbr_blogs.recipe @@ -59,9 +59,9 @@ class HBR(BasicNewsRecipe): def get_browser(self): br = BasicNewsRecipe.get_browser(self) br.open(self.LOGIN_URL) - br.select_form(name='signInForm') - br['signInForm:username'] = self.username - br['signInForm:password'] = self.password + br.select_form(name='signin-form') + br['signin-form:username'] = self.username + br['signin-form:password'] = self.password raw = br.submit().read() if 'My Account' not in raw: raise Exception('Failed to login, are you sure your username and password are correct?') From d3674afdddfaf49a0639c9b9ee7e74c4374ce74c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 28 Apr 2011 16:04:10 -0600 Subject: [PATCH 28/29] Fix HBR Blogs --- recipes/hbr_blogs.recipe | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/recipes/hbr_blogs.recipe b/recipes/hbr_blogs.recipe index 2846ba79a3..bd72a95ebf 100644 --- a/recipes/hbr_blogs.recipe +++ b/recipes/hbr_blogs.recipe @@ -1,9 +1,6 @@ from calibre.web.feeds.news import BasicNewsRecipe import re -# Needed for BLOGs -from calibre.web.feeds import Feed - class HBR(BasicNewsRecipe): title = 'Harvard Business Review Blogs' @@ -32,6 +29,7 @@ class HBR(BasicNewsRecipe): feeds = [('Blog','http://feeds.harvardbusiness.org/harvardbusiness')] oldest_article = 30 max_articles_per_feed = 100 + use_embedded_content = False else: timefmt = ' [%B %Y]' @@ -161,27 +159,13 @@ class HBR(BasicNewsRecipe): return startDate, endDate #------------------------------------------------------------------------------------------------- - def hbr_parse_blogs(self, feeds): - # Do the "official" parse_feeds first - rssFeeds = Feed() - # Use the PARSE_FEEDS method to get a Feeds object of the articles - rssFeeds = BasicNewsRecipe.parse_feeds(self) - - # Create a new feed of the right configuration and append to existing afeeds - self.feed_to_index_append(rssFeeds[:], feeds) - -#------------------------------------------------------------------------------------------------- def parse_index(self): if self.INCLUDE_ARTICLES == True: soup = self.hbr_get_toc() feeds = self.hbr_parse_toc(soup) else: - feeds = [] - - # blog stuff - if self.INCLUDE_BLOGS == True: - self.hbr_parse_blogs(feeds) + return BasicNewsRecipe.parse_index(self) return feeds #------------------------------------------------------------------------------------------------- From fc4b09e5acc2fc323c4e51fea40848723881327d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 28 Apr 2011 16:42:29 -0600 Subject: [PATCH 29/29] Novinky by Tomas Latal --- recipes/novinky.recipe | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 recipes/novinky.recipe diff --git a/recipes/novinky.recipe b/recipes/novinky.recipe new file mode 100644 index 0000000000..7c5186cb99 --- /dev/null +++ b/recipes/novinky.recipe @@ -0,0 +1,42 @@ +__license__ = 'GPL v3' +__copyright__ = '2011, Tomas Latal' + +from calibre.web.feeds.news import BasicNewsRecipe + +class NovinkyCZ(BasicNewsRecipe): + title = 'Novinky' + __author__ = 'Tomas Latal' + __version__ = '1.0' + __date__ = '24 April 2011' + description = 'News from server Novinky.cz' + oldest_article = 1 + max_articles_per_feed = 10 + encoding = 'utf8' + publisher = 'Novinky' + category = 'news, CZ' + language = 'cs' + publication_type = 'newsportal' + no_stylesheets = True + remove_javascript = True + extra_css = 'p.acmDescription{font-style:italic;} p.acmAuthor{font-size:0.8em; color:#707070}' + + feeds = [ + (u'Dom\xe1c\xed', u'http://www.novinky.cz/rss/domaci/'), + (u'Zahrani\u010d\xed', u'http://www.novinky.cz/rss/zahranicni/'), + (u'Krimi', u'http://www.novinky.cz/rss/krimi/'), + (u'Ekonomika', u'http://www.novinky.cz/rss/ekonomika/'), + (u'Finance', u'http://www.novinky.cz/rss/finance/'), + (u'Kultura', u'http://www.novinky.cz/rss/kultura/'), + (u'Koktejl', u'http://www.novinky.cz/rss/koktejl/'), + (u'Internet a PC', u'http://www.novinky.cz/rss/internet-a-pc/'), + (u'Auto-moto', u'http://www.novinky.cz/rss/auto/'), + ] + + remove_tags_before = dict(id='articleContent') + + remove_tags_after = [dict(id='movedArticleAuthors')] + + remove_tags = [ + dict(name='div', attrs={'id':['articleColumnInfo','pictureInnerBox']}), + dict(name='p', attrs={'id':['articleDate']}) + ]