From be026b8d2bf56f1878b7de275c2e5df45a55231f Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Tue, 14 Jun 2011 23:14:00 -0300 Subject: [PATCH 01/21] KTouch - Image file names have changed - fix display of covers --- src/calibre/devices/kobo/driver.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 04fb3c37b0..1efce88f27 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -85,6 +85,16 @@ class KOBO(USBMS): except: self.fwversion = 'unknown' + # Determine Hardware version differences + product_id = self.detected_device.idProduct + debug_print ("device: ", product_id) + if product_id == 0x4161: #Original and KWifi + image_suffix = ' - NickelBookCover.parsed' + else: #KTouch + image_suffix = ' - N3_LIBRARY_FULL.parsed' + + debug_print("Image Suffix: ", image_suffix) + if self.fwversion != '1.0' and self.fwversion != '1.4': self.has_kepubs = True debug_print('Version of firmware: ', self.fwversion, 'Has kepubs:', self.has_kepubs) @@ -125,8 +135,9 @@ class KOBO(USBMS): if idx is not None: bl_cache[lpath] = None if ImageID is not None: - imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - NickelBookCover.parsed') + imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + image_suffix) #print "Image name Normalized: " + imagename + if imagename is not None: bl[idx].thumbnail = ImageWrapper(imagename) if (ContentType != '6' and MimeType != 'Shortcover'): From c9d34ca631a564b1d78c18b9b905ce30c7f662d6 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Tue, 14 Jun 2011 23:23:32 -0300 Subject: [PATCH 02/21] KTouch - Image file names have changed - fix deleting images --- src/calibre/devices/kobo/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 1efce88f27..25ef30edbd 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -297,7 +297,7 @@ class KOBO(USBMS): path_prefix = '.kobo/images/' path = self._main_prefix + path_prefix + ImageID - file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed',) + file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed', ' - N3_LIBRARY_FULL.parsed', ' - N3_LIBRARY_GRID.parsed', ' - N3_LIBRARY_LIST.parsed', ' - N3_SOCIAL_CURRENTREAD.parsed',) for ending in file_endings: fpath = path + ending From e8b723fb9900d8e9e448caf641cc1a6f5c220768 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Wed, 15 Jun 2011 21:53:40 -0300 Subject: [PATCH 03/21] KTouch - Image file names have changed - modify fix for display of covers --- src/calibre/devices/kobo/driver.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 25ef30edbd..6076aa35eb 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -85,16 +85,6 @@ class KOBO(USBMS): except: self.fwversion = 'unknown' - # Determine Hardware version differences - product_id = self.detected_device.idProduct - debug_print ("device: ", product_id) - if product_id == 0x4161: #Original and KWifi - image_suffix = ' - NickelBookCover.parsed' - else: #KTouch - image_suffix = ' - N3_LIBRARY_FULL.parsed' - - debug_print("Image Suffix: ", image_suffix) - if self.fwversion != '1.0' and self.fwversion != '1.4': self.has_kepubs = True debug_print('Version of firmware: ', self.fwversion, 'Has kepubs:', self.has_kepubs) @@ -135,7 +125,11 @@ class KOBO(USBMS): if idx is not None: bl_cache[lpath] = None if ImageID is not None: - imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + image_suffix) + imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - NickelBookCover.parsed') + if not os.path.exists(imagename): + # Try the Touch version if the image does not exist + imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - N3_LIBRARY_FULL.parsed') + #print "Image name Normalized: " + imagename if imagename is not None: From cb66ec304d43609281fe93242b557e018816f568 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Wed, 15 Jun 2011 22:17:52 -0300 Subject: [PATCH 04/21] KTouch - Hide the internal help files that don't exist on the device storage --- src/calibre/devices/kobo/driver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 6076aa35eb..978a848707 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -199,7 +199,9 @@ class KOBO(USBMS): changed = False for i, row in enumerate(cursor): # self.report_progress((i+1) / float(numrows), _('Getting list of books on device...')) - + if row[3].startswith("file:///usr/local/Kobo/help/"): + # These are internal to the Kobo device and do not exist + continue path = self.path_from_contentid(row[3], row[5], row[4], oncard) mime = mime_type_ext(path_to_ext(path)) if path.find('kepub') == -1 else 'application/epub+zip' # debug_print("mime:", mime) From 515bc88d595c93c47912e47eb26d2805c21ce163 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Wed, 15 Jun 2011 22:31:12 -0300 Subject: [PATCH 05/21] KTouch - Why oh why does the content id keep changing format? --- src/calibre/devices/kobo/driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 978a848707..e606f65e38 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -457,7 +457,10 @@ class KOBO(USBMS): path = self._main_prefix + path + '.kobo' # print "Path: " + path elif (ContentType == "6" or ContentType == "10") and MimeType == 'application/x-kobo-epub+zip': - path = self._main_prefix + '.kobo/kepub/' + path + if path.startswith("file:///mnt/onboard/"): + path = self._main_prefix + path.replace("file:///mnt/onboard/", '') + else: + path = self._main_prefix + '.kobo/kepub/' + path # print "Internal: " + path else: # if path.startswith("file:///mnt/onboard/"): From 405db329847653f6ec1867ec9071e24faee32ac8 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Jun 2011 14:50:02 +0100 Subject: [PATCH 06/21] libri.de store plugin --- src/calibre/customize/builtins.py | 11 +++ src/calibre/gui2/store/libri_de_plugin.py | 92 +++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 src/calibre/gui2/store/libri_de_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index b2268d3732..17b7244c58 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1300,6 +1300,16 @@ class StoreLegimiStore(StoreBase): headquarters = 'PL' formats = ['EPUB'] +class StoreLibreDEStore(StoreBase): + name = 'Libri DE' + author = 'Charles Haley' + description = u'Sicher Bücher, Hörbücher und Downloads online bestellen.' + actual_plugin = 'calibre.gui2.store.libri_de_plugin:LibreDEStore' + + headquarters = 'DE' + formats = ['EPUB', 'PDF'] + affiliate = True + class StoreManyBooksStore(StoreBase): name = 'ManyBooks' description = u'Public domain and creative commons works from many sources.' @@ -1450,6 +1460,7 @@ plugins += [ StoreGutenbergStore, StoreKoboStore, StoreLegimiStore, + StoreLibreDEStore, StoreManyBooksStore, StoreMobileReadStore, StoreNextoStore, diff --git a/src/calibre/gui2/store/libri_de_plugin.py b/src/calibre/gui2/store/libri_de_plugin.py new file mode 100644 index 0000000000..edcdff0cf2 --- /dev/null +++ b/src/calibre/gui2/store/libri_de_plugin.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class LibreDEStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + url = 'http://ad.zanox.com/ppc/?18817073C15644254T' + url_details = ('http://ad.zanox.com/ppc/?18845780C1371495675T&ULP=[[' + 'http://www.libri.de/shop/action/productDetails?artiId={0}]]') + + if external or self.config.get('open_external', False): + if detail_item: + url = url_details.format(detail_item) + print(url) + open_url(QUrl(url)) + else: + detail_url = None + if detail_item: + detail_url = url_details.format(detail_item) + print(detail_url) + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = ('http://www.libri.de/shop/action/quickSearch?facetNodeId=6' + '&mainsearchSubmit=Los!&searchString=' + urllib2.quote(query)) + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[contains(@class, "item")]'): + if counter <= 0: + break + + details = data.xpath('./div[@class="beschreibungContainer"]') + if not details: + continue + details = details[0] + id = ''.join(details.xpath('./div[@class="text"]/a/@name')).strip() + if not id: + continue + cover_url = ''.join(details.xpath('./div[@class="bild"]/a/img/@src')) + title = ''.join(details.xpath('./div[@class="text"]/span[@class="titel"]/a/text()')).strip() + author = ''.join(details.xpath('./div[@class="text"]/span[@class="author"]/text()')).strip() + pdf = details.xpath( + 'boolean(.//span[@class="format" and contains(text(), "pdf")]/text())') + epub = details.xpath( + 'boolean(.//span[@class="format" and contains(text(), "epub")]/text())') + mobi = details.xpath( + 'boolean(.//span[@class="format" and contains(text(), "mobipocket")]/text())') + price = (''.join(data.xpath('.//span[@class="preis"]/text()'))).replace('*', '') + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.drm = SearchResult.DRM_UNKNOWN + s.detail_item = id + formats = [] + if epub: + formats.append('ePub') + if pdf: + formats.append('PDF') + if mobi: + formats.append('MOBI') + s.formats = ', '.join(formats) + + yield s From d7d8acb7cc78698315e1dd31629afa5dee4f96ca Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Jun 2011 14:50:52 +0100 Subject: [PATCH 07/21] Remove print statements --- src/calibre/gui2/store/libri_de_plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/calibre/gui2/store/libri_de_plugin.py b/src/calibre/gui2/store/libri_de_plugin.py index edcdff0cf2..ed93eeff0e 100644 --- a/src/calibre/gui2/store/libri_de_plugin.py +++ b/src/calibre/gui2/store/libri_de_plugin.py @@ -30,13 +30,11 @@ class LibreDEStore(BasicStoreConfig, StorePlugin): if external or self.config.get('open_external', False): if detail_item: url = url_details.format(detail_item) - print(url) open_url(QUrl(url)) else: detail_url = None if detail_item: detail_url = url_details.format(detail_item) - print(detail_url) d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(self.config.get('tags', '')) From 757778fbda802c999efdd8cb8d5e2b5a1e0c8647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Sun, 19 Jun 2011 13:37:27 +0200 Subject: [PATCH 08/21] improve many authors handling --- src/calibre/gui2/store/woblink_plugin.py | 2 +- src/calibre/gui2/store/zixo_plugin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/woblink_plugin.py b/src/calibre/gui2/store/woblink_plugin.py index 69be8f2e94..9f2e29e825 100644 --- a/src/calibre/gui2/store/woblink_plugin.py +++ b/src/calibre/gui2/store/woblink_plugin.py @@ -57,7 +57,7 @@ class WoblinkStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//td[@class="w10 va-t"]/a[1]/img/@src')) title = ''.join(data.xpath('.//h3[@class="title"]/a[1]/text()')) - author = ''.join(data.xpath('.//p[@class="author"]/a[1]/text()')) + author = ', '.join(data.xpath('.//p[@class="author"]/a/text()')) price = ''.join(data.xpath('.//div[@class="prices"]/p[1]/span/text()')) price = re.sub('PLN', ' zł', price) price = re.sub('\.', ',', price) diff --git a/src/calibre/gui2/store/zixo_plugin.py b/src/calibre/gui2/store/zixo_plugin.py index 419b87137a..b4e54736c0 100644 --- a/src/calibre/gui2/store/zixo_plugin.py +++ b/src/calibre/gui2/store/zixo_plugin.py @@ -53,7 +53,7 @@ class ZixoStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//a[@class="productThumb"]/img/@src')) title = ''.join(data.xpath('.//a[@class="title"]/text()')) - author = ''.join(data.xpath('.//div[@class="productDescription"]/span[1]/a/text()')) + author = ','.join(data.xpath('.//div[@class="productDescription"]/span[1]/a/text()')) price = ''.join(data.xpath('.//div[@class="priceList"]/span/text()')) price = re.sub('\.', ',', price) From 91fcf1fd1602868b8e7dfe863eead7e54e757416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Sun, 19 Jun 2011 13:42:59 +0200 Subject: [PATCH 09/21] fix authors in legimi store plugin --- src/calibre/gui2/store/legimi_plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/store/legimi_plugin.py b/src/calibre/gui2/store/legimi_plugin.py index 2f69da24e5..792c7db4a7 100644 --- a/src/calibre/gui2/store/legimi_plugin.py +++ b/src/calibre/gui2/store/legimi_plugin.py @@ -58,6 +58,8 @@ class LegimiStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//div[@class="item_cover_container"]/a/img/@src')) title = ''.join(data.xpath('.//div[@class="item_entries"]/h2/a/text()')) author = ''.join(data.xpath('.//div[@class="item_entries"]/span[1]/a/text()')) + author = re.sub(',','',author) + author = re.sub(';',',',author) price = ''.join(data.xpath('.//div[@class="item_entries"]/span[3]/text()')) price = re.sub(r'[^0-9,]*','',price) + ' zł' From d9895e8acc47d2e1efbf33c280341e478fd831a0 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 19 Jun 2011 17:40:55 +0100 Subject: [PATCH 10/21] Fix beam ebooks not finding books if ratings are present --- src/calibre/gui2/store/beam_ebooks_de_plugin.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/store/beam_ebooks_de_plugin.py b/src/calibre/gui2/store/beam_ebooks_de_plugin.py index b589a8c310..6e31a76f8d 100644 --- a/src/calibre/gui2/store/beam_ebooks_de_plugin.py +++ b/src/calibre/gui2/store/beam_ebooks_de_plugin.py @@ -51,19 +51,19 @@ class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): if counter <= 0: break - id = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/@href')).strip() + id = ''.join(data.xpath('./tr/td[1]/a/@href')).strip() if not id: continue id = id[7:] cover_url = ''.join(data.xpath('./tr/td[1]/a/img/@src')) if cover_url: cover_url = 'http://www.beam-ebooks.de' + cover_url - title = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/b/text()')) - author = ' '.join(data.xpath('./tr/td/div[@class="stil2"]/' - 'child::b/text()' - '|' - './tr/td/div[@class="stil2"]/' - 'child::strong/text()')) + temp = ''.join(data.xpath('./tr/td[1]/a/img/@alt')) + colon = temp.find(':') + if not temp.startswith('eBook') or colon < 0: + continue + author = temp[5:colon] + title = temp[colon+1:] price = ''.join(data.xpath('./tr/td[3]/text()')) pdf = data.xpath( 'boolean(./tr/td[3]/a/img[contains(@alt, "PDF")]/@alt)') From d88510f2247771cf4a077c20fddb53a8556b1843 Mon Sep 17 00:00:00 2001 From: GRiker Date: Mon, 20 Jun 2011 07:21:04 -0600 Subject: [PATCH 11/21] further revision to generateThumbnail --- src/calibre/library/catalog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 006b381214..8508fb266f 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -4728,7 +4728,7 @@ Author '{0}': pass else: # uuid found in cache with matching crc - thumb_data = zf.read(title['uuid']) + thumb_data = zf.read(title['uuid']+cover_crc) with open(os.path.join(image_dir, thumb_file), 'wb') as f: f.write(thumb_data) return From 9887c222072f7deb50bb8718efd006a48e44a383 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Jun 2011 09:50:33 -0600 Subject: [PATCH 12/21] Add descriptions to builtin plugins --- src/calibre/customize/builtins.py | 30 ++++++++++++++++++++- src/calibre/customize/conversion.py | 4 +++ src/calibre/ebooks/epub/fix/epubcheck.py | 4 +++ src/calibre/ebooks/epub/fix/unmanifested.py | 4 +++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index d1c5b6ccd5..62ad977c88 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -762,99 +762,127 @@ plugins += input_profiles + output_profiles class ActionAdd(InterfaceActionBase): name = 'Add Books' actual_plugin = 'calibre.gui2.actions.add:AddAction' + description = _('Add books to calibre or the connected device') class ActionFetchAnnotations(InterfaceActionBase): name = 'Fetch Annotations' actual_plugin = 'calibre.gui2.actions.annotate:FetchAnnotationsAction' + description = _('Fetch annotations from a connected Kindle (experimental)') class ActionGenerateCatalog(InterfaceActionBase): name = 'Generate Catalog' actual_plugin = 'calibre.gui2.actions.catalog:GenerateCatalogAction' + description = _('Generate a catalog of the books in your calibre library') class ActionConvert(InterfaceActionBase): name = 'Convert Books' actual_plugin = 'calibre.gui2.actions.convert:ConvertAction' + description = _('Convert books to various ebook formats') class ActionDelete(InterfaceActionBase): name = 'Remove Books' actual_plugin = 'calibre.gui2.actions.delete:DeleteAction' + description = _('Delete books from your calibre library or connected device') class ActionEditMetadata(InterfaceActionBase): name = 'Edit Metadata' actual_plugin = 'calibre.gui2.actions.edit_metadata:EditMetadataAction' + description = _('Edit the metadata of books in your calibre library') class ActionView(InterfaceActionBase): name = 'View' actual_plugin = 'calibre.gui2.actions.view:ViewAction' + description = _('Read books in your calibre library') class ActionFetchNews(InterfaceActionBase): name = 'Fetch News' actual_plugin = 'calibre.gui2.actions.fetch_news:FetchNewsAction' + description = _('Download news from the internet in ebook form') class ActionSaveToDisk(InterfaceActionBase): name = 'Save To Disk' actual_plugin = 'calibre.gui2.actions.save_to_disk:SaveToDiskAction' + description = _('Export books from your calibre library to the hard disk') class ActionShowBookDetails(InterfaceActionBase): name = 'Show Book Details' actual_plugin = 'calibre.gui2.actions.show_book_details:ShowBookDetailsAction' + description = _('Show book details in a separate popup') class ActionRestart(InterfaceActionBase): name = 'Restart' actual_plugin = 'calibre.gui2.actions.restart:RestartAction' + description = _('Restart calibre') class ActionOpenFolder(InterfaceActionBase): name = 'Open Folder' actual_plugin = 'calibre.gui2.actions.open:OpenFolderAction' + description = _('Open the folder that contains the book files in your' + ' calibre library') class ActionSendToDevice(InterfaceActionBase): name = 'Send To Device' actual_plugin = 'calibre.gui2.actions.device:SendToDeviceAction' + description = _('Send books to the connected device') class ActionConnectShare(InterfaceActionBase): name = 'Connect Share' actual_plugin = 'calibre.gui2.actions.device:ConnectShareAction' + description = _('Send books via email or the web also connect to iTunes or' + ' folders on your computer as if they are devices') class ActionHelp(InterfaceActionBase): name = 'Help' actual_plugin = 'calibre.gui2.actions.help:HelpAction' + description = _('Browse the calibre User Manual') class ActionPreferences(InterfaceActionBase): name = 'Preferences' actual_plugin = 'calibre.gui2.actions.preferences:PreferencesAction' + description = _('Customize calibre') class ActionSimilarBooks(InterfaceActionBase): name = 'Similar Books' actual_plugin = 'calibre.gui2.actions.similar_books:SimilarBooksAction' + description = _('Easily find books similar to the currently selected one') class ActionChooseLibrary(InterfaceActionBase): name = 'Choose Library' actual_plugin = 'calibre.gui2.actions.choose_library:ChooseLibraryAction' + description = _('Switch between different calibre libraries and perform' + ' maintenance on them') class ActionAddToLibrary(InterfaceActionBase): name = 'Add To Library' actual_plugin = 'calibre.gui2.actions.add_to_library:AddToLibraryAction' + description = _('Copy books from the devce to your calibre library') class ActionEditCollections(InterfaceActionBase): name = 'Edit Collections' actual_plugin = 'calibre.gui2.actions.edit_collections:EditCollectionsAction' + description = _('Edit the collections in which books are placed on your device') class ActionCopyToLibrary(InterfaceActionBase): name = 'Copy To Library' actual_plugin = 'calibre.gui2.actions.copy_to_library:CopyToLibraryAction' + description = _('Copy a book from one calibre library to another') class ActionTweakEpub(InterfaceActionBase): name = 'Tweak ePub' actual_plugin = 'calibre.gui2.actions.tweak_epub:TweakEpubAction' + description = _('Make small twekas to epub files in your calibre library') class ActionNextMatch(InterfaceActionBase): name = 'Next Match' actual_plugin = 'calibre.gui2.actions.next_match:NextMatchAction' + description = _('Find the next or previous match when searching in ' + 'your calibre library in highlight mode') class ActionStore(InterfaceActionBase): name = 'Store' author = 'John Schember' actual_plugin = 'calibre.gui2.actions.store:StoreAction' + description = _('Search for books from different book sellers') def customization_help(self, gui=False): return 'Customize the behavior of the store search.' @@ -870,7 +898,7 @@ class ActionStore(InterfaceActionBase): class ActionPluginUpdater(InterfaceActionBase): name = 'Plugin Updater' author = 'Grant Drake' - description = 'Queries the MobileRead forums for updates to plugins to install' + description = _('Get new calibre plugins or update your existing ones') actual_plugin = 'calibre.gui2.actions.plugin_updates:PluginUpdaterAction' plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog, diff --git a/src/calibre/customize/conversion.py b/src/calibre/customize/conversion.py index b77ac81587..88f19df7af 100644 --- a/src/calibre/customize/conversion.py +++ b/src/calibre/customize/conversion.py @@ -259,6 +259,10 @@ class OutputFormatPlugin(Plugin): #: (option_name, recommended_value, recommendation_level) recommendations = set([]) + @property + def description(self): + return _('Convert ebooks to the %s format'%self.file_type) + def __init__(self, *args): Plugin.__init__(self, *args) self.report_progress = DummyReporter() diff --git a/src/calibre/ebooks/epub/fix/epubcheck.py b/src/calibre/ebooks/epub/fix/epubcheck.py index 81f4ce4d80..9e812e1cf4 100644 --- a/src/calibre/ebooks/epub/fix/epubcheck.py +++ b/src/calibre/ebooks/epub/fix/epubcheck.py @@ -26,6 +26,10 @@ class Epubcheck(ePubFixer): 'significant changes to your epub, complain to the epubcheck ' 'project.') + @property + def description(self): + return self.long_description + @property def fix_name(self): return 'epubcheck' diff --git a/src/calibre/ebooks/epub/fix/unmanifested.py b/src/calibre/ebooks/epub/fix/unmanifested.py index da7a9a9d0e..98fdd4615f 100644 --- a/src/calibre/ebooks/epub/fix/unmanifested.py +++ b/src/calibre/ebooks/epub/fix/unmanifested.py @@ -22,6 +22,10 @@ class Unmanifested(ePubFixer): 'the manifest or delete them as specified by the ' 'delete unmanifested option.') + @property + def description(self): + return self.long_description + @property def fix_name(self): return 'unmanifested' From 9672083e842fbf4fc28bfe0bb20228ae76239b52 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Jun 2011 09:55:03 -0600 Subject: [PATCH 13/21] Frontline by DM. Fixes #799802 (New recipe for Frontline magazine) --- recipes/frontlineonnet.recipe | 81 +++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 recipes/frontlineonnet.recipe diff --git a/recipes/frontlineonnet.recipe b/recipes/frontlineonnet.recipe new file mode 100644 index 0000000000..3b65e4bb18 --- /dev/null +++ b/recipes/frontlineonnet.recipe @@ -0,0 +1,81 @@ +__license__ = 'GPL v3' +__copyright__ = '2011, Darko Miletic ' +''' +frontlineonnet.com +''' + +import re +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe + +class Frontlineonnet(BasicNewsRecipe): + title = 'Frontline' + __author__ = 'Darko Miletic' + description = "India's national magazine" + publisher = 'Frontline' + category = 'news, politics, India' + no_stylesheets = True + delay = 1 + INDEX = 'http://frontlineonnet.com/' + use_embedded_content = False + encoding = 'cp1252' + language = 'en_IN' + publication_type = 'magazine' + masthead_url = 'http://frontlineonnet.com/images/newfline.jpg' + extra_css = """ + body{font-family: Verdana,Arial,Helvetica,sans-serif} + img{margin-top:0.5em; margin-bottom: 0.7em; display: block} + """ + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'linearize_tables' : True + } + + preprocess_regexps = [ + (re.compile(r'.*?title', re.DOTALL|re.IGNORECASE),lambda match: '') + ,(re.compile(r'', re.DOTALL|re.IGNORECASE),lambda match: '
') + ,(re.compile(r'', re.DOTALL|re.IGNORECASE),lambda match: '
') + ,(re.compile(r'
', re.DOTALL|re.IGNORECASE),lambda match: '
') + ,(re.compile(r'
', re.DOTALL|re.IGNORECASE),lambda match: '') + ] + + keep_only_tags= [ + dict(name='font', attrs={'class':'storyhead'}) + ,dict(attrs={'class':'byline'}) + ] + remove_attributes=['size','noshade','border'] + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll('img'): + if not item.has_key('alt'): + item['alt'] = 'image' + return soup + + def parse_index(self): + articles = [] + soup = self.index_to_soup(self.INDEX) + for feed_link in soup.findAll('a',href=True): + if feed_link['href'].startswith('stories/'): + url = self.INDEX + feed_link['href'] + title = self.tag_to_string(feed_link) + date = strftime(self.timefmt) + articles.append({ + 'title' :title + ,'date' :date + ,'url' :url + ,'description':'' + }) + return [('Frontline', articles)] + + def print_version(self, url): + return "http://www.hinduonnet.com/thehindu/thscrip/print.pl?prd=fline&file=" + url.rpartition('/')[2] + + def image_url_processor(self, baseurl, url): + return url.replace('../images/', self.INDEX + 'images/').strip() From 47415caf3b46ac8a9d7de1d8f8e41ff2e3217e08 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Jun 2011 12:40:25 -0600 Subject: [PATCH 14/21] Print out job sarted notification for device jobs to the debug queue --- src/calibre/gui2/device.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 99f795cdda..9f6381b859 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -49,6 +49,9 @@ class DeviceJob(BaseJob): # {{{ self._aborted = False def start_work(self): + if DEBUG: + prints('Job:', self.id, self.description, 'started', + safe_encode=True) self.start_time = time.time() self.job_manager.changed_queue.put(self) From 8e22f5c4325362dc234f58a30af6b7d0e61d3848 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Jun 2011 13:40:04 -0600 Subject: [PATCH 15/21] More device job sequencing debug printouts --- src/calibre/gui2/device.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 9f6381b859..a527cc2e27 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -58,10 +58,17 @@ class DeviceJob(BaseJob): # {{{ def job_done(self): self.duration = time.time() - self.start_time self.percent = 1 + if DEBUG: + prints('DeviceJob:', self.id, self.description, + 'done, calling callback', safe_encode=True) + try: self.callback_on_done(self) except: pass + if DEBUG: + prints('DeviceJob:', self.id, self.description, + 'callback returned', safe_encode=True) self.job_manager.changed_queue.put(self) def report_progress(self, percent, msg=''): From 15d8272efe25bae56731d49dd46fbbf4377b84d4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Jun 2011 15:54:32 -0600 Subject: [PATCH 16/21] ... --- src/calibre/library/database2.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 67c67b1ff7..51ada244e3 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -201,16 +201,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): dbprefs = DBPrefs(self) for key in default_prefs: # be sure that prefs not to be copied are listed below - if key in ['news_to_be_synced']: + if key in frozenset(['news_to_be_synced']): continue - try: - dbprefs[key] = default_prefs[key] - except: - pass # ignore options that don't exist anymore - fmvals = [f for f in default_prefs['field_metadata'].values() if f['is_custom']] - for f in fmvals: - self.create_custom_column(f['label'], f['name'], f['datatype'], - f['is_multiple'] is not None, f['is_editable'], f['display']) + dbprefs[key] = default_prefs[key] + if 'field_metadata' in default_prefs: + fmvals = [f for f in default_prefs['field_metadata'].values() if f['is_custom']] + for f in fmvals: + self.create_custom_column(f['label'], f['name'], f['datatype'], + f['is_multiple'] is not None, f['is_editable'], f['display']) self.initialize_dynamic() def get_property(self, idx, index_is_id=False, loc=-1): From 50dadb45cf0c5e80e16fcbcb9b193dcdb88356c8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Jun 2011 16:13:18 -0600 Subject: [PATCH 17/21] Start work on new db backend --- src/calibre/db/__init__.py | 67 ++++++ src/calibre/db/backend.py | 404 +++++++++++++++++++++++++++++++++++++ 2 files changed, 471 insertions(+) create mode 100644 src/calibre/db/__init__.py create mode 100644 src/calibre/db/backend.py diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py new file mode 100644 index 0000000000..4384cab2da --- /dev/null +++ b/src/calibre/db/__init__.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +''' +Rewrite of the calibre database backend. + +Broad Objectives: + + * Use the sqlite db only as a datastore. i.e. do not do + sorting/searching/concatenation or anything else in sqlite. Instead + mirror the sqlite tables in memory, create caches and lookup maps from + them and create a set_* API that updates the memory caches and the sqlite + correctly. + + * Move from keeping a list of books in memory as a cache to a per table + cache. This allows much faster search and sort operations at the expense + of slightly slower lookup operations. That slowdown can be mitigated by + keeping lots of maps and updating them in the set_* API. Also + get_categories becomes blazingly fast. + + * Separate the database layer from the cache layer more cleanly. Rather + than having the db layer refer to the cache layer and vice versa, the + cache layer will refer to the db layer only and the new API will be + defined on the cache layer. + + * Get rid of index_is_id and other poor design decisions + + * Minimize the API as much as possible and define it cleanly + + * Do not change the on disk format of metadata.db at all (this is for + backwards compatibility) + + * Get rid of the need for a separate db access thread by switching to apsw + to access sqlite, which is thread safe + + * The new API will have methods to efficiently do bulk operations and will + use shared/exclusive/pending locks to serialize access to the in-mem data + structures. Use the same locking scheme as sqlite itself does. + +How this will proceed: + + 1. Create the new API + 2. Create a test suite for it + 3. Write a replacement for LibraryDatabase2 that uses the new API + internally + 4. Lots of testing of calibre with the new LibraryDatabase2 + 5. Gradually migrate code to use the (much faster) new api wherever possible (the new api + will be exposed via db.new_api) + + I plan to work on this slowly, in parallel to normal calibre development + work. + +Various things that require other things before they can be migrated: + 1. From initialize_dynamic(): set_saved_searches, + load_user_template_functions. Also add custom + columns/categories/searches info into + self.field_metadata. Finally, implement metadata dirtied + functionality. + +''' diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py new file mode 100644 index 0000000000..4e6c028b93 --- /dev/null +++ b/src/calibre/db/backend.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +# Imports {{{ +import os, shutil, uuid, json +from functools import partial + +import apsw + +from calibre import isbytestring, force_unicode, prints +from calibre.constants import (iswindows, filesystem_encoding, + preferred_encoding) +from calibre.ptempfile import PersistentTemporaryFile +from calibre.library.schema_upgrades import SchemaUpgrade +from calibre.library.field_metadata import FieldMetadata +from calibre.ebooks.metadata import title_sort, author_to_author_sort +from calibre.utils.icu import strcmp +from calibre.utils.config import to_json, from_json, prefs, tweaks +from calibre.utils.date import utcfromtimestamp +# }}} + +''' +Differences in semantics from pysqlite: + + 1. execute/executemany/executescript operate in autocommit mode + +''' + +class DynamicFilter(object): # {{{ + + 'No longer used, present for legacy compatibility' + + def __init__(self, name): + self.name = name + self.ids = frozenset([]) + + def __call__(self, id_): + return int(id_ in self.ids) + + def change(self, ids): + self.ids = frozenset(ids) +# }}} + +class DBPrefs(dict): # {{{ + + 'Store preferences as key:value pairs in the db' + + def __init__(self, db): + dict.__init__(self) + self.db = db + self.defaults = {} + self.disable_setting = False + for key, val in self.db.conn.get('SELECT key,val FROM preferences'): + try: + val = self.raw_to_object(val) + except: + prints('Failed to read value for:', key, 'from db') + continue + dict.__setitem__(self, key, val) + + def raw_to_object(self, raw): + if not isinstance(raw, unicode): + raw = raw.decode(preferred_encoding) + return json.loads(raw, object_hook=from_json) + + def to_raw(self, val): + return json.dumps(val, indent=2, default=to_json) + + def __getitem__(self, key): + try: + return dict.__getitem__(self, key) + except KeyError: + return self.defaults[key] + + def __delitem__(self, key): + dict.__delitem__(self, key) + self.db.conn.execute('DELETE FROM preferences WHERE key=?', (key,)) + + def __setitem__(self, key, val): + if self.disable_setting: + return + raw = self.to_raw(val) + self.db.conn.execute('INSERT OR REPLACE INTO preferences (key,val) VALUES (?,?)', (key, + raw)) + dict.__setitem__(self, key, val) + + def set(self, key, val): + self.__setitem__(key, val) + +# }}} + +# Extra collators {{{ +def pynocase(one, two, encoding='utf-8'): + if isbytestring(one): + try: + one = one.decode(encoding, 'replace') + except: + pass + if isbytestring(two): + try: + two = two.decode(encoding, 'replace') + except: + pass + return cmp(one.lower(), two.lower()) + +def _author_to_author_sort(x): + if not x: return '' + return author_to_author_sort(x.replace('|', ',')) + +def icu_collator(s1, s2): + return strcmp(force_unicode(s1, 'utf-8'), force_unicode(s2, 'utf-8')) +# }}} + +class Connection(apsw.Connection): # {{{ + + BUSY_TIMEOUT = 2000 # milliseconds + + def __init__(self, path): + apsw.Connection.__init__(self, path) + + self.setbusytimeout(self.BUSY_TIMEOUT) + self.execute('pragma cache_size=5000') + self.conn.execute('pragma temp_store=2') + + encoding = self.execute('pragma encoding').fetchone()[0] + self.conn.create_collation('PYNOCASE', partial(pynocase, + encoding=encoding)) + + self.conn.create_function('title_sort', 1, title_sort) + self.conn.create_function('author_to_author_sort', 1, + _author_to_author_sort) + + self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4())) + + # Dummy functions for dynamically created filters + self.conn.create_function('books_list_filter', 1, lambda x: 1) + self.conn.create_collation('icucollate', icu_collator) + + def create_dynamic_filter(self, name): + f = DynamicFilter(name) + self.conn.create_function(name, 1, f) + + def get(self, *args, **kw): + ans = self.cursor().execute(*args) + if kw.get('all', True): + return ans.fetchall() + for row in ans: + return ans[0] + + def execute(self, sql, bindings=None): + cursor = self.cursor() + return cursor.execute(sql, bindings) + + def executemany(self, sql, sequence_of_bindings): + return self.cursor().executemany(sql, sequence_of_bindings) + + def executescript(self, sql): + with self: + # Use an explicit savepoint so that even if this is called + # while a transaction is active, it is atomic + return self.cursor().execute(sql) +# }}} + +class DB(SchemaUpgrade): + + PATH_LIMIT = 40 if iswindows else 100 + WINDOWS_LIBRARY_PATH_LIMIT = 75 + + # Initialize database {{{ + + def __init__(self, library_path, default_prefs=None, read_only=False): + try: + if isbytestring(library_path): + library_path = library_path.decode(filesystem_encoding) + except: + import traceback + traceback.print_exc() + + self.field_metadata = FieldMetadata() + + self.library_path = os.path.abspath(library_path) + self.dbpath = os.path.join(library_path, 'metadata.db') + self.dbpath = os.environ.get('CALIBRE_OVERRIDE_DATABASE_PATH', + self.dbpath) + + if iswindows and len(self.library_path) + 4*self.PATH_LIMIT + 10 > 259: + raise ValueError(_( + 'Path to library too long. Must be less than' + ' %d characters.')%(259-4*self.PATH_LIMIT-10)) + exists = self._exists = os.path.exists(self.dbpath) + if not exists: + # Be more strict when creating new libraries as the old calculation + # allowed for max path lengths of 265 chars. + if (iswindows and len(self.library_path) > + self.WINDOWS_LIBRARY_PATH_LIMIT): + raise ValueError(_( + 'Path to library too long. Must be less than' + ' %d characters.')%self.WINDOWS_LIBRARY_PATH_LIMIT) + + if read_only and os.path.exists(self.dbpath): + # Work on only a copy of metadata.db to ensure that + # metadata.db is not changed + pt = PersistentTemporaryFile('_metadata_ro.db') + pt.close() + shutil.copyfile(self.dbpath, pt.name) + self.dbpath = pt.name + + self.is_case_sensitive = (not iswindows and + not os.path.exists(self.dbpath.replace('metadata.db', + 'MeTAdAtA.dB'))) + + self._conn = None + + if self.user_version == 0: + self.initialize_database() + + SchemaUpgrade.__init__(self) + # Guarantee that the library_id is set + self.library_id + + self.initialize_prefs(default_prefs) + + # Fix legacy triggers and columns + self.conn.executescript(''' + DROP TRIGGER IF EXISTS author_insert_trg; + CREATE TEMP TRIGGER author_insert_trg + AFTER INSERT ON authors + BEGIN + UPDATE authors SET sort=author_to_author_sort(NEW.name) WHERE id=NEW.id; + END; + DROP TRIGGER IF EXISTS author_update_trg; + CREATE TEMP TRIGGER author_update_trg + BEFORE UPDATE ON authors + BEGIN + UPDATE authors SET sort=author_to_author_sort(NEW.name) + WHERE id=NEW.id AND name <> NEW.name; + END; + UPDATE authors SET sort=author_to_author_sort(name) WHERE sort IS NULL; + ''') + + def initialize_prefs(self, default_prefs): + self.prefs = DBPrefs(self) + + if default_prefs is not None and not self._exists: + # Only apply default prefs to a new database + for key in default_prefs: + # be sure that prefs not to be copied are listed below + if key not in frozenset(['news_to_be_synced']): + self.prefs[key] = default_prefs[key] + if 'field_metadata' in default_prefs: + fmvals = [f for f in default_prefs['field_metadata'].values() + if f['is_custom']] + for f in fmvals: + self.create_custom_column(f['label'], f['name'], + f['datatype'], f['is_multiple'] is not None, + f['is_editable'], f['display']) + + defs = self.prefs.defaults + defs['gui_restriction'] = defs['cs_restriction'] = '' + defs['categories_using_hierarchy'] = [] + defs['column_color_rules'] = [] + + # Migrate the bool tristate tweak + defs['bools_are_tristate'] = \ + tweaks.get('bool_custom_columns_are_tristate', 'yes') == 'yes' + if self.prefs.get('bools_are_tristate') is None: + self.prefs.set('bools_are_tristate', defs['bools_are_tristate']) + + # Migrate column coloring rules + if self.prefs.get('column_color_name_1', None) is not None: + from calibre.library.coloring import migrate_old_rule + old_rules = [] + for i in range(1, 6): + col = self.prefs.get('column_color_name_'+str(i), None) + templ = self.prefs.get('column_color_template_'+str(i), None) + if col and templ: + try: + del self.prefs['column_color_name_'+str(i)] + rules = migrate_old_rule(self.field_metadata, templ) + for templ in rules: + old_rules.append((col, templ)) + except: + pass + if old_rules: + self.prefs['column_color_rules'] += old_rules + + # Migrate saved search and user categories to db preference scheme + def migrate_preference(key, default): + oldval = prefs[key] + if oldval != default: + self.prefs[key] = oldval + prefs[key] = default + if key not in self.prefs: + self.prefs[key] = default + + migrate_preference('user_categories', {}) + migrate_preference('saved_searches', {}) + + # migrate grouped_search_terms + if self.prefs.get('grouped_search_terms', None) is None: + try: + ogst = tweaks.get('grouped_search_terms', {}) + ngst = {} + for t in ogst: + ngst[icu_lower(t)] = ogst[t] + self.prefs.set('grouped_search_terms', ngst) + except: + pass + + # Rename any user categories with names that differ only in case + user_cats = self.prefs.get('user_categories', []) + catmap = {} + for uc in user_cats: + ucl = icu_lower(uc) + if ucl not in catmap: + catmap[ucl] = [] + catmap[ucl].append(uc) + cats_changed = False + for uc in catmap: + if len(catmap[uc]) > 1: + prints('found user category case overlap', catmap[uc]) + cat = catmap[uc][0] + suffix = 1 + while icu_lower((cat + unicode(suffix))) in catmap: + suffix += 1 + prints('Renaming user category %s to %s'%(cat, cat+unicode(suffix))) + user_cats[cat + unicode(suffix)] = user_cats[cat] + del user_cats[cat] + cats_changed = True + if cats_changed: + self.prefs.set('user_categories', user_cats) + + @property + def conn(self): + if self._conn is None: + self._conn = apsw.Connection(self.dbpath) + if self._exists and self.user_version == 0: + self._conn.close() + os.remove(self.dbpath) + self._conn = apsw.Connection(self.dbpath) + return self._conn + + @dynamic_property + def user_version(self): + doc = 'The user version of this database' + + def fget(self): + return self.conn.get('pragma user_version;', all=False) + + def fset(self, val): + self.conn.execute('pragma user_version=%d'%int(val)) + + return property(doc=doc, fget=fget, fset=fset) + + def initialize_database(self): + metadata_sqlite = P('metadata_sqlite.sql', data=True, + allow_user_override=False).decode('utf-8') + self.conn.executescript(metadata_sqlite) + if self.user_version == 0: + self.user_version = 1 + # }}} + + # Database layer API {{{ + + @classmethod + def exists_at(cls, path): + return path and os.path.exists(os.path.join(path, 'metadata.db')) + + @dynamic_property + def library_id(self): + doc = ('The UUID for this library. As long as the user only operates' + ' on libraries with calibre, it will be unique') + + def fget(self): + if getattr(self, '_library_id_', None) is None: + ans = self.conn.get('SELECT uuid FROM library_id', all=False) + if ans is None: + ans = str(uuid.uuid4()) + self.library_id = ans + else: + self._library_id_ = ans + return self._library_id_ + + def fset(self, val): + self._library_id_ = unicode(val) + self.conn.execute(''' + DELETE FROM library_id; + INSERT INTO library_id (uuid) VALUES (?); + ''', self._library_id_) + + return property(doc=doc, fget=fget, fset=fset) + + def last_modified(self): + ''' Return last modified time as a UTC datetime object ''' + return utcfromtimestamp(os.stat(self.dbpath).st_mtime) + + # }}} + From a34a2a0e0574dfb9d7b4ae6374ed2034e2bf9d85 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 20 Jun 2011 19:01:12 -0400 Subject: [PATCH 18/21] Fix bug #799367: Could not download ebooks from store --- src/calibre/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 3a35feb66f..663b4d8a50 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -591,8 +591,10 @@ def get_download_filename(url, cookie_file=None): cj.load(cookie_file) br.set_cookiejar(cj) + last_part_name = '' try: with closing(br.open(url)) as r: + last_part_name = r.geturl().split('/')[-1] disposition = r.info().get('Content-disposition', '') for p in disposition.split(';'): if 'filename' in p: @@ -612,7 +614,7 @@ def get_download_filename(url, cookie_file=None): traceback.print_exc() if not filename: - filename = r.geturl().split('/')[-1] + filename = last_part_name return filename From 83634d8a542aadfefbd74767aac15f6286af5a22 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Mon, 20 Jun 2011 23:33:54 -0300 Subject: [PATCH 19/21] Expired books should be reported as a Collection - The next server sync should delete them --- src/calibre/devices/kobo/driver.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index e606f65e38..6a9ea56ceb 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -100,7 +100,7 @@ class KOBO(USBMS): for idx,b in enumerate(bl): bl_cache[b.lpath] = idx - def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType): + def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType, expired): changed = False try: lpath = path.partition(self.normalize_path(prefix))[2] @@ -118,6 +118,11 @@ class KOBO(USBMS): elif readstatus == 3: playlist_map[lpath]= "Closed" + # Related to a bug in the Kobo firmware that leaves an expired row for deleted books + # this shows an expired Collection so the user can decide to delete the book + if expired == 3: + playlist_map[lpath] = "Expired" + path = self.normalize_path(path) # print "Normalized FileName: " + path @@ -131,7 +136,8 @@ class KOBO(USBMS): imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - N3_LIBRARY_FULL.parsed') #print "Image name Normalized: " + imagename - + if not os.path.exists(imagename): + debug_print("Strange - The image name does not exist - title: ", title) if imagename is not None: bl[idx].thumbnail = ImageWrapper(imagename) if (ContentType != '6' and MimeType != 'Shortcover'): @@ -192,7 +198,7 @@ class KOBO(USBMS): self.dbversion = result[0] query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ - 'ImageID, ReadStatus from content where BookID is Null' + 'ImageID, ReadStatus, ___ExpirationStatus from content where BookID is Null' cursor.execute (query) @@ -207,10 +213,10 @@ class KOBO(USBMS): # debug_print("mime:", mime) if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"): - changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4]) + changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8]) # print "shortbook: " + path elif oncard == 'carda' and row[3].startswith("file:///mnt/sd/"): - changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4]) + changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8]) if changed: need_sync = True From 43c9ebe77f3294f14f13036c59d1c58a290f346a Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Tue, 21 Jun 2011 00:06:25 -0300 Subject: [PATCH 20/21] Emulate the Kobo device delete more closely. Leave the Book record, the next server sync should delete them --- src/calibre/devices/kobo/driver.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 6a9ea56ceb..b523b7f296 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -280,8 +280,12 @@ class KOBO(USBMS): cursor.execute('delete from content_keys where volumeid = ?', t) # Delete the chapters associated with the book next - t = (ContentID,ContentID,) - cursor.execute('delete from content where BookID = ? or ContentID = ?', t) + t = (ContentID,) + # Kobo does not delete the Book row (ie the row where the BookID is Null) + # The next server sync should remove the row + cursor.execute('delete from content where BookID = ?', t) + cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0, ___ExpirationStatus=3 ' \ + 'where BookID is Null and ContentID =?',t) connection.commit() From f9dc7d978011f09fc1ef046da316b3ee38871fc2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Jun 2011 21:48:18 -0600 Subject: [PATCH 21/21] EOD on the new db api --- resources/metadata_sqlite.sql | 6 +- src/calibre/db/backend.py | 39 +++++++++- src/calibre/db/tables.py | 143 ++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 src/calibre/db/tables.py diff --git a/resources/metadata_sqlite.sql b/resources/metadata_sqlite.sql index 9c4f666449..aa29d4b8de 100644 --- a/resources/metadata_sqlite.sql +++ b/resources/metadata_sqlite.sql @@ -13,8 +13,10 @@ CREATE TABLE books ( id INTEGER PRIMARY KEY AUTOINCREMENT, isbn TEXT DEFAULT "" COLLATE NOCASE, lccn TEXT DEFAULT "" COLLATE NOCASE, path TEXT NOT NULL DEFAULT "", - flags INTEGER NOT NULL DEFAULT 1 - , uuid TEXT, has_cover BOOL DEFAULT 0, last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00"); + flags INTEGER NOT NULL DEFAULT 1, + uuid TEXT, + has_cover BOOL DEFAULT 0, + last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00"); CREATE TABLE books_authors_link ( id INTEGER PRIMARY KEY, book INTEGER NOT NULL, author INTEGER NOT NULL, diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 4e6c028b93..159612e52d 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -23,6 +23,8 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.utils.icu import strcmp from calibre.utils.config import to_json, from_json, prefs, tweaks from calibre.utils.date import utcfromtimestamp +from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, + SizeTable, FormatsTable, AuthorsTable, IdentifiersTable) # }}} ''' @@ -167,7 +169,7 @@ class Connection(apsw.Connection): # {{{ return self.cursor().execute(sql) # }}} -class DB(SchemaUpgrade): +class DB(object, SchemaUpgrade): PATH_LIMIT = 40 if iswindows else 100 WINDOWS_LIBRARY_PATH_LIMIT = 75 @@ -400,5 +402,40 @@ class DB(SchemaUpgrade): ''' Return last modified time as a UTC datetime object ''' return utcfromtimestamp(os.stat(self.dbpath).st_mtime) + def read_tables(self): + tables = {} + for col in ('title', 'sort', 'author_sort', 'series_index', 'comments', + 'timestamp', 'published', 'uuid', 'path', 'cover', + 'last_modified'): + metadata = self.field_metadata[col].copy() + if metadata['table'] is None: + metadata['table'], metadata['column'] == 'books', ('has_cover' + if col == 'cover' else col) + tables[col] = OneToOneTable(col, metadata) + + for col in ('series', 'publisher', 'rating'): + tables[col] = ManyToOneTable(col, self.field_metadata[col].copy()) + + for col in ('authors', 'tags', 'formats', 'identifiers'): + cls = { + 'authors':AuthorsTable, + 'formats':FormatsTable, + 'identifiers':IdentifiersTable, + }.get(col, ManyToManyTable) + tables[col] = cls(col, self.field_metadata[col].copy()) + + tables['size'] = SizeTable('size', self.field_metadata['size'].copy()) + + with self.conn: # Use a single transaction, to ensure nothing modifies + # the db while we are reading + for table in tables.itervalues(): + try: + table.read() + except: + prints('Failed to read table:', table.name) + raise + + return tables + # }}} diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py new file mode 100644 index 0000000000..7240b3ec6e --- /dev/null +++ b/src/calibre/db/tables.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from datetime import datetime + +from dateutil.tz import tzoffset + +from calibre.constants import plugins +from calibre.utils.date import parse_date, local_tz +from calibre.ebooks.metadata import author_to_author_sort + +_c_speedup = plugins['speedup'][0] + +def _c_convert_timestamp(val): + if not val: + return None + try: + ret = _c_speedup.parse_date(val.strip()) + except: + ret = None + if ret is None: + return parse_date(val, as_utc=False) + year, month, day, hour, minutes, seconds, tzsecs = ret + return datetime(year, month, day, hour, minutes, seconds, + tzinfo=tzoffset(None, tzsecs)).astimezone(local_tz) + +class Table(object): + + def __init__(self, name, metadata): + self.name, self.metadata = name, metadata + + # self.adapt() maps values from the db to python objects + self.adapt = \ + { + 'datetime': _c_convert_timestamp, + 'bool': bool + }.get( + metadata['datatype'], lambda x: x) + if name == 'authors': + # Legacy + self.adapt = lambda x: x.replace('|', ',') if x else None + +class OneToOneTable(Table): + + def read(self, db): + self.book_col_map = {} + idcol = 'id' if self.metadata['table'] == 'books' else 'book' + for row in db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol, + self.metadata['column'], self.metadata['table'])): + self.book_col_map[row[0]] = self.adapt(row[1]) + +class SizeTable(OneToOneTable): + + def read(self, db): + self.book_col_map = {} + for row in db.conn.execute( + 'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data ' + 'WHERE data.book=books.id) FROM books'): + self.book_col_map[row[0]] = self.adapt(row[1]) + +class ManyToOneTable(Table): + + def read(self, db): + self.id_map = {} + self.extra_map = {} + self.col_book_map = {} + self.book_col_map = {} + self.read_id_maps(db) + self.read_maps(db) + + def read_id_maps(self, db): + for row in db.conn.execute('SELECT id, {0} FROM {1}'.format( + self.metadata['name'], self.metadata['table'])): + if row[1]: + self.id_map[row[0]] = self.adapt(row[1]) + + def read_maps(self, db): + for row in db.conn.execute( + 'SELECT book, {0} FROM books_{1}_link'.format( + self.metadata['link_column'], self.metadata['table'])): + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + self.book_col_map[row[0]] = row[1] + +class ManyToManyTable(ManyToOneTable): + + def read_maps(self, db): + for row in db.conn.execute( + 'SELECT book, {0} FROM books_{1}_link'.format( + self.metadata['link_column'], self.metadata['table'])): + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + if row[0] not in self.book_col_map: + self.book_col_map[row[0]] = [] + self.book_col_map[row[0]].append(row[1]) + +class AuthorsTable(ManyToManyTable): + + def read_id_maps(self, db): + for row in db.conn.execute( + 'SELECT id, name, sort FROM authors'): + self.id_map[row[0]] = row[1] + self.extra_map[row[0]] = (row[2] if row[2] else + author_to_author_sort(row[1])) + +class FormatsTable(ManyToManyTable): + + def read_id_maps(self, db): + pass + + def read_maps(self, db): + for row in db.conn.execute('SELECT book, format, name FROM data'): + if row[1] is not None: + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + if row[0] not in self.book_col_map: + self.book_col_map[row[0]] = [] + self.book_col_map[row[0]].append((row[1], row[2])) + +class IdentifiersTable(ManyToManyTable): + + def read_id_maps(self, db): + pass + + def read_maps(self, db): + for row in db.conn.execute('SELECT book, type, val FROM identifiers'): + if row[1] is not None and row[2] is not None: + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + if row[0] not in self.book_col_map: + self.book_col_map[row[0]] = [] + self.book_col_map[row[0]].append((row[1], row[2])) +