From 45704e36c5786b34adf698fc7941177367be9dbc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 18 Jun 2013 09:00:46 +0530 Subject: [PATCH 0001/1154] Add Kobo Aura HD to Welcome Wizard --- src/calibre/gui2/wizard/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/wizard/__init__.py b/src/calibre/gui2/wizard/__init__.py index 798ac5faca..f813eed892 100644 --- a/src/calibre/gui2/wizard/__init__.py +++ b/src/calibre/gui2/wizard/__init__.py @@ -139,7 +139,7 @@ class Kobo(Device): id = 'kobo' class KoboVox(Kobo): - name = 'Kobo Vox' + name = 'Kobo Vox and Kobo Aura HD' output_profile = 'tablet' id = 'kobo_vox' From 77baa05d3dbc3f13ae1d4aae1214775b2eb47c2f Mon Sep 17 00:00:00 2001 From: GRiker Date: Tue, 18 Jun 2013 09:14:34 -0600 Subject: [PATCH 0002/1154] When in synchronous mode (direct to iBooks), disable PDF transfers, as we can't update metadata in iTunes. Not sure when this started, but as of 11.0.4 it's broken. --- src/calibre/devices/apple/driver.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 66d27bad2b..aa90e2a9ef 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -1073,6 +1073,24 @@ class ITUNES(DriverBase): self.plugboards = plugboards self.plugboard_func = pb_func + def settings(self): + ''' + iBooks won't accept PDFs through the automation interface (11.0.4), not sure + when that became a problem. If we're directly connected to the iDevice, + remove PDF from list of supported formats. + ''' + if self.verbose: + logger().info("%s.settings()" % self.__class__.__name__) + + # If direct connection, remove PDF support + if self.manual_sync_mode: + opts = super(ITUNES, self).settings() + if 'pdf' in opts.format_map: + opts.format_map.remove('pdf') + return opts + else: + return super(ITUNES, self).settings() + def shutdown(self): if False and self.verbose: logger().info("%s.shutdown()\n" % self.__class__.__name__) @@ -3014,6 +3032,7 @@ class ITUNES(DriverBase): lb_added.year.set(metadata_x.pubdate.year) if db_added: + # This will fail for PDF files db_added.name.set(metadata_x.title) db_added.album.set(metadata_x.title) db_added.artist.set(authors_to_string(metadata_x.authors)) From ccaa960edf9733e78028cb50c73b805790196a3e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Jun 2013 10:02:20 +0530 Subject: [PATCH 0003/1154] pep8 --- src/calibre/gui2/dialogs/plugin_updater.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/dialogs/plugin_updater.py b/src/calibre/gui2/dialogs/plugin_updater.py index 3820169876..df76bec27d 100644 --- a/src/calibre/gui2/dialogs/plugin_updater.py +++ b/src/calibre/gui2/dialogs/plugin_updater.py @@ -254,7 +254,7 @@ Platforms: Windows, OSX, Linux; History: Yes; return self.installed_version is not None def is_upgrade_available(self): - return self.is_installed() and (self.installed_version < self.available_version \ + return self.is_installed() and (self.installed_version < self.available_version or self.is_deprecated) def is_valid_platform(self): @@ -317,7 +317,7 @@ class DisplayPluginModel(QAbstractTableModel): def data(self, index, role): if not index.isValid(): - return NONE; + return NONE row, col = index.row(), index.column() if row < 0 or row >= self.rowCount(): return NONE @@ -357,7 +357,7 @@ class DisplayPluginModel(QAbstractTableModel): else: return self._get_status_tooltip(display_plugin) elif role == Qt.ForegroundRole: - if col != 1: # Never change colour of the donation column + if col != 1: # Never change colour of the donation column if display_plugin.is_deprecated: return QVariant(QBrush(Qt.blue)) if display_plugin.is_disabled(): @@ -417,7 +417,7 @@ class DisplayPluginModel(QAbstractTableModel): icon_name = 'plugin_upgrade_invalid.png' else: icon_name = 'plugin_upgrade_ok.png' - else: # A plugin available not currently installed + else: # A plugin available not currently installed if display_plugin.is_valid_to_install(): icon_name = 'plugin_new_valid.png' else: @@ -429,11 +429,11 @@ class DisplayPluginModel(QAbstractTableModel): return QVariant(_('This plugin has been deprecated and should be uninstalled')+'\n\n'+ _('Right-click to see more options')) if not display_plugin.is_valid_platform(): - return QVariant(_('This plugin can only be installed on: %s') % \ + return QVariant(_('This plugin can only be installed on: %s') % ', '.join(display_plugin.platforms)+'\n\n'+ _('Right-click to see more options')) if numeric_version < display_plugin.calibre_required_version: - return QVariant(_('You must upgrade to at least Calibre %s before installing this plugin') % \ + return QVariant(_('You must upgrade to at least Calibre %s before installing this plugin') % self._get_display_version(display_plugin.calibre_required_version)+'\n\n'+ _('Right-click to see more options')) if display_plugin.installed_version < display_plugin.available_version: @@ -687,7 +687,7 @@ class PluginUpdaterDialog(SizePersistedDialog): def _install_clicked(self): display_plugin = self._selected_display_plugin() - if not question_dialog(self, _('Install %s')%display_plugin.name, '

' + \ + if not question_dialog(self, _('Install %s')%display_plugin.name, '

' + _('Installing plugins is a security risk. ' 'Plugins can contain a virus/malware. ' 'Only install it if you got it from a trusted source.' @@ -886,3 +886,4 @@ class PluginUpdaterDialog(SizePersistedDialog): pt.write(raw) pt.close() return pt.name + From 07c935b700bd6e7d35f3e587caa1a189b1a49669 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Jun 2013 10:12:03 +0530 Subject: [PATCH 0004/1154] Do not show builtin plugins in the get new plugins dialog If a builtin plugin with the same name as a third party plugin exists, then the builtin plagin was displayed in the get new plugins dialog as installed (happened with the new DOCX Input plugin). --- src/calibre/gui2/dialogs/plugin_updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/plugin_updater.py b/src/calibre/gui2/dialogs/plugin_updater.py index df76bec27d..c5d79218f9 100644 --- a/src/calibre/gui2/dialogs/plugin_updater.py +++ b/src/calibre/gui2/dialogs/plugin_updater.py @@ -89,7 +89,7 @@ def get_installed_plugin_status(display_plugin): display_plugin.installed_version = None display_plugin.plugin = None for plugin in initialized_plugins(): - if plugin.name == display_plugin.name: + if plugin.name == display_plugin.name and plugin.plugin_path is not None: display_plugin.plugin = plugin display_plugin.installed_version = plugin.version break From 0e14d36438eeb7ed8f39610267a1c78b7f98889f Mon Sep 17 00:00:00 2001 From: David Forrester Date: Wed, 19 Jun 2013 14:23:43 +1000 Subject: [PATCH 0005/1154] SQL delete needs firmware check for older Kobo firmare Kobo driver: Fix a regression when deleting empty shelves on Kobo devices with older firmware. Fixes #1192441 [Private bug](https://bugs.launchpad.net/calibre/+bug/1192441) As reported here, http://www.mobileread.com/forums/showthread.php?t=214760, if the Kobo device is using firmware before 2.5.0, it doesn't have the Activity table. The delete from this table when maintaining shelves needs a version check around it. --- src/calibre/devices/kobo/driver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index cddf6a561f..cb325efb07 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -1880,7 +1880,7 @@ class KOBOTOUCH(KOBO): # Remove any entries for the Activity table - removes tile from new home page if self.has_activity_table(): - debug_print('KoboTouch:delete_via_sql: detete from Activity') + debug_print('KoboTouch:delete_via_sql: delete from Activity') cursor.execute('delete from Activity where Id =?', t) connection.commit() @@ -2391,7 +2391,8 @@ class KOBOTOUCH(KOBO): cursor = connection.cursor() cursor.execute(delete_query) cursor.execute(update_query) - cursor.execute(delete_activity_query) + if self.has_activity_table(): + cursor.execute(delete_activity_query) connection.commit() cursor.close() From 8bd6cc840c8460279d1413cdf5ed141f0c12a9a4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Jun 2013 11:36:33 +0530 Subject: [PATCH 0006/1154] DOCX metadata: Be more intelligent for covers DOCX metadata: When reading covers from DOCX files use the first image as specified in the actual markup instead of just the first image in the container. --- src/calibre/ebooks/metadata/docx.py | 40 ++++++++++++++++++----------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/calibre/ebooks/metadata/docx.py b/src/calibre/ebooks/metadata/docx.py index ea34d27d3a..2c8b91bc70 100644 --- a/src/calibre/ebooks/metadata/docx.py +++ b/src/calibre/ebooks/metadata/docx.py @@ -8,29 +8,39 @@ __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' from calibre.ebooks.docx.container import DOCX +from calibre.ebooks.docx.names import XPath, get -from calibre.utils.zipfile import ZipFile from calibre.utils.magick.draw import identify_data +images = XPath('//*[name()="w:drawing" or name()="w:pict"]/descendant::*[(name()="a:blip" and @r:embed) or (name()="v:imagedata" and @r:id)][1]') + +def get_cover(docx): + doc = docx.document + rid_map = docx.document_relationships[0] + for image in images(doc): + rid = get(image, 'r:embed') or get(image, 'r:id') + if rid in rid_map: + try: + raw = docx.read(rid_map[rid]) + width, height, fmt = identify_data(raw) + except Exception: + continue + if 0.8 <= height/width <= 1.8 and height*width >= 160000: + return (fmt, raw) + def get_metadata(stream): c = DOCX(stream, extract=False) mi = c.metadata + try: + cdata = get_cover(c) + except Exception: + cdata = None + import traceback + traceback.print_exc() c.close() stream.seek(0) - cdata = None - with ZipFile(stream, 'r') as zf: - for zi in zf.infolist(): - ext = zi.filename.rpartition('.')[-1].lower() - if cdata is None and ext in {'jpeg', 'jpg', 'png', 'gif'}: - raw = zf.read(zi) - try: - width, height, fmt = identify_data(raw) - except: - continue - if 0.8 <= height/width <= 1.8 and height*width >= 160000: - cdata = (fmt, raw) - if cdata is not None: - mi.cover_data = cdata + if cdata is not None: + mi.cover_data = cdata return mi From 8020f489ca9eb51f3b997aba384f08bd4143ebcb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Jun 2013 12:39:20 +0530 Subject: [PATCH 0007/1154] pep8 --- src/calibre/web/feeds/templates.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/calibre/web/feeds/templates.py b/src/calibre/web/feeds/templates.py index a22a79ef20..68af525cfd 100644 --- a/src/calibre/web/feeds/templates.py +++ b/src/calibre/web/feeds/templates.py @@ -13,7 +13,7 @@ from lxml.html.builder import HTML, HEAD, TITLE, STYLE, DIV, BODY, \ from calibre import preferred_encoding, strftime, isbytestring -def CLASS(*args, **kwargs): # class is a reserved word in Python +def CLASS(*args, **kwargs): # class is a reserved word in Python kwargs['class'] = ' '.join(args) return kwargs @@ -26,7 +26,7 @@ class Template(object): self.html_lang = lang def generate(self, *args, **kwargs): - if not kwargs.has_key('style'): + if 'style' not in kwargs: kwargs['style'] = '' for key in kwargs.keys(): if isbytestring(kwargs[key]): @@ -152,8 +152,8 @@ class FeedTemplate(Template): body.append(div) if getattr(feed, 'image', None): div.append(DIV(IMG( - alt = feed.image_alt if feed.image_alt else '', - src = feed.image_url + alt=feed.image_alt if feed.image_alt else '', + src=feed.image_url ), CLASS('calibre_feed_image'))) if getattr(feed, 'description', None): @@ -261,8 +261,8 @@ class TouchscreenIndexTemplate(Template): for i, feed in enumerate(feeds): if feed: tr = TR() - tr.append(TD( CLASS('calibre_rescale_120'), A(feed.title, href='feed_%d/index.html'%i))) - tr.append(TD( '%s' % len(feed.articles), style="text-align:right")) + tr.append(TD(CLASS('calibre_rescale_120'), A(feed.title, href='feed_%d/index.html'%i))) + tr.append(TD('%s' % len(feed.articles), style="text-align:right")) toc.append(tr) div = DIV( masthead_p, @@ -307,7 +307,7 @@ class TouchscreenFeedTemplate(Template): if f > 0: link = A(CLASS('feed_link'), trim_title(feeds[f-1].title), - href = '../feed_%d/index.html' % int(f-1)) + href='../feed_%d/index.html' % int(f-1)) navbar_tr.append(TD(CLASS('feed_prev'),link)) # Up to Sections @@ -319,13 +319,12 @@ class TouchscreenFeedTemplate(Template): if f < len(feeds)-1: link = A(CLASS('feed_link'), trim_title(feeds[f+1].title), - href = '../feed_%d/index.html' % int(f+1)) + href='../feed_%d/index.html' % int(f+1)) navbar_tr.append(TD(CLASS('feed_next'),link)) navbar_t.append(navbar_tr) top_navbar = navbar_t bottom_navbar = copy.copy(navbar_t) - #print "\n%s\n" % etree.tostring(navbar_t, pretty_print=True) - + # print "\n%s\n" % etree.tostring(navbar_t, pretty_print=True) # Build the page head = HEAD(TITLE(feed.title)) @@ -342,8 +341,8 @@ class TouchscreenFeedTemplate(Template): if getattr(feed, 'image', None): div.append(DIV(IMG( - alt = feed.image_alt if feed.image_alt else '', - src = feed.image_url + alt=feed.image_alt if feed.image_alt else '', + src=feed.image_url ), CLASS('calibre_feed_image'))) if getattr(feed, 'description', None): @@ -411,6 +410,7 @@ class TouchscreenNavBarTemplate(Template): navbar_tr.append(TD(CLASS('article_next'),link)) navbar_t.append(navbar_tr) navbar.append(navbar_t) - #print "\n%s\n" % etree.tostring(navbar, pretty_print=True) + # print "\n%s\n" % etree.tostring(navbar, pretty_print=True) self.root = HTML(head, BODY(navbar)) + From adcc1739a65a5f5dc2a1bc6b0f13531534a00c98 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Jun 2013 13:12:14 +0530 Subject: [PATCH 0008/1154] News download: "downloaded from" for touchscreens News download: Add the "downloaded from" link at the bottom of every article when using a touchscreen output profile (like the Tablet profile). --- src/calibre/web/feeds/templates.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/calibre/web/feeds/templates.py b/src/calibre/web/feeds/templates.py index 68af525cfd..3ee90c43a6 100644 --- a/src/calibre/web/feeds/templates.py +++ b/src/calibre/web/feeds/templates.py @@ -387,6 +387,14 @@ class TouchscreenNavBarTemplate(Template): navbar_t = TABLE(CLASS('touchscreen_navbar')) navbar_tr = TR() + if bottom and not url.startswith('file://'): + navbar.append(HR()) + text = 'This article was downloaded by ' + p = PT(text, STRONG(__appname__), A(url, href=url), + style='text-align:left; max-width: 100%; overflow: hidden;') + p[0].tail = ' from ' + navbar.append(p) + navbar.append(BR()) # | Previous if art > 0: link = A(CLASS('article_link'),_('Previous'),href='%s../article_%d/index.html'%(prefix, art-1)) From 91b9bc6f0d9030d8e62b3a471dae6fa81c10d981 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Jun 2013 09:05:57 +0530 Subject: [PATCH 0009/1154] When searching allow use of uppercase locations When searching allow use of uppercase location names, such as AUTHOR instead of author, automatically lowercasing them. Fixes #1192785 [search errors are replicated automatically](https://bugs.launchpad.net/calibre/+bug/1192785) --- src/calibre/utils/search_query_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index a4e88d021f..745e01670f 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -240,8 +240,8 @@ class Parser(object): # the search string is something like 'author: "foo"' because it # will be interpreted as 'author:"foo"'. I am choosing to accept the # possible error. The expression should be written '"author:" foo' - if len(words) > 1 and words[0] in self.locations: - loc = words[0] + if len(words) > 1 and words[0].lower() in self.locations: + loc = words[0].lower() words = words[1:] if len(words) == 1 and self.token_type() == self.QUOTED_WORD: return ['token', loc, self.token(advance=True)] From 0f2d569f1624f7934984d8c3ee99d74934a4f479 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Jun 2013 09:19:16 +0530 Subject: [PATCH 0010/1154] Add a test for uppercase locations --- src/calibre/utils/search_query_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index 745e01670f..2682088681 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -697,6 +697,7 @@ class Tester(SearchQueryParser): tests = { 'Dysfunction' : set([348]), 'title:Dysfunction' : set([348]), + 'Title:Dysfunction' : set([348]), 'title:Dysfunction OR author:Laurie': set([348, 444]), '(tag:txt or tag:pdf)': set([33, 258, 354, 305, 242, 51, 55, 56, 154]), '(tag:txt OR tag:pdf) and author:Tolstoy': set([55, 56]), From 8f9528e517367041a216a98f5b5a6800bb9526f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Jun 2013 09:19:35 +0530 Subject: [PATCH 0011/1154] ... --- session.vim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/session.vim b/session.vim index a513afc2ec..a786e71451 100644 --- a/session.vim +++ b/session.vim @@ -25,7 +25,7 @@ fun! CalibreLog() " Setup buffers to edit the calibre changelog and version info prior to " making a release. enew - read ! git log --first-parent "--pretty=\%an:::\%n\%s\%n\%b\%n" -500 + read ! git log "--pretty=\%an:::\%n\%s\%n\%b\%n" -500 setl nomodifiable noswapfile buftype=nofile hi def link au Keyword syntax match au /^.*:::$/ From 2f37913801aa0758154711a0d8cff4dd69861ddc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Jun 2013 10:02:52 +0530 Subject: [PATCH 0012/1154] Confirm when dropping files onto Book Details When dropping files onto the Book Details panel, ask for confirmation before adding the files to the book. The confirmation can be disabled. --- src/calibre/gui2/actions/add.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index e18f5fe77c..4071ad1468 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -14,6 +14,7 @@ from calibre import as_unicode from calibre.gui2 import (error_dialog, choose_files, choose_dir, warning_dialog, info_dialog) from calibre.gui2.dialogs.add_empty_book import AddEmptyBookDialog +from calibre.gui2.dialogs.confirm_delete import confirm from calibre.gui2.dialogs.progress import ProgressDialog from calibre.gui2.widgets import IMAGE_EXTENSIONS from calibre.ebooks import BOOK_EXTENSIONS @@ -119,7 +120,6 @@ class AddAction(InterfaceAction): if current_idx.isValid(): view.model().current_changed(current_idx, current_idx) - def add_recursive(self, single): root = choose_dir(self.gui, 'recursive book import root dir dialog', 'Select root folder') @@ -205,7 +205,6 @@ class AddAction(InterfaceAction): 'authors'])) return - mi = MetaInformation(None) mi.isbn = x['isbn'] if self.isbn_add_tags: @@ -227,7 +226,8 @@ class AddAction(InterfaceAction): return db = self.gui.library_view.model().db current_idx = self.gui.library_view.currentIndex() - if not current_idx.isValid(): return + if not current_idx.isValid(): + return cid = db.id(current_idx.row()) from calibre.gui2.dnd import DownloadDialog d = DownloadDialog(url, fname, self.gui) @@ -235,7 +235,7 @@ class AddAction(InterfaceAction): if d.err is None: self.files_dropped_on_book(None, [d.fpath], cid=cid) - def files_dropped_on_book(self, event, paths, cid=None): + def files_dropped_on_book(self, event, paths, cid=None, do_confirm=True): accept = False if self.gui.current_view() is not self.gui.library_view: return @@ -243,8 +243,10 @@ class AddAction(InterfaceAction): cover_changed = False current_idx = self.gui.library_view.currentIndex() if cid is None: - if not current_idx.isValid(): return + if not current_idx.isValid(): + return cid = db.id(current_idx.row()) if cid is None else cid + formats = [] for path in paths: ext = os.path.splitext(path)[1].lower() if ext: @@ -257,10 +259,18 @@ class AddAction(InterfaceAction): db.set_cover(cid, pmap) cover_changed = True elif ext in BOOK_EXTENSIONS: - db.add_format_with_hooks(cid, ext, path, index_is_id=True) + formats.append((ext, path)) accept = True if accept and event is not None: event.accept() + if do_confirm and formats: + if not confirm( + _('You have dropped some files onto the book %s. This will' + ' add or replace the files for this book. Do you want to proceed?') % db.title(cid, index_is_id=True), + 'confirm_drop_on_book', parent=self.gui): + formats = [] + for ext, path in formats: + db.add_format_with_hooks(cid, ext, path, index_is_id=True) if current_idx.isValid(): self.gui.library_view.model().current_changed(current_idx, current_idx) if cover_changed: @@ -277,10 +287,9 @@ class AddAction(InterfaceAction): to_device = allow_device and self.gui.stack.currentIndex() != 0 self._add_books(books, to_device) if to_device: - self.gui.status_bar.show_message(\ + self.gui.status_bar.show_message( _('Uploading books to device.'), 2000) - def add_filesystem_book(self, paths, allow_device=True): self._add_filesystem_book(paths, allow_device=allow_device) @@ -453,3 +462,4 @@ class AddAction(InterfaceAction): self._adder.add(ok_paths) + From f964f512cf93d0a1c1fece37d933d1c00f2a7a56 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Jun 2013 10:11:38 +0530 Subject: [PATCH 0013/1154] Fix unable to change case for search queries Fix unable to change the case of a previously used search because of the search history. --- src/calibre/gui2/search_box.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index 6cfa3d0fec..85ddf533c4 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -197,7 +197,7 @@ class SearchBox2(QComboBox): # {{{ self.search.emit(text) if store_in_history: - idx = self.findText(text, Qt.MatchFixedString) + idx = self.findText(text, Qt.MatchFixedString|Qt.MatchCaseSensitive) self.block_signals(True) if idx < 0: self.insertItem(0, text) From b7d7a98fa65eaa07d2225e74bcc8c4c4a7ebe365 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Jun 2013 10:20:39 +0530 Subject: [PATCH 0014/1154] HTMLZ Output: Fix handling of images with URL unsafe filenames Fixes #1192687 [htmlz output not rewriting img and urls](https://bugs.launchpad.net/calibre/+bug/1192687) --- src/calibre/ebooks/htmlz/oeb2html.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/htmlz/oeb2html.py b/src/calibre/ebooks/htmlz/oeb2html.py index c6a7276f6c..b162d9e19c 100644 --- a/src/calibre/ebooks/htmlz/oeb2html.py +++ b/src/calibre/ebooks/htmlz/oeb2html.py @@ -18,7 +18,7 @@ from urlparse import urldefrag from calibre import prepare_string_for_xml from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace,\ - OEB_IMAGES, XLINK, rewrite_links + OEB_IMAGES, XLINK, rewrite_links, urlnormalize from calibre.ebooks.oeb.stylizer import Stylizer from calibre.utils.logging import default_log @@ -100,7 +100,7 @@ class OEB2HTML(object): def rewrite_link(self, url, page=None): if not page: return url - abs_url = page.abshref(url) + abs_url = page.abshref(urlnormalize(url)) if abs_url in self.images: return 'images/%s' % self.images[abs_url] if abs_url in self.links: From 59d1054cbff4007b46aecd0d54ebdc1df9ae4152 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Jun 2013 11:50:19 +0530 Subject: [PATCH 0015/1154] DOCX Input: Add support for images used as bullets --- src/calibre/ebooks/docx/images.py | 7 ++++--- src/calibre/ebooks/docx/numbering.py | 31 ++++++++++++++++++++++++---- src/calibre/ebooks/docx/to_html.py | 4 ++-- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/calibre/ebooks/docx/images.py b/src/calibre/ebooks/docx/images.py index e24b550797..ea3685316f 100644 --- a/src/calibre/ebooks/docx/images.py +++ b/src/calibre/ebooks/docx/images.py @@ -100,11 +100,12 @@ class Images(object): def __call__(self, relationships_by_id): self.rid_map = relationships_by_id - def generate_filename(self, rid, base=None): + def generate_filename(self, rid, base=None, rid_map=None): if rid in self.used: return self.used[rid] - raw = self.docx.read(self.rid_map[rid]) - base = base or ascii_filename(self.rid_map[rid].rpartition('/')[-1]).replace(' ', '_') or 'image' + rid_map = self.rid_map if rid_map is None else rid_map + raw = self.docx.read(rid_map[rid]) + base = base or ascii_filename(rid_map[rid].rpartition('/')[-1]).replace(' ', '_') or 'image' ext = what(None, raw) or base.rpartition('.')[-1] or 'jpeg' base = base.rpartition('.')[0] if not base: diff --git a/src/calibre/ebooks/docx/numbering.py b/src/calibre/ebooks/docx/numbering.py index 0178df3227..2bf86eea27 100644 --- a/src/calibre/ebooks/docx/numbering.py +++ b/src/calibre/ebooks/docx/numbering.py @@ -40,13 +40,14 @@ class Level(object): self.paragraph_style = self.character_style = None self.is_numbered = False self.num_template = None + self.pic_id = None if lvl is not None: self.read_from_xml(lvl) def copy(self): ans = Level() - for x in ('restart', 'start', 'fmt', 'para_link', 'paragraph_style', 'character_style', 'is_numbered', 'num_template'): + for x in ('restart', 'pic_id', 'start', 'fmt', 'para_link', 'paragraph_style', 'character_style', 'is_numbered', 'num_template'): setattr(ans, x, getattr(self, x)) return ans @@ -80,6 +81,8 @@ class Level(object): if val == 'bullet': self.is_numbered = False self.fmt = {'\uf0a7':'square', 'o':'circle'}.get(lt, 'disc') + for lpid in XPath('./w:lvlPicBulletId[@w:val]')(lvl): + self.pic_id = get(lpid, 'w:val') else: self.is_numbered = True self.fmt = STYLE_MAP.get(val, 'decimal') @@ -103,6 +106,19 @@ class Level(object): else: self.character_style.update(ps) + def css(self, images, pic_map, rid_map): + ans = {'list-style-type': self.fmt} + if self.pic_id: + rid = pic_map.get(self.pic_id, None) + if rid: + try: + fname = images.generate_filename(rid, rid_map=rid_map) + except Exception: + fname = None + else: + ans['list-style-image'] = 'url("images/%s")' % fname + return ans + class NumberingDefinition(object): def __init__(self, parent=None): @@ -127,9 +143,16 @@ class Numbering(object): self.definitions = {} self.instances = {} self.counters = {} + self.pic_map = {} - def __call__(self, root, styles): + def __call__(self, root, styles, rid_map): ' Read all numbering style definitions ' + self.rid_map = rid_map + for npb in XPath('./w:numPicBullet[@w:numPicBulletId]')(root): + npbid = get(npb, 'w:numPicBulletId') + for idata in XPath('descendant::v:imagedata[@r:id]')(npb): + rid = get(idata, 'r:id') + self.pic_map[npbid] = rid lazy_load = {} for an in XPath('./w:abstractNum[@w:abstractNumId]')(root): an_id = get(an, 'w:abstractNumId') @@ -198,7 +221,7 @@ class Numbering(object): if (restart is None and ilvl == levelnum + 1) or restart == levelnum + 1: counter[ilvl] = lvl.start - def apply_markup(self, items, body, styles, object_map): + def apply_markup(self, items, body, styles, object_map, images): for p, num_id, ilvl in items: d = self.instances.get(num_id, None) if d is not None: @@ -232,7 +255,7 @@ class Numbering(object): if has_template: wrap.set('lvlid', lvlid) else: - wrap.set('class', styles.register({'list-style-type': lvl.fmt}, 'list')) + wrap.set('class', styles.register(lvl.css(images, self.pic_map, self.rid_map), 'list')) parent.insert(idx, wrap) last_val = None for child in current_run: diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index 963d1fc6c8..79020d9c0a 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -137,7 +137,7 @@ class Convert(object): except (TypeError, ValueError): lvl = 0 numbered.append((html_obj, num_id, lvl)) - self.numbering.apply_markup(numbered, self.body, self.styles, self.object_map) + self.numbering.apply_markup(numbered, self.body, self.styles, self.object_map, self.images) self.apply_frames() if len(self.body) > 0: @@ -263,7 +263,7 @@ class Convert(object): except KeyError: self.log.warn('Numbering styles %s do not exist' % nname) else: - numbering(fromstring(raw), self.styles) + numbering(fromstring(raw), self.styles, self.docx.get_relationships(nname)[0]) self.styles.resolve_numbering(numbering) From c8b9d624cdf2c4e8cc821f5704b2a9a6d1a0a8fa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Jun 2013 12:02:12 +0530 Subject: [PATCH 0016/1154] Fix caching of images in docx to use filenames instead of possibly non-unique relationship ids. --- src/calibre/ebooks/docx/images.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/docx/images.py b/src/calibre/ebooks/docx/images.py index ea3685316f..85e957a589 100644 --- a/src/calibre/ebooks/docx/images.py +++ b/src/calibre/ebooks/docx/images.py @@ -101,10 +101,11 @@ class Images(object): self.rid_map = relationships_by_id def generate_filename(self, rid, base=None, rid_map=None): - if rid in self.used: - return self.used[rid] rid_map = self.rid_map if rid_map is None else rid_map - raw = self.docx.read(rid_map[rid]) + fname = rid_map[rid] + if fname in self.used: + return self.used[fname] + raw = self.docx.read(fname) base = base or ascii_filename(rid_map[rid].rpartition('/')[-1]).replace(' ', '_') or 'image' ext = what(None, raw) or base.rpartition('.')[-1] or 'jpeg' base = base.rpartition('.')[0] @@ -118,7 +119,7 @@ class Images(object): n, e = base.rpartition('.')[0::2] name = '%s-%d.%s' % (n, c, e) c += 1 - self.used[rid] = name + self.used[fname] = name with open(os.path.join(self.dest_dir, name), 'wb') as f: f.write(raw) self.all_images.add('images/' + name) From 6aea840b3e998ee335ade1815d949894747a46f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Jun 2013 12:03:45 +0530 Subject: [PATCH 0017/1154] Ebook-viewer: Show docx cover, if present --- src/calibre/ebooks/oeb/iterator/book.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/oeb/iterator/book.py b/src/calibre/ebooks/oeb/iterator/book.py index 4ebd543aab..d3cedce688 100644 --- a/src/calibre/ebooks/oeb/iterator/book.py +++ b/src/calibre/ebooks/oeb/iterator/book.py @@ -143,7 +143,7 @@ class EbookIterator(BookmarksMixin): cover = self.opf.cover if cover and self.ebook_ext in {'lit', 'mobi', 'prc', 'opf', 'fb2', - 'azw', 'azw3'}: + 'azw', 'azw3', 'docx'}: cfile = os.path.join(self.base, 'calibre_iterator_cover.html') rcpath = os.path.relpath(cover, self.base).replace(os.sep, '/') chtml = (TITLEPAGE%prepare_string_for_xml(rcpath, True)).encode('utf-8') From ee47445d99c7db10a53b62fd278cd25e53c42be5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Jun 2013 08:35:02 +0530 Subject: [PATCH 0018/1154] Update The Walrus Mag Fixes #1193021 [Update recipe The Walrus Mag](https://bugs.launchpad.net/calibre/+bug/1193021) --- recipes/walrusmag.recipe | 47 ++++++---------------------------------- 1 file changed, 7 insertions(+), 40 deletions(-) diff --git a/recipes/walrusmag.recipe b/recipes/walrusmag.recipe index 5c10100fe4..72b110b046 100644 --- a/recipes/walrusmag.recipe +++ b/recipes/walrusmag.recipe @@ -1,46 +1,13 @@ from calibre.web.feeds.news import BasicNewsRecipe -class AdvancedUserRecipe1282101454(BasicNewsRecipe): - title = 'The Walrus Mag' +class AdvancedUserRecipe1371743239(BasicNewsRecipe): + title = u'The Walrus Mag' + description = 'National general interest magazine about Canada and its place in the world' language = 'en' - __author__ = 'TonytheBookworm' - description = 'national general interest magazine about Canada and its place in the world' - publisher = 'Tony Stegall' - category = 'Canada, news' - oldest_article = 365 + __author__ = 'Bert' + oldest_article = 34 max_articles_per_feed = 100 + auto_cleanup = True - masthead_url = 'http://www.walrusmagazine.com/images/wordmark.png' - keep_only_tags = [ - dict(name='h1'), - dict(name='div', attrs={'id':['prbody']}) - # ,dict(attrs={'id':['cxArticleText','cxArticleBodyText']}) - ] - feeds = [ - ('Walrus Magazine', 'http://feeds.feedburner.com/WalrusFeatureArticles?format=xml'), - - ] - - - - - def print_version(self, url): - split1 = url.split("/articles/") - #print 'THE SPLIT IS: ', split1 - url1 = split1[0] - #print 'url1 is: ',url1 - url2 = split1[1] - #print 'url2 is: ',url2 - - - #need to convert to print_version - #originalversion is : http://www.walrusmagazine.com/articles/2010.09-frontier-no-one-can-hear-you-scream/ - #printversion should be: http://www.walrusmagazine.com/print/2010.09-frontier-no-one-can-hear-you-scream/ - - - - - print_url = url1 + '/print/' + url2 - #print 'THIS URL WILL PRINT: ', print_url # this is a test string to see what the url is it will return - return print_url + feeds = [(u'The Walrus Mag', u'http://thewalrus.ca/feed/')] From 10bbb183372a3d98d5174726b1e068f5bd085ead Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Jun 2013 08:40:46 +0530 Subject: [PATCH 0019/1154] ... --- manual/develop.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual/develop.rst b/manual/develop.rst index a939a442b4..9e5b47e8e7 100644 --- a/manual/develop.rst +++ b/manual/develop.rst @@ -49,7 +49,7 @@ All the |app| python code is in the ``calibre`` package. This package contains t * Metadata reading, writing, and downloading is all in ``ebooks.metadata`` * Conversion happens in a pipeline, for the structure of the pipeline, see :ref:`conversion-introduction`. The pipeline consists of an input - plugin, various transforms and an output plugin. The that code constructs + plugin, various transforms and an output plugin. The code that constructs and drives the pipeline is in :file:`plumber.py`. The pipeline works on a representation of an ebook that is like an unzipped epub, with manifest, spine, toc, guide, html content, etc. The From b8753d0b95db8b64436c450b46ac9c6ac6353cb2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Jun 2013 09:11:09 +0530 Subject: [PATCH 0020/1154] version 0.9.36 --- Changelog.yaml | 59 ++++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index 8462264e38..f952b961e2 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,65 @@ # new recipes: # - title: +- version: 0.9.36 + date: 2013-06-21 + + new features: + - title: "DOCX Input: Support for Table of Contents created using the Word Table of Contents tool. calibre now first looks for such a Table of Contents and only if one is not found does it generate a ToC from headings." + + - title: "DOCX Input: Add support for images used as bullets in lists" + + - title: "DOCX Input: If a large image that looks like a cover is present at the start of the document, remove it and use it as the cover of the output ebook. This can be turned off under the DOCX Input section of the conversion dialog." + + - title: "When dropping files onto the Book Details panel, ask for confirmation before adding the files to the book. The confirmation can be disabled." + + - title: "News download: Add the 'downloaded from' link at the bottom of every article when using a touchscreen output profile (like the Tablet profile)." + + - title: "E-book viewer: Change the bookmark button to always popup a menu when clicked, makes accessing existing bookmarks easier." + + - title: "After a bulk metadata download, focus the review button on the popup notification, instead of the OK button." + tickets: [1190931] + + bug fixes: + - title: "DOCX Input: Hide text that has been marked as not being visible in the web view in Word." + + - title: "DOCX Input: When converting docx files with large numbers of unnamed images, do not crash on windows." + tickets: [1191354] + + - title: "DOCX Input: Add support for the Word setting 'No space between paragraphs with the same style'." + tickets: [119100] + + - title: "MOBI Output: Fix rendering of SVG images that embed large raster images in 64bit calibre installs." + tickets: [1191020] + + - title: "HTMLZ Output: Fix handling of images with URL unsafe filenames." + tickets: [1192687] + + - title: "Fix unable to change the case of a previously used search because of the search history." + + - title: "When searching allow use of uppercase location names, such as AUTHOR instead of author, automatically lowercasing them." + tickets: [1192785] + + - title: "DOCX metadata: When reading covers from DOCX files use the first image as specified in the actual markup instead of just the first image in the container." + + - title: "Kobo driver: Fix a regression when deleting empty shelves on Kobo devices with older firmware." + tickets: [1192441] + + - title: "Do not show builtin plugins in the get new plugins dialog If a builtin plugin with the same name as a third party plugin exists, then the builtin plagin was displayed in the get new plugins dialog as installed (happened with the new DOCX Input plugin)." + + - title: "Apple driver: When in synchronous mode (direct to iBooks), disable PDF transfers, as we can't update metadata in iTunes. Not sure when this started, but as of iTunes 11.0.4 it's broken." + + - title: "Get Books: Fix error when using internal browser on some systems" + tickets: [1191199] + + improved recipes: + - The Walrus Mag + - Various Polish news sources + + new recipes: + - title: Various Polish news sources + author: fenuks + - version: 0.9.35 date: 2013-06-14 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index fb2743fd7c..6834a4e66d 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 9, 35) +numeric_version = (0, 9, 36) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " From 27678a85c6342aeb48e56b3464e0fdc84ed0f374 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Jun 2013 16:33:43 +0530 Subject: [PATCH 0021/1154] Add tests for set_metadata() --- src/calibre/db/cache.py | 36 +++++++++++++++++++++++------ src/calibre/db/tests/base.py | 6 ++--- src/calibre/db/tests/writing.py | 40 +++++++++++++++++++++++++++++++-- src/calibre/db/write.py | 2 +- 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index b79ff2a31b..88f06b43ba 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -843,9 +843,25 @@ class Cache(object): @write_api def set_metadata(self, book_id, mi, ignore_errors=False, force_changes=False, set_title=True, set_authors=True): - if callable(getattr(mi, 'to_book_metadata', None)): + ''' + Set metadata for the book `id` from the `Metadata` object `mi` + + Setting force_changes=True will force set_metadata to update fields even + if mi contains empty values. In this case, 'None' is distinguished from + 'empty'. If mi.XXX is None, the XXX is not replaced, otherwise it is. + The tags, identifiers, and cover attributes are special cases. Tags and + identifiers cannot be set to None so then will always be replaced if + force_changes is true. You must ensure that mi contains the values you + want the book to have. Covers are always changed if a new cover is + provided, but are never deleted. Also note that force_changes has no + effect on setting title or authors. + ''' + + try: # Handle code passing in an OPF object instead of a Metadata object mi = mi.to_book_metadata() + except (AttributeError, TypeError): + pass def set_field(name, val, **kwargs): self._set_field(name, {book_id:val}, **kwargs) @@ -864,7 +880,7 @@ class Cache(object): set_field('authors', authors, do_path_update=False) if path_changed: - self._update_path((book_id,)) + self._update_path({book_id}) def protected_set_field(name, val, **kwargs): try: @@ -890,12 +906,16 @@ class Cache(object): if cdata is not None: self._set_cover({book_id: cdata}) - for field in ('title_sort', 'author_sort', 'publisher', 'series', - 'tags', 'comments', 'languages', 'pubdate'): + for field in ('author_sort', 'publisher', 'series', 'tags', 'comments', + 'languages', 'pubdate'): val = mi.get(field, None) if (force_changes and val is not None) or not mi.is_null(field): protected_set_field(field, val) + val = mi.get('title_sort', None) + if (force_changes and val is not None) or not mi.is_null('title_sort'): + protected_set_field('sort', val) + # identifiers will always be replaced if force_changes is True mi_idents = mi.get_identifiers() if force_changes: @@ -917,9 +937,11 @@ class Cache(object): val = mi.get(key, None) if force_changes or val is not None: protected_set_field(key, val) - extra = mi.get_extra(key) - if extra is not None: - protected_set_field(key+'_index', extra) + idx = key + '_index' + if idx in self.fields: + extra = mi.get_extra(key) + if extra is not None or force_changes: + protected_set_field(idx, extra) # }}} diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index cc8da89b05..b57b017ba3 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -78,7 +78,7 @@ class BaseTest(unittest.TestCase): def cloned_library(self): return self.clone_library(self.library_path) - def compare_metadata(self, mi1, mi2): + def compare_metadata(self, mi1, mi2, exclude=()): allfk1 = mi1.all_field_keys() allfk2 = mi2.all_field_keys() self.assertEqual(allfk1, allfk2) @@ -88,7 +88,7 @@ class BaseTest(unittest.TestCase): 'ondevice_col', 'last_modified', 'has_cover', 'cover_data'}.union(allfk1) for attr in all_keys: - if attr == 'user_metadata': + if attr == 'user_metadata' or attr in exclude: continue attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr) if attr == 'formats': @@ -97,7 +97,7 @@ class BaseTest(unittest.TestCase): attr1, attr2 = set(attr1), set(attr2) self.assertEqual(attr1, attr2, '%s not the same: %r != %r'%(attr, attr1, attr2)) - if attr.startswith('#'): + if attr.startswith('#') and attr + '_index' not in exclude: attr1, attr2 = mi1.get_extra(attr), mi2.get_extra(attr) self.assertEqual(attr1, attr2, '%s {#extra} not the same: %r != %r'%(attr, attr1, attr2)) diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 6d3169b905..5d04c11def 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -376,7 +376,43 @@ class WritingTest(BaseTest): self.assertTrue(old.has_cover(book_id)) # }}} - def test_set_metadata(self): + def test_set_metadata(self): # {{{ ' Test setting of metadata ' - self.assertTrue(False, 'TODO: test set_metadata()') + ae = self.assertEqual + cache = self.init_cache(self.cloned_library) + + # Check that changing title/author updates the path + mi = cache.get_metadata(1) + old_path = cache.field_for('path', 1) + old_title, old_author = mi.title, mi.authors[0] + ae(old_path, '%s/%s (1)' % (old_author, old_title)) + mi.title, mi.authors = 'New Title', ['New Author'] + cache.set_metadata(1, mi) + ae(cache.field_for('path', 1), '%s/%s (1)' % (mi.authors[0], mi.title)) + p = cache.format_abspath(1, 'FMT1') + self.assertTrue(mi.authors[0] in p and mi.title in p) + + # Compare old and new set_metadata() + db = self.init_old(self.cloned_library) + mi = db.get_metadata(1, index_is_id=True, get_cover=True, cover_as_data=True) + mi2 = db.get_metadata(3, index_is_id=True, get_cover=True, cover_as_data=True) + db.set_metadata(2, mi) + db.set_metadata(1, mi2, force_changes=True) + oldmi = db.get_metadata(2, index_is_id=True, get_cover=True, cover_as_data=True) + oldmi2 = db.get_metadata(1, index_is_id=True, get_cover=True, cover_as_data=True) + db.close() + del db + cache = self.init_cache(self.cloned_library) + cache.set_metadata(2, mi) + nmi = cache.get_metadata(2, get_cover=True, cover_as_data=True) + ae(oldmi.cover_data, nmi.cover_data) + self.compare_metadata(nmi, oldmi, exclude={'last_modified', 'format_metadata'}) + cache.set_metadata(1, mi2, force_changes=True) + nmi2 = cache.get_metadata(1, get_cover=True, cover_as_data=True) + # The new code does not allow setting of #series_index to None, instead + # it is reset to 1.0 + ae(nmi2.get_extra('#series'), 1.0) + self.compare_metadata(nmi2, oldmi2, exclude={'last_modified', 'format_metadata', '#series_index'}) + + # }}} diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py index 7fdb2070c0..9bae3e6abb 100644 --- a/src/calibre/db/write.py +++ b/src/calibre/db/write.py @@ -212,7 +212,7 @@ def custom_series_index(book_id_val_map, db, field, *args): ids = series_field.ids_for_book(book_id) if ids: sequence.append((sidx, book_id, ids[0])) - field.table.book_col_map[book_id] = sidx + field.table.book_col_map[book_id] = sidx if sequence: db.conn.executemany('UPDATE %s SET %s=? WHERE book=? AND value=?'%( field.metadata['table'], field.metadata['column']), sequence) From 4a105ef459fe5204d6218c4b2bdf8084ba5dc78c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Jun 2013 16:40:47 +0530 Subject: [PATCH 0022/1154] ... --- src/calibre/db/tests/legacy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 48408d6105..af6b977aef 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -120,7 +120,8 @@ class LegacyTest(BaseTest): for attr in dir(db): if attr in SKIP_ATTRS: continue - self.assertTrue(hasattr(ndb, attr), 'The attribute %s is missing' % attr) + if not hasattr(ndb, attr): + raise AssertionError('The attribute %s is missing' % attr) obj, nobj = getattr(db, attr), getattr(ndb, attr) if attr not in SKIP_ARGSPEC: try: From a2aad03af904e174463464a9139247e665ca685e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Jun 2013 22:25:37 +0530 Subject: [PATCH 0023/1154] Stop using googlecode for file hosting --- setup/hosting.py | 44 ++++++++++++++++++++++++++++++++++++-------- setup/upload.py | 12 ++++++++++-- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/setup/hosting.py b/setup/hosting.py index 8707388181..9853f181a4 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -10,13 +10,13 @@ __docformat__ = 'restructuredtext en' import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib from argparse import ArgumentParser, FileType from subprocess import check_call -from tempfile import NamedTemporaryFile#, mkdtemp +from tempfile import NamedTemporaryFile from collections import OrderedDict import mechanize from lxml import html -def login_to_google(username, password): +def login_to_google(username, password): # {{{ br = mechanize.Browser() br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')] @@ -30,15 +30,16 @@ def login_to_google(username, password): x = re.search(br'(?is).*?', raw) if x is not None: print ('Title of post login page: %s'%x.group()) - #open('/tmp/goog.html', 'wb').write(raw) + # open('/tmp/goog.html', 'wb').write(raw) raise ValueError(('Failed to login to google with credentials: %s %s' '\nGoogle sometimes requires verification when logging in from a ' 'new IP address. Use lynx to login and supply the verification, ' 'at: lynx -accept_all_cookies https://accounts.google.com/ServiceLogin?service=code') %(username, password)) return br +# }}} -class ReadFileWithProgressReporting(file): # {{{ +class ReadFileWithProgressReporting(file): # {{{ def __init__(self, path, mode='rb'): file.__init__(self, path, mode) @@ -101,7 +102,7 @@ class Base(object): # {{{ #}}} -class GoogleCode(Base):# {{{ +class GoogleCode(Base): # {{{ def __init__(self, # A mapping of filenames to file descriptions. The descriptions are @@ -141,7 +142,7 @@ class GoogleCode(Base):# {{{ # The pattern to match filenames for the files being uploaded and # extract version information from them. Must have a named group # named version - filename_pattern=r'{appname}-(?:portable-installer-)?(?P.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' + filename_pattern=r'{appname}-(?:portable-installer-)?(?P.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' # noqa ): self.username, self.password, = username, password @@ -227,7 +228,8 @@ class GoogleCode(Base):# {{{ paths = eval(raw) if raw else {} paths.update(self.paths) rem = [x for x in paths if self.version not in x] - for x in rem: paths.pop(x) + for x in rem: + paths.pop(x) raw = ['%r : %r,'%(k, v) for k, v in paths.items()] raw = '{\n\n%s\n\n}\n'%('\n'.join(raw)) with NamedTemporaryFile() as t: @@ -347,7 +349,7 @@ class GoogleCode(Base):# {{{ # }}} -class SourceForge(Base): # {{{ +class SourceForge(Base): # {{{ # Note that you should manually ssh once to username,project@frs.sourceforge.net # on the staging server so that the host key is setup @@ -378,6 +380,28 @@ class SourceForge(Base): # {{{ # }}} +def upload_to_servers(files, version): # {{{ + for server, rdir in {'files':'/usr/share/nginx/html'}.iteritems(): + print('Uploading to server:', server) + server = '%s.calibre-ebook.com' % server + rdir = '%s/%s/' % (rdir, version) + for x in files: + start = time.time() + print ('Uploading', x) + for i in range(5): + try: + check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x, + 'root@%s:%s'%(server, rdir)]) + except KeyboardInterrupt: + raise SystemExit(1) + except: + print ('\nUpload failed, trying again in 30 seconds') + time.sleep(30) + else: + break + print ('Uploaded in', int(time.time() - start), 'seconds\n\n') +# }}} + # CLI {{{ def cli_parser(): epilog='Copyright Kovid Goyal 2012' @@ -409,6 +433,7 @@ def cli_parser(): sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge', epilog=epilog) cron = subparsers.add_parser('cron', help='Call script from cron') + subparsers.add_parser('calibre', help='Upload to calibre file servers') a = gc.add_argument @@ -471,8 +496,11 @@ def main(args=None): sf() elif args.service == 'cron': login_to_google(args.username, args.password) + elif args.service == 'calibre': + upload_to_servers(ofiles, args.version) if __name__ == '__main__': main() # }}} + diff --git a/setup/upload.py b/setup/upload.py index 673f9f4679..1c7348bfe9 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -81,7 +81,7 @@ class ReUpload(Command): # {{{ # Data {{{ def get_google_data(): - with open(os.path.expanduser('~/work/kde/conf/googlecodecalibre'), 'rb') as f: + with open(os.path.expanduser('~/work/env/private/googlecodecalibre'), 'rb') as f: gc_password, ga_un, pw = f.read().strip().split('|') return { @@ -111,6 +111,9 @@ def sf_cmdline(ver, sdata): return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'], sdata['username']] +def calibre_cmdline(ver): + return [__appname__, ver, 'fmap', 'calibre'] + def run_remote_upload(args): print 'Running remotely:', ' '.join(args) subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST), @@ -133,7 +136,8 @@ class UploadInstallers(Command): # {{{ try: self.upload_to_staging(tdir, files) self.upload_to_sourceforge() - self.upload_to_google(opts.replace) + self.upload_to_calibre() + # self.upload_to_google(opts.replace) finally: shutil.rmtree(tdir, ignore_errors=True) @@ -170,6 +174,10 @@ class UploadInstallers(Command): # {{{ sdata = get_sourceforge_data() args = sf_cmdline(__version__, sdata) run_remote_upload(args) + + def upload_to_calibre(self): + run_remote_upload(calibre_cmdline(__version__)) + # }}} class UploadUserManual(Command): # {{{ From b06d080987897347494bbc202872e70e6fbb5bae Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Jun 2013 07:05:24 +0530 Subject: [PATCH 0024/1154] Update La Nacion (Costa Rica) --- recipes/la_nacion_cr.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/la_nacion_cr.recipe b/recipes/la_nacion_cr.recipe index ae320064d6..99b927edbb 100644 --- a/recipes/la_nacion_cr.recipe +++ b/recipes/la_nacion_cr.recipe @@ -20,7 +20,7 @@ class crnews(BasicNewsRecipe): no_stylesheets = True - feeds = [(u'Portada', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=portada'), (u'Ultima Hora', u'http://www.nacion.com/Generales/RSS/UltimaHoraRss.aspx'), (u'Nacionales', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=elpais'), (u'Entretenimiento', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=entretenimiento'), (u'Sucesos', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=sucesos'), (u'Deportes', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=deportes'), (u'Internacionales', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=mundo'), (u'Economia', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=economia'), (u'Aldea Global', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=aldeaglobal'), (u'Tecnologia', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=tecnologia'), (u'Opinion', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=opinion')] + feeds = [(u'Portada', u'http://www.nacion.com/rss/'), (u'Ultima Hora', u'http://www.nacion.com/rss/latest/'), (u'Nacionales', u'http://www.nacion.com/rss/nacional/'), (u'Entretenimiento', u'http://www.nacion.com/rss/ocio/'), (u'Sucesos', u'http://www.nacion.com/rss/sucesos/'), (u'Deportes', u'http://www.nacion.com/rss/deportes/'), (u'Internacionales', u'http://www.nacion.com/rss/mundo/'), (u'Economia', u'http://www.nacion.com/rss/economia/'), (u'Vivir', u'http://www.nacion.com/rss/vivir/'), (u'Tecnologia', u'http://www.nacion.com/rss/tecnologia/'), (u'Opinion', u'http://www.nacion.com/rss/opinion/')] def get_cover_url(self): index = 'http://kiosko.net/cr/np/cr_nacion.html' From ec2bd359abbaa941dd7cc505b45d21753d914ade Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Jun 2013 07:30:27 +0530 Subject: [PATCH 0025/1154] Copy calibre releases to my backup hdd --- setup/upload.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup/upload.py b/setup/upload.py index 1c7348bfe9..a113eef703 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -133,15 +133,18 @@ class UploadInstallers(Command): # {{{ files = {x:installer_description(x) for x in all_possible.intersection(available)} tdir = mkdtemp() + backup = os.path.join('/mnt/external/calibre/%s' % __version__) + if not os.path.exists(backup): + os.mkdir(backup) try: - self.upload_to_staging(tdir, files) + self.upload_to_staging(tdir, backup, files) self.upload_to_sourceforge() self.upload_to_calibre() # self.upload_to_google(opts.replace) finally: shutil.rmtree(tdir, ignore_errors=True) - def upload_to_staging(self, tdir, files): + def upload_to_staging(self, tdir, backup, files): os.mkdir(tdir+'/dist') hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'hosting.py') @@ -149,6 +152,7 @@ class UploadInstallers(Command): # {{{ for f in files: shutil.copyfile(f, os.path.join(tdir, f)) + shutil.copyfile(f, os.path.join(backup, f)) with open(os.path.join(tdir, 'fmap'), 'wb') as fo: for f, desc in files.iteritems(): From bf1055abd97934eacc3c015769a9a82cd54b362a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Jun 2013 07:42:30 +0530 Subject: [PATCH 0026/1154] fetch-ebbok-metadata: Fix --opf argument requiring a value --- src/calibre/ebooks/metadata/sources/cli.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/cli.py b/src/calibre/ebooks/metadata/sources/cli.py index f8b9c6b7a9..b20c6a5bfa 100644 --- a/src/calibre/ebooks/metadata/sources/cli.py +++ b/src/calibre/ebooks/metadata/sources/cli.py @@ -34,9 +34,9 @@ def option_parser(): parser.add_option('-i', '--isbn', help='Book ISBN') parser.add_option('-v', '--verbose', default=False, action='store_true', help='Print the log to the console (stderr)') - parser.add_option('-o', '--opf', help='Output the metadata in OPF format') + parser.add_option('-o', '--opf', help='Output the metadata in OPF format instead of human readable text.', action='store_true', default=False) parser.add_option('-c', '--cover', - help='Specify a filename. The cover, if available, will be saved to it') + help='Specify a filename. The cover, if available, will be saved to it. Without this option, no cover will be downloaded.') parser.add_option('-d', '--timeout', default='30', help='Timeout in seconds. Default is 30') @@ -71,16 +71,14 @@ def main(args=sys.argv): if opts.cover and results: cover = download_cover(log, title=opts.title, authors=authors, identifiers=result.identifiers, timeout=int(opts.timeout)) - if cover is None: + if cover is None and not opts.opf: prints('No cover found', file=sys.stderr) else: save_cover_data_to(cover[-1], opts.cover) result.cover = cf = opts.cover - log = buf.getvalue() - result = (metadata_to_opf(result) if opts.opf else unicode(result).encode('utf-8')) @@ -95,3 +93,4 @@ def main(args=sys.argv): if __name__ == '__main__': sys.exit(main()) + From 6a134427c84491d7d5df488a3eb1aa55c68a7109 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Jun 2013 07:59:40 +0530 Subject: [PATCH 0027/1154] Update Frontline --- recipes/frontlineonnet.recipe | 51 +++++++++++++++++------------------ 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/recipes/frontlineonnet.recipe b/recipes/frontlineonnet.recipe index dc1d16cfd4..73d866c3b3 100644 --- a/recipes/frontlineonnet.recipe +++ b/recipes/frontlineonnet.recipe @@ -46,35 +46,34 @@ class Frontlineonnet(BasicNewsRecipe): keep_only_tags= [ dict(name='div', attrs={'id':'content'}) - #,dict(attrs={'class':'byline'}) ] - #remove_attributes=['size','noshade','border'] - - #def preprocess_html(self, soup): - #for item in soup.findAll(style=True): - #del item['style'] - #for item in soup.findAll('img'): - #if not item.has_key('alt'): - #item['alt'] = 'image' - #return soup + remove_attributes=['size','noshade','border'] def parse_index(self): articles = [] + current_section = None + feeds = [] soup = self.index_to_soup(self.INDEX) - for feed_link in soup.findAll('div', id='headseccol'): - a = feed_link.find('a', href=True) - title = self.tag_to_string(a) - url = a['href'] - articles.append({ - 'title' :title - ,'date' :'' - ,'url' :url - ,'description':'' - }) - return [('Frontline', articles)] + for h3 in soup.findAll('h3'): + if h3.get('class', None) == 'artListSec': + if articles: + feeds.append((current_section, articles)) + articles = [] + current_section = self.tag_to_string(h3).strip() + self.log(current_section) + elif h3.get('id', None) in {'headseccol', 'headsec'}: + a = h3.find('a', href=True) + if a is not None: + title = self.tag_to_string(a) + url = a['href'] + articles.append({ + 'title' :title + ,'date' :'' + ,'url' :url + ,'description':'' + }) + self.log('\t', title, url) + if articles: + feeds.append((current_section, articles)) + return feeds - #def print_version(self, url): - #return "http://www.hinduonnet.com/thehindu/thscrip/print.pl?prd=fline&file=" + url.rpartition('/')[2] - - #def image_url_processor(self, baseurl, url): - #return url.replace('../images/', self.INDEX + 'images/').strip() From 2ee5ad2e301c7acfb7a3b137a998b71d8ac87d52 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Jun 2013 08:30:59 +0530 Subject: [PATCH 0028/1154] Ensure dist files have correct permissions --- setup/upload.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/upload.py b/setup/upload.py index a113eef703..784c0cf9f8 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -151,8 +151,10 @@ class UploadInstallers(Command): # {{{ shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py')) for f in files: - shutil.copyfile(f, os.path.join(tdir, f)) - shutil.copyfile(f, os.path.join(backup, f)) + for x in (tdir, backup): + dest = os.path.join(x, f) + shutil.copyfile(f, dest) + os.chmod(dest, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH) with open(os.path.join(tdir, 'fmap'), 'wb') as fo: for f, desc in files.iteritems(): From 8042ad136fa7d11f3eb3e69c357ba3e88a0d1b41 Mon Sep 17 00:00:00 2001 From: GRiker Date: Sat, 22 Jun 2013 09:28:37 -0700 Subject: [PATCH 0029/1154] Improved error handling for float values with non-period decimal points, bad datetime.datetime values. --- src/calibre/devices/idevice/parse_xml.py | 30 +++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/calibre/devices/idevice/parse_xml.py b/src/calibre/devices/idevice/parse_xml.py index 80d8ebaccd..6982b5a213 100755 --- a/src/calibre/devices/idevice/parse_xml.py +++ b/src/calibre/devices/idevice/parse_xml.py @@ -192,22 +192,30 @@ class XmlPropertyListParser(object): pattern = XmlPropertyListParser.DATETIME_PATTERN match = pattern.match(content) if not match: - raise PropertyListParseError("Failed to parse datetime '%s'" % content) + print("XmlPropertyListParser() ERROR: error parsing %s as datetime" % repr(content)) + #raise PropertyListParseError("Failed to parse datetime '%s'" % content) + d = datetime.datetime.today() + else: + groups, components = match.groupdict(), [] + for key in units: + value = groups[key] + if value is None: + break + components.append(int(value)) + while len(components) < 3: + components.append(1) - groups, components = match.groupdict(), [] - for key in units: - value = groups[key] - if value is None: - break - components.append(int(value)) - while len(components) < 3: - components.append(1) + d = datetime.datetime(*components) - d = datetime.datetime(*components) self._push_value(d) def _parse_real(self, name, content): - self._push_value(float(content)) + content = content.replace(',', '.') + try: + self._push_value(float(content)) + except: + print("XmlPropertyListParser() WARNING: error converting %s to float" % repr(content)) + self._push_value(0.0) def _parse_integer(self, name, content): self._push_value(int(content)) From 4fe86065f9ecef37c1c3ef9a85170bc36e5bfacf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 23 Jun 2013 09:09:45 +0530 Subject: [PATCH 0030/1154] Fix scanning for books on Aluratek Color Fix a regression that broke scanning for books on all devices that used the Aluratek Color driver. Fixes #1192940 [No longer seeing books on Odys Leon (Aluratek colour)](https://bugs.launchpad.net/calibre/+bug/1192940) --- src/calibre/devices/misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 4a2e6aa864..b20ec3ca6e 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -211,6 +211,7 @@ class ALURATEK_COLOR(USBMS): VENDOR_NAME = ['USB_2.0', 'EZREADER', 'C4+'] WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['USB_FLASH_DRIVER', '.', 'TOUCH'] SCAN_FROM_ROOT = True + SUPPORTS_SUB_DIRS_FOR_SCAN = True class TREKSTOR(USBMS): From c8889644bd77beee40423f608301071be1743f77 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 23 Jun 2013 10:47:29 +0530 Subject: [PATCH 0031/1154] Remove obsolete recipes --- recipes/living_digital.recipe | 16 ---------------- recipes/pc_quest_india.recipe | 16 ---------------- 2 files changed, 32 deletions(-) delete mode 100644 recipes/living_digital.recipe delete mode 100644 recipes/pc_quest_india.recipe diff --git a/recipes/living_digital.recipe b/recipes/living_digital.recipe deleted file mode 100644 index 2fcf50dfab..0000000000 --- a/recipes/living_digital.recipe +++ /dev/null @@ -1,16 +0,0 @@ -from calibre.web.feeds.news import CalibrePeriodical - -class LivingDigital(CalibrePeriodical): - - title = 'Living Digital' - calibre_periodicals_slug = 'living-digital' - - description = ''' - Catch the latest buzz in the digital world with Living Digital. Enjoy - reviews, news, features and recommendations on a wide range of consumer - technology products - from smartphones to flat panel TVs, netbooks to - cameras, and many more consumer lifestyle gadgets. To subscribe, visit - calibre - Periodicals. - ''' - language = 'en_IN' diff --git a/recipes/pc_quest_india.recipe b/recipes/pc_quest_india.recipe deleted file mode 100644 index e45272a2df..0000000000 --- a/recipes/pc_quest_india.recipe +++ /dev/null @@ -1,16 +0,0 @@ -from calibre.web.feeds.news import CalibrePeriodical - -class PCQ(CalibrePeriodical): - - title = 'PCQuest' - calibre_periodicals_slug = 'pc-quest-india' - - description = ''' - Buying a tech product? Seeking a tech solution? Consult PCQuest, India's - market-leading selection and implementation guide for the latest - technologies: servers, business apps, security, open source, gadgets and - more. To subscribe visit, calibre - Periodicals. - ''' - language = 'en_IN' From e8e4dbc35b0a3070c446b5562e4508b18ad346c8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 24 Jun 2013 09:56:07 +0530 Subject: [PATCH 0032/1154] Update Miradasalsur. Fixes #1193922 [Updated recipe for Miradas al Sur](https://bugs.launchpad.net/calibre/+bug/1193922) --- recipes/icons/miradasalsur.png | Bin 0 -> 1120 bytes recipes/miradasalsur.recipe | 97 ++++++++++++++++----------------- 2 files changed, 47 insertions(+), 50 deletions(-) create mode 100644 recipes/icons/miradasalsur.png diff --git a/recipes/icons/miradasalsur.png b/recipes/icons/miradasalsur.png new file mode 100644 index 0000000000000000000000000000000000000000..9cb7d033addb5d783f10eda8764433907d27df0c GIT binary patch literal 1120 zcmdr~?OT&o7{8xAVo zdH?_w9Tl+}d->RMC$ZH8%Mt)ch+mNyIf8hsh{cLr*l;w9c#g->D8h~xkbpZp5)k4L zgCQ;)UIgKydMh8$qmqt1K>$B6f%o zForc(*VvM>?df^WN53N%H(XF|&N(_#S=*m?jA{J!Rmthzk}|4@9a#{UMWlc9UlaZMF)CeHN3}Ez;U*F54p4{zOf{u zE2-(tE;XvcOyh%*xt~Lh;f8w;<=Peq=}gUX_H?|u)BKpwyo^g3)R%OFrfqMjQ8}+& zG1p#y)8wstu`=BpzwJ4wn3Gkugsokm?c1@xiS&J$oc25ipUSUq&%wu2bnTOLbY-;> zeEylAG*of1S*&iAg+2vUrzWcZyx}Q1tutzabyq>FTTov{ZujJ{C!lVTg}1o{b;WG4 zR9~_t>Mia zKm`z&Yl$m{wn@e&*?K6JR`Na@jI$4`87^3>@v z*!|_6oV{P|%l$epzo2mcfg<>1d)lYz8J~T=W9P2SEd3YRu Date: Mon, 24 Jun 2013 10:18:24 +0530 Subject: [PATCH 0033/1154] News download: Respect more default conversion settings News download: Apply the default page margin conversion settings. Also, when converting to PDF, apply the pdf conversion defaults. Fixes #1193763 [Private bug](https://bugs.launchpad.net/calibre/+bug/1193763) --- src/calibre/gui2/tools.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index eda60a4fec..32e3174c8d 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -24,7 +24,7 @@ from calibre.ebooks.conversion.config import GuiRecommendations, \ load_defaults, load_specifics, save_specifics from calibre.gui2.convert import bulk_defaults_for_input_format -def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ +def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ out_format=None, show_no_format_warning=True): changed = False jobs = [] @@ -47,7 +47,7 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ result = d.exec_() if result == QDialog.Accepted: - #if not convert_existing(parent, db, [book_id], d.output_format): + # if not convert_existing(parent, db, [book_id], d.output_format): # continue mi = db.get_metadata(book_id, True) @@ -116,7 +116,6 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ msg = _('This book has no actual ebook files') res.append('%s - %s'%(title, msg)) - msg = '%s' % '\n'.join(res) warning_dialog(parent, _('Could not convert some books'), _('Could not convert %(num)d of %(tot)d books, because no supported source' @@ -254,7 +253,7 @@ class QueueBulk(QProgressDialog): # }}} -def fetch_scheduled_recipe(arg): # {{{ +def fetch_scheduled_recipe(arg): # {{{ fmt = prefs['output_format'].lower() # Never use AZW3 for periodicals... if fmt == 'azw3': @@ -266,6 +265,10 @@ def fetch_scheduled_recipe(arg): # {{{ if 'output_profile' in ps: recs.append(('output_profile', ps['output_profile'], OptionRecommendation.HIGH)) + for edge in ('left', 'top', 'bottom', 'right'): + edge = 'margin_' + edge + if edge in ps: + recs.append((edge, ps[edge], OptionRecommendation.HIGH)) lf = load_defaults('look_and_feel') if lf.get('base_font_size', 0.0) != 0.0: @@ -283,18 +286,24 @@ def fetch_scheduled_recipe(arg): # {{{ if epub.get('epub_flatten', False): recs.append(('epub_flatten', True, OptionRecommendation.HIGH)) + if fmt == 'pdf': + pdf = load_defaults('pdf_output') + from calibre.customize.ui import plugin_for_output_format + p = plugin_for_output_format('pdf') + for opt in p.options: + recs.append(opt.name, pdf.get(opt.name, opt.recommended_value), OptionRecommendation.HIGH) + args = [arg['recipe'], pt.name, recs] if arg['username'] is not None: recs.append(('username', arg['username'], OptionRecommendation.HIGH)) if arg['password'] is not None: recs.append(('password', arg['password'], OptionRecommendation.HIGH)) - return 'gui_convert', args, _('Fetch news from ')+arg['title'], fmt.upper(), [pt] # }}} -def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{ +def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{ from calibre.gui2.dialogs.catalog import Catalog # Build the Catalog dialog in gui2.dialogs.catalog @@ -354,7 +363,7 @@ def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{ d.catalog_title # }}} -def convert_existing(parent, db, book_ids, output_format): # {{{ +def convert_existing(parent, db, book_ids, output_format): # {{{ already_converted_ids = [] already_converted_titles = [] for book_id in book_ids: @@ -372,3 +381,4 @@ def convert_existing(parent, db, book_ids, output_format): # {{{ return book_ids # }}} + From f58b8aee4fd223ad6f2434389de276956765eb88 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 24 Jun 2013 10:21:42 +0530 Subject: [PATCH 0034/1154] Update taz.de (RSS) Fixes #18 (RSS Recipe broken: Taz RSS) --- recipes/taz_rss.recipe | 1 - 1 file changed, 1 deletion(-) diff --git a/recipes/taz_rss.recipe b/recipes/taz_rss.recipe index 90cf27a303..3ccbe2a4f1 100644 --- a/recipes/taz_rss.recipe +++ b/recipes/taz_rss.recipe @@ -18,7 +18,6 @@ class TazRSSRecipe(BasicNewsRecipe): feeds = [(u'TAZ main feed', u'http://www.taz.de/rss.xml')] keep_only_tags = [dict(name='div', attrs={'class': 'sect sect_article'})] - remove_tags_after = dict(name='div', attrs={'class': 'rack'}) remove_tags = [ dict(name=['div'], attrs={'class': 'artikelwerbung'}), dict(name=['ul'], attrs={'class': 'toolbar'}),] From d2292a759d5718ea6cb82360d7e3026652020f18 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 24 Jun 2013 18:04:07 +0530 Subject: [PATCH 0035/1154] Upload installers to downloadbestsoftware.com as well --- setup/hosting.py | 26 ++++++++++++++++++++++++++ setup/upload.py | 6 ++++++ 2 files changed, 32 insertions(+) diff --git a/setup/hosting.py b/setup/hosting.py index 9853f181a4..76ab3992a0 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -402,6 +402,29 @@ def upload_to_servers(files, version): # {{{ print ('Uploaded in', int(time.time() - start), 'seconds\n\n') # }}} +def upload_to_dbs(files, version): # {{{ + print('Uploading to downloadbestsoftware.com') + server = 'www.downloadbestsoft-mirror1.com' + rdir = 'release/' + check_call(['ssh', 'kovid@%s' % server, 'rm -f release/*']) + for x in files: + start = time.time() + print ('Uploading', x) + for i in range(5): + try: + check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x, + 'kovid@%s:%s'%(server, rdir)]) + except KeyboardInterrupt: + raise SystemExit(1) + except: + print ('\nUpload failed, trying again in 30 seconds') + time.sleep(30) + else: + break + print ('Uploaded in', int(time.time() - start), 'seconds\n\n') + check_call(['ssh', 'kovid@%s' % server, '/home/kovid/uploadFiles']) +# }}} + # CLI {{{ def cli_parser(): epilog='Copyright Kovid Goyal 2012' @@ -434,6 +457,7 @@ def cli_parser(): epilog=epilog) cron = subparsers.add_parser('cron', help='Call script from cron') subparsers.add_parser('calibre', help='Upload to calibre file servers') + subparsers.add_parser('dbs', help='Upload to downloadbestsoftware.com') a = gc.add_argument @@ -498,6 +522,8 @@ def main(args=None): login_to_google(args.username, args.password) elif args.service == 'calibre': upload_to_servers(ofiles, args.version) + elif args.service == 'dbs': + upload_to_dbs(ofiles, args.version) if __name__ == '__main__': main() diff --git a/setup/upload.py b/setup/upload.py index 784c0cf9f8..8a4e467dd0 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -114,6 +114,9 @@ def sf_cmdline(ver, sdata): def calibre_cmdline(ver): return [__appname__, ver, 'fmap', 'calibre'] +def dbs_cmdline(ver): + return [__appname__, ver, 'fmap', 'dbs'] + def run_remote_upload(args): print 'Running remotely:', ' '.join(args) subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST), @@ -140,6 +143,7 @@ class UploadInstallers(Command): # {{{ self.upload_to_staging(tdir, backup, files) self.upload_to_sourceforge() self.upload_to_calibre() + self.upload_to_dbs() # self.upload_to_google(opts.replace) finally: shutil.rmtree(tdir, ignore_errors=True) @@ -184,6 +188,8 @@ class UploadInstallers(Command): # {{{ def upload_to_calibre(self): run_remote_upload(calibre_cmdline(__version__)) + def upload_to_dbs(self): + run_remote_upload(dbs_cmdline(__version__)) # }}} class UploadUserManual(Command): # {{{ From 7089c66a986ec8741216449d76d6fd79e4afdc9d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 24 Jun 2013 18:45:05 +0530 Subject: [PATCH 0036/1154] LRF Output: Fix " entities in attribute values causing problems --- src/calibre/ebooks/lrf/html/convert_from.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/lrf/html/convert_from.py b/src/calibre/ebooks/lrf/html/convert_from.py index c755cabd92..2b9a3617dc 100644 --- a/src/calibre/ebooks/lrf/html/convert_from.py +++ b/src/calibre/ebooks/lrf/html/convert_from.py @@ -104,7 +104,7 @@ class HTMLConverter(object): # Replace entities (re.compile(ur'&(\S+?);'), partial(entity_to_unicode, - exceptions=['lt', 'gt', 'amp'])), + exceptions=['lt', 'gt', 'amp', 'quot'])), # Remove comments from within style tags as they can mess up BeatifulSoup (re.compile(r'()', re.IGNORECASE|re.DOTALL), strip_style_comments), From 8f30c17486701bf2cc29a84c98af5336de75fc56 Mon Sep 17 00:00:00 2001 From: GRiker Date: Mon, 24 Jun 2013 14:14:08 -0700 Subject: [PATCH 0037/1154] Fixed typo(?) in set_metadata() for touched files. --- src/calibre/utils/podofo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/utils/podofo/__init__.py b/src/calibre/utils/podofo/__init__.py index 13c12a9bb3..a0b5d85331 100644 --- a/src/calibre/utils/podofo/__init__.py +++ b/src/calibre/utils/podofo/__init__.py @@ -36,7 +36,7 @@ def set_metadata(stream, mi): except WorkerError as e: raise Exception('Failed to set PDF metadata: %s'%e.orig_tb) if touched: - with open(os.path.join(tdir, u'output.pdf'), 'rb') as f: + with open(os.path.join(tdir, u'input.pdf'), 'rb') as f: f.seek(0, 2) if f.tell() > 100: f.seek(0) From 5b07091d5939c0f25e3122d66817ed86f22968e1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Jun 2013 09:12:04 +0530 Subject: [PATCH 0038/1154] Make the manual self contained --- manual/resources/simple_donate_button.gif | Bin 0 -> 2132 bytes manual/templates/layout.html | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 manual/resources/simple_donate_button.gif diff --git a/manual/resources/simple_donate_button.gif b/manual/resources/simple_donate_button.gif new file mode 100644 index 0000000000000000000000000000000000000000..42dd2c3c88f13d0a8eebeecfff7088effdcf8566 GIT binary patch literal 2132 zcmb`G`#;kQ1AxD~glRIYNRqj2<~D{D)!c7WG$hAm8*`@_38`j7t~K}N3}Zx5jU^@J z)?#LbbO@cdP;YhW{m_+9y}s``|HJ!yo}ZuJ9xoqH#{)D5pbPu}fOCvP4utbgc4vK$ zq_`hs`5Z|nIc9~EqmoYer-qSP5gyD)zgS98LP$_@*l`BsWMXJkCe1TD+T%i^+i99# zR*YX(bTGv;Cd4N;%rA!GnGot5Pj+D)_qh;3$)z5nMpLM?kaLW%vx$+}xv8g4(a)sP zax)mG&m^*PlQPm7x!DQXrA+I(Y!ksb)5=Wq+8n&#f;l_RiJRnHa_Z2f47*a+!OGKi zwP%T(WY^+UuTqv*L1s#6PEy_}7B?fk^laKd4OGHY8>rrK?IOB8e@|DLp12IvU9Q_( zg_cz5J*?l;lxHqr6R#E+wHISM%M3*Y=GThx;&KB?rNKZA=HV60a2+5O0P-g2kN_oZ z*bC-C!7>_BIAq(t!BU>o^He6W1p72$K7n>4VL*K~d7( z7S`j=W24uDWj6yyL{X#HPs(pZD!M6?lCYWHpg-<~EZz@!c01z5ov_(^;Y;_!RtCd{ zMf9<2(ekdC@f*o=14(1ISfIicoc4P=68u3L{7DwFJR0^%9{FwZ_{P)Y>yx3so>9I{ zo&5Pc9Gs7MIZS&kjsEup(Ao<`!W@4QjW)#SeaiYT5)Zy z%N=aWog7LIIhMO0F7!B3|MdQO0#MlkrV)DHVn1zb6)k-D27@+IxWgzX4_DFh=%TiICT6@_G-Q?Wy-3fe zr&U&kqg)S{^%ysl>mMk5?mn)7o0r^VreZMqI0>n!MZPH9xxf1u!_9;EE9v!8Sz1U8*lGxF@r!iSDh&A`=! zA1TpyowwHS_>&#GUk|3jNa8%7NXRWAOUd})v z%!i#}<*NLF9OpYHoRiAQ>C_CjpvYO`GWl z^MV|;V?v;F0MlbUgv#^STc}4Lw?8>6@X|l?Zf$70wm0*!vej(z_%2;4h7AR9Boc~< zWU~Ra@H{CD2@qgvNvmnzQdVxDjr3wDPf0h&CbM8r-O< z5u_$u!yT{`tbg4ZBMkJ?Pv391U+NUI=8jg$O}ZC*_YhlY4h02_ z2}*X37~o_U2~Hh?o84|k6?O2nrvhy}aE2NMMI0E83b*YVMjnVKsXY!T#0JpRGD&d5 zWj4%iei&stz|}M+L2$g+$gMzcDDo3PbQ)+z)XNQ2>Iiw;bRcShLf5qqgl#&r1`u=y z2|dZ=+D=g++xae+K6M^YK--f`G$S2iu9xWA{e&%MnuVbeEc=5t-zs<(CwLLR#73x7O_VN>Hn zX6O>H$lXotqRjjgZHe_yKfW=XwdGkEx75EvwmXjbJYm&qT`7KL>b1&zlG*>IVJ-iS zUcM{J`W>cm!(lSc`J3=)PRr=_bQPRF6d#bcOcN?wzSC3cpDM7mYgSLexgrv$@(Pm;G$y93r-%lC)Bga!y^>S_ literal 0 HcmV?d00001 diff --git a/manual/templates/layout.html b/manual/templates/layout.html index b8389b0ac9..188e829469 100644 --- a/manual/templates/layout.html +++ b/manual/templates/layout.html @@ -62,7 +62,7 @@

- +

From 230ef564eef44b81bc004f1cd21016c3c012adc5 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Tue, 25 Jun 2013 08:20:07 +0200 Subject: [PATCH 0039/1154] Fix bug #1193763 - Save to device template generates exception for custom column names. --- src/calibre/gui2/dialogs/template_dialog.py | 2 ++ src/calibre/gui2/preferences/save_template.py | 5 +++-- src/calibre/gui2/preferences/saving.py | 3 ++- src/calibre/gui2/preferences/sending.py | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index 2bafc2812a..3db6e37eb0 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -262,6 +262,8 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): self.mi.rating = 4.0 self.mi.tags = [_('Tag 1'), _('Tag 2')] self.mi.languages = ['eng'] + if fm is not None: + self.mi.set_all_user_metadata(fm.custom_field_metadata()) # Remove help icon on title bar icon = self.windowIcon() diff --git a/src/calibre/gui2/preferences/save_template.py b/src/calibre/gui2/preferences/save_template.py index 627c4c7fa9..145e014800 100644 --- a/src/calibre/gui2/preferences/save_template.py +++ b/src/calibre/gui2/preferences/save_template.py @@ -24,7 +24,7 @@ class SaveTemplate(QWidget, Ui_Form): Ui_Form.__init__(self) self.setupUi(self) - def initialize(self, name, default, help): + def initialize(self, name, default, help, field_metadata): variables = sorted(FORMAT_ARG_DESCS.keys()) rows = [] for var in variables: @@ -36,6 +36,7 @@ class SaveTemplate(QWidget, Ui_Form): table = u'%s
'%(u'\n'.join(rows)) self.template_variables.setText(table) + self.field_metadata = field_metadata self.opt_template.initialize(name+'_template_history', default, help) self.opt_template.editTextChanged.connect(self.changed) @@ -44,7 +45,7 @@ class SaveTemplate(QWidget, Ui_Form): self.open_editor.clicked.connect(self.do_open_editor) def do_open_editor(self): - t = TemplateDialog(self, self.opt_template.text()) + t = TemplateDialog(self, self.opt_template.text(), fm=self.field_metadata) t.setWindowTitle(_('Edit template')) if t.exec_(): self.opt_template.set_value(t.rule[1]) diff --git a/src/calibre/gui2/preferences/saving.py b/src/calibre/gui2/preferences/saving.py index bd5fcbb078..e1a235803d 100644 --- a/src/calibre/gui2/preferences/saving.py +++ b/src/calibre/gui2/preferences/saving.py @@ -34,7 +34,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): ConfigWidgetBase.initialize(self) self.save_template.blockSignals(True) self.save_template.initialize('save_to_disk', self.proxy['template'], - self.proxy.help('template')) + self.proxy.help('template'), + self.gui.library_view.model().db.field_metadata) self.save_template.blockSignals(False) def restore_defaults(self): diff --git a/src/calibre/gui2/preferences/sending.py b/src/calibre/gui2/preferences/sending.py index 3fce5cb072..bc46ac500b 100644 --- a/src/calibre/gui2/preferences/sending.py +++ b/src/calibre/gui2/preferences/sending.py @@ -44,7 +44,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): ConfigWidgetBase.initialize(self) self.send_template.blockSignals(True) self.send_template.initialize('send_to_device', self.proxy['send_template'], - self.proxy.help('send_template')) + self.proxy.help('send_template'), + self.gui.library_view.model().db.field_metadata) self.send_template.blockSignals(False) def restore_defaults(self): From 2096dce1cdcea621f314145f71d5ad5a7390a13e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Jun 2013 13:09:51 +0530 Subject: [PATCH 0040/1154] Move User Manual and staging to the download server --- setup/hosting.py | 113 ++++++++++++++++++++++++++++++++++++++++++++--- setup/upload.py | 11 +++-- 2 files changed, 112 insertions(+), 12 deletions(-) diff --git a/setup/hosting.py b/setup/hosting.py index 76ab3992a0..1e78f4694d 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -7,16 +7,14 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib +import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil from argparse import ArgumentParser, FileType from subprocess import check_call from tempfile import NamedTemporaryFile from collections import OrderedDict -import mechanize -from lxml import html - def login_to_google(username, password): # {{{ + import mechanize br = mechanize.Browser() br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')] @@ -246,6 +244,7 @@ class GoogleCode(Base): # {{{ return login_to_google(self.username, self.gmail_password) def get_files_hosted_by_google_code(self): + from lxml import html self.info('Getting existing files in google code:', self.gc_project) raw = urllib2.urlopen(self.files_list).read() root = html.fromstring(raw) @@ -380,11 +379,111 @@ class SourceForge(Base): # {{{ # }}} +def generate_index(): # {{{ + os.chdir('/srv/download') + releases = set() + for x in os.listdir('.'): + if os.path.isdir(x) and '.' in x: + releases.add(tuple((int(y) for y in x.split('.')))) + rmap = OrderedDict() + for rnum in sorted(releases, reverse=True): + series = rnum[:2] if rnum[0] == 0 else rnum[:1] + if series not in rmap: + rmap[series] = [] + rmap[series].append(rnum) + + template = '''\n {title}

{title}

{msg}

{body} ''' # noqa + style = ''' + body { font-family: sans-serif; background-color: #eee; } + a { text-decoration: none; } + a:visited { color: blue } + a:hover { color: red } + ul { list-style-type: none } + li { padding-bottom: 1ex } + dd li { text-indent: 0; margin: 0 } + dd ul { padding: 0; margin: 0 } + dt { font-weight: bold } + dd { margin-bottom: 2ex } + ''' + body = [] + for series in rmap: + body.append('
  • {0}.x\xa0\xa0\xa0[{1} releases]
  • '.format( # noqa + '.'.join(map(type(''), series)), len(rmap[series]))) + body = '
      {0}
    '.format(' '.join(body)) + index = template.format(title='Previous calibre releases', style=style, msg='Choose a series of calibre releases', body=body) + with open('index.html', 'wb') as f: + f.write(index.encode('utf-8')) + + for series, releases in rmap.iteritems(): + sname = '.'.join(map(type(''), series)) + body = [ + '
  • {0}
  • '.format('.'.join(map(type(''), r))) + for r in releases] + body = '
      {0}
    '.format(' '.join(body)) + index = template.format(title='Previous calibre releases (%s.x)' % sname, style=style, + msg='Choose a calibre release', body=body) + with open('%s.html' % sname, 'wb') as f: + f.write(index.encode('utf-8')) + + for r in releases: + rname = '.'.join(map(type(''), r)) + os.chdir(rname) + try: + body = [] + files = os.listdir('.') + windows = [x for x in files if x.endswith('.msi')] + if windows: + windows = ['
  • {1}
  • '.format( + x, 'Windows 64-bit Installer' if '64bit' in x else 'Windows 32-bit Installer') + for x in windows] + body.append('
    Windows
      {0}
    '.format(' '.join(windows))) + portable = [x for x in files if '-portable-' in x] + if portable: + body.append('
    Calibre Portable
    {1}
    '.format( + portable[0], 'Calibre Portable Installer')) + osx = [x for x in files if x.endswith('.dmg')] + if osx: + body.append('
    Apple Mac
    {1}
    '.format( + osx[0], 'OS X Disk Image (.dmg)')) + linux = [x for x in files if x.endswith('.bz2')] + if linux: + linux = ['
  • {1}
  • '.format( + x, 'Linux 64-bit binary' if 'x86_64' in x else 'Linux 32-bit binary') + for x in linux] + body.append('
    Linux
      {0}
    '.format(' '.join(linux))) + source = [x for x in files if x.endswith('.xz') or x.endswith('.gz')] + if source: + body.append('
    Source Code
    {1}
    '.format( + source[0], 'Source code (all platforms)')) + + body = '
    {0}
    '.format(''.join(body)) + index = template.format(title='calibre release (%s)' % rname, style=style, + msg='', body=body) + with open('index.html', 'wb') as f: + f.write(index.encode('utf-8')) + finally: + os.chdir('..') + +# }}} + def upload_to_servers(files, version): # {{{ - for server, rdir in {'files':'/usr/share/nginx/html'}.iteritems(): + base = '/srv/download/' + dest = os.path.join(base, version) + if not os.path.exists(dest): + os.mkdir(dest) + for src in files: + shutil.copyfile(src, os.path.join(dest, os.path.basename(src))) + generate_index() + + for server, rdir in {'files':'/srv/download/'}.iteritems(): print('Uploading to server:', server) server = '%s.calibre-ebook.com' % server - rdir = '%s/%s/' % (rdir, version) + # Copy the generated index files + print ('Copying generated index') + check_call(['rsync', '-hzr', '-e', 'ssh -x', '--include', '*.html', + '--filter', '-! */', base, 'root@%s:%s' % (server, rdir)]) + # Copy the release files + rdir = '%s%s/' % (rdir, version) for x in files: start = time.time() print ('Uploading', x) @@ -400,6 +499,7 @@ def upload_to_servers(files, version): # {{{ else: break print ('Uploaded in', int(time.time() - start), 'seconds\n\n') + # }}} def upload_to_dbs(files, version): # {{{ @@ -530,3 +630,4 @@ if __name__ == '__main__': # }}} + diff --git a/setup/upload.py b/setup/upload.py index 8a4e467dd0..639a2e98d5 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -19,10 +19,9 @@ from setup import Command, __version__, installer_name, __appname__ PREFIX = "/var/www/calibre-ebook.com" DOWNLOADS = PREFIX+"/htdocs/downloads" BETAS = DOWNLOADS +'/betas' -USER_MANUAL = '/var/www/localhost/htdocs/' HTML2LRF = "calibre/ebooks/lrf/html/demo" TXT2LRF = "src/calibre/ebooks/lrf/txt/demo" -STAGING_HOST = '67.207.135.179' +STAGING_HOST = 'download.calibre-ebook.com' STAGING_USER = 'root' STAGING_DIR = '/root/staging' @@ -141,8 +140,8 @@ class UploadInstallers(Command): # {{{ os.mkdir(backup) try: self.upload_to_staging(tdir, backup, files) - self.upload_to_sourceforge() self.upload_to_calibre() + self.upload_to_sourceforge() self.upload_to_dbs() # self.upload_to_google(opts.replace) finally: @@ -219,9 +218,9 @@ class UploadUserManual(Command): # {{{ for x in glob.glob(self.j(path, '*')): self.build_plugin_example(x) - check_call(' '.join(['rsync', '-z', '-r', '--progress', - 'manual/.build/html/', - 'bugs:%s'%USER_MANUAL]), shell=True) + for host in ('download', 'files'): + check_call(' '.join(['rsync', '-z', '-r', '--progress', + 'manual/.build/html/', '%s:/srv/manual/' % host]), shell=True) # }}} class UploadDemo(Command): # {{{ From e4876b579eba58402b507b58557a8c92db5037f4 Mon Sep 17 00:00:00 2001 From: Hakan Tandogan Date: Tue, 25 Jun 2013 19:47:48 +0200 Subject: [PATCH 0041/1154] On book merge, merge identifiers as well. In case of conflict, the target wins. --- src/calibre/gui2/actions/edit_metadata.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index e5a9bfbc7d..84b1d367c6 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -399,8 +399,7 @@ class EditMetadataAction(InterfaceAction): if safe_merge: if not confirm('

    '+_( 'Book formats and metadata from the selected books ' - 'will be added to the first selected book (%s). ' - 'ISBN will not be merged.

    ' + 'will be added to the first selected book (%s).
    ' 'The second and subsequently selected books will not ' 'be deleted or changed.

    ' 'Please confirm you want to proceed.')%title @@ -413,7 +412,7 @@ class EditMetadataAction(InterfaceAction): 'Book formats from the selected books will be merged ' 'into the first selected book (%s). ' 'Metadata in the first selected book will not be changed. ' - 'Author, Title, ISBN and all other metadata will not be merged.

    ' + 'Author, Title and all other metadata will not be merged.

    ' 'After merger the second and subsequently ' 'selected books, with any metadata they have will be deleted.

    ' 'All book formats of the first selected book will be kept ' @@ -427,8 +426,7 @@ class EditMetadataAction(InterfaceAction): else: if not confirm('

    '+_( 'Book formats and metadata from the selected books will be merged ' - 'into the first selected book (%s). ' - 'ISBN will not be merged.

    ' + 'into the first selected book (%s).
    ' 'After merger the second and ' 'subsequently selected books will be deleted.

    ' 'All book formats of the first selected book will be kept ' @@ -490,11 +488,13 @@ class EditMetadataAction(InterfaceAction): def merge_metadata(self, dest_id, src_ids): db = self.gui.library_view.model().db dest_mi = db.get_metadata(dest_id, index_is_id=True) + merged_identifiers = db.get_identifiers(dest_id, index_is_id=True) orig_dest_comments = dest_mi.comments dest_cover = db.cover(dest_id, index_is_id=True) had_orig_cover = bool(dest_cover) for src_id in src_ids: src_mi = db.get_metadata(src_id, index_is_id=True) + if src_mi.comments and orig_dest_comments != src_mi.comments: if not dest_mi.comments: dest_mi.comments = src_mi.comments @@ -523,7 +523,15 @@ class EditMetadataAction(InterfaceAction): if not dest_mi.series: dest_mi.series = src_mi.series dest_mi.series_index = src_mi.series_index + + src_identifiers = db.get_identifiers(src_id, index_is_id=True) + src_identifiers.update(merged_identifiers) + merged_identifiers = src_identifiers.copy() + db.set_metadata(dest_id, dest_mi, ignore_errors=False) + + db.set_identifiers(dest_id, merged_identifiers) + if not had_orig_cover and dest_cover: db.set_cover(dest_id, dest_cover) From 7b6a742f2542ba5265898785a5dace388c11e5f5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jun 2013 00:07:00 +0530 Subject: [PATCH 0042/1154] ... --- src/calibre/gui2/actions/edit_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index babd690384..729de33c7f 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -426,7 +426,7 @@ class EditMetadataAction(InterfaceAction): else: if not confirm('

    '+_( 'Book formats and metadata from the selected books will be merged ' - 'into the first selected book (%s).
    ' + 'into the first selected book (%s).

    ' 'After merger the second and ' 'subsequently selected books will be deleted.

    ' 'All book formats of the first selected book will be kept ' From 30cea5df3a8f5b3c125912854b78699f9cbd4219 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jun 2013 10:40:00 +0530 Subject: [PATCH 0043/1154] ... --- src/calibre/web/fetch/javascript.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/web/fetch/javascript.py b/src/calibre/web/fetch/javascript.py index 56460c18bf..6e9ef86ff1 100644 --- a/src/calibre/web/fetch/javascript.py +++ b/src/calibre/web/fetch/javascript.py @@ -128,6 +128,8 @@ def download_resources(browser, resource_cache, output_dir): else: img_counter += 1 ext = what(None, raw) or 'jpg' + if ext == 'jpeg': + ext = 'jpg' # Apparently Moon+ cannot handle .jpeg href = 'img_%d.%s' % (img_counter, ext) dest = os.path.join(output_dir, href) resource_cache[h] = dest From 03452d2a038873d8df345512c7433019ada7efa2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jun 2013 14:59:46 +0530 Subject: [PATCH 0044/1154] Conversion: Add option to embed all referenced fonts Conversion: Add an option to embed all fonts that are referenced in the input document but are not already embedded. This will search your system for the referenced font, and if found, the font will be embedded. Only works if the output format supports font embedding (for example: EPUB or AZW3). --- src/calibre/ebooks/conversion/cli.py | 5 +- src/calibre/ebooks/conversion/plumber.py | 17 + .../ebooks/oeb/transforms/embed_fonts.py | 233 +++++++++++ src/calibre/ebooks/oeb/transforms/flatcss.py | 2 +- src/calibre/ebooks/oeb/transforms/subset.py | 208 +++++----- src/calibre/gui2/convert/look_and_feel.py | 2 +- src/calibre/gui2/convert/look_and_feel.ui | 371 +++++++++--------- 7 files changed, 561 insertions(+), 277 deletions(-) create mode 100644 src/calibre/ebooks/oeb/transforms/embed_fonts.py diff --git a/src/calibre/ebooks/conversion/cli.py b/src/calibre/ebooks/conversion/cli.py index f2e5f4e3c9..a0abebc5fe 100644 --- a/src/calibre/ebooks/conversion/cli.py +++ b/src/calibre/ebooks/conversion/cli.py @@ -136,7 +136,7 @@ def add_pipeline_options(parser, plumber): [ 'base_font_size', 'disable_font_rescaling', 'font_size_mapping', 'embed_font_family', - 'subset_embedded_fonts', + 'subset_embedded_fonts', 'embed_all_fonts', 'line_height', 'minimum_line_height', 'linearize_tables', 'extra_css', 'filter_css', @@ -320,7 +320,7 @@ def main(args=sys.argv): opts.search_replace = read_sr_patterns(opts.search_replace, log) recommendations = [(n.dest, getattr(opts, n.dest), - OptionRecommendation.HIGH) \ + OptionRecommendation.HIGH) for n in parser.options_iter() if n.dest] plumber.merge_ui_recommendations(recommendations) @@ -342,3 +342,4 @@ def main(args=sys.argv): if __name__ == '__main__': sys.exit(main()) + diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 1f459229c8..a96574e904 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -205,6 +205,16 @@ OptionRecommendation(name='embed_font_family', 'with some output formats, principally EPUB and AZW3.') ), +OptionRecommendation(name='embed_all_fonts', + recommended_value=False, level=OptionRecommendation.LOW, + help=_( + 'Embed every font that is referenced in the input document ' + 'but not already embedded. This will search your system for the ' + 'fonts, and if found, they will be embedded. Embedding will only work ' + 'if the format you are converting to supports embedded fonts, such as ' + 'EPUB, AZW3 or PDF.' + )), + OptionRecommendation(name='subset_embedded_fonts', recommended_value=False, level=OptionRecommendation.LOW, help=_( @@ -965,6 +975,9 @@ OptionRecommendation(name='search_replace', if self.for_regex_wizard and hasattr(self.opts, 'no_process'): self.opts.no_process = True self.flush() + if self.opts.embed_all_fonts or self.opts.embed_font_family: + # Start the threaded font scanner now, for performance + from calibre.utils.fonts.scanner import font_scanner # noqa import cssutils, logging cssutils.log.setLevel(logging.WARN) get_types_map() # Ensure the mimetypes module is intialized @@ -1129,6 +1142,10 @@ OptionRecommendation(name='search_replace', RemoveFakeMargins()(self.oeb, self.log, self.opts) RemoveAdobeMargins()(self.oeb, self.log, self.opts) + if self.opts.embed_all_fonts: + from calibre.ebooks.oeb.transforms.embed_fonts import EmbedFonts + EmbedFonts()(self.oeb, self.log, self.opts) + if self.opts.subset_embedded_fonts and self.output_plugin.file_type != 'pdf': from calibre.ebooks.oeb.transforms.subset import SubsetFonts SubsetFonts()(self.oeb, self.log, self.opts) diff --git a/src/calibre/ebooks/oeb/transforms/embed_fonts.py b/src/calibre/ebooks/oeb/transforms/embed_fonts.py new file mode 100644 index 0000000000..027b8af1de --- /dev/null +++ b/src/calibre/ebooks/oeb/transforms/embed_fonts.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +import logging +from collections import defaultdict + +import cssutils +from lxml import etree + +from calibre import guess_type +from calibre.ebooks.oeb.base import XPath, CSS_MIME, XHTML +from calibre.ebooks.oeb.transforms.subset import get_font_properties, find_font_face_rules, elem_style +from calibre.utils.filenames import ascii_filename +from calibre.utils.fonts.scanner import font_scanner, NoFonts + +def used_font(style, embedded_fonts): + ff = [unicode(f) for f in style.get('font-family', []) if unicode(f).lower() not in { + 'serif', 'sansserif', 'sans-serif', 'fantasy', 'cursive', 'monospace'}] + if not ff: + return False, None + lnames = {unicode(x).lower() for x in ff} + + matching_set = [] + + # Filter on font-family + for ef in embedded_fonts: + flnames = {x.lower() for x in ef.get('font-family', [])} + if not lnames.intersection(flnames): + continue + matching_set.append(ef) + if not matching_set: + return True, None + + # Filter on font-stretch + widths = {x:i for i, x in enumerate(('ultra-condensed', + 'extra-condensed', 'condensed', 'semi-condensed', 'normal', + 'semi-expanded', 'expanded', 'extra-expanded', 'ultra-expanded' + ))} + + width = widths[style.get('font-stretch', 'normal')] + for f in matching_set: + f['width'] = widths[style.get('font-stretch', 'normal')] + + min_dist = min(abs(width-f['width']) for f in matching_set) + if min_dist > 0: + return True, None + nearest = [f for f in matching_set if abs(width-f['width']) == + min_dist] + if width <= 4: + lmatches = [f for f in nearest if f['width'] <= width] + else: + lmatches = [f for f in nearest if f['width'] >= width] + matching_set = (lmatches or nearest) + + # Filter on font-style + fs = style.get('font-style', 'normal') + matching_set = [f for f in matching_set if f.get('font-style', 'normal') == fs] + + # Filter on font weight + fw = int(style.get('font-weight', '400')) + matching_set = [f for f in matching_set if f.get('weight', 400) == fw] + + if not matching_set: + return True, None + return True, matching_set[0] + + +class EmbedFonts(object): + + ''' + Embed all referenced fonts, if found on system. Must be called after CSS flattening. + ''' + + def __call__(self, oeb, log, opts): + self.oeb, self.log, self.opts = oeb, log, opts + self.sheet_cache = {} + self.find_style_rules() + self.find_embedded_fonts() + self.parser = cssutils.CSSParser(loglevel=logging.CRITICAL, log=logging.getLogger('calibre.css')) + self.warned = set() + self.warned2 = set() + + for item in oeb.spine: + if not hasattr(item.data, 'xpath'): + continue + sheets = [] + for href in XPath('//h:link[@href and @type="text/css"]/@href')(item.data): + sheet = self.oeb.manifest.hrefs.get(item.abshref(href), None) + if sheet is not None: + sheets.append(sheet) + if sheets: + self.process_item(item, sheets) + + def find_embedded_fonts(self): + ''' + Find all @font-face rules and extract the relevant info from them. + ''' + self.embedded_fonts = [] + for item in self.oeb.manifest: + if not hasattr(item.data, 'cssRules'): + continue + self.embedded_fonts.extend(find_font_face_rules(item, self.oeb)) + + def find_style_rules(self): + ''' + Extract all font related style information from all stylesheets into a + dict mapping classes to font properties specified by that class. All + the heavy lifting has already been done by the CSS flattening code. + ''' + rules = defaultdict(dict) + for item in self.oeb.manifest: + if not hasattr(item.data, 'cssRules'): + continue + for i, rule in enumerate(item.data.cssRules): + if rule.type != rule.STYLE_RULE: + continue + props = {k:v for k,v in + get_font_properties(rule).iteritems() if v} + if not props: + continue + for sel in rule.selectorList: + sel = sel.selectorText + if sel and sel.startswith('.'): + # We dont care about pseudo-selectors as the worst that + # can happen is some extra characters will remain in + # the font + sel = sel.partition(':')[0] + rules[sel[1:]].update(props) + + self.style_rules = dict(rules) + + def get_page_sheet(self): + if self.page_sheet is None: + manifest = self.oeb.manifest + id_, href = manifest.generate('page_css', 'page_styles.css') + self.page_sheet = manifest.add(id_, href, CSS_MIME, data=self.parser.parseString('', validate=False)) + head = self.current_item.xpath('//*[local-name()="head"][1]') + if head: + href = self.current_item.relhref(href) + l = etree.SubElement(head[0], XHTML('link'), + rel='stylesheet', type=CSS_MIME, href=href) + l.tail = '\n' + else: + self.log.warn('No cannot embed font rules') + return self.page_sheet + + def process_item(self, item, sheets): + ff_rules = [] + self.current_item = item + self.page_sheet = None + for sheet in sheets: + if 'page_css' in sheet.id: + ff_rules.extend(find_font_face_rules(sheet, self.oeb)) + self.page_sheet = sheet + + base = {'font-family':['serif'], 'font-weight': '400', + 'font-style':'normal', 'font-stretch':'normal'} + + for body in item.data.xpath('//*[local-name()="body"]'): + self.find_usage_in(body, base, ff_rules) + + def find_usage_in(self, elem, inherited_style, ff_rules): + style = elem_style(self.style_rules, elem.get('class', '') or '', inherited_style) + for child in elem: + self.find_usage_in(child, style, ff_rules) + has_font, existing = used_font(style, ff_rules) + if not has_font: + return + if existing is None: + in_book = used_font(style, self.embedded_fonts)[1] + if in_book is None: + # Try to find the font in the system + added = self.embed_font(style) + if added is not None: + ff_rules.append(added) + self.embedded_fonts.append(added) + else: + # TODO: Create a page rule from the book rule (cannot use it + # directly as paths might be different) + item = in_book['item'] + sheet = self.parser.parseString(in_book['rule'].cssText, validate=False) + rule = sheet.cssRules[0] + page_sheet = self.get_page_sheet() + href = page_sheet.abshref(item.href) + rule.style.setProperty('src', 'url(%s)' % href) + ff_rules.append(find_font_face_rules(sheet, self.oeb)[0]) + page_sheet.data.insertRule(rule, len(page_sheet.data.cssRules)) + + def embed_font(self, style): + ff = [unicode(f) for f in style.get('font-family', []) if unicode(f).lower() not in { + 'serif', 'sansserif', 'sans-serif', 'fantasy', 'cursive', 'monospace'}] + if not ff: + return + ff = ff[0] + if ff in self.warned: + return + try: + fonts = font_scanner.fonts_for_family(ff) + except NoFonts: + self.log.warn('Failed to find fonts for family:', ff, 'not embedding') + self.warned.add(ff) + return + try: + weight = int(style.get('font-weight', '400')) + except (ValueError, TypeError, AttributeError): + w = style['font-weight'] + if w not in self.warned2: + self.log.warn('Invalid weight in font style: %r' % w) + self.warned2.add(w) + return + for f in fonts: + if f['weight'] == weight and f['font-style'] == style.get('font-style', 'normal') and f['font-stretch'] == style.get('font-stretch', 'normal'): + self.log('Embedding font %s from %s' % (f['full_name'], f['path'])) + data = font_scanner.get_font_data(f) + name = f['full_name'] + ext = 'otf' if f['is_otf'] else 'ttf' + name = ascii_filename(name).replace(' ', '-').replace('(', '').replace(')', '') + fid, href = self.oeb.manifest.generate(id=u'font', href=u'fonts/%s.%s'%(name, ext)) + item = self.oeb.manifest.add(fid, href, guess_type('dummy.'+ext)[0], data=data) + item.unload_data_from_memory() + page_sheet = self.get_page_sheet() + href = page_sheet.relhref(item.href) + css = '''@font-face { font-family: "%s"; font-weight: %s; font-style: %s; font-stretch: %s; src: url(%s) }''' % ( + f['font-family'], f['font-weight'], f['font-style'], f['font-stretch'], href) + sheet = self.parser.parseString(css, validate=False) + page_sheet.data.insertRule(sheet.cssRules[0], len(page_sheet.data.cssRules)) + return find_font_face_rules(sheet, self.oeb)[0] + diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index dd2d20333d..9c08934938 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -194,7 +194,7 @@ class CSSFlattener(object): for i, font in enumerate(faces): ext = 'otf' if font['is_otf'] else 'ttf' fid, href = self.oeb.manifest.generate(id=u'font', - href=u'%s.%s'%(ascii_filename(font['full_name']).replace(u' ', u'-'), ext)) + href=u'fonts/%s.%s'%(ascii_filename(font['full_name']).replace(u' ', u'-'), ext)) item = self.oeb.manifest.add(fid, href, guess_type('dummy.'+ext)[0], data=font_scanner.get_font_data(font)) diff --git a/src/calibre/ebooks/oeb/transforms/subset.py b/src/calibre/ebooks/oeb/transforms/subset.py index 744e37b193..96170bd49c 100644 --- a/src/calibre/ebooks/oeb/transforms/subset.py +++ b/src/calibre/ebooks/oeb/transforms/subset.py @@ -12,6 +12,111 @@ from collections import defaultdict from calibre.ebooks.oeb.base import urlnormalize from calibre.utils.fonts.sfnt.subset import subset, NoGlyphs, UnsupportedFont +def get_font_properties(rule, default=None): + ''' + Given a CSS rule, extract normalized font properties from + it. Note that shorthand font property should already have been expanded + by the CSS flattening code. + ''' + props = {} + s = rule.style + for q in ('font-family', 'src', 'font-weight', 'font-stretch', + 'font-style'): + g = 'uri' if q == 'src' else 'value' + try: + val = s.getProperty(q).propertyValue[0] + val = getattr(val, g) + if q == 'font-family': + val = [x.value for x in s.getProperty(q).propertyValue] + if val and val[0] == 'inherit': + val = None + except (IndexError, KeyError, AttributeError, TypeError, ValueError): + val = None if q in {'src', 'font-family'} else default + if q in {'font-weight', 'font-stretch', 'font-style'}: + val = unicode(val).lower() if (val or val == 0) else val + if val == 'inherit': + val = default + if q == 'font-weight': + val = {'normal':'400', 'bold':'700'}.get(val, val) + if val not in {'100', '200', '300', '400', '500', '600', '700', + '800', '900', 'bolder', 'lighter'}: + val = default + if val == 'normal': + val = '400' + elif q == 'font-style': + if val not in {'normal', 'italic', 'oblique'}: + val = default + elif q == 'font-stretch': + if val not in {'normal', 'ultra-condensed', 'extra-condensed', + 'condensed', 'semi-condensed', 'semi-expanded', + 'expanded', 'extra-expanded', 'ultra-expanded'}: + val = default + props[q] = val + return props + + +def find_font_face_rules(sheet, oeb): + ''' + Find all @font-face rules in the given sheet and extract the relevant info from them. + sheet can be either a ManifestItem or a CSSStyleSheet. + ''' + ans = [] + try: + rules = sheet.data.cssRules + except AttributeError: + rules = sheet.cssRules + + for i, rule in enumerate(rules): + if rule.type != rule.FONT_FACE_RULE: + continue + props = get_font_properties(rule, default='normal') + if not props['font-family'] or not props['src']: + continue + + try: + path = sheet.abshref(props['src']) + except AttributeError: + path = props['src'] + ff = oeb.manifest.hrefs.get(urlnormalize(path), None) + if not ff: + continue + props['item'] = ff + if props['font-weight'] in {'bolder', 'lighter'}: + props['font-weight'] = '400' + props['weight'] = int(props['font-weight']) + props['rule'] = rule + props['chars'] = set() + ans.append(props) + + return ans + + +def elem_style(style_rules, cls, inherited_style): + ''' + Find the effective style for the given element. + ''' + classes = cls.split() + style = inherited_style.copy() + for cls in classes: + style.update(style_rules.get(cls, {})) + wt = style.get('font-weight', None) + pwt = inherited_style.get('font-weight', '400') + if wt == 'bolder': + style['font-weight'] = { + '100':'400', + '200':'400', + '300':'400', + '400':'700', + '500':'700', + }.get(pwt, '900') + elif wt == 'lighter': + style['font-weight'] = { + '600':'400', '700':'400', + '800':'700', '900':'700'}.get(pwt, '100') + + return style + + class SubsetFonts(object): ''' @@ -76,72 +181,15 @@ class SubsetFonts(object): self.log('Reduced total font size to %.1f%% of original'% (totals[0]/totals[1] * 100)) - def get_font_properties(self, rule, default=None): - ''' - Given a CSS rule, extract normalized font properties from - it. Note that shorthand font property should already have been expanded - by the CSS flattening code. - ''' - props = {} - s = rule.style - for q in ('font-family', 'src', 'font-weight', 'font-stretch', - 'font-style'): - g = 'uri' if q == 'src' else 'value' - try: - val = s.getProperty(q).propertyValue[0] - val = getattr(val, g) - if q == 'font-family': - val = [x.value for x in s.getProperty(q).propertyValue] - if val and val[0] == 'inherit': - val = None - except (IndexError, KeyError, AttributeError, TypeError, ValueError): - val = None if q in {'src', 'font-family'} else default - if q in {'font-weight', 'font-stretch', 'font-style'}: - val = unicode(val).lower() if (val or val == 0) else val - if val == 'inherit': - val = default - if q == 'font-weight': - val = {'normal':'400', 'bold':'700'}.get(val, val) - if val not in {'100', '200', '300', '400', '500', '600', '700', - '800', '900', 'bolder', 'lighter'}: - val = default - if val == 'normal': val = '400' - elif q == 'font-style': - if val not in {'normal', 'italic', 'oblique'}: - val = default - elif q == 'font-stretch': - if val not in { 'normal', 'ultra-condensed', 'extra-condensed', - 'condensed', 'semi-condensed', 'semi-expanded', - 'expanded', 'extra-expanded', 'ultra-expanded'}: - val = default - props[q] = val - return props - def find_embedded_fonts(self): ''' Find all @font-face rules and extract the relevant info from them. ''' self.embedded_fonts = [] for item in self.oeb.manifest: - if not hasattr(item.data, 'cssRules'): continue - for i, rule in enumerate(item.data.cssRules): - if rule.type != rule.FONT_FACE_RULE: - continue - props = self.get_font_properties(rule, default='normal') - if not props['font-family'] or not props['src']: - continue - - path = item.abshref(props['src']) - ff = self.oeb.manifest.hrefs.get(urlnormalize(path), None) - if not ff: - continue - props['item'] = ff - if props['font-weight'] in {'bolder', 'lighter'}: - props['font-weight'] = '400' - props['weight'] = int(props['font-weight']) - props['chars'] = set() - props['rule'] = rule - self.embedded_fonts.append(props) + if not hasattr(item.data, 'cssRules'): + continue + self.embedded_fonts.extend(find_font_face_rules(item, self.oeb)) def find_style_rules(self): ''' @@ -151,12 +199,13 @@ class SubsetFonts(object): ''' rules = defaultdict(dict) for item in self.oeb.manifest: - if not hasattr(item.data, 'cssRules'): continue + if not hasattr(item.data, 'cssRules'): + continue for i, rule in enumerate(item.data.cssRules): if rule.type != rule.STYLE_RULE: continue props = {k:v for k,v in - self.get_font_properties(rule).iteritems() if v} + get_font_properties(rule).iteritems() if v} if not props: continue for sel in rule.selectorList: @@ -172,41 +221,17 @@ class SubsetFonts(object): def find_font_usage(self): for item in self.oeb.manifest: - if not hasattr(item.data, 'xpath'): continue + if not hasattr(item.data, 'xpath'): + continue for body in item.data.xpath('//*[local-name()="body"]'): base = {'font-family':['serif'], 'font-weight': '400', 'font-style':'normal', 'font-stretch':'normal'} self.find_usage_in(body, base) - def elem_style(self, cls, inherited_style): - ''' - Find the effective style for the given element. - ''' - classes = cls.split() - style = inherited_style.copy() - for cls in classes: - style.update(self.style_rules.get(cls, {})) - wt = style.get('font-weight', None) - pwt = inherited_style.get('font-weight', '400') - if wt == 'bolder': - style['font-weight'] = { - '100':'400', - '200':'400', - '300':'400', - '400':'700', - '500':'700', - }.get(pwt, '900') - elif wt == 'lighter': - style['font-weight'] = { - '600':'400', '700':'400', - '800':'700', '900':'700'}.get(pwt, '100') - - return style - def used_font(self, style): ''' Given a style find the embedded font that matches it. Returns None if - no match is found ( can happen if not family matches). + no match is found (can happen if no family matches). ''' ff = style.get('font-family', []) lnames = {unicode(x).lower() for x in ff} @@ -222,7 +247,7 @@ class SubsetFonts(object): return None # Filter on font-stretch - widths = {x:i for i, x in enumerate(( 'ultra-condensed', + widths = {x:i for i, x in enumerate(('ultra-condensed', 'extra-condensed', 'condensed', 'semi-condensed', 'normal', 'semi-expanded', 'expanded', 'extra-expanded', 'ultra-expanded' ))} @@ -280,7 +305,7 @@ class SubsetFonts(object): return ans def find_usage_in(self, elem, inherited_style): - style = self.elem_style(elem.get('class', '') or '', inherited_style) + style = elem_style(self.style_rules, elem.get('class', '') or '', inherited_style) for child in elem: self.find_usage_in(child, style) font = self.used_font(style) @@ -290,3 +315,4 @@ class SubsetFonts(object): font['chars'] |= chars + diff --git a/src/calibre/gui2/convert/look_and_feel.py b/src/calibre/gui2/convert/look_and_feel.py index 24ee288cc6..a3e364b9ca 100644 --- a/src/calibre/gui2/convert/look_and_feel.py +++ b/src/calibre/gui2/convert/look_and_feel.py @@ -32,7 +32,7 @@ class LookAndFeelWidget(Widget, Ui_Form): Widget.__init__(self, parent, ['change_justification', 'extra_css', 'base_font_size', 'font_size_mapping', 'line_height', 'minimum_line_height', - 'embed_font_family', 'subset_embedded_fonts', + 'embed_font_family', 'embed_all_fonts', 'subset_embedded_fonts', 'smarten_punctuation', 'unsmarten_punctuation', 'disable_font_rescaling', 'insert_blank_line', 'remove_paragraph_spacing', diff --git a/src/calibre/gui2/convert/look_and_feel.ui b/src/calibre/gui2/convert/look_and_feel.ui index 43736fb1f2..e9d9caeed7 100644 --- a/src/calibre/gui2/convert/look_and_feel.ui +++ b/src/calibre/gui2/convert/look_and_feel.ui @@ -14,6 +14,70 @@ Form + + + + Keep &ligatures + + + + + + + &Linearize tables + + + + + + + Base &font size: + + + opt_base_font_size + + + + + + + &Line size: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + opt_insert_blank_line_size + + + + + + + true + + + + + + + Remove &spacing between paragraphs + + + + + + + &Indent size: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + opt_remove_paragraph_spacing_indent_size + + + @@ -24,6 +88,57 @@ + + + + Insert &blank line between paragraphs + + + + + + + em + + + 1 + + + + + + + Text &justification: + + + opt_change_justification + + + + + + + + + + Smarten &punctuation + + + + + + + &Transliterate unicode characters to ASCII + + + + + + + &UnSmarten punctuation + + + @@ -44,51 +159,6 @@ - - - - % - - - 1 - - - 900.000000000000000 - - - - - - - pt - - - 1 - - - 0.000000000000000 - - - 50.000000000000000 - - - 1.000000000000000 - - - 15.000000000000000 - - - - - - - Font size &key: - - - opt_font_size_mapping - - - @@ -133,56 +203,72 @@ - - - - true - - - - - - - Remove &spacing between paragraphs - - - - - - - &Indent size: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - opt_remove_paragraph_spacing_indent_size - - - - - - - <p>When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent. - - - No change - + + - em + % + + + 1 + + + 900.000000000000000 + + + + + + + pt 1 - -0.100000000000000 + 0.000000000000000 + + + 50.000000000000000 - 0.100000000000000 + 1.000000000000000 + + + 15.000000000000000 - + + + + &Disable font size rescaling + + + + + + + + + + Font size &key: + + + opt_font_size_mapping + + + + + + + &Embed font family: + + + opt_embed_font_family + + + + 0 @@ -300,121 +386,42 @@ - - - - Insert &blank line between paragraphs - - - - + + + <p>When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent. + + + No change + em 1 - - - - - - Text &justification: + + -0.100000000000000 - - opt_change_justification + + 0.100000000000000 - - - - - - - Smarten &punctuation - - - - - - - &Transliterate unicode characters to ASCII - - - - - - - &UnSmarten punctuation - - - - - - - Keep &ligatures - - - - - - - &Linearize tables - - - - - - - Base &font size: - - - opt_base_font_size - - - - - - - &Line size: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - opt_insert_blank_line_size - - - - - - - &Embed font family: - - - opt_embed_font_family - - - - - - - &Disable font size rescaling - - - - - - - + &Subset all embedded fonts + + + + &Embed referenced fonts + + + From 63d133ea5c1a3ee8d4949e95ce9cf8e9e9c9d644 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jun 2013 15:46:41 +0530 Subject: [PATCH 0045/1154] AZW3 Input: Add support for page-progression-direction AZW3 Input: Add support for the page-progression-direction that is used to indicate page turns should happen from right to left. The attribute is passed into EPUB when converting. Fixes #1194766 [Incorrect conversion japanese MOBI](https://bugs.launchpad.net/calibre/+bug/1194766) --- src/calibre/ebooks/metadata/opf2.py | 11 +++++++++++ src/calibre/ebooks/mobi/reader/mobi8.py | 12 +++++++++--- src/calibre/ebooks/mobi/utils.py | 11 ++++++++++- src/calibre/ebooks/oeb/base.py | 3 +++ src/calibre/ebooks/oeb/reader.py | 3 +++ 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index fb80cc8bfe..77e334dd3e 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -1047,6 +1047,14 @@ class OPF(object): # {{{ if raw: return raw.rpartition(':')[-1] + @property + def page_progression_direction(self): + spine = self.XPath('descendant::*[re:match(name(), "spine", "i")][1]')(self.root) + if spine: + for k, v in spine[0].attrib.iteritems(): + if k == 'page-progression-direction' or k.endswith('}page-progression-direction'): + return v + def guess_cover(self): ''' Try to guess a cover. Needed for some old/badly formed OPF files. @@ -1185,6 +1193,7 @@ class OPFCreator(Metadata): ''' Metadata.__init__(self, title='', other=other) self.base_path = os.path.abspath(base_path) + self.page_progression_direction = None if self.application_id is None: self.application_id = str(uuid.uuid4()) if not isinstance(self.toc, TOC): @@ -1356,6 +1365,8 @@ class OPFCreator(Metadata): spine = E.spine() if self.toc is not None: spine.set('toc', 'ncx') + if self.page_progression_direction is not None: + spine.set('page-progression-direction', self.page_progression_direction) if self.spine is not None: for ref in self.spine: if ref.id is not None: diff --git a/src/calibre/ebooks/mobi/reader/mobi8.py b/src/calibre/ebooks/mobi/reader/mobi8.py index aff79d65c2..97d38a9660 100644 --- a/src/calibre/ebooks/mobi/reader/mobi8.py +++ b/src/calibre/ebooks/mobi/reader/mobi8.py @@ -20,7 +20,7 @@ from calibre.ebooks.mobi.reader.ncx import read_ncx, build_toc from calibre.ebooks.mobi.reader.markup import expand_mobi8_markup from calibre.ebooks.metadata.opf2 import Guide, OPFCreator from calibre.ebooks.metadata.toc import TOC -from calibre.ebooks.mobi.utils import read_font_record +from calibre.ebooks.mobi.utils import read_font_record, read_resc_record from calibre.ebooks.oeb.parse_utils import parse_html from calibre.ebooks.oeb.base import XPath, XHTML, xml2text from calibre.utils.imghdr import what @@ -65,6 +65,7 @@ class Mobi8Reader(object): self.mobi6_reader, self.log = mobi6_reader, log self.header = mobi6_reader.book_header self.encrypted_fonts = [] + self.resc_data = {} def __call__(self): self.mobi6_reader.check_for_drm() @@ -389,9 +390,11 @@ class Mobi8Reader(object): data = sec[0] typ = data[:4] href = None - if typ in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n', - b'RESC', b'BOUN', b'FDST', b'DATP', b'AUDI', b'VIDE'}: + if typ in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n', b'BOUN', + b'FDST', b'DATP', b'AUDI', b'VIDE'}: pass # Ignore these records + elif typ == b'RESC': + self.resc_data = read_resc_record(data) elif typ == b'FONT': font = read_font_record(data) href = "fonts/%05d.%s" % (fname_idx, font['ext']) @@ -452,6 +455,9 @@ class Mobi8Reader(object): opf.create_manifest_from_files_in([os.getcwdu()], exclude=exclude) opf.create_spine(spine) opf.set_toc(toc) + ppd = self.resc_data.get('page-progression-direction', None) + if ppd: + opf.page_progression_direction = ppd with open('metadata.opf', 'wb') as of, open('toc.ncx', 'wb') as ncx: opf.render(of, ncx, 'toc.ncx') diff --git a/src/calibre/ebooks/mobi/utils.py b/src/calibre/ebooks/mobi/utils.py index e9bc4f669f..008b33a0ff 100644 --- a/src/calibre/ebooks/mobi/utils.py +++ b/src/calibre/ebooks/mobi/utils.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import struct, string, zlib, os +import struct, string, zlib, os, re from collections import OrderedDict from io import BytesIO @@ -393,6 +393,15 @@ def mobify_image(data): data = im.export('gif') return data +def read_resc_record(data): + ans = {} + match = re.search(br''']*page-progression-direction=['"](.+?)['"]''', data) + if match is not None: + ppd = match.group(1).lower() + if ppd in {b'ltr', b'rtl'}: + ans['page-progression-direction'] = ppd.decode('ascii') + return ans + # Font records {{{ def read_font_record(data, extent=1040): ''' diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index d4b3a2b7ab..29fc27ee3f 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -1210,6 +1210,7 @@ class Spine(object): def __init__(self, oeb): self.oeb = oeb self.items = [] + self.page_progression_direction = None def _linear(self, linear): if isinstance(linear, basestring): @@ -1896,4 +1897,6 @@ class OEBBook(object): attrib={'media-type': PAGE_MAP_MIME}) spine.attrib['page-map'] = id results[PAGE_MAP_MIME] = (href, self.pages.to_page_map()) + if self.spine.page_progression_direction in {'ltr', 'rtl'}: + spine.attrib['page-progression-direction'] = self.spine.page_progression_direction return results diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index eb7e2eca4c..cb10b4ccce 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -330,6 +330,9 @@ class OEBReader(object): if len(spine) == 0: raise OEBError("Spine is empty") self._spine_add_extra() + for val in xpath(opf, '/o2:package/o2:spine/@page-progression-direction'): + if val in {'ltr', 'rtl'}: + spine.page_progression_direction = val def _guide_from_opf(self, opf): guide = self.oeb.guide From f63f142618a503acd4d8597bc97117d69a93840b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jun 2013 16:55:12 +0530 Subject: [PATCH 0046/1154] PDF Output: Fix add ToC option not being used PDF Output: Fix Table of Contents being added tot he end of the PDF even without the Add Table of Contents option being enabled. Fixes #1194836 [When convert to PDF, it always create TOC at the end](https://bugs.launchpad.net/calibre/+bug/1194836) --- src/calibre/ebooks/pdf/render/from_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/pdf/render/from_html.py b/src/calibre/ebooks/pdf/render/from_html.py index 5b9f58e326..8ea1d8203e 100644 --- a/src/calibre/ebooks/pdf/render/from_html.py +++ b/src/calibre/ebooks/pdf/render/from_html.py @@ -253,7 +253,7 @@ class PDFWriter(QObject): return self.loop.exit(1) try: if not self.render_queue: - if self.toc is not None and len(self.toc) > 0 and not hasattr(self, 'rendered_inline_toc'): + if self.opts.pdf_add_toc and self.toc is not None and len(self.toc) > 0 and not hasattr(self, 'rendered_inline_toc'): return self.render_inline_toc() self.loop.exit() else: From 3743d26d35badcc978a93db0d3d2aa53eae032ee Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 10:14:10 +0530 Subject: [PATCH 0047/1154] Save dist file sizes for bandwidth calculation Also fix a typo in copying dist files to tdir and backup. --- setup/upload.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/setup/upload.py b/setup/upload.py index 639a2e98d5..dd59067c0c 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -134,6 +134,8 @@ class UploadInstallers(Command): # {{{ available = set(glob.glob('dist/*')) files = {x:installer_description(x) for x in all_possible.intersection(available)} + sizes = {os.path.basename(x):os.path.getsize(x) for x in files} + self.record_sizes(sizes) tdir = mkdtemp() backup = os.path.join('/mnt/external/calibre/%s' % __version__) if not os.path.exists(backup): @@ -147,6 +149,11 @@ class UploadInstallers(Command): # {{{ finally: shutil.rmtree(tdir, ignore_errors=True) + def record_sizes(self, sizes): + print ('\nRecording dist sizes') + args = ['%s:%s:%s' % (__version__, fname, size) for fname, size in sizes.iteritems()] + check_call(['ssh', 'divok', 'dist_sizes'] + args) + def upload_to_staging(self, tdir, backup, files): os.mkdir(tdir+'/dist') hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)), @@ -154,9 +161,9 @@ class UploadInstallers(Command): # {{{ shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py')) for f in files: - for x in (tdir, backup): - dest = os.path.join(x, f) - shutil.copyfile(f, dest) + for x in (tdir+'/dist', backup): + dest = os.path.join(x, os.path.basename(f)) + shutil.copy2(f, x) os.chmod(dest, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH) with open(os.path.join(tdir, 'fmap'), 'wb') as fo: From 87dda89378e8a95663897eba720b0ae04d958d7d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 12:41:46 +0530 Subject: [PATCH 0048/1154] Add notes on provisioning a file hosting server --- setup/file_hosting_servers.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 setup/file_hosting_servers.rst diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst new file mode 100644 index 0000000000..7121628744 --- /dev/null +++ b/setup/file_hosting_servers.rst @@ -0,0 +1,32 @@ +Provisioning a file hosting server +==================================== + +Create the ssh authorized keys file. + +Edit /etc/ssh/sshd_config and change PermitRootLogin to without-password. +Restart sshd. + +apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools +chsh -s /bin/zsh + +mkdir -p /root/staging /root/work/vim /srv/download /srv/manual + +scp .zshrc .vimrc server: +scp -r ~/work/vim/zsh-syntax-highlighting server:work/vim + +If the server has a backup hard-disk, mount it at /mnt/backup and edit /etc/fstab so that it is auto-mounted. +Then, add the following to crontab +@daily /usr/bin/rsync -ha /srv /mnt/backup +@daily /usr/bin/rsync -ha /etc /mnt/backup + +Nginx +------ + +Copy over /etc/nginx/sites-available/default from another file server. When +copying, remember to use cat instead of cp to preserve hardlinks (the file is a +hardlink to /etc/nginx/sites-enabled/default) + +rsync /srv from another file server + +service nginx start + From c3009256c498893370e0cc6f61bd018abe17a38e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 13:01:52 +0530 Subject: [PATCH 0049/1154] ToC Editor: Use filenames when generating from files ToC Editor: When generating a ToC from files, if the file has no text, do not skip it. Instead create an entry using the filename of the file. --- src/calibre/ebooks/oeb/polish/toc.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/oeb/polish/toc.py b/src/calibre/ebooks/oeb/polish/toc.py index 8be23bdc38..a364da58f5 100644 --- a/src/calibre/ebooks/oeb/polish/toc.py +++ b/src/calibre/ebooks/oeb/polish/toc.py @@ -281,15 +281,18 @@ def find_text(node): def from_files(container): toc = TOC() - for spinepath in container.spine_items: + for i, spinepath in enumerate(container.spine_items): name = container.abspath_to_name(spinepath) root = container.parsed(name) body = XPath('//h:body')(root) if not body: continue text = find_text(body[0]) - if text: - toc.add(text, name) + if not text: + text = name.rpartition('/')[-1] + if i == 0 and text.rpartition('.')[0].lower() in {'titlepage', 'cover'}: + text = _('Cover') + toc.add(text, name) return toc def add_id(container, name, loc): From f8509fe8260e409c6e9f21d309d7ce8b9fd6a529 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 13:27:31 +0530 Subject: [PATCH 0050/1154] Log the wait before sending email When waiting before sending email, log the wait. Fixes #1195173 [Feature Request - Multiple books emailed to device](https://bugs.launchpad.net/calibre/+bug/1195173) --- src/calibre/gui2/email.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 9ebb94b00a..9b077fa39f 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -92,7 +92,11 @@ class Sendmail(object): raise worker.exception def sendmail(self, attachment, aname, to, subject, text, log): + logged = False while time.time() - self.last_send_time <= self.rate_limit: + if not logged: + log('Waiting %s seconds before sending, to avoid being marked as spam.\nYou can control this delay via Preferences->Tweaks' % self.rate_limit) + logged = True time.sleep(1) try: opts = email_config().parse() From 86691f22a24c96061b1a78f1c332d58b9b52b6db Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 13:28:06 +0530 Subject: [PATCH 0051/1154] ... --- src/calibre/gui2/email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 9b077fa39f..52da1909fe 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -94,7 +94,7 @@ class Sendmail(object): def sendmail(self, attachment, aname, to, subject, text, log): logged = False while time.time() - self.last_send_time <= self.rate_limit: - if not logged: + if not logged and self.rate_limit > 0: log('Waiting %s seconds before sending, to avoid being marked as spam.\nYou can control this delay via Preferences->Tweaks' % self.rate_limit) logged = True time.sleep(1) From 32fccdb9010d161f6e2acbb4b1d66cf99162f57b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 13:29:09 +0530 Subject: [PATCH 0052/1154] pep8 --- src/calibre/gui2/email.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 52da1909fe..f8c7552437 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -32,7 +32,7 @@ class Worker(Thread): self.func, self.args = func, args def run(self): - #time.sleep(1000) + # time.sleep(1000) try: self.func(*self.args) except Exception as e: @@ -46,7 +46,7 @@ class Worker(Thread): class Sendmail(object): MAX_RETRIES = 1 - TIMEOUT = 15 * 60 # seconds + TIMEOUT = 15 * 60 # seconds def __init__(self): self.calculate_rate_limit() @@ -166,7 +166,7 @@ def email_news(mi, remove, get_fmts, done, job_manager): plugboard_email_value = 'email' plugboard_email_formats = ['epub', 'mobi', 'azw3'] -class EmailMixin(object): # {{{ +class EmailMixin(object): # {{{ def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None, do_auto_convert=True, specific_format=None): @@ -208,10 +208,10 @@ class EmailMixin(object): # {{{ if not components: components = [mi.title] subjects.append(os.path.join(*components)) - a = authors_to_string(mi.authors if mi.authors else \ + a = authors_to_string(mi.authors if mi.authors else [_('Unknown')]) - texts.append(_('Attached, you will find the e-book') + \ - '\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' + \ + texts.append(_('Attached, you will find the e-book') + + '\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' + _('in the %s format.') % os.path.splitext(f)[1][1:].upper()) prefix = ascii_filename(t+' - '+a) @@ -231,7 +231,7 @@ class EmailMixin(object): # {{{ auto = [] if _auto_ids != []: for id in _auto_ids: - if specific_format == None: + if specific_format is None: dbfmts = self.library_view.model().db.formats(id, index_is_id=True) formats = [f.lower() for f in (dbfmts.split(',') if dbfmts else [])] @@ -302,8 +302,9 @@ class EmailMixin(object): # {{{ sent_mails = email_news(mi, remove, get_fmts, self.email_sent, self.job_manager) if sent_mails: - self.status_bar.show_message(_('Sent news to')+' '+\ + self.status_bar.show_message(_('Sent news to')+' '+ ', '.join(sent_mails), 3000) # }}} + From 13f31c7839ca98b1663c34a5bf7daef359b21f61 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 17:11:23 +0530 Subject: [PATCH 0053/1154] ... --- setup/file_hosting_servers.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst index 7121628744..563c7bc64a 100644 --- a/setup/file_hosting_servers.rst +++ b/setup/file_hosting_servers.rst @@ -6,6 +6,11 @@ Create the ssh authorized keys file. Edit /etc/ssh/sshd_config and change PermitRootLogin to without-password. Restart sshd. +hostname whatever +Edit /etc/hosts and put in FQDN in the appropriate places, for example:: + 27.0.1.1 download.calibre-ebook.com download + 46.28.49.116 download.calibre-ebook.com download + apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools chsh -s /bin/zsh @@ -15,9 +20,9 @@ scp .zshrc .vimrc server: scp -r ~/work/vim/zsh-syntax-highlighting server:work/vim If the server has a backup hard-disk, mount it at /mnt/backup and edit /etc/fstab so that it is auto-mounted. -Then, add the following to crontab -@daily /usr/bin/rsync -ha /srv /mnt/backup -@daily /usr/bin/rsync -ha /etc /mnt/backup +Then, add the following to crontab:: + @daily /usr/bin/rsync -ha /srv /mnt/backup + @daily /usr/bin/rsync -ha /etc /mnt/backup Nginx ------ From 836074e37d9b3780093b89548c035cdb29603349 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 17:15:52 +0530 Subject: [PATCH 0054/1154] ... --- setup/file_hosting_servers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst index 563c7bc64a..8dd0afe098 100644 --- a/setup/file_hosting_servers.rst +++ b/setup/file_hosting_servers.rst @@ -11,7 +11,7 @@ Edit /etc/hosts and put in FQDN in the appropriate places, for example:: 27.0.1.1 download.calibre-ebook.com download 46.28.49.116 download.calibre-ebook.com download -apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools +apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools mosh chsh -s /bin/zsh mkdir -p /root/staging /root/work/vim /srv/download /srv/manual From 952b95d3ad566dca005863f9403e3a846b5e1e8e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 17:34:01 +0530 Subject: [PATCH 0055/1154] pep8 --- recipes/miradasalsur.recipe | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/recipes/miradasalsur.recipe b/recipes/miradasalsur.recipe index 4794503384..b931fcb1d7 100644 --- a/recipes/miradasalsur.recipe +++ b/recipes/miradasalsur.recipe @@ -4,9 +4,7 @@ sur.infonews.com ''' import datetime -from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe -from calibre.ebooks.BeautifulSoup import Tag class MiradasAlSur(BasicNewsRecipe): title = 'Miradas al Sur' @@ -25,7 +23,7 @@ class MiradasAlSur(BasicNewsRecipe): extra_css = """ body{font-family: Arial,Helvetica,sans-serif} h1{font-family: Georgia,Times,serif} - .field-field-story-author{color: gray; font-size: small} + .field-field-story-author{color: gray; font-size: small} """ conversion_options = { 'comment' : description @@ -34,22 +32,22 @@ class MiradasAlSur(BasicNewsRecipe): , 'language' : language , 'series' : title } - + keep_only_tags = [dict(name='div', attrs={'id':['content-header', 'content-area']})] - remove_tags = [ - dict(name=['link','meta','iframe','embed','object']), + remove_tags = [ + dict(name=['link','meta','iframe','embed','object']), dict(name='form', attrs={'class':'fivestar-widget'}), dict(attrs={'class':lambda x: x and 'terms-inline' in x.split()}) ] feeds = [ - (u'Politica' , u'http://sur.infonews.com/taxonomy/term/1/0/feed' ), - (u'Internacional' , u'http://sur.infonews.com/taxonomy/term/2/0/feed' ), + (u'Politica' , u'http://sur.infonews.com/taxonomy/term/1/0/feed'), + (u'Internacional' , u'http://sur.infonews.com/taxonomy/term/2/0/feed'), (u'Informe Especial' , u'http://sur.infonews.com/taxonomy/term/14/0/feed'), - (u'Delitos y pesquisas', u'http://sur.infonews.com/taxonomy/term/6/0/feed' ), - (u'Lesa Humanidad' , u'http://sur.infonews.com/taxonomy/term/7/0/feed' ), - (u'Cultura' , u'http://sur.infonews.com/taxonomy/term/8/0/feed' ), - (u'Deportes' , u'http://sur.infonews.com/taxonomy/term/9/0/feed' ), + (u'Delitos y pesquisas', u'http://sur.infonews.com/taxonomy/term/6/0/feed'), + (u'Lesa Humanidad' , u'http://sur.infonews.com/taxonomy/term/7/0/feed'), + (u'Cultura' , u'http://sur.infonews.com/taxonomy/term/8/0/feed'), + (u'Deportes' , u'http://sur.infonews.com/taxonomy/term/9/0/feed'), (u'Contratapa' , u'http://sur.infonews.com/taxonomy/term/10/0/feed'), ] @@ -60,10 +58,10 @@ class MiradasAlSur(BasicNewsRecipe): cdate = datetime.date.today() todayweekday = cdate.isoweekday() if (todayweekday != 7): - cdate -= datetime.timedelta(days=todayweekday) - cover_page_url = cdate.strftime('http://sur.infonews.com/ediciones/%Y-%m-%d/tapa'); + cdate -= datetime.timedelta(days=todayweekday) + cover_page_url = cdate.strftime('http://sur.infonews.com/ediciones/%Y-%m-%d/tapa') soup = self.index_to_soup(cover_page_url) cover_item = soup.find('img', attrs={'class':lambda x: x and 'imagecache-tapa_edicion_full' in x.split()}) if cover_item: - cover_url = cover_item['src'] + cover_url = cover_item['src'] return cover_url From 986ab2a0787a538bc5fe0c512da97f982e331dbc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 28 Jun 2013 08:14:25 +0530 Subject: [PATCH 0056/1154] ebook-convert: Add option to read metadata from OPF --- src/calibre/ebooks/conversion/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/conversion/cli.py b/src/calibre/ebooks/conversion/cli.py index a0abebc5fe..f2795005d8 100644 --- a/src/calibre/ebooks/conversion/cli.py +++ b/src/calibre/ebooks/conversion/cli.py @@ -94,6 +94,8 @@ def option_recommendation_to_cli_option(add_option, rec): if opt.long_switch == 'verbose': attrs['action'] = 'count' attrs.pop('type', '') + if opt.name == 'read_metadata_from_opf': + switches.append('--from-opf') if opt.name in DEFAULT_TRUE_OPTIONS and rec.recommended_value is True: switches = ['--disable-'+opt.long_switch] add_option(Option(*switches, **attrs)) @@ -190,7 +192,7 @@ def add_pipeline_options(parser, plumber): ), 'METADATA' : (_('Options to set metadata in the output'), - plumber.metadata_option_names, + plumber.metadata_option_names + ['read_metadata_from_opf'], ), 'DEBUG': (_('Options to help with debugging the conversion'), [ From 837adb4eabda66083840b3fb15b165946b31376a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 28 Jun 2013 08:50:14 +0530 Subject: [PATCH 0057/1154] version 0.9.37 --- Changelog.yaml | 44 ++++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index f952b961e2..7439a02986 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,50 @@ # new recipes: # - title: +- version: 0.9.37 + date: 2013-06-28 + + new features: + - title: "Conversion: Add option to embed all referenced fonts" + type: major + description: "Add an option to embed all fonts that are referenced in the input document but are not already embedded. This will search your system for the referenced font, and if found, the font will be embedded. Only works if the output format supports font embedding (for example: EPUB or AZW3). The option is under the Look & Feel section of the conversion dialog." + + - title: "ToC Editor: When generating a ToC from files, if the file has no text, do not skip it. Instead create an entry using the filename of the file." + + - title: "AZW3 Input: Add support for the page-progression-direction that is used to indicate page turns should happen from right to left. The attribute is passed into EPUB when converting." + tickets: [1194766] + + - title: "ebook-convert: Add a --from-opf option to read metadata from OPF files directly, instead of having to run ebook-meta --from-opf after conversion" + + bug fixes: + - title: "PDF Output: Fix Table of Contents being added to the end of the PDF even without the Add Table of Contents option being enabled." + tickets: [1194836] + + - title: "When auto-merging books on add, also merge identifiers." + + - title: "Fix an error when using the Template Editor to create a template that uses custom columns." + tickets: [1193763] + + - title: "LRF Output: Fix " entities in attribute values causing problems" + + - title: "News download: Apply the default page margin conversion settings. Also, when converting to PDF, apply the pdf conversion defaults." + tickets: [1193912] + + - title: "Fix a regression that broke scanning for books on all devices that used the Aluratek Color driver." + tickets: [1192940] + + - title: "fetch-ebbok-metadata: Fix --opf argument erroneously requiring a value" + + - title: "When waiting before sending email, log the wait." + tickets: [1195173] + + improved recipes: + - taz.de (RSS) + - Miradas al sur + - Frontline + - La Nacion (Costa Rica) + + - version: 0.9.36 date: 2013-06-21 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 6834a4e66d..a4edca6bd5 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 9, 36) +numeric_version = (0, 9, 37) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " From 6579327a6d99635411ee8a7dcf1d143f5fb5a789 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 28 Jun 2013 12:26:56 +0530 Subject: [PATCH 0058/1154] Various minor fixes in the publish process --- setup/file_hosting_servers.rst | 10 ++++++++++ setup/hosting.py | 8 ++++++-- setup/upload.py | 2 -- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst index 8dd0afe098..261241e24d 100644 --- a/setup/file_hosting_servers.rst +++ b/setup/file_hosting_servers.rst @@ -35,3 +35,13 @@ rsync /srv from another file server service nginx start +Services +--------- + +SSH into sourceforge and downloadbestsoftware so that their host keys are +stored. + + ssh -oStrictHostKeyChecking=no kovid@www.downloadbestsoft-mirror1.com + ssh -oStrictHostKeyChecking=no kovidgoyal,calibre@frs.sourceforge.net + ssh -oStrictHostKeyChecking=no files.calibre-ebook.com (and whatever other mirrors are present) + diff --git a/setup/hosting.py b/setup/hosting.py index 1e78f4694d..d97373cdbc 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -473,14 +473,18 @@ def upload_to_servers(files, version): # {{{ os.mkdir(dest) for src in files: shutil.copyfile(src, os.path.join(dest, os.path.basename(src))) - generate_index() + cwd = os.getcwd() + try: + generate_index() + finally: + os.chdir(cwd) for server, rdir in {'files':'/srv/download/'}.iteritems(): print('Uploading to server:', server) server = '%s.calibre-ebook.com' % server # Copy the generated index files print ('Copying generated index') - check_call(['rsync', '-hzr', '-e', 'ssh -x', '--include', '*.html', + check_call(['rsync', '-hza', '-e', 'ssh -x', '--include', '*.html', '--filter', '-! */', base, 'root@%s:%s' % (server, rdir)]) # Copy the release files rdir = '%s%s/' % (rdir, version) diff --git a/setup/upload.py b/setup/upload.py index dd59067c0c..0475773e01 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -255,8 +255,6 @@ class UploadToServer(Command): # {{{ description = 'Upload miscellaneous data to calibre server' def run(self, opts): - check_call('ssh divok rm -f %s/calibre-\*.tar.xz'%DOWNLOADS, shell=True) - # check_call('scp dist/calibre-*.tar.xz divok:%s/'%DOWNLOADS, shell=True) check_call('gpg --armor --detach-sign dist/calibre-*.tar.xz', shell=True) check_call('scp dist/calibre-*.tar.xz.asc divok:%s/signatures/'%DOWNLOADS, From 54a9a7c98e7bbf995201fc6f323ad45cb0ca20f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 28 Jun 2013 12:44:44 +0530 Subject: [PATCH 0059/1154] pep8 and small perf improvement for human_readable() --- src/calibre/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 5e940efcd9..07ad906247 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -310,9 +310,9 @@ def get_parsed_proxy(typ='http', debug=True): proxy = proxies.get(typ, None) if proxy: pattern = re.compile(( - '(?:ptype://)?' \ - '(?:(?P\w+):(?P.*)@)?' \ - '(?P[\w\-\.]+)' \ + '(?:ptype://)?' + '(?:(?P\w+):(?P.*)@)?' + '(?P[\w\-\.]+)' '(?::(?P\d+))?').replace('ptype', typ) ) @@ -535,7 +535,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252', ent = match.group(1) if ent in exceptions: return '&'+ent+';' - if ent in {'apos', 'squot'}: # squot is generated by some broken CMS software + if ent in {'apos', 'squot'}: # squot is generated by some broken CMS software return check("'") if ent == 'hellips': ent = 'hellip' @@ -565,7 +565,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252', return '&'+ent+';' _ent_pat = re.compile(r'&(\S+?);') -xml_entity_to_unicode = partial(entity_to_unicode, result_exceptions = { +xml_entity_to_unicode = partial(entity_to_unicode, result_exceptions={ '"' : '"', "'" : ''', '<' : '<', @@ -670,8 +670,8 @@ def human_readable(size, sep=' '): """ Convert a size in bytes into a human readable form """ divisor, suffix = 1, "B" for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')): - if size < 1024**(i+1): - divisor, suffix = 1024**(i), candidate + if size < (1 << ((i + 1) * 10)): + divisor, suffix = (1 << (i * 10)), candidate break size = str(float(size)/divisor) if size.find(".") > -1: From 4b4b89bc6ebf58602ab2dfca1a3c076298b1deed Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Jun 2013 00:15:38 +0530 Subject: [PATCH 0060/1154] Update iprofessional Fixes #1195826 [Updated recipe for iprofessional](https://bugs.launchpad.net/calibre/+bug/1195826) --- recipes/iprofesional.recipe | 66 +++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/recipes/iprofesional.recipe b/recipes/iprofesional.recipe index e8edbbc7a3..82d1b81674 100644 --- a/recipes/iprofesional.recipe +++ b/recipes/iprofesional.recipe @@ -1,5 +1,4 @@ -__license__ = 'GPL v3' -__copyright__ = '2011, Darko Miletic ' +__copyright__ = '2011-2013, Darko Miletic ' ''' www.iprofesional.com ''' @@ -19,13 +18,15 @@ class iProfesional(BasicNewsRecipe): use_embedded_content = False language = 'es_AR' remove_empty_feeds = True - publication_type = 'nesportal' - masthead_url = 'http://www.iprofesional.com/img/logo-iprofesional.png' + publication_type = 'newsportal' + masthead_url = 'http://www.iprofesional.com/img/header/logoiprofesional.png' extra_css = """ - body{font-family: Arial,Helvetica,sans-serif } + body{font-family: 'Droid Sans',Arial,sans-serif } img{margin-bottom: 0.4em; display:block} - .titulo-interior{font-family: Georgia,"Times New Roman",Times,serif} - .autor-nota{font-size: small; font-weight: bold; font-style: italic; color: gray} + .titulo{font-family: WhitneyBoldWhitneyBold,Arial,Helvetica,sans-serif; color: blue} + .fecha-archivo{font-weight: bold; color: rgb(205, 150, 24)} + .description{font-weight: bold; color: gray } + .firma{font-size: small} """ conversion_options = { @@ -35,27 +36,21 @@ class iProfesional(BasicNewsRecipe): , 'language' : language } - keep_only_tags = [dict(attrs={'class':['fecha','interior-nota']})] - - remove_tags = [ - dict(name=['meta','link','base','embed','object','iframe']) - ,dict(attrs={'class':['menu-imprimir','guardarNota','IN-widget','fin','permalink']}) - ] - remove_attributes=['lang','xmlns:og','xmlns:fb'] - + keep_only_tags = [dict(attrs={'class':'desarrollo'})] + remove_tags = [dict(name=['meta','link','base','embed','object','iframe'])] feeds = [ (u'Ultimas noticias' , u'http://feeds.feedburner.com/iprofesional-principales-noticias') - ,(u'Finanzas' , u'http://feeds.feedburner.com/iprofesional-finanzas' ) - ,(u'Impuestos' , u'http://feeds.feedburner.com/iprofesional-impuestos' ) - ,(u'Negocios' , u'http://feeds.feedburner.com/iprofesional-economia' ) - ,(u'Comercio Exterior' , u'http://feeds.feedburner.com/iprofesional-comercio-exterior' ) - ,(u'Tecnologia' , u'http://feeds.feedburner.com/iprofesional-tecnologia' ) - ,(u'Management' , u'http://feeds.feedburner.com/iprofesional-managment' ) - ,(u'Marketing' , u'http://feeds.feedburner.com/iprofesional-marketing' ) - ,(u'Legales' , u'http://feeds.feedburner.com/iprofesional-legales' ) - ,(u'Autos' , u'http://feeds.feedburner.com/iprofesional-autos' ) - ,(u'Vinos' , u'http://feeds.feedburner.com/iprofesional-vinos-bodegas' ) + ,(u'Finanzas' , u'http://feeds.feedburner.com/iprofesional-finanzas') + ,(u'Impuestos' , u'http://feeds.feedburner.com/iprofesional-impuestos') + ,(u'Negocios' , u'http://feeds.feedburner.com/iprofesional-economia') + ,(u'Comercio Exterior' , u'http://feeds.feedburner.com/iprofesional-comercio-exterior') + ,(u'Tecnologia' , u'http://feeds.feedburner.com/iprofesional-tecnologia') + ,(u'Management' , u'http://feeds.feedburner.com/iprofesional-managment') + ,(u'Marketing' , u'http://feeds.feedburner.com/iprofesional-marketing') + ,(u'Legales' , u'http://feeds.feedburner.com/iprofesional-legales') + ,(u'Autos' , u'http://feeds.feedburner.com/iprofesional-autos') + ,(u'Vinos' , u'http://feeds.feedburner.com/iprofesional-vinos-bodegas') ] def preprocess_html(self, soup): @@ -64,16 +59,17 @@ class iProfesional(BasicNewsRecipe): for item in soup.findAll('a'): limg = item.find('img') if item.string is not None: - str = item.string - item.replaceWith(str) + str = item.string + item.replaceWith(str) else: - if limg: - item.name = 'div' - item.attrs = [] - else: - str = self.tag_to_string(item) - item.replaceWith(str) + if limg: + item.name = 'div' + item.attrs = [] + else: + str = self.tag_to_string(item) + item.replaceWith(str) for item in soup.findAll('img'): - if not item.has_key('alt'): - item['alt'] = 'image' + if 'alt' not in item: + item['alt'] = 'image' return soup + From 9e857d1ed7a44bc6e56d125faae626486a0d6830 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Jun 2013 09:39:36 +0530 Subject: [PATCH 0061/1154] DOCX Input: Support horizontal rules DOCX Input: Add support for horizontal rules created by typing three hyphens and pressing enter. --- src/calibre/ebooks/docx/cleanup.py | 13 +++++++++++++ src/calibre/ebooks/docx/images.py | 22 +++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/docx/cleanup.py b/src/calibre/ebooks/docx/cleanup.py index a55f8449d8..10bfd9a78f 100644 --- a/src/calibre/ebooks/docx/cleanup.py +++ b/src/calibre/ebooks/docx/cleanup.py @@ -8,6 +8,8 @@ __copyright__ = '2013, Kovid Goyal ' import os +from calibre.ebooks.docx.names import ancestor + def mergeable(previous, current): if previous.tail or current.tail: return False @@ -97,6 +99,16 @@ def before_count(root, tag, limit=10): return limit def cleanup_markup(log, root, styles, dest_dir, detect_cover): + # Move


    s outside paragraphs, if possible. + for hr in root.xpath('//span/hr'): + p = ancestor(hr, 'p') + descendants = tuple(p.iterdescendants()) + if descendants[-1] is hr: + parent = p.getparent() + idx = parent.index(p) + parent.insert(idx+1, hr) + hr.tail = '\n\t' + # Merge consecutive spans that have the same styling current_run = [] for span in root.xpath('//span'): @@ -165,3 +177,4 @@ def cleanup_markup(log, root, styles, dest_dir, detect_cover): return path + diff --git a/src/calibre/ebooks/docx/images.py b/src/calibre/ebooks/docx/images.py index 85e957a589..b0a5348d90 100644 --- a/src/calibre/ebooks/docx/images.py +++ b/src/calibre/ebooks/docx/images.py @@ -8,7 +8,7 @@ __copyright__ = '2013, Kovid Goyal ' import os -from lxml.html.builder import IMG +from lxml.html.builder import IMG, HR from calibre.ebooks.docx.names import XPath, get, barename from calibre.utils.filenames import ascii_filename @@ -163,6 +163,26 @@ class Images(object): yield ans def pict_to_html(self, pict, page): + # First see if we have an
    + is_hr = len(pict) == 1 and get(pict[0], 'o:hr') in {'t', 'true'} + if is_hr: + style = {} + hr = HR() + try: + pct = float(get(pict[0], 'o:hrpct')) + except (ValueError, TypeError, AttributeError): + pass + else: + if pct > 0: + style['width'] = '%.3g%%' % pct + align = get(pict[0], 'o:hralign', 'center') + if align in {'left', 'right'}: + style['margin-left'] = '0' if align == 'left' else 'auto' + style['margin-right'] = 'auto' if align == 'left' else '0' + if style: + hr.set('style', '; '.join(('%s:%s' % (k, v) for k, v in style.iteritems()))) + yield hr + for imagedata in XPath('descendant::v:imagedata[@r:id]')(pict): rid = get(imagedata, 'r:id') if rid in self.rid_map: From fa2f92e96cb17740e19630bd290c47cc3f902e47 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Jun 2013 10:42:18 +0530 Subject: [PATCH 0062/1154] ... --- src/calibre/ebooks/conversion/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/conversion/cli.py b/src/calibre/ebooks/conversion/cli.py index f2795005d8..45b7841347 100644 --- a/src/calibre/ebooks/conversion/cli.py +++ b/src/calibre/ebooks/conversion/cli.py @@ -27,7 +27,7 @@ specified as the first two arguments to the command. The output ebook format is guessed from the file extension of \ output_file. output_file can also be of the special format .EXT where \ EXT is the output file extension. In this case, the name of the output \ -file is derived the name of the input file. Note that the filenames must \ +file is derived from the name of the input file. Note that the filenames must \ not start with a hyphen. Finally, if output_file has no extension, then \ it is treated as a directory and an "open ebook" (OEB) consisting of HTML \ files is written to that directory. These files are the files that would \ From 373b3c7a222f11e59b40dee03327d8e0599f065c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Jun 2013 12:30:56 +0530 Subject: [PATCH 0063/1154] Fix #22 (Remove double article entries which are already part in other rss feeds of the same recipe) --- recipes/taz_rss.recipe | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/recipes/taz_rss.recipe b/recipes/taz_rss.recipe index 3ccbe2a4f1..0535b6ef3a 100644 --- a/recipes/taz_rss.recipe +++ b/recipes/taz_rss.recipe @@ -1,23 +1,43 @@ -__license__ = 'GPL v3' -__copyright__ = '2010, Alexander Schremmer ' +__license__ = 'GPL v3' +__copyright__ = '2013, Alexander Schremmer , Robert Riemann ' +import re from calibre.web.feeds.news import BasicNewsRecipe class TazRSSRecipe(BasicNewsRecipe): - title = u'Taz.de (die tageszeitung) RSS Feed - German' - __author__ = 'Alexander Schremmer' + title = u'Taz - die Tageszeitung' + description = u'Taz.de - die tageszeitung' + __author__ = 'Alexander Schremmer, Robert Riemann' language = 'de' lang = 'de-DE' oldest_article = 7 max_articles_per_feed = 100 publisher = 'taz Entwicklungs GmbH & Co. Medien KG' + # masthead_url = u'http://galeria-autonomica.de/wp-content/uploads/a_taz-logo.gif' + masthead_url = u'http://upload.wikimedia.org/wikipedia/de/thumb/1/15/Die-Tageszeitung-Logo.svg/500px-Die-Tageszeitung-Logo.svg.png' conversion_options = {'publisher': publisher, 'language': lang, } - - feeds = [(u'TAZ main feed', u'http://www.taz.de/rss.xml')] + feeds = [ + (u'Schlagzeilen', u'http://www.taz.de/!p3270;rss/'), + (u'Politik', u'http://www.taz.de/Politik/!p2;rss/'), + (u'Zukunft', u'http://www.taz.de/Zukunft/!p4;rss/'), + (u'Netz', u'http://www.taz.de/Netz/!p5;rss/'), + (u'Debatte', u'http://www.taz.de/Debatte/!p9;rss/'), + (u'Leben', u'http://www.taz.de/Leben/!p10;rss/'), + (u'Sport', u'http://www.taz.de/Sport/!p12;rss/'), + (u'Wahrheit', u'http://www.taz.de/Wahrheit/!p13;rss/'), + (u'Berlin', u'http://www.taz.de/Berlin/!p14;rss/'), + (u'Nord', u'http://www.taz.de/Nord/!p11;rss/') + ] keep_only_tags = [dict(name='div', attrs={'class': 'sect sect_article'})] remove_tags = [ - dict(name=['div'], attrs={'class': 'artikelwerbung'}), - dict(name=['ul'], attrs={'class': 'toolbar'}),] + dict(name=['div'], attrs={'class': 'artikelwerbung'}), + dict(name=['ul'], attrs={'class': 'toolbar'}), + # remove: taz paywall + dict(name=['div'], attrs={'id': 'tzi_paywall'}), + # remove: Artikel zum Thema (not working on Kindle) + dict(name=['div'], attrs={'class': re.compile(r".*\bsect_seealso\b.*")}), + dict(name=['div'], attrs={'class': 'sectfoot'}) + ] From e8ada93ddad20094425cfd037a96f675a4f9e70b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Sun, 30 Jun 2013 00:58:24 +0200 Subject: [PATCH 0064/1154] deduplicate code snippet --- recipes/gosc_niedzielny.recipe | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/recipes/gosc_niedzielny.recipe b/recipes/gosc_niedzielny.recipe index ba2280c2a5..7ff2d48f84 100644 --- a/recipes/gosc_niedzielny.recipe +++ b/recipes/gosc_niedzielny.recipe @@ -47,13 +47,7 @@ class GN(BasicNewsRecipe): return feeds def find_articles(self, main_block): - for a in main_block.findAll('div', attrs={'class':'prev_doc2'}): - art = a.find('a') - yield { - 'title' : self.tag_to_string(art), - 'url' : 'http://www.gosc.pl' + art['href'] - } - for a in main_block.findAll('div', attrs={'class':'sr-document'}): + for a in main_block.findAll('div', attrs={'class':['prev_doc2', 'sr-document']}): art = a.find('a') yield { 'title' : self.tag_to_string(art), From 3db0f1895a2547d75d3015fcea54f1647b5f2afd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 30 Jun 2013 10:45:57 +0530 Subject: [PATCH 0065/1154] ... --- src/calibre/ebooks/oeb/transforms/embed_fonts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/oeb/transforms/embed_fonts.py b/src/calibre/ebooks/oeb/transforms/embed_fonts.py index 027b8af1de..879e05da8f 100644 --- a/src/calibre/ebooks/oeb/transforms/embed_fonts.py +++ b/src/calibre/ebooks/oeb/transforms/embed_fonts.py @@ -197,7 +197,7 @@ class EmbedFonts(object): if not ff: return ff = ff[0] - if ff in self.warned: + if ff in self.warned or ff == 'inherit': return try: fonts = font_scanner.fonts_for_family(ff) From 0223f60c9d28fc4433e63233e91230b8ddb6f708 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 30 Jun 2013 20:03:57 +0530 Subject: [PATCH 0066/1154] ... --- src/calibre/ebooks/conversion/plumber.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index a96574e904..14b5482a04 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -134,8 +134,7 @@ OptionRecommendation(name='output_profile', help=_('Specify the output profile. The output profile ' 'tells the conversion system how to optimize the ' 'created document for the specified device. In some cases, ' - 'an output profile is required to produce documents that ' - 'will work on a device. For example EPUB on the SONY reader. ' + 'an output profile can be used to optimize the output for a particular device, but this is rarely necessary. ' 'Choices are:') + ', '.join([x.short_name for x in output_profiles()]) ), From 47626ee0cccb1ae7199b254ed01b7cea1f26f5b5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 1 Jul 2013 09:03:55 +0530 Subject: [PATCH 0067/1154] ... --- setup/file_hosting_servers.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst index 261241e24d..c72998958e 100644 --- a/setup/file_hosting_servers.rst +++ b/setup/file_hosting_servers.rst @@ -11,6 +11,10 @@ Edit /etc/hosts and put in FQDN in the appropriate places, for example:: 27.0.1.1 download.calibre-ebook.com download 46.28.49.116 download.calibre-ebook.com download +dpkg-reconfigure tzdata +set timezone to Asia/Kolkata +service cron restart + apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools mosh chsh -s /bin/zsh From 10c4c8f69f7b2904350e7f474ef14ce77da8b172 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 1 Jul 2013 12:05:28 +0530 Subject: [PATCH 0068/1154] ... --- src/calibre/gui2/tag_browser/ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/tag_browser/ui.py b/src/calibre/gui2/tag_browser/ui.py index 79d4a85f84..8760ccd23b 100644 --- a/src/calibre/gui2/tag_browser/ui.py +++ b/src/calibre/gui2/tag_browser/ui.py @@ -334,7 +334,7 @@ class TagBrowserWidget(QWidget): # {{{ search_layout = QHBoxLayout() self._layout.addLayout(search_layout) self.item_search = HistoryLineEdit(parent) - self.item_search.setMinimumContentsLength(10) + self.item_search.setMinimumContentsLength(5) self.item_search.setSizeAdjustPolicy(self.item_search.AdjustToMinimumContentsLengthWithIcon) try: self.item_search.lineEdit().setPlaceholderText( From 59346348c5ab9db1510e6d022d36b8a19078aa44 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 1 Jul 2013 13:46:43 +0530 Subject: [PATCH 0069/1154] ... --- src/calibre/library/catalogs/epub_mobi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/catalogs/epub_mobi.py b/src/calibre/library/catalogs/epub_mobi.py index 4984fbf9e2..673d764593 100644 --- a/src/calibre/library/catalogs/epub_mobi.py +++ b/src/calibre/library/catalogs/epub_mobi.py @@ -149,7 +149,7 @@ class EPUB_MOBI(CatalogPlugin): default=None, dest='output_profile', action=None, - help=_("Specifies the output profile. In some cases, an output profile is required to optimize the catalog for the device. For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n" + help=_("Specifies the output profile. In some cases, an output profile is required to optimize the catalog for the device. For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n" "Default: '%default'\n" "Applies to: AZW3, ePub, MOBI output formats")), Option('--prefix-rules', From 144f66c13b6e020dee1a6b225a728967d7dc21a1 Mon Sep 17 00:00:00 2001 From: GRiker Date: Mon, 1 Jul 2013 03:04:32 -0600 Subject: [PATCH 0070/1154] Added mkdir() to libiMobileDevice --- .../devices/idevice/libimobiledevice.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/calibre/devices/idevice/libimobiledevice.py b/src/calibre/devices/idevice/libimobiledevice.py index ca6ed57a77..ba0f4d0f38 100644 --- a/src/calibre/devices/idevice/libimobiledevice.py +++ b/src/calibre/devices/idevice/libimobiledevice.py @@ -418,6 +418,14 @@ class libiMobileDevice(): if False: self._idevice_set_debug_level(DEBUG) + def mkdir(self, path): + ''' + Mimic mkdir(), creating a directory at path. Does not create + intermediate folders + ''' + self._log_location("'%s'" % path) + return self._afc_make_directory(path) + def mount_ios_app(self, app_name=None, app_id=None): ''' Convenience method to get iDevice ready to talk to app_name or app_id @@ -1007,6 +1015,27 @@ class libiMobileDevice(): self.log(" %s: %s" % (key, file_stats[key])) return file_stats + def _afc_make_directory(self, path): + ''' + Creates a directory on the device. Does not create intermediate dirs. + + Args: + client: (AFC_CLIENT_T) The client to use to make a directory + dir: (const char *) The directory's fully-qualified path + + Result: + error: AFC_E_SUCCESS on success or an AFC_E_* error value + ''' + self._log_location("%s" % repr(path)) + + error = self.lib.afc_make_directory(byref(self.afc), + str(path)) & 0xFFFF + if error: + if self.verbose: + self.log(" ERROR: %s" % self._afc_error(error)) + + return error + def _afc_read_directory(self, directory=''): ''' Gets a directory listing of the directory requested From 9952abad4aa8a973a5ed06a3a9575ecda1a446e0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 1 Jul 2013 15:08:54 +0530 Subject: [PATCH 0071/1154] Polish: Add option to embed referenced fonts Book polishing: Add option to embed all referenced fonts when polishing books using the "Polish Books" tool. Fixes #1196038 [[enhancement] embed font without conversion](https://bugs.launchpad.net/calibre/+bug/1196038) --- resources/compiled_coffeescript.zip | Bin 71177 -> 71881 bytes src/calibre/ebooks/oeb/polish/embed.py | 158 ++++++++++++++++++ .../ebooks/oeb/polish/font_stats.coffee | 12 ++ src/calibre/ebooks/oeb/polish/main.py | 18 +- src/calibre/ebooks/oeb/polish/stats.py | 70 ++++++-- src/calibre/gui2/actions/polish.py | 2 + 6 files changed, 245 insertions(+), 15 deletions(-) create mode 100644 src/calibre/ebooks/oeb/polish/embed.py diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index cb48a58bd2714e22fdeb5e827bef753f51998001..e092b53157f3c30f95712db0b597358b744dc08a 100644 GIT binary patch delta 454 zcmeBN!*X&Xi$s7oGm8iV2+S(D=k%>F=<5+a28JVi+qKvkr*co#KP6R1p{O(mN*3!V6qi)yq)u<(W0dBy=28HI#JtH91>`25 zwG*3se=7H6PAi+~feRRi?{vGRjS#z{MzK1~FPeAu~-u1L7P_g=(mUFuBCM;%JCmEYO;gqSDmK z{)eR||49^{-0+=!^80f3=@SGPg{H3)Vq891w}q#XZz~@o8@n_klL#}mFkxVjpI#u$ zC^Oxfol#_ZurQ+*6C?k02T?}R=~ILmO_^@1KsXPC87-JL=z%$tAINh|Hx~gaHRyy0 zBvf%tZvqMEOoj-=v~Wy64HCG$VET6vMt3HK6(9*$W&=*f>5ig|B9jw-aftxjQrDU#ThxKFBD@mWx6Fgy+E8%boyH{Mhm9(@(_-VIHM7heiex0 z$)w!^5xf8ryftz1_XV=k6(tz8nB-?dB$6Z;4Vd(QK{zWV7 + ans = {} + for node in document.getElementsByTagName('*') + rules = document.defaultView.getMatchedCSSRules(node, '') + if rules + for rule in rules + style = rule.style + family = style.getPropertyValue('font-family') + if family + ans[family] = true + py_bridge.value = ans + if window? window.font_stats = new FontStats() diff --git a/src/calibre/ebooks/oeb/polish/main.py b/src/calibre/ebooks/oeb/polish/main.py index 08b5004c91..ff46288643 100644 --- a/src/calibre/ebooks/oeb/polish/main.py +++ b/src/calibre/ebooks/oeb/polish/main.py @@ -14,6 +14,7 @@ from functools import partial from calibre.ebooks.oeb.polish.container import get_container from calibre.ebooks.oeb.polish.stats import StatsCollector from calibre.ebooks.oeb.polish.subset import subset_all_fonts +from calibre.ebooks.oeb.polish.embed import embed_all_fonts from calibre.ebooks.oeb.polish.cover import set_cover from calibre.ebooks.oeb.polish.replace import smarten_punctuation from calibre.ebooks.oeb.polish.jacket import ( @@ -21,6 +22,7 @@ from calibre.ebooks.oeb.polish.jacket import ( from calibre.utils.logging import Log ALL_OPTS = { + 'embed': False, 'subset': False, 'opf': None, 'cover': None, @@ -47,6 +49,12 @@ changes needed for the desired effect.

    Note that polishing only works on files in the %s formats.

    \ ''')%_(' or ').join('%s'%x for x in SUPPORTED), +'embed': _('''\ +

    Embed all fonts that are referenced in the document and are not already embedded. +This will scan your computer for the fonts, and if they are found, they will be +embedded into the document.

    +'''), + 'subset': _('''\

    Subsetting fonts means reducing an embedded font to contain only the characters used from that font in the book. This @@ -118,8 +126,8 @@ def polish(file_map, opts, log, report): ebook = get_container(inbook, log) jacket = None - if opts.subset: - stats = StatsCollector(ebook) + if opts.subset or opts.embed: + stats = StatsCollector(ebook, do_embed=opts.embed) if opts.opf: rt(_('Updating metadata')) @@ -159,6 +167,11 @@ def polish(file_map, opts, log, report): smarten_punctuation(ebook, report) report('') + if opts.embed: + rt(_('Embedding referenced fonts')) + embed_all_fonts(ebook, stats, report) + report('') + if opts.subset: rt(_('Subsetting embedded fonts')) subset_all_fonts(ebook, stats.font_stats, report) @@ -197,6 +210,7 @@ def option_parser(): parser = OptionParser(usage=USAGE) a = parser.add_option o = partial(a, default=False, action='store_true') + o('--embed-fonts', '-e', dest='embed', help=CLI_HELP['embed']) o('--subset-fonts', '-f', dest='subset', help=CLI_HELP['subset']) a('--cover', '-c', help=_( 'Path to a cover image. Changes the cover specified in the ebook. ' diff --git a/src/calibre/ebooks/oeb/polish/stats.py b/src/calibre/ebooks/oeb/polish/stats.py index d4a5c96111..77b99ff9b6 100644 --- a/src/calibre/ebooks/oeb/polish/stats.py +++ b/src/calibre/ebooks/oeb/polish/stats.py @@ -7,10 +7,11 @@ __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import json, sys, os +import json, sys, os, logging from urllib import unquote +from collections import defaultdict -from cssutils import parseStyle +from cssutils import CSSParser from PyQt4.Qt import (pyqtProperty, QString, QEventLoop, Qt, QSize, QTimer, pyqtSlot) from PyQt4.QtWebKit import QWebPage, QWebView @@ -41,14 +42,14 @@ def normalize_font_properties(font): 'extra-expanded', 'ultra-expanded'}: val = 'normal' font['font-stretch'] = val + return font -widths = {x:i for i, x in enumerate(( 'ultra-condensed', +widths = {x:i for i, x in enumerate(('ultra-condensed', 'extra-condensed', 'condensed', 'semi-condensed', 'normal', 'semi-expanded', 'expanded', 'extra-expanded', 'ultra-expanded' ))} def get_matching_rules(rules, font): - normalize_font_properties(font) matches = [] # Filter on family @@ -100,7 +101,7 @@ def get_matching_rules(rules, font): return m return [] -class Page(QWebPage): # {{{ +class Page(QWebPage): # {{{ def __init__(self, log): self.log = log @@ -157,10 +158,12 @@ class Page(QWebPage): # {{{ class StatsCollector(object): - def __init__(self, container): + def __init__(self, container, do_embed=False): self.container = container self.log = self.logger = container.log + self.do_embed = do_embed must_use_qt() + self.parser = CSSParser(loglevel=logging.CRITICAL, log=logging.getLogger('calibre.css')) self.loop = QEventLoop() self.view = QWebView() @@ -173,6 +176,10 @@ class StatsCollector(object): self.render_queue = list(container.spine_items) self.font_stats = {} + self.font_usage_map = {} + self.font_spec_map = {} + self.font_rule_map = {} + self.all_font_rules = {} QTimer.singleShot(0, self.render_book) @@ -235,27 +242,35 @@ class StatsCollector(object): rules = [] for rule in font_face_rules: ff = rule.get('font-family', None) - if not ff: continue - style = parseStyle('font-family:%s'%ff, validate=False) + if not ff: + continue + style = self.parser.parseStyle('font-family:%s'%ff, validate=False) ff = [x.value for x in style.getProperty('font-family').propertyValue] if not ff or ff[0] == 'inherit': continue rule['font-family'] = frozenset(icu_lower(f) for f in ff) src = rule.get('src', None) - if not src: continue - style = parseStyle('background-image:%s'%src, validate=False) + if not src: + continue + style = self.parser.parseStyle('background-image:%s'%src, validate=False) src = style.getProperty('background-image').propertyValue[0].uri name = self.href_to_name(src, '@font-face rule') + if name is None: + continue rule['src'] = name normalize_font_properties(rule) rule['width'] = widths[rule['font-stretch']] rule['weight'] = int(rule['font-weight']) rules.append(rule) - if not rules: + if not rules and not self.do_embed: return + self.font_rule_map[self.container.abspath_to_name(self.current_item)] = rules + for rule in rules: + self.all_font_rules[rule['src']] = rule + for rule in rules: if rule['src'] not in self.font_stats: self.font_stats[rule['src']] = set() @@ -265,19 +280,48 @@ class StatsCollector(object): if not isinstance(font_usage, list): raise Exception('Unknown error occurred while reading font usage') exclude = {'\n', '\r', '\t'} + self.font_usage_map[self.container.abspath_to_name(self.current_item)] = fu = defaultdict(dict) + bad_fonts = {'serif', 'sans-serif', 'monospace', 'cursive', 'fantasy', 'sansserif', 'inherit'} for font in font_usage: text = set() for t in font['text']: text |= frozenset(t) text.difference_update(exclude) - if not text: continue + if not text: + continue + normalize_font_properties(font) for rule in get_matching_rules(rules, font): self.font_stats[rule['src']] |= text + if self.do_embed: + ff = [icu_lower(x) for x in font.get('font-family', [])] + if ff and ff[0] not in bad_fonts: + keys = {'font-weight', 'font-style', 'font-stretch', 'font-family'} + key = frozenset(((k, ff[0] if k == 'font-family' else v) for k, v in font.iteritems() if k in keys)) + val = fu[key] + if not val: + val.update({k:(font[k][0] if k == 'font-family' else font[k]) for k in keys}) + val['text'] = set() + val['text'] |= text + self.font_usage_map[self.container.abspath_to_name(self.current_item)] = dict(fu) + + if self.do_embed: + self.page.evaljs('window.font_stats.get_font_families()') + font_families = self.page.bridge_value + if not isinstance(font_families, dict): + raise Exception('Unknown error occurred while reading font families') + self.font_spec_map[self.container.abspath_to_name(self.current_item)] = fs = set() + for raw in font_families.iterkeys(): + style = self.parser.parseStyle('font-family:' + raw, validate=False).getProperty('font-family') + for x in style.propertyValue: + x = x.value + if x and x.lower() not in bad_fonts: + fs.add(x) if __name__ == '__main__': from calibre.ebooks.oeb.polish.container import get_container from calibre.utils.logging import default_log default_log.filter_level = default_log.DEBUG ebook = get_container(sys.argv[-1], default_log) - print (StatsCollector(ebook).font_stats) + print (StatsCollector(ebook, do_embed=True).font_stats) + diff --git a/src/calibre/gui2/actions/polish.py b/src/calibre/gui2/actions/polish.py index eb21fb2626..0f21807afb 100644 --- a/src/calibre/gui2/actions/polish.py +++ b/src/calibre/gui2/actions/polish.py @@ -45,6 +45,7 @@ class Polish(QDialog): # {{{ ORIGINAL_* format before running it.

    ''') ), + 'embed':_('

    Embed referenced fonts

    %s')%HELP['embed'], 'subset':_('

    Subsetting fonts

    %s')%HELP['subset'], 'smarten_punctuation': @@ -75,6 +76,7 @@ class Polish(QDialog): # {{{ count = 0 self.all_actions = OrderedDict([ + ('embed', _('&Embed all referenced fonts')), ('subset', _('&Subset all embedded fonts')), ('smarten_punctuation', _('Smarten &punctuation')), ('metadata', _('Update &metadata in the book files')), From c3cc6a2278facf35cfbc260e8b1e9fd978008913 Mon Sep 17 00:00:00 2001 From: GRiker Date: Mon, 1 Jul 2013 03:46:32 -0600 Subject: [PATCH 0072/1154] Revert "Fixed typo(?) in set_metadata() for touched files." This reverts commit 8f30c17486701bf2cc29a84c98af5336de75fc56. --- src/calibre/utils/podofo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/utils/podofo/__init__.py b/src/calibre/utils/podofo/__init__.py index a0b5d85331..13c12a9bb3 100644 --- a/src/calibre/utils/podofo/__init__.py +++ b/src/calibre/utils/podofo/__init__.py @@ -36,7 +36,7 @@ def set_metadata(stream, mi): except WorkerError as e: raise Exception('Failed to set PDF metadata: %s'%e.orig_tb) if touched: - with open(os.path.join(tdir, u'input.pdf'), 'rb') as f: + with open(os.path.join(tdir, u'output.pdf'), 'rb') as f: f.seek(0, 2) if f.tell() > 100: f.seek(0) From 6af87c05e31d886101feb065cd373021472ba888 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 1 Jul 2013 15:33:20 +0530 Subject: [PATCH 0073/1154] Driver for PocketBook Surfpad 2 Fixes #1182850 [Private bug](https://bugs.launchpad.net/calibre/+bug/1182850) --- src/calibre/devices/android/driver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index a39c190d05..31b60389ad 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -96,7 +96,7 @@ class ANDROID(USBMS): # Google 0x18d1 : { - 0x0001 : [0x0223, 0x230, 0x9999], + 0x0001 : [0x0222, 0x0223, 0x230, 0x9999], 0x0002 : [0x9999], 0x0003 : [0x0230, 0x9999], 0x4e11 : [0x0100, 0x226, 0x227], @@ -219,7 +219,7 @@ class ANDROID(USBMS): 'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD', 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0', 'COBY_MID', 'VS', 'AINOL', 'TOPWISE', 'PAD703', 'NEXT8D12', - 'MEDIATEK', 'KEENHI', 'TECLAST', 'SURFTAB', 'XENTA',] + 'MEDIATEK', 'KEENHI', 'TECLAST', 'SURFTAB', 'XENTA', 'OBREEY_S'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'A953', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', @@ -241,7 +241,7 @@ class ANDROID(USBMS): 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E', 'NOVO7', 'MB526', '_USB#WYK7MSF8KE', 'TABLET_PC', 'F', 'MT65XX_MS', 'ICS', 'E400', '__FILE-STOR_GADG', 'ST80208-1', 'GT-S5660M_CARD', 'XT894', '_USB', - 'PROD_TAB13-201', + 'PROD_TAB13-201', 'URFPAD2', ] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', @@ -254,7 +254,7 @@ class ANDROID(USBMS): 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E', 'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD', 'ST80208-1', 'XT894', - '_USB', 'PROD_TAB13-201', + '_USB', 'PROD_TAB13-201', 'URFPAD2' ] OSX_MAIN_MEM = 'Android Device Main Memory' From e83738653ca3688569f37ef1b8f5bf3fe70060ad Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 2 Jul 2013 09:14:21 +0530 Subject: [PATCH 0074/1154] Fix #25 (bugfix: pass book_id to FileType plugins on postimport instead of __builtins__.id()) --- src/calibre/library/database2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 98e0190ecd..435d8edeeb 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1500,8 +1500,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): format = os.path.splitext(npath)[-1].lower().replace('.', '').upper() stream = lopen(npath, 'rb') format = check_ebook_format(stream, format) - retval = self.add_format(index, format, stream, replace=replace, - index_is_id=index_is_id, path=path, notify=notify) + id = index if index_is_id else self.id(index) + retval = self.add_format(id, format, stream, replace=replace, + index_is_id=True, path=path, notify=notify) run_plugins_on_postimport(self, id, format) return retval From acbd785af90eaac58432bceae8e9fb9171b8d5c1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 2 Jul 2013 09:24:09 +0530 Subject: [PATCH 0075/1154] Fix docstring of is_image_collection --- src/calibre/customize/conversion.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/calibre/customize/conversion.py b/src/calibre/customize/conversion.py index fe0d563206..38ffcef71f 100644 --- a/src/calibre/customize/conversion.py +++ b/src/calibre/customize/conversion.py @@ -77,7 +77,7 @@ class OptionRecommendation(object): self.option.choices: raise ValueError('OpRec: %s: Recommended value not in choices'% self.option.name) - if not (isinstance(self.recommended_value, (int, float, str, unicode))\ + if not (isinstance(self.recommended_value, (int, float, str, unicode)) or self.recommended_value is None): raise ValueError('OpRec: %s:'%self.option.name + repr(self.recommended_value) + @@ -139,8 +139,10 @@ class InputFormatPlugin(Plugin): file_types = set([]) #: If True, this input plugin generates a collection of images, - #: one per HTML file. You can obtain access to the images via - #: convenience method, :meth:`get_image_collection`. + #: one per HTML file. This can be set dynamically, in the convert method + #: if the input files can be both image collections and non-image collections. + #: If you set this to True, you must implement the get_images() method that returns + #: a list of images. is_image_collection = False #: Number of CPU cores used by this plugin @@ -238,7 +240,6 @@ class InputFormatPlugin(Plugin): ret = self.convert(stream, options, file_ext, log, accelerators) - return ret def postprocess_book(self, oeb, opts, log): @@ -313,7 +314,6 @@ class OutputFormatPlugin(Plugin): Plugin.__init__(self, *args) self.report_progress = DummyReporter() - def convert(self, oeb_book, output, input_plugin, opts, log): ''' Render the contents of `oeb_book` (which is an instance of @@ -363,3 +363,4 @@ class OutputFormatPlugin(Plugin): + From 3c81080eadc76b5fd40a5320ee207e65f96017fb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 2 Jul 2013 11:17:08 +0530 Subject: [PATCH 0076/1154] Democracy Now by Antoine Beaupre --- recipes/democracy_now.recipe | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 recipes/democracy_now.recipe diff --git a/recipes/democracy_now.recipe b/recipes/democracy_now.recipe new file mode 100644 index 0000000000..f7868c19dd --- /dev/null +++ b/recipes/democracy_now.recipe @@ -0,0 +1,45 @@ +# vim:fileencoding=utf-8 +from calibre.web.feeds.news import BasicNewsRecipe + +class DemocracyNowRecipe(BasicNewsRecipe): + title = u'Democracy now!' + __author__ = u'Antoine Beaupré' + description = 'A daily TV/radio news program, hosted by Amy Goodman and Juan Gonzalez, airing on over 1,100 stations, pioneering the largest community media collaboration in the United States.' # noqa + language = 'en' + cover_url = 'http://www.democracynow.org/images/dn-logo-for-podcast.png' + + oldest_article = 1 + max_articles_per_feed = 10 + publication_type = 'magazine' + + auto_cleanup = False + use_embedded_content = False + no_stylesheets = True + remove_javascript = True + + feeds = [ + (u'Daily news', u'http://www.democracynow.org/democracynow.rss')] + + keep_only_tags = [dict(name='div', attrs={'id': 'page'}), ] + remove_tags = [dict(name='div', attrs={'id': 'topics_list'}), + dict(name='div', attrs={'id': 'header'}), + dict(name='div', attrs={'id': 'footer'}), + dict(name='div', attrs={'id': 'right'}), + dict(name='div', attrs={'id': 'left-panel'}), + dict(name='div', attrs={'id': 'top-video-content'}), + dict(name='div', attrs={'id': 'google-news-date'}), + dict(name='div', attrs={'id': 'story-donate'}), + dict( + name='div', attrs={'id': 'transcript-expand-collapse'}), + dict(name='span', attrs={'class': 'show-links'}), + dict(name='span', attrs={'class': 'storyNav'}), + dict(name='div', attrs={'class': 'headline_share'}), + dict(name='div', attrs={'class': 'mediaBar'}), + dict(name='div', attrs={'class': 'shareAndPrinterBar'}), + dict(name='div', attrs={'class': 'utility-navigation'}), + dict(name='div', attrs={'class': 'bottomContentNav'}), + dict(name='div', attrs={'class': 'recentShows'}), + dict( + name='div', attrs={'class': 'printer-and-transcript-links'}), + ] + From 19016a109d651c93a50beee6a2743de5f24fe737 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 2 Jul 2013 11:22:53 +0530 Subject: [PATCH 0077/1154] Add warning about font licensing to embed options --- src/calibre/ebooks/conversion/plumber.py | 3 ++- src/calibre/ebooks/oeb/polish/main.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 14b5482a04..5778bbbabc 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -211,7 +211,8 @@ OptionRecommendation(name='embed_all_fonts', 'but not already embedded. This will search your system for the ' 'fonts, and if found, they will be embedded. Embedding will only work ' 'if the format you are converting to supports embedded fonts, such as ' - 'EPUB, AZW3 or PDF.' + 'EPUB, AZW3 or PDF. Please ensure that you have the proper license for embedding ' + 'the fonts used in this book.' )), OptionRecommendation(name='subset_embedded_fonts', diff --git a/src/calibre/ebooks/oeb/polish/main.py b/src/calibre/ebooks/oeb/polish/main.py index ff46288643..69d03098c7 100644 --- a/src/calibre/ebooks/oeb/polish/main.py +++ b/src/calibre/ebooks/oeb/polish/main.py @@ -53,6 +53,7 @@ changes needed for the desired effect.

    Embed all fonts that are referenced in the document and are not already embedded. This will scan your computer for the fonts, and if they are found, they will be embedded into the document.

    +

    Please ensure that you have the proper license for embedding the fonts used in this book.

    '''), 'subset': _('''\ From 9394f87c14df4f866d50f32350863cafb145a96b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 2 Jul 2013 11:23:59 +0530 Subject: [PATCH 0078/1154] ... --- src/calibre/ebooks/conversion/plumber.py | 2 +- src/calibre/ebooks/oeb/polish/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 5778bbbabc..d4bdeb4562 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -212,7 +212,7 @@ OptionRecommendation(name='embed_all_fonts', 'fonts, and if found, they will be embedded. Embedding will only work ' 'if the format you are converting to supports embedded fonts, such as ' 'EPUB, AZW3 or PDF. Please ensure that you have the proper license for embedding ' - 'the fonts used in this book.' + 'the fonts used in this document.' )), OptionRecommendation(name='subset_embedded_fonts', diff --git a/src/calibre/ebooks/oeb/polish/main.py b/src/calibre/ebooks/oeb/polish/main.py index 69d03098c7..c5a7d4db6d 100644 --- a/src/calibre/ebooks/oeb/polish/main.py +++ b/src/calibre/ebooks/oeb/polish/main.py @@ -53,7 +53,7 @@ changes needed for the desired effect.

    Embed all fonts that are referenced in the document and are not already embedded. This will scan your computer for the fonts, and if they are found, they will be embedded into the document.

    -

    Please ensure that you have the proper license for embedding the fonts used in this book.

    +

    Please ensure that you have the proper license for embedding the fonts used in this document.

    '''), 'subset': _('''\ From f5db5d9c00fcbc47358d992b7e9d8b62ca7fbf76 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 2 Jul 2013 18:37:21 +0530 Subject: [PATCH 0079/1154] Driver for Trekstor Pyrus Maxi Fixes #1196931 [Device not recogized](https://bugs.launchpad.net/calibre/+bug/1196931) --- src/calibre/devices/misc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index b20ec3ca6e..e35db8f03d 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -227,16 +227,17 @@ class TREKSTOR(USBMS): VENDOR_ID = [0x1e68] PRODUCT_ID = [0x0041, 0x0042, 0x0052, 0x004e, 0x0056, 0x0067, # This is for the Pyrus Mini + 0x006f, # This is for the Pyrus Maxi 0x003e, # This is for the EBOOK_PLAYER_5M https://bugs.launchpad.net/bugs/792091 0x5cL, # This is for the 4ink http://www.mobileread.com/forums/showthread.php?t=191318 ] - BCD = [0x0002, 0x100] + BCD = [0x0002, 0x100, 0x0222] EBOOK_DIR_MAIN = 'Ebooks' VENDOR_NAME = 'TREKSTOR' WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['EBOOK_PLAYER_7', - 'EBOOK_PLAYER_5M', 'EBOOK-READER_3.0', 'EREADER_PYRUS', 'PYRUS_MINI'] + 'EBOOK_PLAYER_5M', 'EBOOK-READER_3.0', 'EREADER_PYRUS', 'PYRUS_MINI', 'PYRUS_MAXI'] SUPPORTS_SUB_DIRS = True SUPPORTS_SUB_DIRS_DEFAULT = False From e8839bc8dc5bbf96ead2c12bc303c99ed9447c88 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 08:12:33 +0530 Subject: [PATCH 0080/1154] Prefix version tags with v --- setup/publish.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/publish.py b/setup/publish.py index ac56942273..40d002ee8a 100644 --- a/setup/publish.py +++ b/setup/publish.py @@ -113,6 +113,6 @@ class TagRelease(Command): def run(self, opts): self.info('Tagging release') - subprocess.check_call('git tag -a {0} -m "version-{0}"'.format(__version__).split()) - subprocess.check_call('git push origin {0}'.format(__version__).split()) + subprocess.check_call('git tag -a v{0} -m "version-{0}"'.format(__version__).split()) + subprocess.check_call('git push origin v{0}'.format(__version__).split()) From 3b4094a890120df33b78d073bfa61b41261d30ff Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 10:58:48 +0530 Subject: [PATCH 0081/1154] DOCX: Handle hyperlinks created as fields See https://bugs.launchpad.net/calibre/+bug/1196728 for an example. --- src/calibre/ebooks/docx/fields.py | 106 +++++++++++++++++++++++++++++ src/calibre/ebooks/docx/to_html.py | 22 ++++++ 2 files changed, 128 insertions(+) create mode 100644 src/calibre/ebooks/docx/fields.py diff --git a/src/calibre/ebooks/docx/fields.py b/src/calibre/ebooks/docx/fields.py new file mode 100644 index 0000000000..9b0d053cd0 --- /dev/null +++ b/src/calibre/ebooks/docx/fields.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +import re + +from calibre.ebooks.docx.names import XPath, get + +class Field(object): + + def __init__(self, start): + self.start = start + self.end = None + self.contents = [] + self.instructions = [] + + def add_instr(self, elem): + raw = elem.text + if not raw: + return + name, rest = raw.strip().partition(' ')[0::2] + self.instructions.append((name, rest.strip())) + +WORD, FLAG = 0, 1 +scanner = re.Scanner([ + (r'\\\S{1}', lambda s, t: (t, FLAG)), # A flag of the form \x + (r'"[^"]*"', lambda s, t: (t[1:-1], WORD)), # Quoted word + (r'[^\s\\"]\S*', lambda s, t: (t, WORD)), # A non-quoted word, must not start with a backslash or a space or a quote + (r'\s+', None), +], flags=re.DOTALL) + + +def parse_hyperlink(raw, log): + ans = {} + last_option = None + for token, token_type in scanner.scan(raw)[0]: + if not ans: + if token_type is not WORD: + log('Invalid hyperlink, first token is not a URL (%s)' % raw) + return ans + ans['url'] = token + if token_type is FLAG: + last_option = {'l':'anchor', 'm':'image-map', 'n':'target', 'o':'title', 't':'target'}.get(token[1], None) + if last_option is not None: + ans[last_option] = None + elif token_type is WORD: + if last_option is not None: + ans[last_option] = token + return ans + + +class Fields(object): + + def __init__(self): + self.fields = [] + + def __call__(self, doc, log): + stack = [] + for elem in XPath( + '//*[name()="w:p" or name()="w:r" or name()="w:instrText" or (name()="w:fldChar" and (@w:fldCharType="begin" or @w:fldCharType="end"))]')(doc): + if elem.tag.endswith('}fldChar'): + typ = get(elem, 'w:fldCharType') + if typ == 'begin': + stack.append(Field(elem)) + self.fields.append(stack[-1]) + else: + try: + stack.pop().end = elem + except IndexError: + pass + elif elem.tag.endswith('}instrText'): + if stack: + stack[-1].add_instr(elem) + else: + if stack: + stack[-1].contents.append(elem) + + # Parse hyperlink fields + self.hyperlink_fields = [] + for field in self.fields: + if len(field.instructions) == 1 and field.instructions[0][0] == 'HYPERLINK': + hl = parse_hyperlink(field.instructions[0][1], log) + if hl: + if 'target' in hl and hl['target'] is None: + hl['target'] = '_blank' + all_runs = [] + current_runs = [] + # We only handle spans in a single paragraph + # being wrapped in + for x in field.contents: + if x.tag.endswith('}p'): + if current_runs: + all_runs.append(current_runs) + current_runs = [] + elif x.tag.endswith('}r'): + current_runs.append(x) + if current_runs: + all_runs.append(current_runs) + for runs in all_runs: + self.hyperlink_fields.append((hl, runs)) + + diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index 79020d9c0a..647b021205 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -26,6 +26,7 @@ from calibre.ebooks.docx.footnotes import Footnotes from calibre.ebooks.docx.cleanup import cleanup_markup from calibre.ebooks.docx.theme import Theme from calibre.ebooks.docx.toc import create_toc +from calibre.ebooks.docx.fields import Fields from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.utils.localization import canonicalize_lang, lang_as_iso639_1 @@ -52,6 +53,7 @@ class Convert(object): self.body = BODY() self.theme = Theme() self.tables = Tables() + self.fields = Fields() self.styles = Styles(self.tables) self.images = Images() self.object_map = OrderedDict() @@ -79,6 +81,7 @@ class Convert(object): def __call__(self): doc = self.docx.document relationships_by_id, relationships_by_type = self.docx.document_relationships + self.fields(doc, self.log) self.read_styles(relationships_by_type) self.images(relationships_by_id) self.layers = OrderedDict() @@ -396,6 +399,25 @@ class Convert(object): # hrefs that point nowhere give epubcheck a hernia. The element # should be styled explicitly by Word anyway. # span.set('href', '#') + rmap = {v:k for k, v in self.object_map.iteritems()} + for hyperlink, runs in self.fields.hyperlink_fields: + spans = [rmap[r] for r in runs if r in rmap] + if not spans: + continue + if len(spans) > 1: + span = self.wrap_elems(spans, SPAN()) + span.tag = 'a' + tgt = hyperlink.get('target', None) + if tgt: + span.set('target', tgt) + tt = hyperlink.get('title', None) + if tt: + span.set('title', tt) + url = hyperlink['url'] + if url in self.anchor_map: + span.set('href', '#' + self.anchor_map[url]) + continue + span.set('href', url) def convert_run(self, run): ans = SPAN() From 80f3e7f8674a9069ca6894e033d9afe279c44e96 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 11:23:26 +0530 Subject: [PATCH 0082/1154] DOCX: Insert page breaks at the start of every new section See https://bugs.launchpad.net/calibre/+bug/1196728 for an example --- src/calibre/ebooks/docx/styles.py | 5 +++++ src/calibre/ebooks/docx/to_html.py | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/docx/styles.py b/src/calibre/ebooks/docx/styles.py index 21f45616fa..4572eb59f2 100644 --- a/src/calibre/ebooks/docx/styles.py +++ b/src/calibre/ebooks/docx/styles.py @@ -403,6 +403,11 @@ class Styles(object): ps.margin_top = 0 last_para = p + def apply_section_page_breaks(self, paras): + for p in paras: + ps = self.resolve_paragraph(p) + ps.pageBreakBefore = True + def register(self, css, prefix): h = hash(frozenset(css.iteritems())) ans, _ = self.classes.get(h, (None, None)) diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index 647b021205..1fdd24267a 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -100,6 +100,9 @@ class Convert(object): self.body.append(p) paras.append(wp) self.styles.apply_contextual_spacing(paras) + # Apply page breaks at the start of every section, except the first + # section (since that will be the start of the file) + self.styles.apply_section_page_breaks(self.section_starts[1:]) notes_header = None if self.footnotes.has_notes: @@ -180,6 +183,7 @@ class Convert(object): def read_page_properties(self, doc): current = [] self.page_map = OrderedDict() + self.section_starts = [] for p in descendants(doc, 'w:p', 'w:tbl'): if p.tag.endswith('}tbl'): @@ -189,8 +193,10 @@ class Convert(object): sect = tuple(descendants(p, 'w:sectPr')) if sect: pr = PageProperties(sect) - for x in current + [p]: + paras = current + [p] + for x in paras: self.page_map[x] = pr + self.section_starts.append(paras[0]) current = [] else: current.append(p) From 584beceee347f2a18c70728bdd4830381fabe85c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 11:24:12 +0530 Subject: [PATCH 0083/1154] DOCX: handle bookmarks defined at the paragraph level See https://bugs.launchpad.net/calibre/+bug/1196728 for an example. --- src/calibre/ebooks/docx/to_html.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index 1fdd24267a..fae521d807 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -99,6 +99,7 @@ class Convert(object): p = self.convert_p(wp) self.body.append(p) paras.append(wp) + self.read_block_anchors(doc) self.styles.apply_contextual_spacing(paras) # Apply page breaks at the start of every section, except the first # section (since that will be the start of the file) @@ -296,6 +297,22 @@ class Convert(object): opf.render(of, ncx, 'toc.ncx') return os.path.join(self.dest_dir, 'metadata.opf') + def read_block_anchors(self, doc): + doc_anchors = frozenset(XPath('./w:body/w:bookmarkStart[@w:name]')(doc)) + if doc_anchors: + current_bm = None + rmap = {v:k for k, v in self.object_map.iteritems()} + for p in descendants(doc, 'w:p', 'w:bookmarkStart[@w:name]'): + if p.tag.endswith('}p'): + if current_bm and p in rmap: + para = rmap[p] + if 'id' not in para.attrib: + para.set('id', generate_anchor(current_bm, frozenset(self.anchor_map.itervalues()))) + self.anchor_map[current_bm] = para.get('id') + current_bm = None + elif p in doc_anchors: + current_bm = get(p, 'w:name') + def convert_p(self, p): dest = P() self.object_map[dest] = p From 26e23ac7a6f793bde281517ea8ade1e4c878263f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 12:15:55 +0530 Subject: [PATCH 0084/1154] Splitting: Handle the tail of the split point correctly EPUB/AZW3 Output: Fix splitting on page-break-after with plain text immediately following the split point causing the text to be added before rather than after the split point. --- src/calibre/ebooks/oeb/transforms/split.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/calibre/ebooks/oeb/transforms/split.py b/src/calibre/ebooks/oeb/transforms/split.py index 605a58a31f..36fe6b3167 100644 --- a/src/calibre/ebooks/oeb/transforms/split.py +++ b/src/calibre/ebooks/oeb/transforms/split.py @@ -339,6 +339,8 @@ class FlowSplitter(object): # We want to keep the descendants of the split point in # Tree 1 keep_descendants = True + # We want the split point element, but not its tail + elem.tail = '\n' continue if hit_split_point: @@ -357,6 +359,18 @@ class FlowSplitter(object): for elem in tuple(body2.iterdescendants()): if elem is split_point2: if not before: + # Keep the split point element's tail, if it contains non-whitespace + # text + tail = elem.tail + if tail and not tail.isspace(): + parent = elem.getparent() + idx = parent.index(elem) + if idx == 0: + parent.text = (parent.text or '') + tail + else: + sib = parent[idx-1] + sib.tail = (sib.tail or '') + tail + # Remove the element itself nix_element(elem) break if elem in ancestors: From bbaf2ff574d19981e79a6bd2fcd6bf147a44d4ac Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 13:05:47 +0530 Subject: [PATCH 0085/1154] DOCX: Hyperlinked images DOCX Input: Add support for clickable (hyperlinked) images --- src/calibre/ebooks/docx/container.py | 2 +- src/calibre/ebooks/docx/images.py | 19 ++++++++++++++++--- src/calibre/ebooks/docx/to_html.py | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/docx/container.py b/src/calibre/ebooks/docx/container.py index 68f74a3c82..deaf5bd4d0 100644 --- a/src/calibre/ebooks/docx/container.py +++ b/src/calibre/ebooks/docx/container.py @@ -183,7 +183,7 @@ class DOCX(object): root = fromstring(raw) for item in root.xpath('//*[local-name()="Relationships"]/*[local-name()="Relationship" and @Type and @Target]'): target = item.get('Target') - if item.get('TargetMode', None) != 'External': + if item.get('TargetMode', None) != 'External' and not target.startswith('#'): target = '/'.join((base, target.lstrip('/'))) typ = item.get('Type') Id = item.get('Id') diff --git a/src/calibre/ebooks/docx/images.py b/src/calibre/ebooks/docx/images.py index b0a5348d90..3be3d51c05 100644 --- a/src/calibre/ebooks/docx/images.py +++ b/src/calibre/ebooks/docx/images.py @@ -96,6 +96,7 @@ class Images(object): self.used = {} self.names = set() self.all_images = set() + self.links = [] def __call__(self, relationships_by_id): self.rid_map = relationships_by_id @@ -125,8 +126,18 @@ class Images(object): self.all_images.add('images/' + name) return name - def pic_to_img(self, pic, alt=None): + def pic_to_img(self, pic, alt, parent): name = None + link = None + for hl in XPath('descendant::a:hlinkClick[@r:id]')(parent): + link = {'id':get(hl, 'r:id')} + tgt = hl.get('tgtFrame', None) + if tgt: + link['target'] = tgt + title = hl.get('tooltip', None) + if title: + link['title'] = title + for pr in XPath('descendant::pic:cNvPr')(pic): name = pr.get('name', None) if name: @@ -138,6 +149,8 @@ class Images(object): src = self.generate_filename(rid, name) img = IMG(src='images/%s' % src) img.set('alt', alt or 'Image') + if link is not None: + self.links.append((img, link)) return img def drawing_to_html(self, drawing, page): @@ -145,7 +158,7 @@ class Images(object): for inline in XPath('./wp:inline')(drawing): style, alt = get_image_properties(inline) for pic in XPath('descendant::pic:pic')(inline): - ans = self.pic_to_img(pic, alt) + ans = self.pic_to_img(pic, alt, inline) if ans is not None: if style: ans.set('style', '; '.join('%s: %s' % (k, v) for k, v in style.iteritems())) @@ -156,7 +169,7 @@ class Images(object): style, alt = get_image_properties(anchor) self.get_float_properties(anchor, style, page) for pic in XPath('descendant::pic:pic')(anchor): - ans = self.pic_to_img(pic, alt) + ans = self.pic_to_img(pic, alt, anchor) if ans is not None: if style: ans.set('style', '; '.join('%s: %s' % (k, v) for k, v in style.iteritems())) diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index fae521d807..26e50f9b9d 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -442,6 +442,27 @@ class Convert(object): continue span.set('href', url) + for img, link in self.images.links: + parent = img.getparent() + idx = parent.index(img) + a = A(img) + a.tail, img.tail = img.tail, None + parent.insert(idx, a) + tgt = link.get('target', None) + if tgt: + a.set('target', tgt) + tt = link.get('title', None) + if tt: + a.set('title', tt) + rid = link['id'] + if rid in relationships_by_id: + dest = relationships_by_id[rid] + if dest.startswith('#'): + if dest[1:] in self.anchor_map: + a.set('href', '#' + self.anchor_map[dest[1:]]) + else: + a.set('href', dest) + def convert_run(self, run): ans = SPAN() self.object_map[ans] = run From c8c3741d342cd644781137bc26811f08e1efcb0d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 13:36:30 +0530 Subject: [PATCH 0086/1154] DOCX: Handle redundant bookmarks DOCX Input: Fix links pointing to locations in the same document that contain multiple, redundant bookmarks not working. --- src/calibre/ebooks/docx/to_html.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index 26e50f9b9d..01808657ea 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -342,7 +342,13 @@ class Convert(object): elif x.tag.endswith('}bookmarkStart'): anchor = get(x, 'w:name') if anchor and anchor not in self.anchor_map: + old_anchor = current_anchor self.anchor_map[anchor] = current_anchor = generate_anchor(anchor, frozenset(self.anchor_map.itervalues())) + if old_anchor is not None: + # The previous anchor was not applied to any element + for a, t in tuple(self.anchor_map.iteritems()): + if t == old_anchor: + self.anchor_map[a] = current_anchor elif x.tag.endswith('}hyperlink'): current_hyperlink = x From 36386d06cc954909a173736f1e08a6460084ef0f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 14:42:08 +0530 Subject: [PATCH 0087/1154] PDF Output: Fix mangling of links PDF Output: Fix links that point to URLs with query parameters being mangled by the conversion process. Fixes #1197006 [Broken links in PDF in Adobe reader.](https://bugs.launchpad.net/calibre/+bug/1197006) --- src/calibre/ebooks/pdf/render/links.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/pdf/render/links.py b/src/calibre/ebooks/pdf/render/links.py index 500bbbf6c1..6ec7c500a8 100644 --- a/src/calibre/ebooks/pdf/render/links.py +++ b/src/calibre/ebooks/pdf/render/links.py @@ -8,9 +8,8 @@ __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' import os -from future_builtins import map -from urlparse import urlparse, urlunparse -from urllib2 import quote, unquote +from urlparse import urlparse +from urllib2 import unquote from calibre.ebooks.pdf.render.common import Array, Name, Dictionary, String @@ -84,10 +83,6 @@ class Links(object): action = Dictionary({ 'Type':Name('Action'), 'S':Name('URI'), }) - parts = (x.encode('utf-8') if isinstance(x, type(u'')) else - x for x in purl) - url = urlunparse(map(quote, map(unquote, - parts))).decode('ascii') action['URI'] = String(url) annot['A'] = action if 'A' in annot or 'Dest' in annot: From a149790b93f554038fb6b28a2dfe229d3ac9c9c5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 15:17:46 +0530 Subject: [PATCH 0088/1154] ... --- src/calibre/ebooks/pdf/render/links.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/ebooks/pdf/render/links.py b/src/calibre/ebooks/pdf/render/links.py index 6ec7c500a8..4d0d588771 100644 --- a/src/calibre/ebooks/pdf/render/links.py +++ b/src/calibre/ebooks/pdf/render/links.py @@ -83,6 +83,8 @@ class Links(object): action = Dictionary({ 'Type':Name('Action'), 'S':Name('URI'), }) + # Do not try to normalize/quote/unquote this URL as if it + # has a query part, it will get corrupted action['URI'] = String(url) annot['A'] = action if 'A' in annot or 'Dest' in annot: From a3adb69d944dc2535895eedf9d0159302a07e839 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 21:18:32 +0530 Subject: [PATCH 0089/1154] Metadata download dialog: Prevent the buttons from being re-ordered when the Next button is clicked. --- src/calibre/gui2/metadata/single_download.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py index 3e9bb87687..ed378745a5 100644 --- a/src/calibre/gui2/metadata/single_download.py +++ b/src/calibre/gui2/metadata/single_download.py @@ -1006,7 +1006,7 @@ class FullFetch(QDialog): # {{{ l.addWidget(self.bb) self.bb.rejected.connect(self.reject) self.bb.accepted.connect(self.accept) - self.next_button = self.bb.addButton(_('Next'), self.bb.AcceptRole) + self.next_button = self.bb.addButton(_('Next'), self.bb.ActionRole) self.next_button.setDefault(True) self.next_button.setEnabled(False) self.next_button.setIcon(QIcon(I('ok.png'))) @@ -1019,7 +1019,7 @@ class FullFetch(QDialog): # {{{ self.log_button = self.bb.addButton(_('View log'), self.bb.ActionRole) self.log_button.clicked.connect(self.view_log) self.log_button.setIcon(QIcon(I('debug.png'))) - self.ok_button.setVisible(False) + self.ok_button.setEnabled(False) self.prev_button.setVisible(False) self.identify_widget = IdentifyWidget(self.log, self) @@ -1044,7 +1044,7 @@ class FullFetch(QDialog): # {{{ def book_selected(self, book, caches): self.next_button.setVisible(False) - self.ok_button.setVisible(True) + self.ok_button.setEnabled(True) self.prev_button.setVisible(True) self.book = book self.stack.setCurrentIndex(1) @@ -1055,8 +1055,9 @@ class FullFetch(QDialog): # {{{ def back_clicked(self): self.next_button.setVisible(True) - self.ok_button.setVisible(False) + self.ok_button.setEnabled(False) self.prev_button.setVisible(False) + self.next_button.setFocus() self.stack.setCurrentIndex(0) self.covers_widget.cancel() self.covers_widget.reset_covers() @@ -1081,6 +1082,7 @@ class FullFetch(QDialog): # {{{ self.next_button.setEnabled(True) def next_clicked(self, *args): + gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry()) self.identify_widget.get_result() def ok_clicked(self, *args): From 509cc82d805e7c49057ddba6aaf1109e982b7226 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 21:58:44 +0530 Subject: [PATCH 0090/1154] Allow running python setup.py develop froma git checkout --- setup/install.py | 2 +- setup/translations.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/setup/install.py b/setup/install.py index b1698d88ed..d1bb2058f1 100644 --- a/setup/install.py +++ b/setup/install.py @@ -56,7 +56,7 @@ class Develop(Command): short_description = 'Setup a development environment for calibre' MODE = 0o755 - sub_commands = ['build', 'resources', 'iso639', 'gui',] + sub_commands = ['build', 'resources', 'gui',] def add_postinstall_options(self, parser): parser.add_option('--make-errors-fatal', action='store_true', default=False, diff --git a/setup/translations.py b/setup/translations.py index 17c8d10018..e8b0fecdf1 100644 --- a/setup/translations.py +++ b/setup/translations.py @@ -21,7 +21,12 @@ def qt_sources(): class POT(Command): # {{{ description = 'Update the .pot translation template and upload it' - LP_BASE = os.path.join(os.path.dirname(os.path.dirname(Command.SRC)), 'calibre-translations') + LP_BASE = os.path.join(os.path.dirname(Command.SRC)) + if not os.path.exists(os.path.join(LP_BASE, 'setup', 'iso_639')): + # We are in a git checkout, translations are assumed to be in a + # directory called calibre-translations at the same level as the + # calibre directory. + LP_BASE = os.path.join(os.path.dirname(os.path.dirname(Command.SRC)), 'calibre-translations') LP_SRC = os.path.join(LP_BASE, 'src') LP_PATH = os.path.join(LP_SRC, os.path.join(__appname__, 'translations')) LP_ISO_PATH = os.path.join(LP_BASE, 'setup', 'iso_639') From bd4e828668d2e76dae900ce3530db27482787dda Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 3 Jul 2013 22:23:49 +0530 Subject: [PATCH 0091/1154] Restore building iso639 in develop as it is needed --- setup/install.py | 2 +- setup/iso_639_3.xml | 39178 ++++++++++++++++++++++++++++++++++++++++ setup/translations.py | 11 +- 3 files changed, 39186 insertions(+), 5 deletions(-) create mode 100644 setup/iso_639_3.xml diff --git a/setup/install.py b/setup/install.py index d1bb2058f1..b1698d88ed 100644 --- a/setup/install.py +++ b/setup/install.py @@ -56,7 +56,7 @@ class Develop(Command): short_description = 'Setup a development environment for calibre' MODE = 0o755 - sub_commands = ['build', 'resources', 'gui',] + sub_commands = ['build', 'resources', 'iso639', 'gui',] def add_postinstall_options(self, parser): parser.add_option('--make-errors-fatal', action='store_true', default=False, diff --git a/setup/iso_639_3.xml b/setup/iso_639_3.xml new file mode 100644 index 0000000000..6b94a3850b --- /dev/null +++ b/setup/iso_639_3.xml @@ -0,0 +1,39178 @@ + + + + + + + +]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/setup/translations.py b/setup/translations.py index e8b0fecdf1..786d44a6d6 100644 --- a/setup/translations.py +++ b/setup/translations.py @@ -322,21 +322,24 @@ class GetTranslations(Translations): # {{{ class ISO639(Command): # {{{ - description = 'Compile translations for ISO 639 codes' + description = 'Compile language code maps for performance' DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization', 'iso639.pickle') def run(self, opts): - src = POT.LP_ISO_PATH + src = self.j(self.d(self.SRC), 'setup', 'iso_639_3.xml') if not os.path.exists(src): raise Exception(src + ' does not exist') dest = self.DEST + base = self.d(dest) + if not os.path.exists(base): + os.makedirs(base) if not self.newer(dest, [src, __file__]): self.info('Pickled code is up to date') return self.info('Pickling ISO-639 codes to', dest) from lxml import etree - root = etree.fromstring(open(self.j(src, 'iso_639_3.xml'), 'rb').read()) + root = etree.fromstring(open(src, 'rb').read()) by_2 = {} by_3b = {} by_3t = {} @@ -350,7 +353,7 @@ class ISO639(Command): # {{{ threet = x.get('id') threeb = x.get('part2_code', None) if threeb is None: - # Only recognize langauges in ISO-639-2 + # Only recognize languages in ISO-639-2 continue name = x.get('name') From 32fa93d5847ab3daf3ec0c9d2ade969b812c827d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Thu, 4 Jul 2013 00:02:19 +0200 Subject: [PATCH 0092/1154] update Woblink plugin for website changes --- .../gui2/store/stores/woblink_plugin.py | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/src/calibre/gui2/store/stores/woblink_plugin.py b/src/calibre/gui2/store/stores/woblink_plugin.py index 9b99271192..6eb598d65b 100644 --- a/src/calibre/gui2/store/stores/woblink_plugin.py +++ b/src/calibre/gui2/store/stores/woblink_plugin.py @@ -56,20 +56,20 @@ class WoblinkStore(BasicStoreConfig, StorePlugin): counter = max_results with closing(br.open(url, timeout=timeout)) as f: doc = html.fromstring(f.read()) - for data in doc.xpath('//div[@class="book-item backgroundmix"]'): + for data in doc.xpath('//div[@class="nw_katalog_lista_ksiazka"]'): if counter <= 0: break - id = ''.join(data.xpath('.//td[@class="w10 va-t mYHaveItYes"]/a[1]/@href')) + id = ''.join(data.xpath('.//div[@class="nw_katalog_lista_ksiazka_okladka nw_okladka"]/a[1]/@href')) if not id: continue - cover_url = ''.join(data.xpath('.//td[@class="w10 va-t mYHaveItYes"]/a[1]/img/@src')) - title = ''.join(data.xpath('.//h2[@class="title"]/a[1]/text()')) - author = ', '.join(data.xpath('.//td[@class="va-t"]/h3/a/text()')) - price = ''.join(data.xpath('.//div[@class="prices"]/span[1]/strong/span/text()')) + cover_url = ''.join(data.xpath('.//div[@class="nw_katalog_lista_ksiazka_okladka nw_okladka"]/a[1]/img/@src')) + title = ''.join(data.xpath('.//h2[@class="nw_katalog_lista_ksiazka_detale_tytul"]/a[1]/text()')) + author = ', '.join(data.xpath('.//h3[@class="nw_katalog_lista_ksiazka_detale_autor"]/a/text()')) + price = ''.join(data.xpath('.//div[@class="nw_katalog_lista_ksiazka_opcjezakupu_cena"]/span/text()')) price = re.sub('\.', ',', price) - formats = [ form[8:-4].split('.')[0] for form in data.xpath('.//p[3]/img/@src')] + formats = ', '.join(data.xpath('.//p[@class="nw_katalog_lista_ksiazka_detale_formaty"]/span/text()')) s = SearchResult() s.cover_url = 'http://woblink.com' + cover_url @@ -77,25 +77,15 @@ class WoblinkStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price + ' zł' s.detail_item = id.strip() + s.formats = formats - if 'epub_drm' in formats: + if 'EPUB DRM' in formats: s.drm = SearchResult.DRM_LOCKED - s.formats = 'EPUB' - - counter -= 1 - yield s - elif 'pdf' in formats: - s.drm = SearchResult.DRM_LOCKED - s.formats = 'PDF' counter -= 1 yield s else: s.drm = SearchResult.DRM_UNLOCKED - if 'MOBI_nieb' in formats: - formats.remove('MOBI_nieb') - formats.append('MOBI') - s.formats = ', '.join(formats).upper() counter -= 1 yield s From 2aa8f23d6c0b3ab585897d148c086899d80d02c6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 09:13:37 +0530 Subject: [PATCH 0093/1154] When parsing fields, handle escapes --- src/calibre/ebooks/docx/fields.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/ebooks/docx/fields.py b/src/calibre/ebooks/docx/fields.py index 9b0d053cd0..91dcd87596 100644 --- a/src/calibre/ebooks/docx/fields.py +++ b/src/calibre/ebooks/docx/fields.py @@ -37,7 +37,9 @@ scanner = re.Scanner([ def parse_hyperlink(raw, log): ans = {} last_option = None + raw = raw.replace('\\\\', '\x01').replace('\\"', '\x02') for token, token_type in scanner.scan(raw)[0]: + token = token.replace('\x01', '\\').replace('\x02', '"') if not ans: if token_type is not WORD: log('Invalid hyperlink, first token is not a URL (%s)' % raw) From 852bd4945394242940948e4ed2e0efb759445ce4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 09:54:53 +0530 Subject: [PATCH 0094/1154] MOBI Input: Fix regression parsing ' MOBI Input: Fix a regression that broke parsing of MOBI files with malformed markup that also used entities for apostrophes. Fixes #1197585 [Private bug](https://bugs.launchpad.net/calibre/+bug/1197585) --- src/calibre/utils/soupparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/utils/soupparser.py b/src/calibre/utils/soupparser.py index 403f57baad..efbcf8b970 100644 --- a/src/calibre/utils/soupparser.py +++ b/src/calibre/utils/soupparser.py @@ -62,7 +62,7 @@ def _parse(source, beautifulsoup, makeelement, **bsargs): if makeelement is None: makeelement = html.html_parser.makeelement if 'convertEntities' not in bsargs: - bsargs['convertEntities'] = 'html' + bsargs['convertEntities'] = 'xhtml' # Changed by Kovid, otherwise ' is mangled, see https://bugs.launchpad.net/calibre/+bug/1197585 tree = beautifulsoup(source, **bsargs) root = _convert_tree(tree, makeelement) # from ET: wrap the document in a html root element, if necessary From ef4efd57688b3cebe801557a6fe28f9997af04a5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 11:11:22 +0530 Subject: [PATCH 0095/1154] Refactor creation of hardlinks on windows After creating the hardlink, open and close the file, to ensure that the directory entry for the file contains the correct file size, see http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx --- src/calibre/utils/filenames.py | 45 ++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index 8c4b7edfe5..54ce568539 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -70,9 +70,11 @@ def shorten_components_to(length, components, more_to_take=0): else: if x is components[-1]: b, e = os.path.splitext(x) - if e == '.': e = '' + if e == '.': + e = '' r = shorten_component(b, delta)+e - if r.startswith('.'): r = x[0]+r + if r.startswith('.'): + r = x[0]+r else: r = shorten_component(x, delta) r = r.strip() @@ -115,7 +117,7 @@ def is_case_sensitive(path): os.remove(f1) return is_case_sensitive -def case_preserving_open_file(path, mode='wb', mkdir_mode=0777): +def case_preserving_open_file(path, mode='wb', mkdir_mode=0o777): ''' Open the file pointed to by path with the specified mode. If any directories in path do not exist, they are created. Returns the @@ -211,7 +213,8 @@ def samefile_windows(src, dst): handles = [] def get_fileid(x): - if isbytestring(x): x = x.decode(filesystem_encoding) + if isbytestring(x): + x = x.decode(filesystem_encoding) try: h = win32file.CreateFile(x, 0, 0, None, win32file.OPEN_EXISTING, win32file.FILE_FLAG_BACKUP_SEMANTICS, 0) @@ -254,6 +257,24 @@ def samefile(src, dst): os.path.normcase(os.path.abspath(dst))) return samestring +def windows_hardlink(src, dest): + import win32file, pywintypes + msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) + try: + win32file.CreateHardLink(dest, src) + except pywintypes.error as e: + raise Exception(msg % e) + # We open and close dest, to ensure its directory entry is updated + # see http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx + h = win32file.CreateFile( + dest, 0, win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE | win32file.FILE_SHARE_DELETE, + None, win32file.OPEN_EXISTING, 0, None) + sz = win32file.GetFileSize(h) + win32file.CloseHandle(h) + + if sz != os.path.getsize(src): + raise Exception(msg % ('hardlink size: %d not the same as source size' % sz)) + class WindowsAtomicFolderMove(object): ''' @@ -270,14 +291,16 @@ class WindowsAtomicFolderMove(object): import win32file, winerror from pywintypes import error - if isbytestring(path): path = path.decode(filesystem_encoding) + if isbytestring(path): + path = path.decode(filesystem_encoding) if not os.path.exists(path): return for x in os.listdir(path): f = os.path.normcase(os.path.abspath(os.path.join(path, x))) - if not os.path.isfile(f): continue + if not os.path.isfile(f): + continue try: # Ensure the file is not read-only win32file.SetFileAttributes(f, win32file.FILE_ATTRIBUTE_NORMAL) @@ -315,9 +338,7 @@ class WindowsAtomicFolderMove(object): else: raise ValueError(u'The file %r does not exist'%path) try: - win32file.CreateHardLink(dest, path) - if os.path.getsize(dest) != os.path.getsize(path): - raise Exception('This apparently can happen on network shares. Sigh.') + windows_hardlink(path, dest) return except: pass @@ -355,10 +376,8 @@ class WindowsAtomicFolderMove(object): def hardlink_file(src, dest): if iswindows: - import win32file - win32file.CreateHardLink(dest, src) - if os.path.getsize(dest) != os.path.getsize(src): - raise Exception('This apparently can happen on network shares. Sigh.') + windows_hardlink(src, dest) return os.link(src, dest) + From c950518da25890d5a9f30b1f18b7fb1cbe21f15e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 12:18:24 +0530 Subject: [PATCH 0096/1154] Avoid unnecessary string formatting --- src/calibre/utils/filenames.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index 54ce568539..d6193c068b 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -259,10 +259,10 @@ def samefile(src, dst): def windows_hardlink(src, dest): import win32file, pywintypes - msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) try: win32file.CreateHardLink(dest, src) except pywintypes.error as e: + msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) raise Exception(msg % e) # We open and close dest, to ensure its directory entry is updated # see http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx @@ -273,6 +273,7 @@ def windows_hardlink(src, dest): win32file.CloseHandle(h) if sz != os.path.getsize(src): + msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) raise Exception(msg % ('hardlink size: %d not the same as source size' % sz)) class WindowsAtomicFolderMove(object): From c7033beb98340ed2577e9ff6a6515a0f77cebf6c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 12:46:12 +0530 Subject: [PATCH 0097/1154] Make ads in user manual async --- manual/templates/layout.html | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/manual/templates/layout.html b/manual/templates/layout.html index 188e829469..ff2e7b0113 100644 --- a/manual/templates/layout.html +++ b/manual/templates/layout.html @@ -16,16 +16,13 @@
    {% if not embedded %}
    - - + +
    {% endif %} From 7e4a124d527d7fd408209dfe70917c0c2a8f456d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 13:00:08 +0530 Subject: [PATCH 0098/1154] Ensure we dont leak a file handle --- src/calibre/utils/filenames.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index d6193c068b..d756978040 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -269,8 +269,10 @@ def windows_hardlink(src, dest): h = win32file.CreateFile( dest, 0, win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE | win32file.FILE_SHARE_DELETE, None, win32file.OPEN_EXISTING, 0, None) - sz = win32file.GetFileSize(h) - win32file.CloseHandle(h) + try: + sz = win32file.GetFileSize(h) + finally: + win32file.CloseHandle(h) if sz != os.path.getsize(src): msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) From f1ce0eb75c38f0f4fc3f4dea50f77d6666766925 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 13:16:09 +0530 Subject: [PATCH 0099/1154] Fix test failures due to file locking on windows --- src/calibre/db/legacy.py | 4 +++- src/calibre/db/tests/legacy.py | 32 +++++++++++++++++++------------- src/calibre/db/tests/writing.py | 3 +++ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 874707fa2e..287d2d47db 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -64,9 +64,11 @@ class LibraryDatabase(object): self.backend.close() def break_cycles(self): + delattr(self.backend, 'field_metadata') self.data.cache.backend = None self.data.cache = None - self.data = self.backend = self.new_api = self.field_metadata = self.prefs = self.listeners = self.refresh_ondevice = None + for x in ('data', 'backend', 'new_api', 'listeners',): + delattr(self, x) # Library wide properties {{{ @property diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index af6b977aef..1fe719e31e 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -117,18 +117,24 @@ class LegacyTest(BaseTest): '__init__', } - for attr in dir(db): - if attr in SKIP_ATTRS: - continue - if not hasattr(ndb, attr): - raise AssertionError('The attribute %s is missing' % attr) - obj, nobj = getattr(db, attr), getattr(ndb, attr) - if attr not in SKIP_ARGSPEC: - try: - argspec = inspect.getargspec(obj) - except TypeError: - pass - else: - self.assertEqual(argspec, inspect.getargspec(nobj), 'argspec for %s not the same' % attr) + try: + for attr in dir(db): + if attr in SKIP_ATTRS: + continue + if not hasattr(ndb, attr): + raise AssertionError('The attribute %s is missing' % attr) + obj, nobj = getattr(db, attr), getattr(ndb, attr) + if attr not in SKIP_ARGSPEC: + try: + argspec = inspect.getargspec(obj) + except TypeError: + pass + else: + self.assertEqual(argspec, inspect.getargspec(nobj), 'argspec for %s not the same' % attr) + finally: + for db in (ndb, db): + db.close() + db.break_cycles() + # }}} diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 5d04c11def..597c98a771 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -374,6 +374,9 @@ class WritingTest(BaseTest): ae(cache.field_for('cover', book_id), 1) ae(old.cover(book_id, index_is_id=True), img, 'Cover was not set correctly for book %d' % book_id) self.assertTrue(old.has_cover(book_id)) + old.close() + old.break_cycles() + del old # }}} def test_set_metadata(self): # {{{ From 4bba2cbf926bfaee59d97576fea6a70a29d9a282 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 08:01:15 +0530 Subject: [PATCH 0100/1154] Initial implementation of add_format() --- src/calibre/db/backend.py | 18 +++++++++++++ src/calibre/db/cache.py | 54 ++++++++++++++++++++++++++++++++++++++- src/calibre/db/tables.py | 19 +++++++++++--- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index c6aa2e646f..97dc673ecc 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1059,6 +1059,24 @@ class DB(object): if wam is not None: wam.close_handles() + def add_format(self, book_id, fmt, stream, title, author, path): + fname = self.construct_file_name(book_id, title, author) + path = os.path.join(self.library_path, path) + fmt = ('.' + fmt.lower()) if fmt else '' + dest = os.path.join(path, fname + fmt) + if not os.path.exists(path): + os.makedirs(path) + size = 0 + + if (not getattr(stream, 'name', False) or not samefile(dest, stream.name)): + with lopen(dest, 'wb') as f: + shutil.copyfileobj(stream, f) + size = f.tell() + elif os.path.exists(dest): + size = os.path.getsize(dest) + + return size, fname + def update_path(self, book_id, title, author, path_field, formats_field): path = self.construct_path_name(book_id, title, author) current_path = path_field.for_book(book_id) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 88f06b43ba..4f7de11269 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -7,12 +7,13 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback, random +import os, traceback, random, shutil from io import BytesIO from collections import defaultdict from functools import wraps, partial from calibre.constants import iswindows +from calibre.customize.ui import run_plugins_on_import, run_plugins_on_postimport from calibre.db import SPOOL_SIZE from calibre.db.categories import get_categories from calibre.db.locking import create_locks @@ -22,6 +23,7 @@ from calibre.db.search import Search from calibre.db.tables import VirtualTable from calibre.db.write import get_series_values from calibre.db.lazy import FormatMetadata, FormatsList +from calibre.ebooks import check_ebook_format from calibre.ebooks.metadata import string_to_authors from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf @@ -51,6 +53,18 @@ def wrap_simple(lock, func): return func(*args, **kwargs) return ans +def run_import_plugins(path_or_stream, fmt): + fmt = fmt.lower() + if hasattr(path_or_stream, 'seek'): + path_or_stream.seek(0) + pt = PersistentTemporaryFile('_import_plugin.'+fmt) + shutil.copyfileobj(path_or_stream, pt, 1024**2) + pt.close() + path = pt.name + else: + path = path_or_stream + return run_plugins_on_import(path, fmt) + class Cache(object): @@ -943,6 +957,43 @@ class Cache(object): if extra is not None or force_changes: protected_set_field(idx, extra) + @write_api + def add_format(self, book_id, fmt, stream_or_path, replace=True, run_hooks=True, dbapi=None): + if run_hooks: + # Run import plugins + npath = run_import_plugins(stream_or_path, fmt) + fmt = os.path.splitext(npath)[-1].lower().replace('.', '').upper() + stream_or_path = lopen(npath, 'rb') + fmt = check_ebook_format(stream_or_path, fmt) + + fmt = (fmt or '').upper() + self.format_metadata_cache[book_id].pop(fmt, None) + try: + name = self.fields['formats'].format_fname(book_id, fmt) + except: + name = None + + if name and not replace: + return False + + path = self._field_for('path', book_id).replace('/', os.sep) + title = self._field_for('title', book_id, default_value=_('Unknown')) + author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0] + stream = stream_or_path if hasattr(stream_or_path, 'read') else lopen(stream_or_path, 'rb') + size, fname = self.backend.add_format(book_id, fmt, stream, title, author, path) + del stream + + max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend) + self.fields['size'].table.update_size(book_id, max_size) + self._update_last_modified((book_id,)) + + if run_hooks: + # Run post import plugins + run_plugins_on_postimport(dbapi or self, book_id, fmt) + stream_or_path.close() + + return True + # }}} class SortKey(object): # {{{ @@ -959,3 +1010,4 @@ class SortKey(object): # {{{ return 0 # }}} + diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 83d4b23712..4dba10abff 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -8,6 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' from datetime import datetime +from collections import defaultdict from dateutil.tz import tzoffset @@ -98,6 +99,9 @@ class SizeTable(OneToOneTable): 'WHERE data.book=books.id) FROM books'): self.book_col_map[row[0]] = self.unserialize(row[1]) + def update_size(self, book_id, size): + self.book_col_map[book_id] = size + class UUIDTable(OneToOneTable): def read(self, db): @@ -194,8 +198,9 @@ class FormatsTable(ManyToManyTable): pass def read_maps(self, db): - self.fname_map = {} - for row in db.conn.execute('SELECT book, format, name FROM data'): + self.fname_map = defaultdict(dict) + self.size_map = defaultdict(dict) + for row in db.conn.execute('SELECT book, format, name, uncompressed_size FROM data'): if row[1] is not None: fmt = row[1].upper() if fmt not in self.col_book_map: @@ -204,9 +209,8 @@ class FormatsTable(ManyToManyTable): if row[0] not in self.book_col_map: self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append(fmt) - if row[0] not in self.fname_map: - self.fname_map[row[0]] = {} self.fname_map[row[0]][fmt] = row[2] + self.size_map[row[0]][fmt] = row[3] for key in tuple(self.book_col_map.iterkeys()): self.book_col_map[key] = tuple(sorted(self.book_col_map[key])) @@ -216,6 +220,13 @@ class FormatsTable(ManyToManyTable): db.conn.execute('UPDATE data SET name=? WHERE book=? AND format=?', (fname, book_id, fmt)) + def update_fmt(self, book_id, fmt, fname, size, db): + self.fname_map[book_id][fmt] = fname + self.size_map[book_id][fmt] = size + db.conn.execute('INSERT OR REPLACE INTO data (book,format,uncompressed_size,name) VALUES (?,?,?,?)', + (book_id, fmt, size, fname)) + return max(self.size_map[book_id].itervalues()) + class IdentifiersTable(ManyToManyTable): def read_id_maps(self, db): From 93264786e1bb9fb4d14b1f76bf536ffd47b21a67 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 08:03:47 +0530 Subject: [PATCH 0101/1154] Fix detection of SD Card in some PRS-T2N devices Fixes #1197970 [Sony PRS-T2 SD card not recognized](https://bugs.launchpad.net/calibre/+bug/1197970) --- src/calibre/devices/prst1/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 0431ca7bfd..9c76eb096f 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -53,7 +53,7 @@ class PRST1(USBMS): r'(PRS-T(1|2|2N)&)' ) WINDOWS_CARD_A_MEM = re.compile( - r'(PRS-T(1|2|2N)__SD&)' + r'(PRS-T(1|2|2N)_{1,2}SD&)' ) MAIN_MEMORY_VOLUME_LABEL = 'SONY Reader Main Memory' STORAGE_CARD_VOLUME_LABEL = 'SONY Reader Storage Card' From 849465ccce3832958251d5ee2c69c350b67f6c10 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 08:27:29 +0530 Subject: [PATCH 0102/1154] version 0.9.38 --- Changelog.yaml | 50 ++++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index 7439a02986..f68617fb3a 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,56 @@ # new recipes: # - title: +- version: 0.9.38 + date: 2013-07-05 + + new features: + - title: "Book polishing: Add option to embed all referenced fonts when polishing books using the 'Polish Books' tool." + tickets: [1196038] + + - title: "DOCX Input: Add support for clickable (hyperlinked) images" + tickets: [1196728] + + - title: "DOCX Input: Insert page breaks at the start of every new section" + tickets: [1196728] + + - title: "Drivers for Trekstor Pyrus Maxi and PocketBook Surfpad 2" + tickets: [1196931, 1182850] + + - title: "DOCX Input: Add support for horizontal rules created by typing three hyphens and pressing enter." + + bug fixes: + - title: "Fix detection of SD Card in some PRS-T2N devices" + tickets: [1197970] + + - title: "MOBI Input: Fix a regression that broke parsing of MOBI files with malformed markup that also used entities for apostrophes." + ticket: [1197585] + + - title: "Get Books: Update Woblink store plugin" + + - title: "Metadata download dialog: Prevent the buttons from being re-ordered when the Next button is clicked." + + - title: "PDF Output: Fix links that point to URLs with query parameters being mangled by the conversion process." + tickets: [1197006] + + - title: "DOCX Input: Fix links pointing to locations in the same document that contain multiple, redundant bookmarks not working." + + - title: "EPUB/AZW3 Output: Fix splitting on page-break-after with plain text immediately following the split point causing the text to be added before rather than after the split point." + tickets: [1196728] + + - title: "DOCX Input: handle bookmarks defined at the paragraph level" + tickets: [1196728] + + - title: "DOCX Input: Handle hyperlinks created as fields" + tickets: [1196728] + + improved recipes: + - iprofessional + + new recipes: + - title: Democracy Now + author: Antoine Beaupre + - version: 0.9.37 date: 2013-06-28 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index a4edca6bd5..99146e206c 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 9, 37) +numeric_version = (0, 9, 38) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " From 8c1bb3d4cb6655a7b6a3d58fbad35c66bb869050 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 10:12:23 +0530 Subject: [PATCH 0103/1154] Add tests for add_format() --- src/calibre/db/tables.py | 13 ++++ src/calibre/db/tests/add_remove.py | 101 +++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 src/calibre/db/tests/add_remove.py diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 4dba10abff..140c95eb88 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -221,6 +221,19 @@ class FormatsTable(ManyToManyTable): (fname, book_id, fmt)) def update_fmt(self, book_id, fmt, fname, size, db): + fmts = list(self.book_col_map.get(book_id, [])) + try: + fmts.remove(fmt) + except ValueError: + pass + fmts.append(fmt) + self.book_col_map[book_id] = tuple(fmts) + + try: + self.col_book_map[fmt].add(book_id) + except KeyError: + self.col_book_map[fmt] = {book_id} + self.fname_map[book_id][fmt] = fname self.size_map[book_id][fmt] = size db.conn.execute('INSERT OR REPLACE INTO data (book,format,uncompressed_size,name) VALUES (?,?,?,?)', diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py new file mode 100644 index 0000000000..c5b165bd02 --- /dev/null +++ b/src/calibre/db/tests/add_remove.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from io import BytesIO +from tempfile import NamedTemporaryFile + +from calibre.db.tests.base import BaseTest +from calibre.ptempfile import PersistentTemporaryFile + +def import_test(replacement_data, replacement_fmt=None): + def func(path, fmt): + if not path.endswith('.'+fmt.lower()): + raise AssertionError('path extension does not match format') + ext = (replacement_fmt or fmt).lower() + with PersistentTemporaryFile('.'+ext) as f: + f.write(replacement_data) + return f.name + return func + +class AddRemoveTest(BaseTest): + + def test_add_format(self): # {{{ + 'Test adding formats to an existing book record' + af, ae, at = self.assertFalse, self.assertEqual, self.assertTrue + + cache = self.init_cache() + table = cache.fields['formats'].table + NF = b'test_add_formatxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + + # Test that replace=False works + previous = cache.format(1, 'FMT1') + af(cache.add_format(1, 'FMT1', BytesIO(NF), replace=False)) + ae(previous, cache.format(1, 'FMT1')) + + # Test that replace=True works + lm = cache.field_for('last_modified', 1) + at(cache.add_format(1, 'FMT1', BytesIO(NF), replace=True)) + ae(NF, cache.format(1, 'FMT1')) + ae(cache.format_metadata(1, 'FMT1')['size'], len(NF)) + at(cache.field_for('size', 1) >= len(NF)) + at(cache.field_for('last_modified', 1) > lm) + ae(('FMT2','FMT1'), cache.formats(1)) + at(1 in table.col_book_map['FMT1']) + + # Test adding a format to a record with no formats + at(cache.add_format(3, 'FMT1', BytesIO(NF), replace=True)) + ae(NF, cache.format(3, 'FMT1')) + ae(cache.format_metadata(3, 'FMT1')['size'], len(NF)) + ae(('FMT1',), cache.formats(3)) + at(3 in table.col_book_map['FMT1']) + at(cache.add_format(3, 'FMTX', BytesIO(NF), replace=True)) + at(3 in table.col_book_map['FMTX']) + ae(('FMT1','FMTX'), cache.formats(3)) + + # Test running on import plugins + import calibre.db.cache as c + orig = c.run_plugins_on_import + try: + c.run_plugins_on_import = import_test(b'replacement data') + at(cache.add_format(3, 'REPL', BytesIO(NF))) + ae(b'replacement data', cache.format(3, 'REPL')) + c.run_plugins_on_import = import_test(b'replacement data2', 'REPL2') + with NamedTemporaryFile(suffix='_test_add_format.repl') as f: + f.write(NF) + f.seek(0) + at(cache.add_format(3, 'REPL', BytesIO(NF))) + ae(b'replacement data', cache.format(3, 'REPL')) + ae(b'replacement data2', cache.format(3, 'REPL2')) + + finally: + c.run_plugins_on_import = orig + + # Test adding FMT with path + with NamedTemporaryFile(suffix='_test_add_format.fmt9') as f: + f.write(NF) + f.seek(0) + at(cache.add_format(2, 'FMT9', f)) + ae(NF, cache.format(2, 'FMT9')) + ae(cache.format_metadata(2, 'FMT9')['size'], len(NF)) + at(cache.field_for('size', 2) >= len(NF)) + at(2 in table.col_book_map['FMT9']) + + del cache + # Test that the old interface also shows correct format data + db = self.init_old() + ae(db.formats(3, index_is_id=True), ','.join(['FMT1', 'FMTX', 'REPL', 'REPL2'])) + ae(db.format(3, 'FMT1', index_is_id=True), NF) + ae(db.format(1, 'FMT1', index_is_id=True), NF) + + db.close() + del db + + # }}} + + From df2c497a13afd39b747b3937c857dd0165d44e55 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 11:42:30 +0530 Subject: [PATCH 0104/1154] ... --- setup/translations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/translations.py b/setup/translations.py index 786d44a6d6..3474e82acb 100644 --- a/setup/translations.py +++ b/setup/translations.py @@ -165,7 +165,7 @@ class Translations(POT): # {{{ subprocess.check_call(['msgfmt', '-o', dest, iso639]) elif locale not in ('en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', 'ltg', 'nds', 'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku', - 'fr_CA', 'him', 'jv', 'ka', 'fur', 'ber'): + 'fr_CA', 'him', 'jv', 'ka', 'fur', 'ber', 'my'): self.warn('No ISO 639 translations for locale:', locale) if self.iso639_errors: From 65e31eee1231c1fe1bcc072c7571b7d80c1947ba Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 12:44:19 +0530 Subject: [PATCH 0105/1154] Implement remove_formats() --- src/calibre/db/backend.py | 11 +++++++- src/calibre/db/cache.py | 29 +++++++++++++++++++++- src/calibre/db/tables.py | 25 +++++++++++++++++-- src/calibre/db/tests/add_remove.py | 40 ++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 4 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 97dc673ecc..b8963fc49d 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -26,7 +26,7 @@ from calibre.utils.date import utcfromtimestamp, parse_date from calibre.utils.filenames import (is_case_sensitive, samefile, hardlink_file, ascii_filename, WindowsAtomicFolderMove) from calibre.utils.magick.draw import save_cover_data_to -from calibre.utils.recycle_bin import delete_tree +from calibre.utils.recycle_bin import delete_tree, delete_file from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable, CompositeTable, LanguagesTable, UUIDTable) @@ -940,6 +940,15 @@ class DB(object): def has_format(self, book_id, fmt, fname, path): return self.format_abspath(book_id, fmt, fname, path) is not None + def remove_format(self, book_id, fmt, fname, path): + path = self.format_abspath(book_id, fmt, fname, path) + if path is not None: + try: + delete_file(path) + except: + import traceback + traceback.print_exc() + def copy_cover_to(self, path, dest, windows_atomic_move=None, use_hardlink=False): path = os.path.abspath(os.path.join(self.library_path, path, 'cover.jpg')) if windows_atomic_move is not None: diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 4f7de11269..4c18dde6cd 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -984,7 +984,7 @@ class Cache(object): del stream max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend) - self.fields['size'].table.update_size(book_id, max_size) + self.fields['size'].table.update_sizes({book_id: max_size}) self._update_last_modified((book_id,)) if run_hooks: @@ -994,6 +994,33 @@ class Cache(object): return True + @write_api + def remove_formats(self, formats_map, db_only=False): + table = self.fields['formats'].table + formats_map = {book_id:frozenset((f or '').upper() for f in fmts) for book_id, fmts in formats_map.iteritems()} + size_map = table.remove_formats(formats_map, self.backend) + self.fields['size'].table.update_sizes(size_map) + + for book_id, fmts in formats_map.iteritems(): + for fmt in fmts: + self.format_metadata_cache[book_id].pop(fmt, None) + + if not db_only: + for book_id, fmts in formats_map.iteritems(): + try: + path = self._field_for('path', book_id).replace('/', os.sep) + except: + continue + for fmt in fmts: + try: + name = self.fields['formats'].format_fname(book_id, fmt) + except: + continue + if name and path: + self.backend.remove_format(book_id, fmt, name, path) + + self._update_last_modified(tuple(formats_map.iterkeys())) + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 140c95eb88..fce8d429ba 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -99,8 +99,8 @@ class SizeTable(OneToOneTable): 'WHERE data.book=books.id) FROM books'): self.book_col_map[row[0]] = self.unserialize(row[1]) - def update_size(self, book_id, size): - self.book_col_map[book_id] = size + def update_sizes(self, size_map): + self.book_col_map.update(size_map) class UUIDTable(OneToOneTable): @@ -220,6 +220,26 @@ class FormatsTable(ManyToManyTable): db.conn.execute('UPDATE data SET name=? WHERE book=? AND format=?', (fname, book_id, fmt)) + def remove_formats(self, formats_map, db): + for book_id, fmts in formats_map.iteritems(): + self.book_col_map[book_id] = [fmt for fmt in self.book_col_map.get(book_id, []) if fmt not in fmts] + for m in (self.fname_map, self.size_map): + m[book_id] = {k:v for k, v in m[book_id].iteritems() if k not in fmts} + for fmt in fmts: + try: + self.col_book_map[fmt].discard(book_id) + except KeyError: + pass + db.conn.executemany('DELETE FROM data WHERE book=? AND format=?', + [(book_id, fmt) for book_id, fmts in formats_map.iteritems() for fmt in fmts]) + def zero_max(book_id): + try: + return max(self.size_map[book_id].itervalues()) + except ValueError: + return 0 + + return {book_id:zero_max(book_id) for book_id in formats_map} + def update_fmt(self, book_id, fmt, fname, size, db): fmts = list(self.book_col_map.get(book_id, [])) try: @@ -259,3 +279,4 @@ class LanguagesTable(ManyToManyTable): def read_id_maps(self, db): ManyToManyTable.read_id_maps(self, db) + diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index c5b165bd02..c411daa826 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -7,6 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os from io import BytesIO from tempfile import NamedTemporaryFile @@ -98,4 +99,43 @@ class AddRemoveTest(BaseTest): # }}} + def test_remove_formats(self): # {{{ + 'Test removal of formats from book records' + af, ae, at = self.assertFalse, self.assertEqual, self.assertTrue + + cache = self.init_cache() + + # Test removal of non-existing format does nothing + formats = {bid:tuple(cache.formats(bid)) for bid in (1, 2, 3)} + cache.remove_formats({1:{'NF'}, 2:{'NF'}, 3:{'NF'}}) + nformats = {bid:tuple(cache.formats(bid)) for bid in (1, 2, 3)} + ae(formats, nformats) + + # Test full removal of format + af(cache.format(1, 'FMT1') is None) + at(cache.has_format(1, 'FMT1')) + cache.remove_formats({1:{'FMT1'}}) + at(cache.format(1, 'FMT1') is None) + af(bool(cache.format_metadata(1, 'FMT1'))) + af(bool(cache.format_metadata(1, 'FMT1', allow_cache=False))) + af('FMT1' in cache.formats(1)) + af(cache.has_format(1, 'FMT1')) + + # Test db only removal + at(cache.has_format(1, 'FMT2')) + ap = cache.format_abspath(1, 'FMT2') + if ap and os.path.exists(ap): + cache.remove_formats({1:{'FMT2'}}) + af(bool(cache.format_metadata(1, 'FMT2'))) + af(cache.has_format(1, 'FMT2')) + at(os.path.exists(ap)) + + # Test that the old interface agrees + db = self.init_old() + at(db.format(1, 'FMT1', index_is_id=True) is None) + + db.close() + del db + # }}} + From e7bf1c7b7d399737563e0ce1985df3d1db35306e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 20:32:26 +0530 Subject: [PATCH 0106/1154] Clarify use of OPF in calibredb set_metadata --- src/calibre/library/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index b1131525d8..547cc5bc08 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -543,13 +543,14 @@ def do_set_metadata(db, id, stream): def set_metadata_option_parser(): return get_parser(_( ''' -%prog set_metadata [options] id /path/to/metadata.opf +%prog set_metadata [options] id [/path/to/metadata.opf] Set the metadata stored in the calibre database for the book identified by id from the OPF file metadata.opf. id is an id number from the list command. You can get a quick feel for the OPF format by using the --as-opf switch to the show_metadata command. You can also set the metadata of individual fields with -the --field option. +the --field option. If you use the --field option, there is no need to specify +an OPF file. ''')) def command_set_metadata(args, dbpath): From 9a8d31ee96cba490ace1d4d2d09427833ecf8247 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 20:48:31 +0530 Subject: [PATCH 0107/1154] Implement create_book_entry(), with tests --- src/calibre/db/__init__.py | 5 +- src/calibre/db/backend.py | 2 +- src/calibre/db/cache.py | 110 +++++++++++++++++++++++++++-- src/calibre/db/tables.py | 7 +- src/calibre/db/tests/add_remove.py | 51 ++++++++++++- src/calibre/db/tests/base.py | 2 + src/calibre/db/tests/reading.py | 62 +++++++++++++--- src/calibre/db/tests/writing.py | 4 +- 8 files changed, 223 insertions(+), 20 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 65beebc1fb..eded760cde 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -9,14 +9,15 @@ __docformat__ = 'restructuredtext en' SPOOL_SIZE = 30*1024*1024 -def _get_next_series_num_for_list(series_indices): +def _get_next_series_num_for_list(series_indices, unwrap=True): from calibre.utils.config_base import tweaks from math import ceil, floor if not series_indices: if isinstance(tweaks['series_index_auto_increment'], (int, float)): return float(tweaks['series_index_auto_increment']) return 1.0 - series_indices = [x[0] for x in series_indices] + if unwrap: + series_indices = [x[0] for x in series_indices] if tweaks['series_index_auto_increment'] == 'next': return floor(series_indices[-1]) + 1 if tweaks['series_index_auto_increment'] == 'first_free': diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index b8963fc49d..f998b91ccb 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1088,7 +1088,7 @@ class DB(object): def update_path(self, book_id, title, author, path_field, formats_field): path = self.construct_path_name(book_id, title, author) - current_path = path_field.for_book(book_id) + current_path = path_field.for_book(book_id, default_value='') formats = formats_field.for_book(book_id, default_value=()) fname = self.construct_file_name(book_id, title, author) # Check if the metadata used to construct paths has changed diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 4c18dde6cd..a6647d5027 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -12,9 +12,10 @@ from io import BytesIO from collections import defaultdict from functools import wraps, partial -from calibre.constants import iswindows +from calibre import isbytestring +from calibre.constants import iswindows, preferred_encoding from calibre.customize.ui import run_plugins_on_import, run_plugins_on_postimport -from calibre.db import SPOOL_SIZE +from calibre.db import SPOOL_SIZE, _get_next_series_num_for_list from calibre.db.categories import get_categories from calibre.db.locking import create_locks from calibre.db.errors import NoSuchFormat @@ -24,12 +25,13 @@ from calibre.db.tables import VirtualTable from calibre.db.write import get_series_values from calibre.db.lazy import FormatMetadata, FormatsList from calibre.ebooks import check_ebook_format -from calibre.ebooks.metadata import string_to_authors +from calibre.ebooks.metadata import string_to_authors, author_to_author_sort from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ptempfile import (base_dir, PersistentTemporaryFile, SpooledTemporaryFile) -from calibre.utils.date import now as nowf +from calibre.utils.config import prefs +from calibre.utils.date import now as nowf, utcnow, UNDEFINED_DATE from calibre.utils.icu import sort_key def api(f): @@ -65,6 +67,16 @@ def run_import_plugins(path_or_stream, fmt): path = path_or_stream return run_plugins_on_import(path, fmt) +def _add_newbook_tag(mi): + tags = prefs['new_book_tags'] + if tags: + for tag in [t.strip() for t in tags]: + if tag: + if not mi.tags: + mi.tags = [tag] + elif tag not in mi.tags: + mi.tags.append(tag) + class Cache(object): @@ -1021,6 +1033,95 @@ class Cache(object): self._update_last_modified(tuple(formats_map.iterkeys())) + @read_api + def get_next_series_num_for(self, series): + books = () + sf = self.fields['series'] + if series: + q = icu_lower(series) + for val, book_ids in sf.iter_searchable_values(self._get_metadata, frozenset(self.all_book_ids())): + if q == icu_lower(val): + books = book_ids + break + series_indices = sorted(self._field_for('series_index', book_id) for book_id in books) + return _get_next_series_num_for_list(tuple(series_indices), unwrap=False) + + @read_api + def author_sort_from_authors(self, authors): + '''Given a list of authors, return the author_sort string for the authors, + preferring the author sort associated with the author over the computed + string. ''' + table = self.fields['authors'].table + result = [] + rmap = {icu_lower(v):k for k, v in table.id_map.iteritems()} + for aut in authors: + aid = rmap.get(icu_lower(aut), None) + result.append(author_to_author_sort(aut) if aid is None else table.asort_map[aid]) + return ' & '.join(result) + + @read_api + def has_book(self, mi): + title = mi.title + if title: + if isbytestring(title): + title = title.decode(preferred_encoding, 'replace') + q = icu_lower(title) + for title in self.fields['title'].table.book_col_map.itervalues(): + if q == icu_lower(title): + return True + return False + + @write_api + def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None, apply_import_tags=True, preserve_uuid=False): + if mi.tags: + mi.tags = list(mi.tags) + if apply_import_tags: + _add_newbook_tag(mi) + if not add_duplicates and self._has_book(mi): + return + series_index = (self._get_next_series_num_for(mi.series) if mi.series_index is None else mi.series_index) + if not mi.authors: + mi.authors = (_('Unknown'),) + aus = mi.author_sort if mi.author_sort else self._author_sort_from_authors(mi.authors) + mi.title = mi.title or _('Unknown') + if isbytestring(aus): + aus = aus.decode(preferred_encoding, 'replace') + if isbytestring(mi.title): + mi.title = mi.title.decode(preferred_encoding, 'replace') + conn = self.backend.conn + if force_id is None: + conn.execute('INSERT INTO books(title, series_index, author_sort) VALUES (?, ?, ?)', + (mi.title, series_index, aus)) + else: + conn.execute('INSERT INTO books(id, title, series_index, author_sort) VALUES (?, ?, ?, ?)', + (force_id, mi.title, series_index, aus)) + book_id = conn.last_insert_rowid() + + mi.timestamp = utcnow() if mi.timestamp is None else mi.timestamp + mi.pubdate = UNDEFINED_DATE if mi.pubdate is None else mi.pubdate + if cover is not None: + mi.cover, mi.cover_data = None, (None, cover) + self._set_metadata(book_id, mi, ignore_errors=True) + if preserve_uuid and mi.uuid: + self._set_field('uuid', {book_id:mi.uuid}) + # Update the caches for fields from the books table + self.fields['size'].table.book_col_map[book_id] = 0 + row = next(conn.execute('SELECT sort, series_index, author_sort, uuid, has_cover FROM books WHERE id=?', (book_id,))) + for field, val in zip(('sort', 'series_index', 'author_sort', 'uuid', 'cover'), row): + if field == 'cover': + val = bool(val) + elif field == 'uuid': + self.fields[field].table.uuid_to_id_map[val] = book_id + self.fields[field].table.book_col_map[book_id] = val + + return book_id + + @write_api + def add_books(self, books, add_duplicates=True): + duplicates, ids = [], [] + for mi, format_map in books: + pass + # }}} class SortKey(object): # {{{ @@ -1038,3 +1139,4 @@ class SortKey(object): # {{{ # }}} + diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index fce8d429ba..6f3343ba12 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -110,7 +110,7 @@ class UUIDTable(OneToOneTable): def update_uuid_cache(self, book_id_val_map): for book_id, uuid in book_id_val_map.iteritems(): - self.uuid_to_id_map.pop(self.book_col_map[book_id], None) # discard old uuid + self.uuid_to_id_map.pop(self.book_col_map.get(book_id, None), None) # discard old uuid self.uuid_to_id_map[uuid] = book_id class CompositeTable(OneToOneTable): @@ -192,6 +192,11 @@ class AuthorsTable(ManyToManyTable): author_to_author_sort(row[1])) self.alink_map[row[0]] = row[3] + def set_sort_names(self, aus_map, db): + self.asort_map.update(aus_map) + db.conn.executemany('UPDATE authors SET sort=? WHERE id=?', + [(v, k) for k, v in aus_map.iteritems()]) + class FormatsTable(ManyToManyTable): def read_id_maps(self, db): diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index c411daa826..c5845f01da 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -10,9 +10,11 @@ __docformat__ = 'restructuredtext en' import os from io import BytesIO from tempfile import NamedTemporaryFile +from datetime import timedelta -from calibre.db.tests.base import BaseTest +from calibre.db.tests.base import BaseTest, IMG from calibre.ptempfile import PersistentTemporaryFile +from calibre.utils.date import now, UNDEFINED_DATE def import_test(replacement_data, replacement_fmt=None): def func(path, fmt): @@ -138,4 +140,51 @@ class AddRemoveTest(BaseTest): del db # }}} + def test_create_book_entry(self): # {{{ + 'Test the creation of new book entries' + from calibre.ebooks.metadata.book.base import Metadata + cache = self.init_cache() + mi = Metadata('Created One', authors=('Creator One', 'Creator Two')) + + book_id = cache.create_book_entry(mi) + self.assertIsNot(book_id, None) + + def do_test(cache, book_id): + for field in ('path', 'uuid', 'author_sort', 'timestamp', 'pubdate', 'title', 'authors', 'series_index', 'sort'): + self.assertTrue(cache.field_for(field, book_id)) + for field in ('size', 'cover'): + self.assertFalse(cache.field_for(field, book_id)) + self.assertEqual(book_id, cache.fields['uuid'].table.uuid_to_id_map[cache.field_for('uuid', book_id)]) + self.assertLess(now() - cache.field_for('timestamp', book_id), timedelta(seconds=30)) + self.assertEqual(('Created One', ('Creator One', 'Creator Two')), (cache.field_for('title', book_id), cache.field_for('authors', book_id))) + self.assertEqual(cache.field_for('series_index', book_id), 1.0) + self.assertEqual(cache.field_for('pubdate', book_id), UNDEFINED_DATE) + + do_test(cache, book_id) + # Test that the db contains correct data + cache = self.init_cache() + do_test(cache, book_id) + + self.assertIs(None, cache.create_book_entry(mi, add_duplicates=False), 'Duplicate added incorrectly') + book_id = cache.create_book_entry(mi, cover=IMG) + self.assertIsNot(book_id, None) + self.assertEqual(IMG, cache.cover(book_id)) + + import calibre.db.cache as c + orig = c.prefs + c.prefs = {'new_book_tags':('newbook', 'newbook2')} + try: + book_id = cache.create_book_entry(mi) + self.assertEqual(('newbook', 'newbook2'), cache.field_for('tags', book_id)) + mi.tags = ('one', 'two') + book_id = cache.create_book_entry(mi) + self.assertEqual(('one', 'two') + ('newbook', 'newbook2'), cache.field_for('tags', book_id)) + mi.tags = () + finally: + c.prefs = orig + + mi.uuid = 'a preserved uuid' + book_id = cache.create_book_entry(mi, preserve_uuid=True) + self.assertEqual(mi.uuid, cache.field_for('uuid', book_id)) + # }}} diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index b57b017ba3..b94faf6b28 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -14,6 +14,8 @@ from future_builtins import map rmtree = partial(shutil.rmtree, ignore_errors=True) +IMG = b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00`\x00`\x00\x00\xff\xe1\x00\x16Exif\x00\x00II*\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xdb\x00C\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\xff\xdb\x00C\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\xff\xc0\x00\x11\x08\x00\x01\x00\x01\x03\x01"\x00\x02\x11\x01\x03\x11\x01\xff\xc4\x00\x15\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\xff\xc4\x00\x14\x10\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x11\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\xbf\x80\x01\xff\xd9' # noqa {{{ }}} + class BaseTest(unittest.TestCase): longMessage = True diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 979e2e9247..24d80d33c7 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -15,7 +15,7 @@ from calibre.db.tests.base import BaseTest class ReadingTest(BaseTest): - def test_read(self): # {{{ + def test_read(self): # {{{ 'Test the reading of data from the database' cache = self.init_cache(self.library_path) tests = { @@ -123,7 +123,7 @@ class ReadingTest(BaseTest): book_id, field, expected_val, val)) # }}} - def test_sorting(self): # {{{ + def test_sorting(self): # {{{ 'Test sorting' cache = self.init_cache(self.library_path) for field, order in { @@ -165,7 +165,7 @@ class ReadingTest(BaseTest): ('title', True)]), 'Subsort failed') # }}} - def test_get_metadata(self): # {{{ + def test_get_metadata(self): # {{{ 'Test get_metadata() returns the same data for both backends' from calibre.library.database2 import LibraryDatabase2 old = LibraryDatabase2(self.library_path) @@ -188,7 +188,7 @@ class ReadingTest(BaseTest): self.compare_metadata(mi1, mi2) # }}} - def test_get_cover(self): # {{{ + def test_get_cover(self): # {{{ 'Test cover() returns the same data for both backends' from calibre.library.database2 import LibraryDatabase2 old = LibraryDatabase2(self.library_path) @@ -212,7 +212,7 @@ class ReadingTest(BaseTest): # }}} - def test_searching(self): # {{{ + def test_searching(self): # {{{ 'Test searching returns the same data for both backends' from calibre.library.database2 import LibraryDatabase2 old = LibraryDatabase2(self.library_path) @@ -267,7 +267,7 @@ class ReadingTest(BaseTest): # }}} - def test_get_categories(self): # {{{ + def test_get_categories(self): # {{{ 'Check that get_categories() returns the same data for both backends' from calibre.library.database2 import LibraryDatabase2 old = LibraryDatabase2(self.library_path) @@ -286,9 +286,9 @@ class ReadingTest(BaseTest): oval, nval = getattr(old, attr), getattr(new, attr) if ( (category in {'rating', '#rating'} and attr in {'id_set', 'sort'}) or - (category == 'series' and attr == 'sort') or # Sorting is wrong in old + (category == 'series' and attr == 'sort') or # Sorting is wrong in old (category == 'identifiers' and attr == 'id_set') or - (category == '@Good Series') or # Sorting is wrong in old + (category == '@Good Series') or # Sorting is wrong in old (category == 'news' and attr in {'count', 'id_set'}) or (category == 'formats' and attr == 'id_set') ): @@ -306,7 +306,7 @@ class ReadingTest(BaseTest): # }}} - def test_get_formats(self): # {{{ + def test_get_formats(self): # {{{ 'Test reading ebook formats using the format() method' from calibre.library.database2 import LibraryDatabase2 from calibre.db.cache import NoSuchFormat @@ -343,3 +343,47 @@ class ReadingTest(BaseTest): # }}} + def test_author_sort_for_authors(self): # {{{ + 'Test getting the author sort for authors from the db' + cache = self.init_cache() + table = cache.fields['authors'].table + table.set_sort_names({next(table.id_map.iterkeys()): 'Fake Sort'}, cache.backend) + + authors = tuple(table.id_map.itervalues()) + nval = cache.author_sort_from_authors(authors) + self.assertIn('Fake Sort', nval) + + db = self.init_old() + self.assertEqual(db.author_sort_from_authors(authors), nval) + db.close() + del db + + # }}} + + def test_get_next_series_num(self): # {{{ + 'Test getting the next series number for a series' + cache = self.init_cache() + cache.set_field('series', {3:'test series'}) + cache.set_field('series_index', {3:13}) + table = cache.fields['series'].table + series = tuple(table.id_map.itervalues()) + nvals = {s:cache.get_next_series_num_for(s) for s in series} + db = self.init_old() + self.assertEqual({s:db.get_next_series_num_for(s) for s in series}, nvals) + db.close() + + # }}} + + def test_has_book(self): # {{{ + 'Test detecting duplicates' + from calibre.ebooks.metadata.book.base import Metadata + cache = self.init_cache() + db = self.init_old() + for title in cache.fields['title'].table.book_col_map.itervalues(): + for x in (db, cache): + self.assertTrue(x.has_book(Metadata(title))) + self.assertTrue(x.has_book(Metadata(title.upper()))) + self.assertFalse(x.has_book(Metadata(title + 'XXX'))) + self.assertFalse(x.has_book(Metadata(title[:1]))) + db.close() + # }}} diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 597c98a771..cb525900ee 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -13,7 +13,7 @@ from io import BytesIO from calibre.ebooks.metadata import author_to_author_sort from calibre.utils.date import UNDEFINED_DATE -from calibre.db.tests.base import BaseTest +from calibre.db.tests.base import BaseTest, IMG class WritingTest(BaseTest): @@ -364,8 +364,8 @@ class WritingTest(BaseTest): ae(cache.field_for('cover', 1), 1) ae(cache.set_cover({1:None}), set([1])) ae(cache.field_for('cover', 1), 0) + img = IMG - img = b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00`\x00`\x00\x00\xff\xe1\x00\x16Exif\x00\x00II*\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xdb\x00C\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\xff\xdb\x00C\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\xff\xc0\x00\x11\x08\x00\x01\x00\x01\x03\x01"\x00\x02\x11\x01\x03\x11\x01\xff\xc4\x00\x15\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\xff\xc4\x00\x14\x10\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x11\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\xbf\x80\x01\xff\xd9' # noqa {{{ }}} # Test setting a cover ae(cache.set_cover({bid:img for bid in (1, 2, 3)}), {1, 2, 3}) old = self.init_old() From 42202faae823d3052150af72ea9ce90d2fd71a35 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 6 Jul 2013 09:30:37 +0530 Subject: [PATCH 0108/1154] Metadata download dialog: Have the OK button enabled in the results screen as well. See #1198288 --- src/calibre/gui2/metadata/single_download.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py index ed378745a5..fc883bd88f 100644 --- a/src/calibre/gui2/metadata/single_download.py +++ b/src/calibre/gui2/metadata/single_download.py @@ -1019,7 +1019,6 @@ class FullFetch(QDialog): # {{{ self.log_button = self.bb.addButton(_('View log'), self.bb.ActionRole) self.log_button.clicked.connect(self.view_log) self.log_button.setIcon(QIcon(I('debug.png'))) - self.ok_button.setEnabled(False) self.prev_button.setVisible(False) self.identify_widget = IdentifyWidget(self.log, self) @@ -1044,7 +1043,6 @@ class FullFetch(QDialog): # {{{ def book_selected(self, book, caches): self.next_button.setVisible(False) - self.ok_button.setEnabled(True) self.prev_button.setVisible(True) self.book = book self.stack.setCurrentIndex(1) @@ -1055,7 +1053,6 @@ class FullFetch(QDialog): # {{{ def back_clicked(self): self.next_button.setVisible(True) - self.ok_button.setEnabled(False) self.prev_button.setVisible(False) self.next_button.setFocus() self.stack.setCurrentIndex(0) @@ -1063,11 +1060,14 @@ class FullFetch(QDialog): # {{{ self.covers_widget.reset_covers() def accept(self): - gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry()) - if self.stack.currentIndex() == 1: - return QDialog.accept(self) # Prevent the usual dialog accept mechanisms from working - pass + gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry()) + if DEBUG_DIALOG: + if self.stack.currentIndex() == 2: + return QDialog.accept(self) + else: + if self.stack.currentIndex() == 1: + return QDialog.accept(self) def reject(self): gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry()) @@ -1087,6 +1087,9 @@ class FullFetch(QDialog): # {{{ def ok_clicked(self, *args): self.cover_pixmap = self.covers_widget.cover_pixmap() + if self.stack.currentIndex() == 0: + self.next_clicked() + return if DEBUG_DIALOG: if self.cover_pixmap is not None: self.w = QLabel() From 7c9ae4ebc918d10fe2defb1d294ce8af0ab26b3e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 6 Jul 2013 11:13:55 +0530 Subject: [PATCH 0109/1154] ... --- manual/customize.rst | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/manual/customize.rst b/manual/customize.rst index ceee4ece62..59475e91f2 100644 --- a/manual/customize.rst +++ b/manual/customize.rst @@ -46,17 +46,31 @@ The default values for the tweaks are reproduced below Overriding icons, templates, et cetera ---------------------------------------- -|app| allows you to override the static resources, like icons, templates, javascript, etc. with customized versions that you like. -All static resources are stored in the resources sub-folder of the calibre install location. On Windows, this is usually -:file:`C:/Program Files/Calibre2/resources`. On OS X, :file:`/Applications/calibre.app/Contents/Resources/resources/`. On linux, if you are using the binary installer -from the calibre website it will be :file:`/opt/calibre/resources`. These paths can change depending on where you choose to install |app|. +|app| allows you to override the static resources, like icons, javascript and +templates for the metadata jacket, catalogs, etc. with customized versions that +you like. All static resources are stored in the resources sub-folder of the +calibre install location. On Windows, this is usually :file:`C:/Program Files/Calibre2/resources`. +On OS X, :file:`/Applications/calibre.app/Contents/Resources/resources/`. On linux, if +you are using the binary installer from the calibre website it will be +:file:`/opt/calibre/resources`. These paths can change depending on where you +choose to install |app|. -You should not change the files in this resources folder, as your changes will get overwritten the next time you update |app|. Instead, go to -:guilabel:`Preferences->Advanced->Miscellaneous` and click :guilabel:`Open calibre configuration directory`. In this configuration directory, create a sub-folder called resources and place the files you want to override in it. Place the files in the appropriate sub folders, for example place images in :file:`resources/images`, etc. -|app| will automatically use your custom file in preference to the built-in one the next time it is started. +You should not change the files in this resources folder, as your changes will +get overwritten the next time you update |app|. Instead, go to +:guilabel:`Preferences->Advanced->Miscellaneous` and click +:guilabel:`Open calibre configuration directory`. In this configuration directory, create a +sub-folder called resources and place the files you want to override in it. +Place the files in the appropriate sub folders, for example place images in +:file:`resources/images`, etc. |app| will automatically use your custom file +in preference to the built-in one the next time it is started. -For example, if you wanted to change the icon for the :guilabel:`Remove books` action, you would first look in the built-in resources folder and see that the relevant file is -:file:`resources/images/trash.png`. Assuming you have an alternate icon in PNG format called :file:`mytrash.png` you would save it in the configuration directory as :file:`resources/images/trash.png`. All the icons used by the calibre user interface are in :file:`resources/images` and its sub-folders. +For example, if you wanted to change the icon for the :guilabel:`Remove books` +action, you would first look in the built-in resources folder and see that the +relevant file is :file:`resources/images/trash.png`. Assuming you have an +alternate icon in PNG format called :file:`mytrash.png` you would save it in +the configuration directory as :file:`resources/images/trash.png`. All the +icons used by the calibre user interface are in :file:`resources/images` and +its sub-folders. Customizing |app| with plugins -------------------------------- From 20920b37b2ae8655aeb135f9c82ae6eb58cacf2e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 6 Jul 2013 11:24:19 +0530 Subject: [PATCH 0110/1154] Implement add_books() --- src/calibre/db/cache.py | 11 +++++++++-- src/calibre/db/tests/add_remove.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index a6647d5027..e6044ce6de 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1117,10 +1117,17 @@ class Cache(object): return book_id @write_api - def add_books(self, books, add_duplicates=True): + def add_books(self, books, add_duplicates=True, apply_import_tags=True, preserve_uuid=False, dbapi=None): duplicates, ids = [], [] for mi, format_map in books: - pass + book_id = self._create_book_entry(mi, add_duplicates=add_duplicates, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid) + if book_id is None: + duplicates.append((mi, format_map)) + else: + ids.append(book_id) + for fmt, stream_or_path in format_map.iteritems(): + self._add_format(book_id, fmt, stream_or_path, dbapi=dbapi) + return ids, duplicates # }}} diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index c5845f01da..5744d63635 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -188,3 +188,19 @@ class AddRemoveTest(BaseTest): self.assertEqual(mi.uuid, cache.field_for('uuid', book_id)) # }}} + def test_add_books(self): # {{{ + 'Test the adding of new books' + from calibre.ebooks.metadata.book.base import Metadata + cache = self.init_cache() + mi = Metadata('Created One', authors=('Creator One', 'Creator Two')) + FMT1, FMT2 = b'format1', b'format2' + format_map = {'FMT1':BytesIO(FMT1), 'FMT2':BytesIO(FMT2)} + ids, duplicates = cache.add_books([(mi, format_map)]) + self.assertTrue(len(ids) == 1) + self.assertFalse(duplicates) + book_id = ids[0] + self.assertEqual(set(cache.formats(book_id)), {'FMT1', 'FMT2'}) + self.assertEqual(cache.format(book_id, 'FMT1'), FMT1) + self.assertEqual(cache.format(book_id, 'FMT2'), FMT2) + # }}} + From 0711e03bd4686262ba35b28326897360fd7cb440 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 6 Jul 2013 12:43:02 +0530 Subject: [PATCH 0111/1154] Initial implementation of remove_book(), needs testing --- src/calibre/db/backend.py | 14 +++++- src/calibre/db/cache.py | 13 +++++ src/calibre/db/tables.py | 103 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 125 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index f998b91ccb..d75106209f 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -29,7 +29,7 @@ from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_tree, delete_file from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable, - CompositeTable, LanguagesTable, UUIDTable) + CompositeTable, UUIDTable) # }}} ''' @@ -711,7 +711,6 @@ class DB(object): 'authors':AuthorsTable, 'formats':FormatsTable, 'identifiers':IdentifiersTable, - 'languages':LanguagesTable, }.get(col, ManyToManyTable) tables[col] = cls(col, self.field_metadata[col].copy()) @@ -1165,5 +1164,16 @@ class DB(object): with lopen(path, 'rb') as f: return f.read() + def remove_books(self, path_map, permanent=False): + for book_id, path in path_map.iteritems(): + if path: + path = os.path.join(self.library_path, path) + if os.path.exists(path): + self.rmtree(path, permanent=permanent) + parent = os.path.dirname(path) + if len(os.listdir(parent)) == 0: + self.rmtree(parent, permanent=permanent) + self.conn.executemany( + 'DELETE FROM books WHERE id=?', [(x,) for x in path_map]) # }}} diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index e6044ce6de..ce0582f893 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1129,6 +1129,19 @@ class Cache(object): self._add_format(book_id, fmt, stream_or_path, dbapi=dbapi) return ids, duplicates + @write_api + def remove_books(self, book_ids, permanent=False): + path_map = {} + for book_id in book_ids: + try: + path = self._field_for('path', book_id).replace('/', os.sep) + except: + path = None + path_map[book_id] = path + self.backend.remove_books(path_map, permanent=permanent) + for field in self.fields.itervalues(): + field.table.remove_books(book_ids, self.backend) + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 6f3343ba12..19c4ade10c 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -20,6 +20,10 @@ _c_speedup = plugins['speedup'][0] ONE_ONE, MANY_ONE, MANY_MANY = xrange(3) +class Null: + pass +null = Null() + def _c_convert_timestamp(val): if not val: return None @@ -55,6 +59,9 @@ class Table(object): self.link_table = (link_table if link_table else 'books_%s_link'%self.metadata['table']) + def remove_books(self, book_ids, db): + return set() + class VirtualTable(Table): ''' @@ -83,6 +90,14 @@ class OneToOneTable(Table): self.metadata['column'], self.metadata['table'])): self.book_col_map[row[0]] = self.unserialize(row[1]) + def remove_books(self, book_ids, db): + clean = set() + for book_id in book_ids: + val = self.book_col_map.pop(book_id, null) + if val is not null: + clean.add(val) + return clean + class PathTable(OneToOneTable): def set_path(self, book_id, path, db): @@ -113,6 +128,15 @@ class UUIDTable(OneToOneTable): self.uuid_to_id_map.pop(self.book_col_map.get(book_id, None), None) # discard old uuid self.uuid_to_id_map[uuid] = book_id + def remove_books(self, book_ids, db): + clean = set() + for book_id in book_ids: + val = self.book_col_map.pop(book_id, null) + if val is not null: + self.uuid_to_id_map.pop(val, None) + clean.add(val) + return clean + class CompositeTable(OneToOneTable): def read(self, db): @@ -124,6 +148,9 @@ class CompositeTable(OneToOneTable): self.composite_sort = d.get('composite_sort', False) self.use_decorations = d.get('use_decorations', False) + def remove_books(self, book_ids, db): + return set() + class ManyToOneTable(Table): ''' @@ -156,6 +183,27 @@ class ManyToOneTable(Table): self.col_book_map[row[1]].add(row[0]) self.book_col_map[row[0]] = row[1] + def remove_books(self, book_ids, db): + clean = set() + for book_id in book_ids: + item_id = self.book_col_map.pop(book_id, None) + if item_id is not None: + try: + self.col_book_map[item_id].discard(book_id) + except KeyError: + if self.id_map.pop(item_id, null) is not null: + clean.add(item_id) + else: + if not self.col_book_map[item_id]: + del self.col_book_map[item_id] + if self.id_map.pop(item_id, null) is not null: + clean.add(item_id) + if clean: + db.conn.executemany( + 'DELETE FROM {0} WHERE id=?'.format(self.metadata['table']), + [(x,) for x in clean]) + return clean + class ManyToManyTable(ManyToOneTable): ''' @@ -166,6 +214,7 @@ class ManyToManyTable(ManyToOneTable): table_type = MANY_MANY selectq = 'SELECT book, {0} FROM {1} ORDER BY id' + do_clean_on_remove = True def read_maps(self, db): for row in db.conn.execute( @@ -180,6 +229,27 @@ class ManyToManyTable(ManyToOneTable): for key in tuple(self.book_col_map.iterkeys()): self.book_col_map[key] = tuple(self.book_col_map[key]) + def remove_books(self, book_ids, db): + clean = set() + for book_id in book_ids: + item_ids = self.book_col_map.pop(book_id, ()) + for item_id in item_ids: + try: + self.col_book_map[item_id].discard(book_id) + except KeyError: + if self.id_map.pop(item_id, null) is not null: + clean.add(item_id) + else: + if not self.col_book_map[item_id]: + del self.col_book_map[item_id] + if self.id_map.pop(item_id, null) is not null: + clean.add(item_id) + if clean and self.do_clean_on_remove: + db.conn.executemany( + 'DELETE FROM {0} WHERE id=?'.format(self.metadata['table']), + [(x,) for x in clean]) + return clean + class AuthorsTable(ManyToManyTable): def read_id_maps(self, db): @@ -197,8 +267,17 @@ class AuthorsTable(ManyToManyTable): db.conn.executemany('UPDATE authors SET sort=? WHERE id=?', [(v, k) for k, v in aus_map.iteritems()]) + def remove_books(self, book_ids, db): + clean = ManyToManyTable.remove_books(self, book_ids, db) + for item_id in clean: + self.alink_map.pop(item_id, None) + self.asort_map.pop(item_id, None) + return clean + class FormatsTable(ManyToManyTable): + do_clean_on_remove = False + def read_id_maps(self, db): pass @@ -220,6 +299,13 @@ class FormatsTable(ManyToManyTable): for key in tuple(self.book_col_map.iterkeys()): self.book_col_map[key] = tuple(sorted(self.book_col_map[key])) + def remove_books(self, book_ids, db): + clean = ManyToManyTable.remove_books(self, book_ids, db) + for book_id in book_ids: + self.fname_map.pop(book_id, None) + self.size_map.pop(book_id, None) + return clean + def set_fname(self, book_id, fmt, fname, db): self.fname_map[book_id][fmt] = fname db.conn.execute('UPDATE data SET name=? WHERE book=? AND format=?', @@ -280,8 +366,19 @@ class IdentifiersTable(ManyToManyTable): self.book_col_map[row[0]] = {} self.book_col_map[row[0]][row[1]] = row[2] -class LanguagesTable(ManyToManyTable): + def remove_books(self, book_ids, db): + clean = set() + for book_id in book_ids: + item_map = self.book_col_map.pop(book_id, {}) + for item_id in item_map: + try: + self.col_book_map[item_id].discard(book_id) + except KeyError: + clean.add(item_id) + else: + if not self.col_book_map[item_id]: + del self.col_book_map[item_id] + clean.add(item_id) + return clean - def read_id_maps(self, db): - ManyToManyTable.read_id_maps(self, db) From e836bbd20390dcc669f90cdb06bb9ea8bb5547a4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 6 Jul 2013 19:02:00 +0530 Subject: [PATCH 0112/1154] DOCX Input: Fix no page break being inserted before the last section. Fixes #1198414 [Private bug](https://bugs.launchpad.net/calibre/+bug/1198414) --- src/calibre/ebooks/docx/to_html.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index 01808657ea..be0576d2b9 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -203,6 +203,7 @@ class Convert(object): current.append(p) if current: + self.section_starts.append(current[0]) last = XPath('./w:body/w:sectPr')(doc) pr = PageProperties(last) for x in current: From 75cf58d0f119476da2da9530aa5d85a51b27a82e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jul 2013 08:47:26 +0530 Subject: [PATCH 0113/1154] Glenn Brenwald and Ludwig von Mises Institute by anywho --- recipes/glenn_greenwald.recipe | 10 ++++++++++ recipes/ludwig_mises.recipe | 14 ++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 recipes/glenn_greenwald.recipe create mode 100644 recipes/ludwig_mises.recipe diff --git a/recipes/glenn_greenwald.recipe b/recipes/glenn_greenwald.recipe new file mode 100644 index 0000000000..63ed285e72 --- /dev/null +++ b/recipes/glenn_greenwald.recipe @@ -0,0 +1,10 @@ +from calibre.web.feeds.news import AutomaticNewsRecipe +class BasicUserRecipe1373130920(AutomaticNewsRecipe): + title = u'Glenn Greenwald | guardian.co.uk' + language = 'en_GB' + __author__ = 'anywho' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + + feeds = [(u'Latest', u'http://www.guardian.co.uk/profile/glenn-greenwald/rss')] diff --git a/recipes/ludwig_mises.recipe b/recipes/ludwig_mises.recipe new file mode 100644 index 0000000000..7e46a9a7db --- /dev/null +++ b/recipes/ludwig_mises.recipe @@ -0,0 +1,14 @@ +from calibre.web.feeds.news import AutomaticNewsRecipe + +class BasicUserRecipe1373130372(AutomaticNewsRecipe): + title = u'Ludwig von Mises Institute' + __author__ = 'anywho' + language = 'en' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + + feeds = [(u'Daily Articles (Full text version)', + u'http://feed.mises.org/MisesFullTextArticles'), + (u'Mises Blog Posts', + u'http://mises.org/blog/index.rdf')] From 535cff6f03378a17ef26078a24bba36c5f41ad2b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jul 2013 10:32:20 +0530 Subject: [PATCH 0114/1154] Tests for remove_books() --- src/calibre/db/cache.py | 7 +++++- src/calibre/db/tests/add_remove.py | 39 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index ce0582f893..b94258b1ef 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1140,7 +1140,12 @@ class Cache(object): path_map[book_id] = path self.backend.remove_books(path_map, permanent=permanent) for field in self.fields.itervalues(): - field.table.remove_books(book_ids, self.backend) + try: + table = field.table + except AttributeError: + continue # Some fields like ondevice do not have tables + else: + table.remove_books(book_ids, self.backend) # }}} diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index 5744d63635..5d490a2bcb 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -204,3 +204,42 @@ class AddRemoveTest(BaseTest): self.assertEqual(cache.format(book_id, 'FMT2'), FMT2) # }}} + def test_remove_books(self): # {{{ + 'Test removal of books' + cache = self.init_cache() + af, ae, at = self.assertFalse, self.assertEqual, self.assertTrue + authors = cache.fields['authors'].table + + # Delete a single book, with no formats and check cleaning + self.assertIn(_('Unknown'), set(authors.id_map.itervalues())) + olen = len(authors.id_map) + item_id = {v:k for k, v in authors.id_map.iteritems()}[_('Unknown')] + cache.remove_books((3,)) + for c in (cache, self.init_cache()): + table = c.fields['authors'].table + self.assertNotIn(3, c.all_book_ids()) + self.assertNotIn(_('Unknown'), set(table.id_map.itervalues())) + self.assertNotIn(item_id, table.asort_map) + self.assertNotIn(item_id, table.alink_map) + ae(len(table.id_map), olen-1) + + # Check that files are removed + fmtpath = cache.format_abspath(1, 'FMT1') + bookpath = os.path.dirname(fmtpath) + authorpath = os.path.dirname(bookpath) + item_id = {v:k for k, v in cache.fields['#series'].table.id_map.iteritems()}['My Series Two'] + cache.remove_books((1,), permanent=True) + for x in (fmtpath, bookpath, authorpath): + af(os.path.exists(x)) + for c in (cache, self.init_cache()): + table = c.fields['authors'].table + self.assertNotIn(1, c.all_book_ids()) + self.assertNotIn('Author Two', set(table.id_map.itervalues())) + self.assertNotIn(6, set(c.fields['rating'].table.id_map.itervalues())) + self.assertIn('A Series One', set(c.fields['series'].table.id_map.itervalues())) + self.assertNotIn('My Series Two', set(c.fields['#series'].table.id_map.itervalues())) + self.assertNotIn(item_id, c.fields['#series'].table.col_book_map) + self.assertNotIn(1, c.fields['#series'].table.book_col_map) + # }}} + + From 3b438889a4fecc1de39c7407ef5b30f973d69501 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jul 2013 11:17:57 +0530 Subject: [PATCH 0115/1154] Test emptying db --- src/calibre/db/tests/add_remove.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index 5d490a2bcb..76349df1c5 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -240,6 +240,15 @@ class AddRemoveTest(BaseTest): self.assertNotIn('My Series Two', set(c.fields['#series'].table.id_map.itervalues())) self.assertNotIn(item_id, c.fields['#series'].table.col_book_map) self.assertNotIn(1, c.fields['#series'].table.book_col_map) + + # Test emptying the db + cache.remove_books(cache.all_book_ids(), permanent=True) + for f in ('authors', 'series', '#series', 'tags'): + table = cache.fields[f].table + self.assertFalse(table.id_map) + self.assertFalse(table.book_col_map) + self.assertFalse(table.col_book_map) + # }}} From e64ff83e070b21461e119f913837c7ea361feeff Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jul 2013 11:44:30 +0530 Subject: [PATCH 0116/1154] Legacy implementations of a few direct methods --- src/calibre/db/legacy.py | 3 +++ src/calibre/db/tests/legacy.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 287d2d47db..2ad5da61b8 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -58,6 +58,9 @@ class LibraryDatabase(object): setattr(self, prop, partial(self.get_property, loc=self.FIELD_MAP[fm])) + for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'): + setattr(self, meth, getattr(self.new_api, meth)) + self.last_update_check = self.last_modified() def close(self): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 1fe719e31e..ae99d8190f 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -103,6 +103,22 @@ class LegacyTest(BaseTest): # }}} + def test_legacy_direct(self): # {{{ + 'Test methods that are directly equivalent in the old and new interface' + from calibre.ebooks.metadata.book.base import Metadata + ndb = self.init_legacy() + db = self.init_old() + for meth, args in { + 'get_next_series_num_for': [('A Series One',)], + 'author_sort_from_authors': [(['Author One', 'Author Two', 'Unknown'],)], + 'has_book':[(Metadata('title one'),), (Metadata('xxxx1111'),)], + }.iteritems(): + for a in args: + self.assertEqual(getattr(db, meth)(*a), getattr(ndb, meth)(*a), + 'The method: %s() returned different results for argument %s' % (meth, a)) + db.close() + # }}} + def test_legacy_coverage(self): # {{{ ' Check that the emulation of the legacy interface is (almost) total ' cl = self.cloned_library From 08834f636a73464366be017cb1074b85be72c2cf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jul 2013 22:47:57 +0530 Subject: [PATCH 0117/1154] Update mediapart.fr --- recipes/mediapart.recipe | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/recipes/mediapart.recipe b/recipes/mediapart.recipe index f1e6c87385..a457b713f2 100644 --- a/recipes/mediapart.recipe +++ b/recipes/mediapart.recipe @@ -1,17 +1,18 @@ __license__ = 'GPL v3' -__copyright__ = '2009, Mathieu Godlewski ; 2010-2012, Louis Gesbert ' +__copyright__ = '2009, Mathieu Godlewski ; 2010-2012, Louis Gesbert ; 2013, Malah ' ''' Mediapart ''' -__author__ = '2009, Mathieu Godlewski ; 2010-2012, Louis Gesbert ' +__author__ = '2009, Mathieu Godlewski ; 2010-2012, Louis Gesbert ; 2013, Malah ' +import re from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag from calibre.web.feeds.news import BasicNewsRecipe class Mediapart(BasicNewsRecipe): title = 'Mediapart' - __author__ = 'Mathieu Godlewski, Louis Gesbert' + __author__ = 'Mathieu Godlewski, Louis Gesbert, Malah' description = 'Global news in french from news site Mediapart' oldest_article = 7 language = 'fr' @@ -21,6 +22,7 @@ class Mediapart(BasicNewsRecipe): use_embedded_content = False no_stylesheets = True + masthead_url = 'https://upload.wikimedia.org/wikipedia/fr/2/23/Mediapart.png' cover_url = 'http://static.mediapart.fr/files/pave_mediapart.jpg' feeds = [ @@ -36,18 +38,18 @@ class Mediapart(BasicNewsRecipe): def print_version(self, url): raw = self.browser.open(url).read() soup = BeautifulSoup(raw.decode('utf8', 'replace')) - link = soup.find('a', {'title':'Imprimer'}) + link = soup.find('a', {'href':re.compile('^/print/[0-9]+')}) if link is None: return None - return link['href'] + return 'http://www.mediapart.fr' + link['href'] # -- Handle login def get_browser(self): br = BasicNewsRecipe.get_browser(self) if self.username is not None and self.password is not None: - br.open('http://www.mediapart.fr/') - br.select_form(nr=0) + br.open('http://blogs.mediapart.fr/editions/guide-du-coordonnateur-d-edition') + br.select_form(nr=1) br['name'] = self.username br['pass'] = self.password br.submit() From a41a945e8f33a856d055f9c55c3cc34358b0023d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 8 Jul 2013 10:26:50 +0530 Subject: [PATCH 0118/1154] Add FAQ about windows temp folder permissions --- manual/faq.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/manual/faq.rst b/manual/faq.rst index bdac21a622..64da7cd7ef 100644 --- a/manual/faq.rst +++ b/manual/faq.rst @@ -776,6 +776,29 @@ The only way to find the culprit is to eliminate the programs one by one and see which one is causing the issue. Basically, stop a program, run calibre, check for crashes. If they still happen, stop another program and repeat. + +Using the viewer or doing any conversions results in a permission denied error on windows +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Something on your computer is preventing calibre from accessing its own +temporary files. Most likely the permissions on your Temp folder are incorrect. +Go to the folder file:`C:\\Users\\USERNAME\\AppData\\Local` in Windows +Explorer and then right click on the file:`Temp` folder, select Properties and go to +the Security tab. Make sure that your user account has full control for this +folder. + +Some users have reported that running the following command in an Administrator +Command Prompt fixed their permissions. To get an Administrator Command Prompt +search for cmd.exe in the start menu, then right click on the command prompt +entry and select Run as Administrator:: + icacls "%appdata%\..\Local\Temp" /reset /T + +Alternately, you can run calibre as Administrator, but doing so will cause +some functionality, such as drag and drop to not work. + +Finally, some users have reported that disabling UAC fixes the problem. + + |app| is not starting on OS X? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From a8deb4b1f8dfb768a32b95b1540be32d5d6e871e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 8 Jul 2013 10:30:34 +0530 Subject: [PATCH 0119/1154] Ignore type errors when sorting device collections Invalid data in the device database on sony readers could cause errors when sorting device collections, ignore those errors. --- src/calibre/devices/usbms/books.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index bbbb3938ac..5254a814be 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -283,11 +283,17 @@ class CollectionsBookList(BookList): return -1 if isinstance(x, basestring) and isinstance(y, basestring): x, y = sort_key(force_unicode(x)), sort_key(force_unicode(y)) - c = cmp(x, y) + try: + c = cmp(x, y) + except TypeError: + c = 0 if c != 0: return c # same as above -- no sort_key needed here - return cmp(xx[2], yy[2]) + try: + return cmp(xx[2], yy[2]) + except TypeError: + return 0 for category, lpaths in collections.items(): books = lpaths.values() From fae8aa1405279b630456880f0689f2eece115154 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 8 Jul 2013 11:32:40 +0530 Subject: [PATCH 0120/1154] Dont change into temp dir when downloading single covers Works around the problem with temp dir permissions on some windows computers. --- src/calibre/ebooks/metadata/sources/worker.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/worker.py b/src/calibre/ebooks/metadata/sources/worker.py index 51fb883e7d..1c83f965e1 100644 --- a/src/calibre/ebooks/metadata/sources/worker.py +++ b/src/calibre/ebooks/metadata/sources/worker.py @@ -106,7 +106,6 @@ def single_identify(title, authors, identifiers): r in results], dump_caches(), log.dump() def single_covers(title, authors, identifiers, caches, tdir): - os.chdir(tdir) load_caches(caches) log = GUILog() results = Queue() @@ -126,9 +125,9 @@ def single_covers(title, authors, identifiers, caches, tdir): name += '{%d}'%c[plugin.name] c[plugin.name] += 1 name = '%s,,%s,,%s,,%s.cover'%(name, width, height, fmt) - with open(name, 'wb') as f: + with open(os.path.join(tdir, name), 'wb') as f: f.write(data) - os.mkdir(name+'.done') + os.mkdir(os.path.join(tdir, name+'.done')) return log.dump() From 3a9fa00032fd8dca84848fcc979a5f97f8def534 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 8 Jul 2013 12:24:39 +0530 Subject: [PATCH 0121/1154] Try to automatically fix temp folder permissions on windows --- manual/faq.rst | 4 +++- src/calibre/__init__.py | 12 ++++++++++-- src/calibre/customize/conversion.py | 2 +- src/calibre/ptempfile.py | 13 +++++++++++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/manual/faq.rst b/manual/faq.rst index 64da7cd7ef..e5a6342cf8 100644 --- a/manual/faq.rst +++ b/manual/faq.rst @@ -790,7 +790,9 @@ folder. Some users have reported that running the following command in an Administrator Command Prompt fixed their permissions. To get an Administrator Command Prompt search for cmd.exe in the start menu, then right click on the command prompt -entry and select Run as Administrator:: +entry and select Run as Administrator. At the command prompt type the following +command and press Enter:: + icacls "%appdata%\..\Local\Temp" /reset /T Alternately, you can run calibre as Administrator, but doing so will cause diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 07ad906247..5d938ecc55 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -436,13 +436,21 @@ def fit_image(width, height, pwidth, pheight): class CurrentDir(object): - def __init__(self, path): + def __init__(self, path, workaround_temp_folder_permissions=False): self.path = path self.cwd = None + self.workaround_temp_folder_permissions = workaround_temp_folder_permissions def __enter__(self, *args): self.cwd = os.getcwdu() - os.chdir(self.path) + try: + os.chdir(self.path) + except OSError: + if not self.workaround_temp_folder_permissions: + raise + from calibre.ptempfile import reset_temp_folder_permissions + reset_temp_folder_permissions() + os.chdir(self.path) return self.cwd def __exit__(self, *args): diff --git a/src/calibre/customize/conversion.py b/src/calibre/customize/conversion.py index 38ffcef71f..9a7ed0d24c 100644 --- a/src/calibre/customize/conversion.py +++ b/src/calibre/customize/conversion.py @@ -233,7 +233,7 @@ class InputFormatPlugin(Plugin): # In case stdout is broken pass - with CurrentDir(output_dir): + with CurrentDir(output_dir, workaround_temp_folder_permissions=True): for x in os.listdir('.'): shutil.rmtree(x) if os.path.isdir(x) else os.remove(x) diff --git a/src/calibre/ptempfile.py b/src/calibre/ptempfile.py index 96271fbeaf..f3816f766b 100644 --- a/src/calibre/ptempfile.py +++ b/src/calibre/ptempfile.py @@ -34,6 +34,19 @@ def app_prefix(prefix): return '%s_'%__appname__ return '%s_%s_%s'%(__appname__, __version__, prefix) +def reset_temp_folder_permissions(): + # There are some broken windows installs where the permissions for the temp + # folder are set to not be executable, which means chdir() into temp + # folders fails. Try to fix that by resetting the permissions on the temp + # folder. + global _base_dir + if iswindows and _base_dir: + import subprocess + from calibre import prints + parent = os.path.dirname(_base_dir) + retcode = subprocess.Popen(['icacls.exe', parent, '/reset', '/Q', '/T']).wait() + prints('Trying to reset permissions of temp folder', parent, 'return code:', retcode) + def base_dir(): global _base_dir if _base_dir is not None and not os.path.exists(_base_dir): From af3d990264298697d907769e3e1a2ac777aa4921 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 8 Jul 2013 12:52:51 +0530 Subject: [PATCH 0122/1154] Edelweiss: Workaround broken advanced search Edelweiss metadata download plugin: Workaround for advanced search being broken at the Edelweiss website. --- .../ebooks/metadata/sources/edelweiss.py | 59 +++++++++++++++---- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/edelweiss.py b/src/calibre/ebooks/metadata/sources/edelweiss.py index 27fd296503..fab0b2017d 100644 --- a/src/calibre/ebooks/metadata/sources/edelweiss.py +++ b/src/calibre/ebooks/metadata/sources/edelweiss.py @@ -34,7 +34,7 @@ def astext(node): return etree.tostring(node, method='text', encoding=unicode, with_tail=False).strip() -class Worker(Thread): # {{{ +class Worker(Thread): # {{{ def __init__(self, sku, url, relevance, result_queue, br, timeout, log, plugin): Thread.__init__(self) @@ -154,8 +154,8 @@ class Worker(Thread): # {{{ # remove all attributes from tags desc = re.sub(r'<([a-zA-Z0-9]+)\s[^>]+>', r'<\1>', desc) # Collapse whitespace - #desc = re.sub('\n+', '\n', desc) - #desc = re.sub(' +', ' ', desc) + # desc = re.sub('\n+', '\n', desc) + # desc = re.sub(' +', ' ', desc) # Remove comments desc = re.sub(r'(?s)', '', desc) return sanitize_comments_html(desc) @@ -183,14 +183,14 @@ class Edelweiss(Source): if sku: return 'http://edelweiss.abovethetreeline.com/ProductDetailPage.aspx?sku=%s'%sku - def get_book_url(self, identifiers): # {{{ + def get_book_url(self, identifiers): # {{{ sku = identifiers.get('edelweiss', None) if sku: return 'edelweiss', sku, self._get_book_url(sku) # }}} - def get_cached_cover_url(self, identifiers): # {{{ + def get_cached_cover_url(self, identifiers): # {{{ sku = identifiers.get('edelweiss', None) if not sku: isbn = identifiers.get('isbn', None) @@ -199,7 +199,7 @@ class Edelweiss(Source): return self.cached_identifier_to_cover_url(sku) # }}} - def create_query(self, log, title=None, authors=None, identifiers={}): # {{{ + def create_query(self, log, title=None, authors=None, identifiers={}): # {{{ from urllib import urlencode BASE_URL = 'http://edelweiss.abovethetreeline.com/CatalogOverview.aspx?' params = { @@ -239,9 +239,40 @@ class Edelweiss(Source): params[k] = v.encode('utf-8') return BASE_URL+urlencode(params) + + def create_query2(self, log, title=None, authors=None, identifiers={}): + ''' The edelweiss advanced search appears to be broken, use the keyword search instead, until it is fixed. ''' + from urllib import urlencode + BASE_URL = 'http://edelweiss.abovethetreeline.com/CatalogOverview.aspx?' + params = { + 'group':'search', + 'section':'CatalogOverview', + 'searchType':1, + 'searchOrgID':'', + 'searchCatalogID': '', + 'searchMailingID': '', + 'searchSelect':1, + } + keywords = [] + isbn = check_isbn(identifiers.get('isbn', None)) + if isbn is not None: + keywords.append(isbn) + elif title or authors: + title_tokens = list(self.get_title_tokens(title)) + if title_tokens: + keywords.extend(title_tokens) + author_tokens = self.get_author_tokens(authors, + only_first_author=True) + if author_tokens: + keywords.extend(author_tokens) + if not keywords: + return None + params['keywords'] = (' '.join(keywords)).encode('utf-8') + return BASE_URL+urlencode(params) + # }}} - def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ + def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ identifiers={}, timeout=30): from urlparse import parse_qs @@ -251,11 +282,12 @@ class Edelweiss(Source): entries = [(book_url, identifiers['edelweiss'])] else: entries = [] - query = self.create_query(log, title=title, authors=authors, + query = self.create_query2(log, title=title, authors=authors, identifiers=identifiers) if not query: log.error('Insufficient metadata to construct query') return + log('Using query URL:', query) try: raw = br.open_novisit(query, timeout=timeout).read() except Exception as e: @@ -270,7 +302,8 @@ class Edelweiss(Source): for entry in CSSSelect('div.listRow div.listRowMain')(root): a = entry.xpath('descendant::a[contains(@href, "sku=") and contains(@href, "ProductDetailPage.aspx")]') - if not a: continue + if not a: + continue href = a[0].get('href') prefix, qs = href.partition('?')[0::2] sku = parse_qs(qs).get('sku', None) @@ -288,7 +321,7 @@ class Edelweiss(Source): div = CSSSelect('div.format.attGroup')(entry) text = astext(div[0]).lower() - if 'audio' in text or 'mp3' in text: # Audio-book, ignore + if 'audio' in text or 'mp3' in text: # Audio-book, ignore continue entries.append((self._get_book_url(sku), sku)) @@ -321,7 +354,7 @@ class Edelweiss(Source): # }}} - def download_cover(self, log, result_queue, abort, # {{{ + def download_cover(self, log, result_queue, abort, # {{{ title=None, authors=None, identifiers={}, timeout=30, get_best_cover=False): cached_url = self.get_cached_cover_url(identifiers) if cached_url is None: @@ -381,7 +414,7 @@ if __name__ == '__main__': ), - ( # Pubdate + ( # Pubdate {'title':'The Great Gatsby', 'authors':['F. Scott Fitzgerald']}, [title_test('The great gatsby', exact=True), authors_test(['F. Scott Fitzgerald']), pubdate_test(2004, 9, 29)] @@ -395,3 +428,5 @@ if __name__ == '__main__': test_identify_plugin(Edelweiss.name, tests) + + From c4cb0e445e9d2dc438f878cbe7590ff060b7c9c4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 8 Jul 2013 15:56:09 +0530 Subject: [PATCH 0123/1154] Update amazon metadata download plugin for website changes Amazon metadata download: Update plugin to deal with the new amazon.com website --- src/calibre/ebooks/metadata/sources/amazon.py | 144 +++++++++++++----- 1 file changed, 104 insertions(+), 40 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index eb9e5a18cc..028bad6922 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -19,6 +19,11 @@ from calibre.ebooks.metadata.sources.base import (Source, Option, fixcase, from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.localization import canonicalize_lang +def CSSSelect(expr): + from cssselect import HTMLTranslator + from lxml.etree import XPath + return XPath(HTMLTranslator().css_to_xpath(expr)) + class Worker(Thread): # Get details {{{ ''' @@ -142,6 +147,8 @@ class Worker(Thread): # Get details {{{ starts-with(text(), "Editora:") or \ starts-with(text(), "出版社:")] ''' + self.publisher_names = {'Publisher', 'Verlag', 'Editore', 'Editeur', 'Editor', 'Editora', '出版社'} + self.language_xpath = ''' descendant::*[ starts-with(text(), "Language:") \ @@ -153,6 +160,7 @@ class Worker(Thread): # Get details {{{ or starts-with(text(), "言語") \ ] ''' + self.language_names = {'Language', 'Sprache', 'Lingua', 'Idioma', 'Langue', '言語'} self.ratings_pat = re.compile( r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち|de un máximo de|de) ([\d\.]+)( (stars|Sternen|stelle|estrellas|estrelas)){0,1}') @@ -310,36 +318,44 @@ class Worker(Thread): # Get details {{{ self.log.exception('Error parsing cover for url: %r'%self.url) mi.has_cover = bool(self.cover_url) - pd = root.xpath(self.pd_xpath) - if pd: - pd = pd[0] - + non_hero = CSSSelect('div#bookDetails_container_div div#nonHeroSection')(root) + if non_hero: + # New style markup try: - isbn = self.parse_isbn(pd) - if isbn: - self.isbn = mi.isbn = isbn + self.parse_new_details(root, mi, non_hero[0]) except: - self.log.exception('Error parsing ISBN for url: %r'%self.url) - - try: - mi.publisher = self.parse_publisher(pd) - except: - self.log.exception('Error parsing publisher for url: %r'%self.url) - - try: - mi.pubdate = self.parse_pubdate(pd) - except: - self.log.exception('Error parsing publish date for url: %r'%self.url) - - try: - lang = self.parse_language(pd) - if lang: - mi.language = lang - except: - self.log.exception('Error parsing language for url: %r'%self.url) - + self.log.exception('Failed to parse new-style book details section') else: - self.log.warning('Failed to find product description for url: %r'%self.url) + pd = root.xpath(self.pd_xpath) + if pd: + pd = pd[0] + + try: + isbn = self.parse_isbn(pd) + if isbn: + self.isbn = mi.isbn = isbn + except: + self.log.exception('Error parsing ISBN for url: %r'%self.url) + + try: + mi.publisher = self.parse_publisher(pd) + except: + self.log.exception('Error parsing publisher for url: %r'%self.url) + + try: + mi.pubdate = self.parse_pubdate(pd) + except: + self.log.exception('Error parsing publish date for url: %r'%self.url) + + try: + lang = self.parse_language(pd) + if lang: + mi.language = lang + except: + self.log.exception('Error parsing language for url: %r'%self.url) + + else: + self.log.warning('Failed to find product description for url: %r'%self.url) mi.source_relevance = self.relevance @@ -359,7 +375,13 @@ class Worker(Thread): # Get details {{{ for l in link: return l.get('href').rpartition('/')[-1] + def totext(self, elem): + return self.tostring(elem, encoding=unicode, method='text').strip() + def parse_title(self, root): + h1 = root.xpath('//h1[@id="title"]') + if h1: + return self.totext(h1[0]) tdiv = root.xpath('//h1[contains(@class, "parseasinTitle")]')[0] actual_title = tdiv.xpath('descendant::*[@id="btAsinTitle"]') if actual_title: @@ -373,6 +395,11 @@ class Worker(Thread): # Get details {{{ return ans def parse_authors(self, root): + matches = CSSSelect('#byline .author .contributorNameID')(root) + if matches: + authors = [self.totext(x) for x in matches] + return [a for a in authors if a] + x = '//h1[contains(@class, "parseasinTitle")]/following-sibling::span/*[(name()="a" and @href) or (name()="span" and @class="contributorNameTrigger")]' aname = root.xpath(x) if not aname: @@ -420,8 +447,8 @@ class Worker(Thread): # Get details {{{ # remove all attributes from tags desc = re.sub(r'<([a-zA-Z0-9]+)\s[^>]+>', r'<\1>', desc) # Collapse whitespace - #desc = re.sub('\n+', '\n', desc) - #desc = re.sub(' +', ' ', desc) + # desc = re.sub('\n+', '\n', desc) + # desc = re.sub(' +', ' ', desc) # Remove the notice about text referring to out of print editions desc = re.sub(r'(?s)--This text ref.*?', '', desc) # Remove comments @@ -429,6 +456,17 @@ class Worker(Thread): # Get details {{{ return sanitize_comments_html(desc) def parse_comments(self, root): + ns = CSSSelect('#bookDescription_feature_div noscript')(root) + if ns: + ns = ns[0] + if len(ns) == 0 and ns.text: + import html5lib + # html5lib parsed noscript as CDATA + ns = html5lib.parseFragment('
    %s
    ' % (ns.text), treebuilder='lxml', namespaceHTMLElements=False)[0] + else: + ns.tag = 'div' + return self._render_comments(ns) + ans = '' desc = root.xpath('//div[@id="ps-content"]/div[@class="content"]') if desc: @@ -472,6 +510,37 @@ class Worker(Thread): # Get details {{{ bn = re.sub(r'\.\.jpg$', '.jpg', (sparts[0] + sparts[-1])) return ('/'.join(parts[:-1]))+'/'+bn + def parse_new_details(self, root, mi, non_hero): + table = non_hero.xpath('descendant::table')[0] + for tr in table.xpath('descendant::tr'): + cells = tr.xpath('descendant::td') + if len(cells) == 2: + name = self.totext(cells[0]) + val = self.totext(cells[1]) + if not val: + continue + if name in self.language_names: + ans = self.lang_map.get(val, None) + if not ans: + ans = canonicalize_lang(val) + if ans: + mi.language = ans + elif name in self.publisher_names: + pub = val.partition(';')[0].partition('(')[0].strip() + if pub: + mi.publisher = pub + date = val.rpartition('(')[-1].replace(')', '').strip() + try: + from calibre.utils.date import parse_only_date + date = self.delocalize_datestr(date) + mi.pubdate = parse_only_date(date, assume_utc=True) + except: + self.log.exception('Failed to parse pubdate: %s' % val) + elif name in {'ISBN', 'ISBN-10', 'ISBN-13'}: + ans = check_isbn(val) + if ans: + self.isbn = mi.isbn = ans + def parse_isbn(self, pd): items = pd.xpath( 'descendant::*[starts-with(text(), "ISBN")]') @@ -721,9 +790,9 @@ class Amazon(Source): def title_ok(title): title = title.lower() - bad = ['bulk pack', '[audiobook]', '[audio cd]'] + bad = ['bulk pack', '[audiobook]', '[audio cd]', '(a book companion)', '( slipcase with door )'] if self.domain == 'com': - bad.append('(spanish edition)') + bad.extend(['(%s edition)' % x for x in ('spanish', 'german')]) for x in bad: if x in title: return False @@ -901,14 +970,9 @@ if __name__ == '__main__': # tests {{{ # To run these test use: calibre-debug -e # src/calibre/ebooks/metadata/sources/amazon.py from calibre.ebooks.metadata.sources.test import (test_identify_plugin, - isbn_test, title_test, authors_test, comments_test, series_test) + isbn_test, title_test, authors_test, comments_test) com_tests = [ # {{{ - ( # Has a spanish edition - {'title':'11/22/63'}, - [title_test('11/22/63: A Novel', exact=True), authors_test(['Stephen King']),] - ), - ( # + in title and uses id="main-image" for cover {'title':'C++ Concurrency in Action'}, [title_test('C++ Concurrency in Action: Practical Multithreading', @@ -916,11 +980,10 @@ if __name__ == '__main__': # tests {{{ ] ), - ( # Series + ( # noscript description {'identifiers':{'amazon':'0756407117'}}, [title_test( - "Throne of the Crescent Moon", - exact=True), series_test('Crescent Moon Kingdoms', 1), + "Throne of the Crescent Moon"), comments_test('Makhslood'), ] ), @@ -1054,3 +1117,4 @@ if __name__ == '__main__': # tests {{{ # }}} + From a94539c32b4f2d3a359f733ee0a277af72c8c8ff Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 9 Jul 2013 14:44:46 +0530 Subject: [PATCH 0124/1154] jsbrowser: Fix handling of html with non lxml safe chars --- src/calibre/web/fetch/javascript.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/web/fetch/javascript.py b/src/calibre/web/fetch/javascript.py index 6e9ef86ff1..d7dfcf0a6a 100644 --- a/src/calibre/web/fetch/javascript.py +++ b/src/calibre/web/fetch/javascript.py @@ -145,8 +145,11 @@ def download_resources(browser, resource_cache, output_dir): elem.removeFromDocument() def save_html(browser, output_dir, postprocess_html, url, recursion_level): - html = strip_encoding_declarations(browser.html) import html5lib + from calibre.utils.cleantext import clean_xml_chars + html = strip_encoding_declarations(browser.html) + if isinstance(html, unicode): + html = clean_xml_chars(html) root = html5lib.parse(html, treebuilder='lxml', namespaceHTMLElements=False).getroot() root = postprocess_html(root, url, recursion_level) if root is None: From bba659b852ded2b2e026eb0be58c67b0862cda6f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 9 Jul 2013 14:53:40 +0530 Subject: [PATCH 0125/1154] Add a check for modern WebKit --- src/calibre/constants.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 99146e206c..5c9ecbc832 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -282,3 +282,8 @@ def get_windows_user_locale_name(): return None return u'_'.join(buf.value.split(u'-')[:2]) +def is_modern_webkit(): + # Check if we are using QtWebKit >= 2.3 + from PyQt4.QtWebKit import qWebKitMajorVersion + return qWebKitMajorVersion() >= 537 + From ed55e76ff4557c4c3558b28b6dcf29c452d9e9e1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 9 Jul 2013 15:45:20 +0530 Subject: [PATCH 0126/1154] Update cracked.com --- recipes/cracked_com.recipe | 68 +++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/recipes/cracked_com.recipe b/recipes/cracked_com.recipe index 829299ae17..a702f93433 100644 --- a/recipes/cracked_com.recipe +++ b/recipes/cracked_com.recipe @@ -1,63 +1,55 @@ from calibre.web.feeds.news import BasicNewsRecipe -class Cracked(BasicNewsRecipe): - title = u'Cracked.com' - __author__ = 'UnWeave' - language = 'en' - description = "America's Only HumorSite since 1958" - publisher = 'Cracked' - category = 'comedy, lists' - oldest_article = 3 #days - max_articles_per_feed = 100 - no_stylesheets = True - encoding = 'ascii' - remove_javascript = True - use_embedded_content = False - feeds = [ (u'Articles', u'http://feeds.feedburner.com/CrackedRSS/') ] +class Cracked(BasicNewsRecipe): + title = u'Cracked.com' + __author__ = 'UnWeave' + language = 'en' + description = "America's Only HumorSite since 1958" + publisher = 'Cracked' + category = 'comedy, lists' + oldest_article = 3 # days + max_articles_per_feed = 100 + no_stylesheets = True + encoding = 'ascii' + remove_javascript = True + use_embedded_content = False + # auto_cleanup = True + + feeds = [(u'Articles', u'http://feeds.feedburner.com/CrackedRSS/')] conversion_options = { - 'comment' : description - , 'tags' : category - , 'publisher' : publisher - , 'language' : language - } + 'comment': description, 'tags': category, 'publisher': publisher, 'language': language + } - remove_tags_before = dict(id='PrimaryContent') + # remove_tags_before = dict(id='PrimaryContent') - remove_tags_after = dict(name='div', attrs={'class':'shareBar'}) + keep_only_tags = dict(name='article', attrs={ + 'class': 'module article dropShadowBottomCurved'}) - remove_tags = [ dict(name='div', attrs={'class':['social', - 'FacebookLike', - 'shareBar' - ]}), + # remove_tags_after = dict(name='div', attrs={'class':'shareBar'}) - dict(name='div', attrs={'id':['inline-share-buttons', - ]}), - - dict(name='span', attrs={'class':['views', - 'KonaFilter' - ]}), - #dict(name='img'), - ] + remove_tags = [ + dict(name='section', attrs={'class': ['socialTools', 'quickFixModule']})] def appendPage(self, soup, appendTag, position): # Check if article has multiple pages - pageNav = soup.find('nav', attrs={'class':'PaginationContent'}) + pageNav = soup.find('nav', attrs={'class': 'PaginationContent'}) if pageNav: # Check not at last page - nextPage = pageNav.find('a', attrs={'class':'next'}) + nextPage = pageNav.find('a', attrs={'class': 'next'}) if nextPage: nextPageURL = nextPage['href'] nextPageSoup = self.index_to_soup(nextPageURL) # 8th
    tag contains article content - nextPageContent = nextPageSoup.findAll('section')[7] + nextPageContent = nextPageSoup.findAll('article')[0] newPosition = len(nextPageContent.contents) - self.appendPage(nextPageSoup,nextPageContent,newPosition) + self.appendPage(nextPageSoup, nextPageContent, newPosition) nextPageContent.extract() pageNav.extract() - appendTag.insert(position,nextPageContent) + appendTag.insert(position, nextPageContent) def preprocess_html(self, soup): self.appendPage(soup, soup.body, 3) return soup + From 26f86ca98741327da50c377a7ec4c09280a81949 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 9 Jul 2013 16:54:11 +0530 Subject: [PATCH 0127/1154] jsbrowser(): Fix typo causing some images to not be downloaded Fixes images missing in the Time recipe --- src/calibre/web/jsbrowser/browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/web/jsbrowser/browser.py b/src/calibre/web/jsbrowser/browser.py index 104347bfb2..387b149bb9 100644 --- a/src/calibre/web/jsbrowser/browser.py +++ b/src/calibre/web/jsbrowser/browser.py @@ -571,7 +571,7 @@ class Browser(QObject, FormsMixin): ans[url] = raw urls.discard(url) - while urls and time.time() - start_time > timeout and self.page.ready_state not in {'complete', 'completed'}: + while urls and time.time() - start_time < timeout and self.page.ready_state not in {'complete', 'completed'}: get_resources() if urls: self.run_for_a_time(0.1) From 03041f925e288b081c5868e163e7d670878ba1fc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 9 Jul 2013 17:33:28 +0530 Subject: [PATCH 0128/1154] Tweak to restrict list of output formats Add a tweak to restrict the list of output formats available in the conversion dialog. Go to Preferences->Tweaks to change it. --- resources/default_tweaks.py | 7 +++++++ src/calibre/gui2/convert/bulk.py | 15 ++++++--------- src/calibre/gui2/convert/single.py | 22 +++++++++++++++++----- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index a0e8fafd0f..d8e158f842 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -537,3 +537,10 @@ many_libraries = 10 # highlight with this tweak. Set it to 'transparent' to disable highlighting. highlight_virtual_library = 'yellow' +#: Choose available output formats for conversion +# Restrict the list of available output formats in the conversion dialogs. +# For example, if you only want to convert to EPUB and AZW3, change this to +# restrict_output_formats = ['EPUB', 'AZW3']. The default value of None causes +# all available output formats to be present. +restrict_output_formats = None + diff --git a/src/calibre/gui2/convert/bulk.py b/src/calibre/gui2/convert/bulk.py index b1c3de122b..91efc73ca9 100644 --- a/src/calibre/gui2/convert/bulk.py +++ b/src/calibre/gui2/convert/bulk.py @@ -9,8 +9,7 @@ import shutil from PyQt4.Qt import QString, SIGNAL from calibre.gui2.convert.single import (Config, sort_formats_by_preference, - GroupModel, gprefs) -from calibre.customize.ui import available_output_formats + GroupModel, gprefs, get_output_formats) from calibre.gui2 import ResizableDialog from calibre.gui2.convert.look_and_feel import LookAndFeelWidget from calibre.gui2.convert.heuristics import HeuristicsWidget @@ -43,7 +42,6 @@ class BulkConfig(Config): 'values saved in a previous conversion (if they exist) instead ' 'of using the defaults specified in the Preferences')) - self.connect(self.output_formats, SIGNAL('currentIndexChanged(QString)'), self.setup_pipeline) self.connect(self.groups, SIGNAL('activated(QModelIndex)'), @@ -96,7 +94,8 @@ class BulkConfig(Config): while True: c = self.stack.currentWidget() - if not c: break + if not c: + break self.stack.removeWidget(c) widgets = [lf, hw, ps, sd, toc, sr] @@ -118,17 +117,14 @@ class BulkConfig(Config): except: pass - def setup_output_formats(self, db, preferred_output_format): if preferred_output_format: preferred_output_format = preferred_output_format.lower() - output_formats = sorted(available_output_formats(), - key=lambda x:{'EPUB':'!A', 'MOBI':'!B'}.get(x.upper(), x)) - output_formats.remove('oeb') + output_formats = get_output_formats(preferred_output_format) preferred_output_format = preferred_output_format if \ preferred_output_format and preferred_output_format \ in output_formats else sort_formats_by_preference(output_formats, - prefs['output_format'])[0] + [prefs['output_format']])[0] self.output_formats.addItems(list(map(QString, [x.upper() for x in output_formats]))) self.output_formats.setCurrentIndex(output_formats.index(preferred_output_format)) @@ -149,3 +145,4 @@ class BulkConfig(Config): bytearray(self.saveGeometry()) return ResizableDialog.done(self, r) + diff --git a/src/calibre/gui2/convert/single.py b/src/calibre/gui2/convert/single.py index 1a915288a8..e8342610dd 100644 --- a/src/calibre/gui2/convert/single.py +++ b/src/calibre/gui2/convert/single.py @@ -29,7 +29,7 @@ from calibre.ebooks.conversion.plumber import (Plumber, from calibre.ebooks.conversion.config import delete_specifics from calibre.customize.ui import available_output_formats from calibre.customize.conversion import OptionRecommendation -from calibre.utils.config import prefs +from calibre.utils.config import prefs, tweaks from calibre.utils.logging import Log class NoSupportedInputFormats(Exception): @@ -48,6 +48,20 @@ def sort_formats_by_preference(formats, prefs): return len(prefs) return sorted(formats, key=key) +def get_output_formats(preferred_output_format): + all_formats = {x.upper() for x in available_output_formats()} + all_formats.discard('OEB') + pfo = preferred_output_format.upper() if preferred_output_format else '' + restrict = tweaks['restrict_output_formats'] + if restrict: + fmts = [x.upper() for x in restrict] + if pfo and pfo not in fmts and pfo in all_formats: + fmts.append(pfo) + else: + fmts = list(sorted(all_formats, + key=lambda x:{'EPUB':'!A', 'MOBI':'!B'}.get(x.upper(), x))) + return fmts + class GroupModel(QAbstractListModel): def __init__(self, widgets): @@ -239,15 +253,13 @@ class Config(ResizableDialog, Ui_Dialog): preferred_output_format): if preferred_output_format: preferred_output_format = preferred_output_format.lower() - output_formats = sorted(available_output_formats(), - key=lambda x:{'EPUB':'!A', 'MOBI':'!B'}.get(x.upper(), x)) - output_formats.remove('oeb') + output_formats = get_output_formats(preferred_output_format) input_format, input_formats = get_input_format_for_book(db, book_id, preferred_input_format) preferred_output_format = preferred_output_format if \ preferred_output_format in output_formats else \ sort_formats_by_preference(output_formats, - prefs['output_format'])[0] + [prefs['output_format']])[0] self.input_formats.addItems(list(map(QString, [x.upper() for x in input_formats]))) self.output_formats.addItems(list(map(QString, [x.upper() for x in From dca69aa470dd84d1967146e1f854b3313eb3a4a1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 9 Jul 2013 17:50:04 +0530 Subject: [PATCH 0129/1154] Confirm format override when add files to book When adding formats to an existing book, ask for confirmation if some formats will be overwritten. --- src/calibre/gui2/actions/add.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 4071ad1468..c5a778f965 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -110,6 +110,19 @@ class AddAction(InterfaceAction): return db = view.model().db + if len(ids) == 1: + formats = db.formats(ids[0], index_is_id=True) + if formats: + formats = {x.upper() for x in formats.split(',')} + nformats = {f.rpartition('.')[-1].upper() for f in books} + override = formats.intersection(nformats) + if override: + title = db.title(ids[0], index_is_id=True) + msg = _('The {0} format(s) will be replaced in the book {1}. Are you sure?').format( + ', '.join(override), title) + if not confirm(msg, 'confirm_format_override_on_add', title=_('Are you sure'), parent=self.gui): + return + for id_ in ids: for fpath in books: fmt = os.path.splitext(fpath)[1][1:].upper() From 2544d48fbeb68f46b5f9009a818c254ef7881a59 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 9 Jul 2013 19:00:16 +0530 Subject: [PATCH 0130/1154] A bit of formatting --- manual/conversion.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/manual/conversion.rst b/manual/conversion.rst index c693d0be15..2387fe915a 100644 --- a/manual/conversion.rst +++ b/manual/conversion.rst @@ -772,9 +772,11 @@ size. By default, |app| uses a page size defined by the current :guilabel:`Output profile`. So if your output profile is set to Kindle, |app| will create a PDF with page size suitable for viewing on the small kindle screen. However, if you view this PDF file on a computer screen, then it will -appear to have too large fonts. To create "normal" sized PDFs, use the override -page size option under :guilabel:`PDF Output` in the conversion dialog. +appear to have too large fonts. To create "normal" sized PDFs, use the +:guilabel:`Override page size` option under :guilabel:`PDF Output` in the conversion dialog. +Headers and Footers +^^^^^^^^^^^^^^^^^^^^ You can insert arbitrary headers and footers on each page of the PDF by specifying header and footer templates. Templates are just snippets of HTML code that get rendered in the header and footer locations. For example, to @@ -813,6 +815,9 @@ the page will be used. bottom margins to large enough values, under the Page Setup section of the conversion dialog. +Printable Table of Contents +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + You can also insert a printable Table of Contents at the end of the PDF that lists the page numbers for every section. This is very useful if you intend to print out the PDF to paper. If you wish to use the PDF on an electronic device, From 135a0420b11369a75cbd66a31073e0d9ce5f418d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 9 Jul 2013 21:08:27 +0530 Subject: [PATCH 0131/1154] Driver for Coby Kyros MID1126 Fixes #1199410 [Unrecognized device Coby Kyros MID1126](https://bugs.launchpad.net/calibre/+bug/1199410) --- src/calibre/devices/android/driver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 31b60389ad..1880324fdc 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -107,7 +107,7 @@ class ANDROID(USBMS): 0x0ff9 : [0x0226], 0xc91 : HTC_BCDS, 0xdddd : [0x216], - 0xdeed : [0x231], + 0xdeed : [0x231, 0x226], }, # Samsung @@ -241,7 +241,7 @@ class ANDROID(USBMS): 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E', 'NOVO7', 'MB526', '_USB#WYK7MSF8KE', 'TABLET_PC', 'F', 'MT65XX_MS', 'ICS', 'E400', '__FILE-STOR_GADG', 'ST80208-1', 'GT-S5660M_CARD', 'XT894', '_USB', - 'PROD_TAB13-201', 'URFPAD2', + 'PROD_TAB13-201', 'URFPAD2', 'MID1126', ] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', @@ -254,7 +254,7 @@ class ANDROID(USBMS): 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E', 'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD', 'ST80208-1', 'XT894', - '_USB', 'PROD_TAB13-201', 'URFPAD2' + '_USB', 'PROD_TAB13-201', 'URFPAD2', 'MID1126', ] OSX_MAIN_MEM = 'Android Device Main Memory' From 0f6161e5baf11bc573731c09e387ba9a5d9f1faf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 9 Jul 2013 22:28:14 +0530 Subject: [PATCH 0132/1154] Update Houston Chronicle --- recipes/houston_chronicle.recipe | 221 +++++++++++++++++++++++++++---- 1 file changed, 193 insertions(+), 28 deletions(-) diff --git a/recipes/houston_chronicle.recipe b/recipes/houston_chronicle.recipe index ed430aa45a..d7e2ae14c3 100644 --- a/recipes/houston_chronicle.recipe +++ b/recipes/houston_chronicle.recipe @@ -1,41 +1,206 @@ #!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- +__license__ = 'GPL v3' +__copyright__ = '2013, Dale Furrow dkfurrow@gmail.com' +''' +chron.com +''' +import re, time +from calibre.web.feeds.recipes import BasicNewsRecipe +from calibre.utils.date import dt_factory, local_tz +from datetime import datetime, timedelta, date +from lxml import html -from calibre.web.feeds.news import BasicNewsRecipe class HoustonChronicle(BasicNewsRecipe): - title = u'The Houston Chronicle' + title = u'The Houston Chronicle' description = 'News from Houston, Texas' - __author__ = 'Kovid Goyal' - language = 'en' - timefmt = ' [%a, %d %b, %Y]' + __author__ = 'Dale Furrow' + language = 'en' no_stylesheets = True - use_embedded_content = False + # use_embedded_content = False remove_attributes = ['style'] - auto_cleanup = True - - oldest_article = 3.0 - - #keep_only_tags = {'class':lambda x: x and ('hst-articletitle' in x or - #'hst-articletext' in x or 'hst-galleryitem' in x)} + remove_empty_feeds = True + timefmt = '[%a, %d %b %Y]' + timestampfmt = '%Y%m%d%H%M%S' + ignore_duplicate_articles = {'url'} remove_attributes = ['xmlns'] - feeds = [ - ('News', "http://www.chron.com/rss/feed/News-270.php"), - ('Sports', - 'http://www.chron.com/sports/headlines/collectionRss/Sports-Headlines-Staff-Stories-10767.php'), - ('Neighborhood', - 'http://www.chron.com/rss/feed/Neighborhood-305.php'), - ('Business', 'http://www.chron.com/rss/feed/Business-287.php'), - ('Entertainment', - 'http://www.chron.com/rss/feed/Entertainment-293.php'), - ('Editorials', - 'http://www.chron.com/opinion/editorials/collectionRss/Opinion-Editorials-Headline-List-10567.php'), - ('Life', 'http://www.chron.com/rss/feed/Life-297.php'), - ('Science & Tech', - 'http://www.chron.com/rss/feed/AP-Technology-and-Science-266.php'), - ] + remove_tags = [dict(name='div', attrs={'class':'socialBar'}), + dict(name='div', attrs={'class':re.compile('post-commentmeta')}), + dict(name='div', attrs={'class':re.compile('slideshow_wrapper')}), + dict(name='div', attrs={'class':'entry-summary'}), + dict(name='a', attrs={'rel':'item-license'})] + + baseUrl = 'http://www.chron.com' + + oldest_web_article = 7.0 + + if oldest_web_article is None: + earliest_date = date.today() + else: + earliest_date = date.today() - timedelta(days=oldest_web_article) + + pages = [('news' , '/news/houston-texas/'), + ('business' , '/business/'), + ('opinion', '/opinion/'), + ('sports', '/sports/')] + + def getLinksFromSectionPage(self, sectionUrl): + pageDoc = html.parse(sectionUrl) + els = pageDoc.xpath("""//div[contains(@class, 'scp-item') + or @class='scp-feature' or contains(@class, 'simplelist') + or contains(@class, 'scp-blogpromo')] + //a[@href and not(@target) and not(child::img)]""") + elList = [] + for el in els: + link = el.get('href') + title = el.text + if link[:4] != 'http': + link = self.baseUrl + link + if title is not None: + elList.append((link, el.text)) + return elList + + def getArticleDescriptionFromDoc(self, pageDoc): + descriptionCharsBreak = 140 + descriptionMaxChars = 300 + descXpath = """//div[contains(@class, 'article-body') or + contains(@class, 'resource-content') or contains(@class, 'post')]//p""" + sentenceRegex = re.compile("(\S.+?[.!?])(?=\s+|$)") + + def stringify_children(node): + return ''.join([x for x in node.itertext()]) + try: + els = pageDoc.xpath(descXpath) + outText = "" + ellipsis = "" + for el in els: + sentences = re.findall(sentenceRegex, stringify_children(el)) + for sentence in sentences: + if len(outText) < descriptionCharsBreak: + outText += sentence + " " + else: + if len(outText) > descriptionMaxChars: + ellipsis = "..." + return outText[:descriptionMaxChars] + ellipsis + return outText + except: + self.log('Error on Article Description') + return "" + + def getPublishedTimeFromDoc(self, pageDoc): + regexDateOnly = re.compile("""(?:January|February|March|April| + May|June|July|August|September|October|November| + December)\s[0-9]{1,2},\s20[01][0-9]""") + regextTimeOnly = re.compile("""[0-9]{1,2}:[0-9]{1,2} \w{2}""") + def getRegularTimestamp(dateString): + try: + outDate = datetime.strptime(dateString, "%Y-%m-%dT%H:%M:%SZ") + return outDate + except: + return None + def getDateFromString(inText): + match = re.findall(regexDateOnly, inText) + if match: + try: + outDate = datetime.strptime(match[0], "%B %d, %Y") + match = re.findall(regextTimeOnly, inText) + if match: + outTime = datetime.strptime(match[0], "%I:%M %p") + return datetime.combine(outDate.date(), outTime.time()) + return outDate + except: + return None + else: + return None + el = pageDoc.xpath("//*[@class='timestamp'][1]") + if len(el) == 1: + return getRegularTimestamp(el[0].get('title')) + else: + el = pageDoc.xpath("//*[@class='entry-date' or @class='post-date'][1]") + if len(el) == 1: + return getDateFromString(el[0].text_content()) + else: + return None + + def getAllFeedDataFromPage(self, page): + articles = [] + linkList = self.getLinksFromSectionPage(self.baseUrl + page[1]) + self.log('from section: ', page[0], " found ", len(linkList), " links") + for link in linkList: + try: + articleDoc = html.parse(link[0]) + description = self.getArticleDescriptionFromDoc(articleDoc) + articleDate = self.getPublishedTimeFromDoc(articleDoc) + if articleDate is not None and description is not None and articleDate.date() > self.earliest_date: + dateText = articleDate.strftime('%a, %d %b') + author = articleDate.strftime(self.timestampfmt) + articles.append({'title':link[1], 'url':link[0], + 'description':description, 'date':dateText, 'author':author}) + self.log(page[0] + ": " + link[1] + ', from ' + dateText + + " description of " + str(len(description)) + ' characters at ' + link[0]) + else: + msg = "" + if articleDate is None: + msg = " No Timestamp Found" + else: + msg = " article older than " + str(self.oldest_web_article) + ' days...' + self.log("Skipping article: ", link[0], msg) + except: + print 'error on fetching ' + link[0] + continue + return articles + + def parse_index(self): + + self.timefmt = ' [%a, %d %b, %Y]' + self.log('starting parse_index: ', time.strftime(self.timestampfmt)) + feeds = [] + for page in self.pages: + articles = [] + articles = self.getAllFeedDataFromPage(page) + if articles: + feeds.append((page[0], articles)) + self.log('finished parse_index: ', time.strftime(self.timestampfmt)) + return feeds + + def preprocess_html(self, thisSoup): + baseTags = [] + baseTags.extend(thisSoup.findAll(name='div', attrs={'id':re.compile('post-\d+')})) + baseTags.extend(thisSoup.findAll(name='div', attrs={'class':'hnews hentry item'})) + allTags = [] + allTags.extend(baseTags) + if len(baseTags) > 0: + for tag in baseTags: + allTags.extend(tag.findAll(True)) + paragraphs = thisSoup.findAll(name='p') + for paragraph in paragraphs: + if paragraph not in allTags: + allTags.append(paragraph) + for tag in baseTags: + while tag.parent is not None: + allTags.append(tag) + tag = tag.parent + for tag in thisSoup.findAll(True): + if tag not in allTags: + tag.extract() + return thisSoup + + def populate_article_metadata(self, article, soup, first): + if not first: + return + try: + article.date = time.strptime(article.author, self.timestampfmt) + article.utctime = dt_factory(article.date, assume_utc=False, as_utc=False) + article.localtime = article.utctime.astimezone(local_tz) + except Exception as inst: # remove after debug + self.log('Exception: ', article.title) # remove after debug + self.log(type(inst)) # remove after debug + self.log(inst) # remove after debug + + From b6c0716cff07a99c518032e028d71724cbb5f818 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Wed, 10 Jul 2013 17:44:00 +0200 Subject: [PATCH 0133/1154] Add: 1) send metadata to device while connected 2) ability to manually match books on device with books in library --- src/calibre/customize/builtins.py | 7 +- src/calibre/gui2/__init__.py | 2 +- src/calibre/gui2/actions/match_books.py | 37 +++++ src/calibre/gui2/device.py | 10 +- src/calibre/gui2/dialogs/match_books.py | 197 ++++++++++++++++++++++++ src/calibre/gui2/dialogs/match_books.ui | 138 +++++++++++++++++ src/calibre/gui2/layout.py | 4 + src/calibre/gui2/library/models.py | 6 + src/calibre/gui2/ui.py | 1 + 9 files changed, 398 insertions(+), 4 deletions(-) create mode 100644 src/calibre/gui2/actions/match_books.py create mode 100644 src/calibre/gui2/dialogs/match_books.py create mode 100644 src/calibre/gui2/dialogs/match_books.ui diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index e6073a3bd4..8fce645c6e 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -886,6 +886,11 @@ class ActionEditCollections(InterfaceActionBase): actual_plugin = 'calibre.gui2.actions.edit_collections:EditCollectionsAction' description = _('Edit the collections in which books are placed on your device') +class ActionMatchBooks(InterfaceActionBase): + name = 'Match Books' + actual_plugin = 'calibre.gui2.actions.match_books:MatchBookAction' + description = _('Match book on the devices to books in the library') + class ActionCopyToLibrary(InterfaceActionBase): name = 'Copy To Library' actual_plugin = 'calibre.gui2.actions.copy_to_library:CopyToLibraryAction' @@ -936,7 +941,7 @@ plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog, ActionFetchNews, ActionSaveToDisk, ActionQuickview, ActionPolish, ActionShowBookDetails,ActionRestart, ActionOpenFolder, ActionConnectShare, ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks, - ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary, + ActionAddToLibrary, ActionEditCollections, ActionMatchBooks, ActionChooseLibrary, ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore, ActionPluginUpdater, ActionPickRandom, ActionEditToC] diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index a552ad8594..2fd2135956 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -73,7 +73,7 @@ defs['action-layout-context-menu'] = ( defs['action-layout-context-menu-device'] = ( 'View', 'Save To Disk', None, 'Remove Books', None, - 'Add To Library', 'Edit Collections', + 'Add To Library', 'Edit Collections', 'Match Books' ) defs['action-layout-context-menu-cover-browser'] = ( diff --git a/src/calibre/gui2/actions/match_books.py b/src/calibre/gui2/actions/match_books.py new file mode 100644 index 0000000000..28b6afeba9 --- /dev/null +++ b/src/calibre/gui2/actions/match_books.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from calibre.gui2 import error_dialog +from calibre.gui2.actions import InterfaceAction +from calibre.gui2.dialogs.match_books import MatchBooks + +class MatchBookAction(InterfaceAction): + + name = 'Match Books' + action_spec = (_('Match book to library'), 'book.png', + _('Match this book to a book in the library'), + ()) + dont_add_to = frozenset(['menubar', 'toolbar', 'context-menu', 'toolbar-child']) + action_type = 'current' + + def genesis(self): + self.qaction.triggered.connect(self.match_books_in_library) + + def location_selected(self, loc): + enabled = loc != 'library' + self.qaction.setEnabled(enabled) + + def match_books_in_library(self, *args): + view = self.gui.current_view() + rows = view.selectionModel().selectedRows() + if not rows or len(rows) != 1: + d = error_dialog(self.gui, _('Match books'), _('You must select one book')) + d.exec_() + return + + id_ = view.model().indices(rows)[0] + MatchBooks(self.gui, view, id_).exec_() diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 15dc1f0c0a..d3225e66e7 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1604,6 +1604,10 @@ class DeviceMixin(object): # {{{ except: pass + def update_metadata_on_device(self): + self.set_books_in_library(self.booklists(), reset=True, force_send=True) + self.refresh_ondevice() + def book_on_device(self, id, reset=False): ''' Return an indication of whether the given book represented by its db id @@ -1652,7 +1656,8 @@ class DeviceMixin(object): # {{{ loc[4] |= self.book_db_uuid_path_map[id] return loc - def set_books_in_library(self, booklists, reset=False, add_as_step_to_job=None): + def set_books_in_library(self, booklists, reset=False, add_as_step_to_job=None, + force_send=False): ''' Set the ondevice indications in the device database. This method should be called before book_on_device is called, because @@ -1675,7 +1680,8 @@ class DeviceMixin(object): # {{{ x = x.lower() if x else '' return string_pat.sub('', x) - update_metadata = device_prefs['manage_device_metadata'] == 'on_connect' + update_metadata = ( + device_prefs['manage_device_metadata'] == 'on_connect' or force_send) get_covers = False desired_thumbnail_height = 0 diff --git a/src/calibre/gui2/dialogs/match_books.py b/src/calibre/gui2/dialogs/match_books.py new file mode 100644 index 0000000000..847d1cbc70 --- /dev/null +++ b/src/calibre/gui2/dialogs/match_books.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + + +from PyQt4.Qt import (Qt, QDialog, QAbstractItemView, QTableWidgetItem, + QByteArray) + +from calibre.gui2 import gprefs, error_dialog +from calibre.gui2.dialogs.match_books_ui import Ui_MatchBooks +from calibre.utils.icu import sort_key + +class TableItem(QTableWidgetItem): + ''' + A QTableWidgetItem that sorts on a separate string and uses ICU rules + ''' + + def __init__(self, val, sort, idx=0): + self.sort = sort + self.sort_idx = idx + QTableWidgetItem.__init__(self, val) + self.setFlags(Qt.ItemIsEnabled|Qt.ItemIsSelectable) + + def __ge__(self, other): + l = sort_key(self.sort) + r = sort_key(other.sort) + if l > r: + return 1 + if l == r: + return self.sort_idx >= other.sort_idx + return 0 + + def __lt__(self, other): + l = sort_key(self.sort) + r = sort_key(other.sort) + if l < r: + return 1 + if l == r: + return self.sort_idx < other.sort_idx + return 0 + +class MatchBooks(QDialog, Ui_MatchBooks): + + def __init__(self, gui, view, id_): + QDialog.__init__(self, gui, flags=Qt.Window) + Ui_MatchBooks.__init__(self) + self.setupUi(self) + self.isClosed = False + + self.books_table_column_widths = None + try: + self.books_table_column_widths = \ + gprefs.get('match_books_dialog_books_table_widths', None) + geom = gprefs.get('match_books_dialog_geometry', bytearray('')) + self.restoreGeometry(QByteArray(geom)) + except: + pass + + self.search_text.initialize('match_books_dialog') + + # Remove the help button from the window title bar + icon = self.windowIcon() + self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint)) + self.setWindowIcon(icon) + + self.device_db = view.model().db + self.library_db = gui.library_view.model().db + self.view = view + self.gui = gui + self.current_device_book_id = id_ + self.current_library_book_id = None + + # Set up the books table columns + self.books_table.setSelectionBehavior(QAbstractItemView.SelectRows) + self.books_table.setSelectionMode(QAbstractItemView.SingleSelection) + self.books_table.setColumnCount(3) + t = QTableWidgetItem(_('Title')) + self.books_table.setHorizontalHeaderItem(0, t) + t = QTableWidgetItem(_('Authors')) + self.books_table.setHorizontalHeaderItem(1, t) + t = QTableWidgetItem(_('Series')) + self.books_table.setHorizontalHeaderItem(2, t) + self.books_table_header_height = self.books_table.height() + self.books_table.cellDoubleClicked.connect(self.book_doubleclicked) + self.books_table.cellClicked.connect(self.book_clicked) + self.books_table.sortByColumn(0, Qt.AscendingOrder) + + # get the standard table row height. Do this here because calling + # resizeRowsToContents can word wrap long cell contents, creating + # double-high rows + self.books_table.setRowCount(1) + self.books_table.setItem(0, 0, TableItem('A', '')) + self.books_table.resizeRowsToContents() + self.books_table_row_height = self.books_table.rowHeight(0) + self.books_table.setRowCount(0) + + self.search_button.clicked.connect(self.do_search) + self.search_button.setDefault(False) + self.search_text.lineEdit().returnPressed.connect(self.return_pressed) + + self.buttonBox.accepted.connect(self.accept) + self.buttonBox.rejected.connect(self.reject) + self.ignore_next_key = False + + def return_pressed(self): + self.ignore_next_key = True + self.do_search() + + def keyPressEvent(self, e): + if self.ignore_next_key: + self.ignore_next_key = False + else: + QDialog.keyPressEvent(self, e) + + def do_search(self): + query = unicode(self.search_text.text()) + if not query: + d = error_dialog(self.gui, _('Match books'), + _('You must enter a search expression into the search box')) + d.exec_() + return + books = self.library_db.data.search(query, return_matches=True) + self.books_table.setRowCount(len(books)) + + self.books_table.setSortingEnabled(False) + for row, b in enumerate(books): + mi = self.library_db.get_metadata(b, index_is_id=True, get_user_categories=False) + a = TableItem(mi.title, mi.title_sort) + a.setData(Qt.UserRole, b) + self.books_table.setItem(row, 0, a) + a = TableItem(' & '.join(mi.authors), mi.author_sort) + self.books_table.setItem(row, 1, a) + series = mi.format_field('series')[1] + if series is None: + series = '' + a = TableItem(series, mi.series, mi.series_index) + self.books_table.setItem(row, 2, a) + self.books_table.setRowHeight(row, self.books_table_row_height) + + self.books_table.setSortingEnabled(True) + + # Deal with sizing the table columns. Done here because the numbers are not + # correct until the first paint. + def resizeEvent(self, *args): + QDialog.resizeEvent(self, *args) + if self.books_table_column_widths is not None: + for c,w in enumerate(self.books_table_column_widths): + self.books_table.setColumnWidth(c, w) + else: + # the vertical scroll bar might not be rendered, so might not yet + # have a width. Assume 25. Not a problem because user-changed column + # widths will be remembered + w = self.books_table.width() - 25 - self.books_table.verticalHeader().width() + w /= self.books_table.columnCount() + for c in range(0, self.books_table.columnCount()): + self.books_table.setColumnWidth(c, w) + self.save_state() + + def book_clicked(self, row, column): + self.book_selected = True; + id_ = self.books_table.item(row, 0).data(Qt.UserRole).toInt()[0] + self.current_library_book_id = id_ + + def book_doubleclicked(self, row, column): + self.book_clicked(row, column) + self.accept() + + def save_state(self): + self.books_table_column_widths = [] + for c in range(0, self.books_table.columnCount()): + self.books_table_column_widths.append(self.books_table.columnWidth(c)) + gprefs['match_books_dialog_books_table_widths'] = self.books_table_column_widths + gprefs['match_books_dialog_geometry'] = bytearray(self.saveGeometry()) + self.search_text.save_history() + + def close(self): + self.save_state() + # clean up to prevent memory leaks + self.device_db = self.view = self.gui = None + + def accept(self): + if not self.current_library_book_id: + d = error_dialog(self.gui, _('Match books'), + _('You must select a matching book')) + d.exec_() + return + mi = self.library_db.get_metadata(self.current_library_book_id, + index_is_id=True, get_user_categories=False) + self.device_db[self.current_device_book_id].smart_update(mi, replace_metadata=True) + self.device_db[self.current_device_book_id].in_library_waiting = True + self.save_state() + QDialog.accept(self) + + def reject(self): + self.close() + QDialog.reject(self) \ No newline at end of file diff --git a/src/calibre/gui2/dialogs/match_books.ui b/src/calibre/gui2/dialogs/match_books.ui new file mode 100644 index 0000000000..814b3b831d --- /dev/null +++ b/src/calibre/gui2/dialogs/match_books.ui @@ -0,0 +1,138 @@ + + + MatchBooks + + + + 0 + 0 + 768 + 342 + + + + + 0 + 0 + + + + Match Books + + + + + + + 100 + 0 + + + + + 350 + 0 + + + + QComboBox::AdjustToMinimumContentsLengthWithIcon + + + 30 + + + + + + + Search + + + + + + + Do a search to find the book you want to match + + + + + + + + 4 + 0 + + + + 0 + + + 0 + + + + + + + <p>Be sure to update metadata on the device when you are + finished matching books (Device -> Update Metadata)</p> + + + + + + + + + Qt::Horizontal + + + + 0 + 0 + + + + + + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + false + + + + + + + + + + + HistoryLineEdit + QComboBox +
    calibre/gui2/widgets.h
    +
    +
    + + + buttonBox + rejected() + MatchBooks + reject() + + + 297 + 217 + + + 286 + 234 + + + + +
    diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py index 82e127970c..d8d91b99af 100644 --- a/src/calibre/gui2/layout.py +++ b/src/calibre/gui2/layout.py @@ -25,6 +25,7 @@ class LocationManager(QObject): # {{{ unmount_device = pyqtSignal() location_selected = pyqtSignal(object) configure_device = pyqtSignal() + update_device_metadata = pyqtSignal() def __init__(self, parent=None): QObject.__init__(self, parent) @@ -60,6 +61,9 @@ class LocationManager(QObject): # {{{ a = m.addAction(QIcon(I('config.png')), _('Configure this device')) a.triggered.connect(self._configure_requested) self._mem.append(a) + a = m.addAction(QIcon(I('sync.png')), _('Update metadata on device')) + a.triggered.connect(lambda x : self.update_device_metadata.emit()) + self._mem.append(a) else: ac.setToolTip(tooltip) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 324031aff0..3dc85bf46a 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -1207,6 +1207,8 @@ class DeviceBooksModel(BooksModel): # {{{ self.search_engine = OnDeviceSearch(self) self.editable = ['title', 'authors', 'collections'] self.book_in_library = None + self.sync_icon = QIcon(I('sync.png')) + def counts(self): return Counts(len(self.db), len(self.db), len(self.map)) @@ -1535,6 +1537,8 @@ class DeviceBooksModel(BooksModel): # {{{ elif DEBUG and cname == 'inlibrary': return QVariant(self.db[self.map[row]].in_library) elif role == Qt.ToolTipRole and index.isValid(): + if col == 0 and hasattr(self.db[self.map[row]], 'in_library_waiting'): + return QVariant(_('Waiting for metadata to be updated')) if self.is_row_marked_for_deletion(row): return QVariant(_('Marked for deletion')) if cname in ['title', 'authors'] or (cname == 'collections' and @@ -1543,6 +1547,8 @@ class DeviceBooksModel(BooksModel): # {{{ elif role == Qt.DecorationRole and cname == 'inlibrary': if self.db[self.map[row]].in_library: return QVariant(self.bool_yes_icon) + elif hasattr(self.db[self.map[row]], 'in_library_waiting'): + return QVariant(self.sync_icon) elif self.db[self.map[row]].in_library is not None: return QVariant(self.bool_no_icon) elif role == Qt.TextAlignmentRole: diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index c9f07ef6f6..06cb4d904f 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -293,6 +293,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.location_manager.location_selected.connect(self.location_selected) self.location_manager.unmount_device.connect(self.device_manager.umount_device) self.location_manager.configure_device.connect(self.configure_connected_device) + self.location_manager.update_device_metadata.connect(self.update_metadata_on_device) self.eject_action.triggered.connect(self.device_manager.umount_device) #################### Update notification ################### From a0c9d4621423c690df49ae1f1d203f73a9b42327 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Wed, 10 Jul 2013 17:44:23 +0200 Subject: [PATCH 0134/1154] Make sure eclipse files are ignored --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 090d11fd24..192b503429 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,6 @@ build dist docs resources/localization -resources/images.qrc resources/scripts.pickle resources/ebook-convert-complete.pickle resources/builtin_recipes.xml @@ -42,3 +41,4 @@ calibre_plugins/ recipes/*.mobi recipes/*.epub recipes/debug +/.metadata/ From 6c310278c989ef5039e241d82169d9cc0b3464a2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 11 Jul 2013 10:41:32 +0530 Subject: [PATCH 0135/1154] Log what type of ToC is used --- src/calibre/ebooks/docx/to_html.py | 2 +- src/calibre/ebooks/docx/toc.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index be0576d2b9..05b4b7e9d3 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -279,7 +279,7 @@ class Convert(object): self.styles.resolve_numbering(numbering) def write(self, doc): - toc = create_toc(doc, self.body, self.resolved_link_map, self.styles, self.object_map) + toc = create_toc(doc, self.body, self.resolved_link_map, self.styles, self.object_map, self.log) raw = html.tostring(self.html, encoding='utf-8', doctype='') with open(os.path.join(self.dest_dir, 'index.html'), 'wb') as f: f.write(raw) diff --git a/src/calibre/ebooks/docx/toc.py b/src/calibre/ebooks/docx/toc.py index 5936d34355..314dc2479d 100644 --- a/src/calibre/ebooks/docx/toc.py +++ b/src/calibre/ebooks/docx/toc.py @@ -21,7 +21,7 @@ class Count(object): def __init__(self): self.val = 0 -def from_headings(body): +def from_headings(body, log): ' Create a TOC from headings in the document ' headings = ('h1', 'h2', 'h3') tocroot = TOC() @@ -58,6 +58,7 @@ def from_headings(body): level_prev[i] = None if len(tuple(tocroot.flat())) > 1: + log('Generating Table of Contents from headings') return tocroot def structure_toc(entries): @@ -98,7 +99,7 @@ def link_to_txt(a, styles, object_map): return tostring(a, method='text', with_tail=False, encoding=unicode).strip() -def from_toc(docx, link_map, styles, object_map): +def from_toc(docx, link_map, styles, object_map, log): toc_level = None level = 0 TI = namedtuple('TI', 'text anchor indent') @@ -132,9 +133,10 @@ def from_toc(docx, link_map, styles, object_map): ml = 0 toc.append(TI(txt, href[1:], ml)) if toc: + log('Found Word Table of Contents, using it to generate the Table of Contents') return structure_toc(toc) -def create_toc(docx, body, link_map, styles, object_map): - return from_toc(docx, link_map, styles, object_map) or from_headings(body) +def create_toc(docx, body, link_map, styles, object_map, log): + return from_toc(docx, link_map, styles, object_map, log) or from_headings(body, log) From 278a13abc3300a5cf034e9d2447b070cf5ab97fa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 11 Jul 2013 15:08:20 +0530 Subject: [PATCH 0136/1154] Bulk metadata edit: option to not refresh after edit Bulk metadata edit: Add a checkbox to prevent the refreshing of the book list after the bulk edit. This means that the book list will not be resorted and any existing search/virtual library will not be refreshed. Useful if you have a large library as the refresh can be slow. --- src/calibre/gui2/__init__.py | 1 + src/calibre/gui2/actions/edit_metadata.py | 9 ++++-- src/calibre/gui2/dialogs/metadata_bulk.py | 5 +++ src/calibre/gui2/dialogs/metadata_bulk.ui | 37 ++++++++++++++++------- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 2fd2135956..7b88a943bb 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -110,6 +110,7 @@ defs['bd_overlay_cover_size'] = False defs['tags_browser_category_icons'] = {} defs['cover_browser_reflections'] = True defs['extra_row_spacing'] = 0 +defs['refresh_book_list_on_bulk_edit'] = True del defs # }}} diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 729de33c7f..4817db953c 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -10,7 +10,7 @@ from functools import partial from PyQt4.Qt import QMenu, QModelIndex, QTimer, QIcon -from calibre.gui2 import error_dialog, Dispatcher, question_dialog +from calibre.gui2 import error_dialog, Dispatcher, question_dialog, gprefs from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog from calibre.gui2.dialogs.confirm_delete import confirm from calibre.gui2.dialogs.device_category_editor import DeviceCategoryEditor @@ -366,8 +366,11 @@ class EditMetadataAction(InterfaceAction): self.gui.tags_view.blockSignals(False) if changed: m = self.gui.library_view.model() - m.refresh(reset=False) - m.research() + if gprefs['refresh_book_list_on_bulk_edit']: + m.refresh(reset=False) + m.research() + else: + m.refresh_ids(book_ids) self.gui.tags_view.recount() if self.gui.cover_flow: self.gui.cover_flow.dataChanged() diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 84ca9135f0..0e4b2a1572 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -317,6 +317,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): Ui_MetadataBulkDialog.__init__(self) self.model = model self.db = model.db + self.refresh_book_list.setChecked(gprefs['refresh_book_list_on_bulk_edit']) + self.refresh_book_list.toggled.connect(self.save_refresh_booklist) self.ids = [self.db.id(r) for r in rows] self.box_title.setText('

    ' + _('Editing meta information for %d books') % @@ -380,6 +382,9 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.authors.setFocus(Qt.OtherFocusReason) self.exec_() + def save_refresh_booklist(self, *args): + gprefs['refresh_book_list_on_bulk_edit'] = bool(self.refresh_book_list.isChecked()) + def save_state(self, *args): gprefs['bulk_metadata_window_geometry'] = \ bytearray(self.saveGeometry()) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index b03f3e8e94..2b48e635be 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -45,7 +45,7 @@ 0 0 950 - 576 + 577 @@ -1113,7 +1113,7 @@ not multiple and the destination field is multiple 0 0 934 - 213 + 256 @@ -1170,14 +1170,30 @@ not multiple and the destination field is multiple - - - Qt::Horizontal - - - QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Ok - - + + + + + If enabled, the book list will be re-sorted and any existing +search or Virtual LIbrary will be refreshed after the edit +is completed. This can be slow on large libraries. + + + &Refresh book list after edit + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + @@ -1241,7 +1257,6 @@ not multiple and the destination field is multiple scrollArea central_widget query_field - button_box save_button remove_button search_field From c1be9b1ecdc45ad74ef398d0deb787da7ae6f916 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 11 Jul 2013 16:13:28 +0530 Subject: [PATCH 0137/1154] Legacy wrappers for add_books() and create_book_entry() --- src/calibre/db/legacy.py | 20 ++++++++++++++++- src/calibre/db/tests/legacy.py | 39 ++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 2ad5da61b8..809ec816a8 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -8,6 +8,7 @@ __copyright__ = '2013, Kovid Goyal ' import os, traceback from functools import partial +from future_builtins import zip from calibre.db import _get_next_series_num_for_list, _get_series_values from calibre.db.backend import DB @@ -150,7 +151,7 @@ class LibraryDatabase(object): def path(self, index, index_is_id=False): 'Return the relative path to the directory containing this books files as a unicode string.' book_id = index if index_is_id else self.data.index_to_id(index) - return self.data.cache.field_for('path', book_id).replace('/', os.sep) + return self.new_api.field_for('path', book_id).replace('/', os.sep) def abspath(self, index, index_is_id=False, create_dirs=True): 'Return the absolute path to the directory containing this books files as a unicode string.' @@ -159,6 +160,23 @@ class LibraryDatabase(object): os.makedirs(path) return path + def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None): + return self.new_api.create_book_entry(mi, cover=cover, add_duplicates=add_duplicates, force_id=force_id) + + def add_books(self, paths, formats, metadata, add_duplicates=True, return_ids=False): + books = [(mi, {fmt:path}) for mi, path, fmt in zip(metadata, paths, formats)] + book_ids, duplicates = self.new_api.add_books(books, add_duplicates=add_duplicates, dbapi=self) + if duplicates: + paths, formats, metadata = [], [], [] + for mi, format_map in duplicates: + metadata.append(mi) + for fmt, path in format_map.iteritems(): + formats.append(fmt) + paths.append(path) + duplicates = (paths, formats, metadata) + ids = book_ids if return_ids else len(book_ids) + return duplicates or None, ids + # Private interface {{{ def __iter__(self): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index ae99d8190f..5735d77ff4 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -7,8 +7,27 @@ __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' import inspect +from repr import repr +from functools import partial +from tempfile import NamedTemporaryFile + from calibre.db.tests.base import BaseTest +class ET(object): + + def __init__(self, func_name, args, kwargs={}, old=None, legacy=None): + self.func_name = func_name + self.args, self.kwargs = args, kwargs + self.old, self.legacy = old, legacy + + def __call__(self, test): + old = self.old or test.init_old(test.cloned_library) + legacy = self.legacy or test.init_legacy(test.cloned_library) + oldres = getattr(old, self.func_name)(*self.args, **self.kwargs) + newres = getattr(legacy, self.func_name)(*self.args, **self.kwargs) + test.assertEqual(oldres, newres, 'Equivalence test for %s with args: %s and kwargs: %s failed' % ( + self.func_name, repr(self.args), repr(self.kwargs))) + class LegacyTest(BaseTest): ''' Test the emulation of the legacy interface. ''' @@ -119,6 +138,26 @@ class LegacyTest(BaseTest): db.close() # }}} + def test_legacy_adding_books(self): # {{{ + 'Test various adding books methods' + from calibre.ebooks.metadata.book.base import Metadata + legacy, old = self.init_legacy(self.cloned_library), self.init_old(self.cloned_library) + mi = Metadata('Added Book', authors=('Added Author',)) + with NamedTemporaryFile(suffix='.aff') as f: + f.write(b'xxx') + f.flush() + T = partial(ET, 'add_books', ([f.name], ['AFF'], [mi]), old=old, legacy=legacy) + T()(self) + T(kwargs={'return_ids':True})(self) + T(kwargs={'add_duplicates':False})(self) + + mi.title = 'Added Book2' + T = partial(ET, 'create_book_entry', (mi,), old=old, legacy=legacy) + T() + T({'add_duplicates':False}) + T({'force_id':1000}) + # }}} + def test_legacy_coverage(self): # {{{ ' Check that the emulation of the legacy interface is (almost) total ' cl = self.cloned_library From 1ac522d94e431473272f8481176e25228e24cfc5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 11 Jul 2013 16:58:03 +0530 Subject: [PATCH 0138/1154] Improve coverage test --- src/calibre/db/tests/legacy.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 5735d77ff4..ddf6833bee 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -27,6 +27,13 @@ class ET(object): newres = getattr(legacy, self.func_name)(*self.args, **self.kwargs) test.assertEqual(oldres, newres, 'Equivalence test for %s with args: %s and kwargs: %s failed' % ( self.func_name, repr(self.args), repr(self.kwargs))) + self.retval = newres + return newres + +def compare_argspecs(old, new, attr): + ok = len(old.args) == len(new.args) and len(old.defaults or ()) == len(new.defaults or ()) and old.args[-len(old.defaults or ()):] == new.args[-len(new.defaults or ()):] # noqa + if not ok: + raise AssertionError('The argspec for %s does not match. %r != %r' % (attr, old, new)) class LegacyTest(BaseTest): @@ -169,15 +176,20 @@ class LegacyTest(BaseTest): '_set_title', '_set_custom', '_update_author_in_cache', } SKIP_ARGSPEC = { - '__init__', + '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', } + missing = [] + try: + total = 0 for attr in dir(db): if attr in SKIP_ATTRS: continue + total += 1 if not hasattr(ndb, attr): - raise AssertionError('The attribute %s is missing' % attr) + missing.append(attr) + continue obj, nobj = getattr(db, attr), getattr(ndb, attr) if attr not in SKIP_ARGSPEC: try: @@ -185,11 +197,15 @@ class LegacyTest(BaseTest): except TypeError: pass else: - self.assertEqual(argspec, inspect.getargspec(nobj), 'argspec for %s not the same' % attr) + compare_argspecs(argspec, inspect.getargspec(nobj), attr) finally: for db in (ndb, db): db.close() db.break_cycles() + if missing: + pc = len(missing)/total + raise AssertionError('{0:.1%} of API ({2} attrs) are missing. For example: {1}'.format(pc, missing[0], len(missing))) + # }}} From 213a2136cf520c2d600843456fd237150cb4b13f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 11 Jul 2013 17:18:45 +0530 Subject: [PATCH 0139/1154] Implement import_book() --- src/calibre/db/cache.py | 4 ++-- src/calibre/db/legacy.py | 15 ++++++++++++++- src/calibre/db/tests/legacy.py | 19 +++++++++++++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index b94258b1ef..b8b9ad7436 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1117,7 +1117,7 @@ class Cache(object): return book_id @write_api - def add_books(self, books, add_duplicates=True, apply_import_tags=True, preserve_uuid=False, dbapi=None): + def add_books(self, books, add_duplicates=True, apply_import_tags=True, preserve_uuid=False, run_hooks=True, dbapi=None): duplicates, ids = [], [] for mi, format_map in books: book_id = self._create_book_entry(mi, add_duplicates=add_duplicates, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid) @@ -1126,7 +1126,7 @@ class Cache(object): else: ids.append(book_id) for fmt, stream_or_path in format_map.iteritems(): - self._add_format(book_id, fmt, stream_or_path, dbapi=dbapi) + self._add_format(book_id, fmt, stream_or_path, dbapi=dbapi, run_hooks=run_hooks) return ids, duplicates @write_api diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 809ec816a8..39d15f604e 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -37,7 +37,7 @@ class LibraryDatabase(object): progress_callback=lambda x, y:True, restore_all_prefs=False): self.is_second_db = is_second_db # TODO: Use is_second_db - self.listeners = set([]) + self.listeners = set() backend = self.backend = DB(library_path, default_prefs=default_prefs, read_only=read_only, restore_all_prefs=restore_all_prefs, @@ -177,6 +177,19 @@ class LibraryDatabase(object): ids = book_ids if return_ids else len(book_ids) return duplicates or None, ids + def import_book(self, mi, formats, notify=True, import_hooks=True, apply_import_tags=True, preserve_uuid=False): + format_map = {} + for path in formats: + ext = os.path.splitext(path)[1][1:].upper() + if ext == 'OPF': + continue + format_map[ext] = path + book_ids, duplicates = self.new_api.add_books( + [(mi, format_map)], add_duplicates=True, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid, dbapi=self, run_hooks=import_hooks) + if notify: + self.notify('add', book_ids) + return book_ids[0] + # Private interface {{{ def __iter__(self): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index ddf6833bee..cf3bd93d9d 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -149,14 +149,29 @@ class LegacyTest(BaseTest): 'Test various adding books methods' from calibre.ebooks.metadata.book.base import Metadata legacy, old = self.init_legacy(self.cloned_library), self.init_old(self.cloned_library) - mi = Metadata('Added Book', authors=('Added Author',)) + mi = Metadata('Added Book0', authors=('Added Author',)) with NamedTemporaryFile(suffix='.aff') as f: f.write(b'xxx') f.flush() T = partial(ET, 'add_books', ([f.name], ['AFF'], [mi]), old=old, legacy=legacy) T()(self) - T(kwargs={'return_ids':True})(self) + book_id = T(kwargs={'return_ids':True})(self)[1][0] + self.assertEqual(legacy.new_api.formats(book_id), ('AFF',)) T(kwargs={'add_duplicates':False})(self) + mi.title = 'Added Book1' + mi.uuid = 'uuu' + T = partial(ET, 'import_book', (mi,[f.name]), old=old, legacy=legacy) + book_id = T()(self) + self.assertNotEqual(legacy.uuid(book_id, index_is_id=True), old.uuid(book_id, index_is_id=True)) + book_id = T(kwargs={'preserve_uuid':True})(self) + self.assertEqual(legacy.uuid(book_id, index_is_id=True), old.uuid(book_id, index_is_id=True)) + self.assertEqual(legacy.new_api.formats(book_id), ('AFF',)) + with NamedTemporaryFile(suffix='.opf') as f: + f.write(b'zzzz') + f.flush() + T = partial(ET, 'import_book', (mi,[f.name]), old=old, legacy=legacy) + book_id = T()(self) + self.assertFalse(legacy.new_api.formats(book_id)) mi.title = 'Added Book2' T = partial(ET, 'create_book_entry', (mi,), old=old, legacy=legacy) From 396c4a61313c7fef8e86849913a1e00b4e7fb393 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 11 Jul 2013 17:54:37 +0530 Subject: [PATCH 0140/1154] Finish implementation of the rest of the import books API --- src/calibre/db/adding.py | 102 +++++++++++++++++++++++++++++++ src/calibre/db/legacy.py | 17 ++++++ src/calibre/library/database2.py | 88 ++------------------------ 3 files changed, 125 insertions(+), 82 deletions(-) create mode 100644 src/calibre/db/adding.py diff --git a/src/calibre/db/adding.py b/src/calibre/db/adding.py new file mode 100644 index 0000000000..34ac84c493 --- /dev/null +++ b/src/calibre/db/adding.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +import os +from calibre.ebooks import BOOK_EXTENSIONS + +def find_books_in_directory(dirpath, single_book_per_directory): + dirpath = os.path.abspath(dirpath) + if single_book_per_directory: + formats = [] + for path in os.listdir(dirpath): + path = os.path.abspath(os.path.join(dirpath, path)) + if os.path.isdir(path) or not os.access(path, os.R_OK): + continue + ext = os.path.splitext(path)[1] + if not ext: + continue + ext = ext[1:].lower() + if ext not in BOOK_EXTENSIONS and ext != 'opf': + continue + formats.append(path) + yield formats + else: + books = {} + for path in os.listdir(dirpath): + path = os.path.abspath(os.path.join(dirpath, path)) + if os.path.isdir(path) or not os.access(path, os.R_OK): + continue + ext = os.path.splitext(path)[1] + if not ext: + continue + ext = ext[1:].lower() + if ext not in BOOK_EXTENSIONS: + continue + + key = os.path.splitext(path)[0] + if key not in books: + books[key] = [] + books[key].append(path) + + for formats in books.values(): + yield formats + +def import_book_directory_multiple(db, dirpath, callback=None, + added_ids=None): + from calibre.ebooks.metadata.meta import metadata_from_formats + + duplicates = [] + for formats in find_books_in_directory(dirpath, False): + mi = metadata_from_formats(formats) + if mi.title is None: + continue + if db.has_book(mi): + duplicates.append((mi, formats)) + continue + book_id = db.import_book(mi, formats) + if added_ids is not None: + added_ids.add(book_id) + if callable(callback): + if callback(mi.title): + break + return duplicates + +def import_book_directory(db, dirpath, callback=None, added_ids=None): + from calibre.ebooks.metadata.meta import metadata_from_formats + dirpath = os.path.abspath(dirpath) + formats = find_books_in_directory(dirpath, True) + formats = list(formats)[0] + if not formats: + return + mi = metadata_from_formats(formats) + if mi.title is None: + return + if db.has_book(mi): + return [(mi, formats)] + book_id = db.import_book(mi, formats) + if added_ids is not None: + added_ids.add(book_id) + if callable(callback): + callback(mi.title) + +def recursive_import(db, root, single_book_per_directory=True, + callback=None, added_ids=None): + root = os.path.abspath(root) + duplicates = [] + for dirpath in os.walk(root): + res = (import_book_directory(db, dirpath[0], callback=callback, + added_ids=added_ids) if single_book_per_directory else + import_book_directory_multiple(db, dirpath[0], + callback=callback, added_ids=added_ids)) + if res is not None: + duplicates.extend(res) + if callable(callback): + if callback(''): + break + return duplicates + diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 39d15f604e..e75ea12169 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -11,6 +11,7 @@ from functools import partial from future_builtins import zip from calibre.db import _get_next_series_num_for_list, _get_series_values +from calibre.db.adding import find_books_in_directory, import_book_directory_multiple, import_book_directory, recursive_import from calibre.db.backend import DB from calibre.db.cache import Cache from calibre.db.categories import CATEGORY_SORTS @@ -160,6 +161,7 @@ class LibraryDatabase(object): os.makedirs(path) return path + # Adding books {{{ def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None): return self.new_api.create_book_entry(mi, cover=cover, add_duplicates=add_duplicates, force_id=force_id) @@ -190,6 +192,21 @@ class LibraryDatabase(object): self.notify('add', book_ids) return book_ids[0] + def find_books_in_directory(self, dirpath, single_book_per_directory): + return find_books_in_directory(dirpath, single_book_per_directory) + + def import_book_directory_multiple(self, dirpath, callback=None, + added_ids=None): + return import_book_directory_multiple(self, dirpath, callback=callback, added_ids=added_ids) + + def import_book_directory(self, dirpath, callback=None, added_ids=None): + return import_book_directory(self, dirpath, callback=callback, added_ids=added_ids) + + def recursive_import(self, root, single_book_per_directory=True, + callback=None, added_ids=None): + return recursive_import(self, root, single_book_per_directory=single_book_per_directory, callback=callback, added_ids=added_ids) + # }}} + # Private interface {{{ def __iter__(self): diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 435d8edeeb..0540e8ede4 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -37,11 +37,12 @@ from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp, from calibre.utils.config import prefs, tweaks, from_json, to_json from calibre.utils.icu import sort_key, strcmp, lower from calibre.utils.search_query_parser import saved_searches, set_saved_searches -from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format +from calibre.ebooks import check_ebook_format from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.formatter_functions import load_user_template_functions from calibre.db import _get_next_series_num_for_list, _get_series_values +from calibre.db.adding import find_books_in_directory, import_book_directory_multiple, import_book_directory, recursive_import from calibre.db.errors import NoSuchFormat from calibre.db.lazy import FormatMetadata, FormatsList from calibre.db.categories import Tag, CATEGORY_SORTS @@ -3728,95 +3729,18 @@ books_series_link feeds return len(books) def find_books_in_directory(self, dirpath, single_book_per_directory): - dirpath = os.path.abspath(dirpath) - if single_book_per_directory: - formats = [] - for path in os.listdir(dirpath): - path = os.path.abspath(os.path.join(dirpath, path)) - if os.path.isdir(path) or not os.access(path, os.R_OK): - continue - ext = os.path.splitext(path)[1] - if not ext: - continue - ext = ext[1:].lower() - if ext not in BOOK_EXTENSIONS and ext != 'opf': - continue - formats.append(path) - yield formats - else: - books = {} - for path in os.listdir(dirpath): - path = os.path.abspath(os.path.join(dirpath, path)) - if os.path.isdir(path) or not os.access(path, os.R_OK): - continue - ext = os.path.splitext(path)[1] - if not ext: - continue - ext = ext[1:].lower() - if ext not in BOOK_EXTENSIONS: - continue - - key = os.path.splitext(path)[0] - if key not in books: - books[key] = [] - books[key].append(path) - - for formats in books.values(): - yield formats + return find_books_in_directory(dirpath, single_book_per_directory) def import_book_directory_multiple(self, dirpath, callback=None, added_ids=None): - from calibre.ebooks.metadata.meta import metadata_from_formats - - duplicates = [] - for formats in self.find_books_in_directory(dirpath, False): - mi = metadata_from_formats(formats) - if mi.title is None: - continue - if self.has_book(mi): - duplicates.append((mi, formats)) - continue - book_id = self.import_book(mi, formats) - if added_ids is not None: - added_ids.add(book_id) - if callable(callback): - if callback(mi.title): - break - return duplicates + return import_book_directory_multiple(self, dirpath, callback=callback, added_ids=added_ids) def import_book_directory(self, dirpath, callback=None, added_ids=None): - from calibre.ebooks.metadata.meta import metadata_from_formats - dirpath = os.path.abspath(dirpath) - formats = self.find_books_in_directory(dirpath, True) - formats = list(formats)[0] - if not formats: - return - mi = metadata_from_formats(formats) - if mi.title is None: - return - if self.has_book(mi): - return [(mi, formats)] - book_id = self.import_book(mi, formats) - if added_ids is not None: - added_ids.add(book_id) - if callable(callback): - callback(mi.title) + return import_book_directory(self, dirpath, callback=callback, added_ids=added_ids) def recursive_import(self, root, single_book_per_directory=True, callback=None, added_ids=None): - root = os.path.abspath(root) - duplicates = [] - for dirpath in os.walk(root): - res = (self.import_book_directory(dirpath[0], callback=callback, - added_ids=added_ids) if single_book_per_directory else - self.import_book_directory_multiple(dirpath[0], - callback=callback, added_ids=added_ids)) - if res is not None: - duplicates.extend(res) - if callable(callback): - if callback(''): - break - return duplicates + return recursive_import(self, root, single_book_per_directory=single_book_per_directory, callback=callback, added_ids=added_ids) def add_custom_book_data(self, book_id, name, val): x = self.conn.get('SELECT id FROM books WHERE ID=?', (book_id,), all=False) From bb94f0007b853b39925c0847f929ca7d152dfad6 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Thu, 11 Jul 2013 15:44:38 +0200 Subject: [PATCH 0141/1154] In device view, make "waiting to sync" icon have precedence over "matched" icon --- src/calibre/gui2/library/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 3dc85bf46a..af293b864d 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -1545,10 +1545,10 @@ class DeviceBooksModel(BooksModel): # {{{ self.db.supports_collections()): return QVariant(_("Double click to edit me

    ")) elif role == Qt.DecorationRole and cname == 'inlibrary': - if self.db[self.map[row]].in_library: - return QVariant(self.bool_yes_icon) - elif hasattr(self.db[self.map[row]], 'in_library_waiting'): + if hasattr(self.db[self.map[row]], 'in_library_waiting'): return QVariant(self.sync_icon) + elif self.db[self.map[row]].in_library: + return QVariant(self.bool_yes_icon) elif self.db[self.map[row]].in_library is not None: return QVariant(self.bool_no_icon) elif role == Qt.TextAlignmentRole: From 477767f2ce3d68528f0aee600af7f18a5a14faac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gae=CC=88tan=20Lehmann?= Date: Thu, 11 Jul 2013 15:53:27 +0200 Subject: [PATCH 0142/1154] add acrimed.org recipe --- recipes/acrimed.recipe | 24 ++++++++++++++++++++++++ recipes/icons/acrimed.png | Bin 0 -> 709 bytes 2 files changed, 24 insertions(+) create mode 100644 recipes/acrimed.recipe create mode 100644 recipes/icons/acrimed.png diff --git a/recipes/acrimed.recipe b/recipes/acrimed.recipe new file mode 100644 index 0000000000..66e3e732bb --- /dev/null +++ b/recipes/acrimed.recipe @@ -0,0 +1,24 @@ +__license__ = 'GPL v3' +__copyright__ = '2012' +''' +acrimed.org +''' + +class Acrimed(BasicNewsRecipe): + title = u'Acrimed' + __author__ = 'Gaëtan Lehmann' + oldest_article = 30 + max_articles_per_feed = 100 + auto_cleanup = True + auto_cleanup_keep = '//div[@class="crayon article-chapo-4112 chapo"]' + language = 'fr' + masthead_url = 'http://www.acrimed.org/IMG/siteon0.gif' + feeds = [(u'Acrimed', u'http://www.acrimed.org/spip.php?page=backend')] + + preprocess_regexps = [ + (re.compile(r'(.*) - Acrimed \| Action Critique M.*dias'), lambda m: '' + m.group(1) + ''), + (re.compile(r'

    (.*) - Acrimed \| Action Critique M.*dias

    '), lambda m: '

    ' + m.group(1) + '

    ')] + + extra_css = """ + .chapo{font-style:italic; margin: 1em 0 0.5em} + """ diff --git a/recipes/icons/acrimed.png b/recipes/icons/acrimed.png new file mode 100644 index 0000000000000000000000000000000000000000..b88f368d3ca856f4b1efc14a62323bb1a82264b0 GIT binary patch literal 709 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!63?wyl`GbKJOS+@4BLl<6e(pbstU$g(vPY0F z14ES>14Ba#1H&(%P{RubhEf9thF1v;3|2E37{m+a>nyLa#5=H?a<6x7qxGd4Cpefo5Gc=&?{4}f0L)zwW* zOtiDJQ&Li@tE-zmd-lhVA0;Ft;^PyFi;E{toOtBOk<`@G#KffX^75ZQe+C5jZ```*1afYAx;TbNT=qTpGVG8859@=M0tr!G%}lObH-gwwg)bPT zin4}>ggp3rzjZ_9p8b>Ft2~c6W+K;EpLj7M<_+6j519i#MR|(tS8i5T@y}c{clQbF z*8S5LwYj~#bo2-3o=wlg|C9tB>kBv=6MKDup0w|+AQs){%kDA;nrx1J5yV*bdS=KI zz4o#T!HSOc3tZS4t`?ZgyCdu)KV<<^|GS)jU;a1=?AiCw+Cg#Sj|q9EOgwhI>s34d z{eApme+gq4fA30;uvz-kYuKl3)oJH3e|v$8??TYLB(uX4e&sVxeJRdam;1~O=t*k Wacl6}cIZ7&1B0ilpUXO@geCyLCONwR literal 0 HcmV?d00001 From 937f475a9ef35b062afd93cdcd1457451b400447 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 11 Jul 2013 21:20:20 +0530 Subject: [PATCH 0143/1154] Implement adding of catalogs and news --- src/calibre/db/adding.py | 63 ++++++++++++++++++++++++++++++++++ src/calibre/db/legacy.py | 11 +++++- src/calibre/db/search.py | 8 ++--- src/calibre/db/tests/legacy.py | 19 ++++++++++ 4 files changed, 96 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/adding.py b/src/calibre/db/adding.py index 34ac84c493..d777e7065e 100644 --- a/src/calibre/db/adding.py +++ b/src/calibre/db/adding.py @@ -100,3 +100,66 @@ def recursive_import(db, root, single_book_per_directory=True, break return duplicates +def add_catalog(cache, path, title): + from calibre.ebooks.metadata.book.base import Metadata + from calibre.ebooks.metadata.meta import get_metadata + from calibre.utils.date import utcnow + + fmt = os.path.splitext(path)[1][1:].lower() + with lopen(path, 'rb') as stream, cache.write_lock: + matches = cache._search('title:="%s" and tags:="%s"' % (title.replace('"', '\\"'), _('Catalog')), None) + db_id = None + if matches: + db_id = list(matches)[0] + try: + mi = get_metadata(stream, fmt) + mi.authors = ['calibre'] + except: + mi = Metadata(title, ['calibre']) + mi.title, mi.authors = title, ['calibre'] + mi.tags = [_('Catalog')] + mi.pubdate = mi.timestamp = utcnow() + if fmt == 'mobi': + mi.cover, mi.cover_data = None, (None, None) + if db_id is None: + db_id = cache._create_book_entry(mi, apply_import_tags=False) + else: + cache._set_metadata(db_id, mi) + cache._add_format(db_id, fmt, stream) + + return db_id + +def add_news(cache, path, arg): + from calibre.ebooks.metadata.meta import get_metadata + from calibre.utils.date import utcnow + + fmt = os.path.splitext(getattr(path, 'name', path))[1][1:].lower() + stream = path if hasattr(path, 'read') else lopen(path, 'rb') + stream.seek(0) + mi = get_metadata(stream, fmt, use_libprs_metadata=False, + force_read_metadata=True) + # Force the author to calibre as the auto delete of old news checks for + # both the author==calibre and the tag News + mi.authors = ['calibre'] + stream.seek(0) + with cache.write_lock: + if mi.series_index is None: + mi.series_index = cache._get_next_series_num_for(mi.series) + mi.tags = [_('News')] + if arg['add_title_tag']: + mi.tags += [arg['title']] + if arg['custom_tags']: + mi.tags += arg['custom_tags'] + if mi.pubdate is None: + mi.pubdate = utcnow() + if mi.timestamp is None: + mi.timestamp = utcnow() + + db_id = cache._create_book_entry(mi, apply_import_tags=False) + cache._add_format(db_id, fmt, stream) + + if not hasattr(path, 'read'): + stream.close() + return db_id + + diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index e75ea12169..b9f23cc5d7 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -11,7 +11,9 @@ from functools import partial from future_builtins import zip from calibre.db import _get_next_series_num_for_list, _get_series_values -from calibre.db.adding import find_books_in_directory, import_book_directory_multiple, import_book_directory, recursive_import +from calibre.db.adding import ( + find_books_in_directory, import_book_directory_multiple, + import_book_directory, recursive_import, add_catalog, add_news) from calibre.db.backend import DB from calibre.db.cache import Cache from calibre.db.categories import CATEGORY_SORTS @@ -205,6 +207,13 @@ class LibraryDatabase(object): def recursive_import(self, root, single_book_per_directory=True, callback=None, added_ids=None): return recursive_import(self, root, single_book_per_directory=single_book_per_directory, callback=callback, added_ids=added_ids) + + def add_catalog(self, path, title): + return add_catalog(self.new_api, path, title) + + def add_news(self, path, arg): + return add_news(self.new_api, path, arg) + # }}} # Private interface {{{ diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 4a6eace0f7..7b4ad90bc3 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -526,7 +526,7 @@ class Parser(SearchQueryParser): if dt == 'bool': return self.bool_search(icu_lower(query), partial(self.field_iter, location, candidates), - self.dbcache.pref('bools_are_tristate')) + self.dbcache._pref('bools_are_tristate')) # special case: colon-separated fields such as identifiers. isbn # is a special case within the case @@ -630,7 +630,7 @@ class Parser(SearchQueryParser): if len(query) < 2: return matches - user_cats = self.dbcache.pref('user_categories') + user_cats = self.dbcache._pref('user_categories') c = set(candidates) if query.startswith('.'): @@ -674,7 +674,7 @@ class Search(object): if search_restriction: q = u'(%s) and (%s)' % (search_restriction, query) - all_book_ids = dbcache.all_book_ids(type=set) + all_book_ids = dbcache._all_book_ids(type=set) if not q: return all_book_ids @@ -686,7 +686,7 @@ class Search(object): # takes 0.000975 seconds and restoring it from a pickle takes # 0.000974 seconds. sqp = Parser( - dbcache, all_book_ids, dbcache.pref('grouped_search_terms'), + dbcache, all_book_ids, dbcache._pref('grouped_search_terms'), self.date_search, self.num_search, self.bool_search, self.keypair_search, prefs['limit_search_columns'], diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index cf3bd93d9d..af16c7c5eb 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -178,6 +178,25 @@ class LegacyTest(BaseTest): T() T({'add_duplicates':False}) T({'force_id':1000}) + + with NamedTemporaryFile(suffix='.txt') as f: + f.write(b'tttttt') + f.seek(0) + bid = legacy.add_catalog(f.name, 'My Catalog') + cache = legacy.new_api + self.assertEqual(cache.formats(bid), ('TXT',)) + self.assertEqual(cache.field_for('title', bid), 'My Catalog') + self.assertEqual(cache.field_for('authors', bid), ('calibre',)) + self.assertEqual(cache.field_for('tags', bid), (_('Catalog'),)) + self.assertTrue(bid < legacy.add_catalog(f.name, 'Something else')) + self.assertEqual(legacy.add_catalog(f.name, 'My Catalog'), bid) + + bid = legacy.add_news(f.name, {'title':'Events', 'add_title_tag':True, 'custom_tags':('one', 'two')}) + self.assertEqual(cache.formats(bid), ('TXT',)) + self.assertEqual(cache.field_for('authors', bid), ('calibre',)) + self.assertEqual(cache.field_for('tags', bid), (_('News'), 'Events', 'one', 'two')) + + old.close() # }}} def test_legacy_coverage(self): # {{{ From a8dba2f02072207174bbae65ed01c51a3ecab93b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 11 Jul 2013 21:25:06 +0530 Subject: [PATCH 0144/1154] ... --- src/calibre/db/tests/legacy.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index af16c7c5eb..ef47f5d7bf 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -183,6 +183,7 @@ class LegacyTest(BaseTest): f.write(b'tttttt') f.seek(0) bid = legacy.add_catalog(f.name, 'My Catalog') + self.assertEqual(old.add_catalog(f.name, 'My Catalog'), bid) cache = legacy.new_api self.assertEqual(cache.formats(bid), ('TXT',)) self.assertEqual(cache.field_for('title', bid), 'My Catalog') @@ -190,6 +191,7 @@ class LegacyTest(BaseTest): self.assertEqual(cache.field_for('tags', bid), (_('Catalog'),)) self.assertTrue(bid < legacy.add_catalog(f.name, 'Something else')) self.assertEqual(legacy.add_catalog(f.name, 'My Catalog'), bid) + self.assertEqual(old.add_catalog(f.name, 'My Catalog'), bid) bid = legacy.add_news(f.name, {'title':'Events', 'add_title_tag':True, 'custom_tags':('one', 'two')}) self.assertEqual(cache.formats(bid), ('TXT',)) From 26d20cdf6baf6d37d2fef66a504228911957f6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Thu, 11 Jul 2013 22:30:26 +0200 Subject: [PATCH 0145/1154] update empik plugin for website change --- src/calibre/gui2/store/stores/empik_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/stores/empik_plugin.py b/src/calibre/gui2/store/stores/empik_plugin.py index c771722120..b716567e54 100644 --- a/src/calibre/gui2/store/stores/empik_plugin.py +++ b/src/calibre/gui2/store/stores/empik_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 2 # Needed for dynamic plugin loading +store_version = 3 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2011-2013, Tomasz Długosz ' @@ -51,7 +51,7 @@ class EmpikStore(BasicStoreConfig, StorePlugin): if not id: continue - cover_url = ''.join(data.xpath('.//div[@class="productBox-450Pic"]/a/img/@data-original')) + cover_url = ''.join(data.xpath('.//div[@class="productBox-450Pic"]/a/img/@src')) title = ''.join(data.xpath('.//a[@class="productBox-450Title"]/text()')) title = re.sub(r' \(ebook\)', '', title) author = ''.join(data.xpath('.//div[@class="productBox-450Author"]/a/text()')) From 90599f9c46a4afbf8556aadac8616fc7c2c89774 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 08:16:55 +0530 Subject: [PATCH 0146/1154] version 0.9.39 --- Changelog.yaml | 39 +++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index f68617fb3a..db25f77a8d 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,45 @@ # new recipes: # - title: +- version: 0.9.39 + date: 2013-07-12 + + new features: + - title: "Bulk metadata edit: Add a checkbox to prevent the refreshing of the book list after the bulk edit. This means that the book list will not be resorted and any existing search/virtual library will not be refreshed. Useful if you have a large library as the refresh can be slow." + + - title: "Allow manually marking a book in the calibre library as being on the device. To do so click the device icon in calibre, then right click on the book you want marked and choose 'Match book to library'. Once you are done marking all the books, right click the device icon and choose 'Update cached metadata'" + + - title: "Driver for Coby Kyros MID1126" + tickets: [1199410] + + - title: "When adding formats to an existing book, by right clicking the add books button, ask for confirmation if some formats will be overwritten." + + - title: "Add a tweak to restrict the list of output formats available in the conversion dialog. Go to Preferences->Tweaks to change it." + + bug fixes: + - title: "Amazon metadata download: Update plugin to deal with the new amazon.com website" + + - title: "Edelweiss metadata download plugin: Workaround for advanced search being broken at the Edelweiss website." + + - title: "Invalid data in the device database on sony readers could cause errors when sorting device collections, ignore those errors." + + - title: "DOCX Input: Fix no page break being inserted before the last section." + tickets: [1198414] + + - title: "Metadata download dialog: Have the OK button enabled in the results screen as well." + tickets: [1198288] + + - title: "Get Books: Update empik store plugin" + + improved recipes: + - Houston Chronicle + - cracked.com + - mediapart.fr + + new recipes: + - title: Glenn Brenwald and Ludwig von Mises Institute + author: anywho + - version: 0.9.38 date: 2013-07-05 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 5c9ecbc832..18b4e3d238 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 9, 38) +numeric_version = (0, 9, 39) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " From 6c9a6c1020f0641fa7ee7757c0d151de28298891 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 10:23:38 +0530 Subject: [PATCH 0147/1154] Implement custom book data API --- src/calibre/db/backend.py | 41 ++++++++++++++++++++++++++++++++++ src/calibre/db/cache.py | 30 +++++++++++++++++++++++++ src/calibre/db/legacy.py | 23 +++++++++++++++++++ src/calibre/db/tests/legacy.py | 31 +++++++++++++++++++++++++ 4 files changed, 125 insertions(+) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index d75106209f..8ebdc4a154 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1175,5 +1175,46 @@ class DB(object): self.rmtree(parent, permanent=permanent) self.conn.executemany( 'DELETE FROM books WHERE id=?', [(x,) for x in path_map]) + + def add_custom_data(self, name, val_map, delete_first): + if delete_first: + self.conn.execute('DELETE FROM books_plugin_data WHERE name=?', (name, )) + self.conn.executemany( + 'INSERT OR REPLACE INTO books_plugin_data (book, name, val) VALUES (?, ?, ?)', + [(book_id, name, json.dumps(val, default=to_json)) + for book_id, val in val_map.iteritems()]) + + def get_custom_book_data(self, name, book_ids, default=None): + book_ids = frozenset(book_ids) + def safe_load(val): + try: + return json.loads(val, object_hook=from_json) + except: + return default + + if len(book_ids) == 1: + bid = next(iter(book_ids)) + ans = {book_id:safe_load(val) for book_id, val in + self.conn.execute('SELECT book, val FROM books_plugin_data WHERE book=? AND name=?', (bid, name))} + return ans or {bid:default} + + ans = {} + for book_id, val in self.conn.execute( + 'SELECT book, val FROM books_plugin_data WHERE name=?', (name,)): + if not book_ids or book_id in book_ids: + val = safe_load(val) + ans[book_id] = val + return ans + + def delete_custom_book_data(self, name, book_ids): + if book_ids: + self.conn.executemany('DELETE FROM books_plugin_data WHERE book=? AND name=?', + [(book_id, name) for book_id in book_ids]) + else: + self.conn.execute('DELETE FROM books_plugin_data WHERE name=?', (name,)) + + def get_ids_for_custom_book_data(self, name): + return frozenset(r[0] for r in self.conn.execute('SELECT book FROM books_plugin_data WHERE name=?', (name,))) + # }}} diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index b8b9ad7436..49f5bd24ec 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1147,6 +1147,36 @@ class Cache(object): else: table.remove_books(book_ids, self.backend) + @write_api + def add_custom_book_data(self, name, val_map, delete_first=False): + ''' Add data for name where val_map is a map of book_ids to values. If + delete_first is True, all previously stored data for name will be + removed. ''' + missing = frozenset(val_map) - self._all_book_ids() + if missing: + raise ValueError('add_custom_book_data: no such book_ids: %d'%missing) + self.backend.add_custom_data(name, val_map, delete_first) + + @read_api + def get_custom_book_data(self, name, book_ids=(), default=None): + ''' Get data for name. By default returns data for all book_ids, pass + in a list of book ids if you only want some data. Returns a map of + book_id to values. If a particular value could not be decoded, uses + default for it. ''' + return self.backend.get_custom_book_data(name, book_ids, default) + + @write_api + def delete_custom_book_data(self, name, book_ids=()): + ''' Delete data for name. By default deletes all data, if you only want + to delete data for some book ids, pass in a list of book ids. ''' + self.backend.delete_custom_book_data(name, book_ids) + + @read_api + def get_ids_for_custom_book_data(self, name): + ''' Return the set of book ids for which name has data. ''' + return self.backend.get_ids_for_custom_book_data(name) + + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index b9f23cc5d7..a12e949b55 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -216,6 +216,29 @@ class LibraryDatabase(object): # }}} + # Custom data {{{ + def add_custom_book_data(self, book_id, name, val): + self.new_api.add_custom_book_data(name, {book_id:val}) + + def add_multiple_custom_book_data(self, name, val_map, delete_first=False): + self.new_api.add_custom_book_data(name, val_map, delete_first=delete_first) + + def get_custom_book_data(self, book_id, name, default=None): + return self.new_api.get_custom_book_data(name, book_ids={book_id}, default=default).get(book_id, default) + + def get_all_custom_book_data(self, name, default=None): + return self.new_api.get_custom_book_data(name, default=default) + + def delete_custom_book_data(self, book_id, name): + self.new_api.delete_custom_book_data(name, book_ids=(book_id,)) + + def delete_all_custom_book_data(self, name): + self.new_api.delete_custom_book_data(name) + + def get_ids_for_custom_book_data(self, name): + return list(self.new_api.get_ids_for_custom_book_data(name)) + # }}} + # Private interface {{{ def __iter__(self): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index ef47f5d7bf..a94e59d17f 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -245,3 +245,34 @@ class LegacyTest(BaseTest): # }}} + def test_legacy_custom_data(self): # {{{ + 'Test the API for custom data storage' + legacy, old = self.init_legacy(self.cloned_library), self.init_old(self.cloned_library) + for name in ('name1', 'name2', 'name3'): + T = partial(ET, 'add_custom_book_data', old=old, legacy=legacy) + T((1, name, 'val1'))(self) + T((2, name, 'val2'))(self) + T((3, name, 'val3'))(self) + T = partial(ET, 'get_ids_for_custom_book_data', old=old, legacy=legacy) + T((name,))(self) + T = partial(ET, 'get_custom_book_data', old=old, legacy=legacy) + T((1, name, object())) + T((9, name, object())) + T = partial(ET, 'get_all_custom_book_data', old=old, legacy=legacy) + T((name, object())) + T((name+'!', object())) + T = partial(ET, 'delete_custom_book_data', old=old, legacy=legacy) + T((name, 1)) + T = partial(ET, 'get_all_custom_book_data', old=old, legacy=legacy) + T((name, object())) + T = partial(ET, 'delete_all_custom_book_data', old=old, legacy=legacy) + T((name)) + T = partial(ET, 'get_all_custom_book_data', old=old, legacy=legacy) + T((name, object())) + + T = partial(ET, 'add_multiple_custom_book_data', old=old, legacy=legacy) + T(('n', {1:'val1', 2:'val2'}))(self) + T = partial(ET, 'get_all_custom_book_data', old=old, legacy=legacy) + T(('n', object())) + old.close() + # }}} From f9bd87d785693e06f263a79c288a179fa54fd7ae Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 10:29:56 +0530 Subject: [PATCH 0148/1154] Ignore unused feeds API --- src/calibre/db/tests/legacy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index a94e59d17f..dd9cbc2dc5 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -210,6 +210,8 @@ class LegacyTest(BaseTest): SKIP_ATTRS = { 'TCat_Tag', '_add_newbook_tag', '_clean_identifier', '_library_id_', '_set_authors', '_set_title', '_set_custom', '_update_author_in_cache', + # Feeds are now stored in the config folder + 'get_feeds', 'get_feed', 'update_feed', 'remove_feeds', 'add_feed', 'set_feeds', } SKIP_ARGSPEC = { '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', @@ -241,7 +243,7 @@ class LegacyTest(BaseTest): if missing: pc = len(missing)/total - raise AssertionError('{0:.1%} of API ({2} attrs) are missing. For example: {1}'.format(pc, missing[0], len(missing))) + raise AssertionError('{0:.1%} of API ({2} attrs) are missing. For example: {1}'.format(pc, ', '.join(missing[:5]), len(missing))) # }}} From 4a2a4a54d471731bdff97733f9605f24759230ef Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 11:04:47 +0530 Subject: [PATCH 0149/1154] Add legacy add_format API --- src/calibre/db/legacy.py | 10 ++++++++++ src/calibre/db/tests/legacy.py | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index a12e949b55..e1e533266f 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -214,6 +214,16 @@ class LibraryDatabase(object): def add_news(self, path, arg): return add_news(self.new_api, path, arg) + def add_format(self, index, fmt, stream, index_is_id=False, path=None, notify=True, replace=True, copy_function=None): + ''' path and copy_function are ignored by the new API ''' + book_id = index if index_is_id else self.data.index_to_id(index) + return self.new_api.add_format(book_id, fmt, stream, replace=replace, run_hooks=False, dbapi=self) + + def add_format_with_hooks(self, index, fmt, fpath, index_is_id=False, path=None, notify=True, replace=True): + ''' path is ignored by the new API ''' + book_id = index if index_is_id else self.data.index_to_id(index) + return self.new_api.add_format(book_id, fmt, fpath, replace=replace, run_hooks=True, dbapi=self) + # }}} # Custom data {{{ diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index dd9cbc2dc5..ced21f479d 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -7,6 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' import inspect +from io import BytesIO from repr import repr from functools import partial from tempfile import NamedTemporaryFile @@ -166,6 +167,11 @@ class LegacyTest(BaseTest): book_id = T(kwargs={'preserve_uuid':True})(self) self.assertEqual(legacy.uuid(book_id, index_is_id=True), old.uuid(book_id, index_is_id=True)) self.assertEqual(legacy.new_api.formats(book_id), ('AFF',)) + + T = partial(ET, 'add_format', old=old, legacy=legacy) + T((0, 'AFF', BytesIO(b'fffff')))(self) + T((0, 'AFF', BytesIO(b'fffff')))(self) + T((0, 'AFF', BytesIO(b'fffff')), {'replace':True})(self) with NamedTemporaryFile(suffix='.opf') as f: f.write(b'zzzz') f.flush() From 6a724d931f9856ce89396c8488cfd2d3811cc95f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 11:10:35 +0530 Subject: [PATCH 0150/1154] ... --- src/calibre/db/legacy.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index e1e533266f..54d0887954 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -217,12 +217,22 @@ class LibraryDatabase(object): def add_format(self, index, fmt, stream, index_is_id=False, path=None, notify=True, replace=True, copy_function=None): ''' path and copy_function are ignored by the new API ''' book_id = index if index_is_id else self.data.index_to_id(index) - return self.new_api.add_format(book_id, fmt, stream, replace=replace, run_hooks=False, dbapi=self) + try: + return self.new_api.add_format(book_id, fmt, stream, replace=replace, run_hooks=False, dbapi=self) + except: + raise + else: + self.notify('metadata', [book_id]) def add_format_with_hooks(self, index, fmt, fpath, index_is_id=False, path=None, notify=True, replace=True): ''' path is ignored by the new API ''' book_id = index if index_is_id else self.data.index_to_id(index) - return self.new_api.add_format(book_id, fmt, fpath, replace=replace, run_hooks=True, dbapi=self) + try: + return self.new_api.add_format(book_id, fmt, fpath, replace=replace, run_hooks=True, dbapi=self) + except: + raise + else: + self.notify('metadata', [book_id]) # }}} From 0a72140cac41989951f2101bcc4df2b4cef01f1e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 12:48:21 +0530 Subject: [PATCH 0151/1154] Various library wide properties --- src/calibre/db/cache.py | 24 ++++++++++++++++++++++++ src/calibre/db/legacy.py | 14 ++++++++++++++ src/calibre/db/tests/legacy.py | 23 ++++++++++++++++++++--- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 49f5bd24ec..a322c9b6d7 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -360,6 +360,30 @@ class Cache(object): ''' return frozenset(iter(self.fields[name])) + @read_api + def all_field_names(self, field): + ''' Frozen set of all fields names (should only be used for many-one and many-many fields) ''' + try: + return frozenset(self.fields[field].table.id_map.itervalues()) + except AttributeError: + raise ValueError('%s is not a many-one or many-many field' % field) + + @read_api + def get_usage_count_by_id(self, field): + try: + return {k:len(v) for k, v in self.fields[field].table.col_book_map.iteritems()} + except AttributeError: + raise ValueError('%s is not a many-one or many-many field' % field) + + @read_api + def get_id_map(self, field): + try: + return self.fields[field].table.id_map.copy() + except AttributeError: + if field == 'title': + return self.fields[field].table.book_col_map.copy() + raise ValueError('%s is not a many-one or many-many field' % field) + @read_api def author_data(self, author_id): ''' diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 54d0887954..5f04dd18b2 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -65,6 +65,14 @@ class LibraryDatabase(object): for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'): setattr(self, meth, getattr(self.new_api, meth)) + for field in ('authors', 'tags', 'publisher', 'series'): + name = field[:-1] if field in {'authors', 'tags'} else field + setattr(self, 'all_%s_names' % name, partial(self.new_api.all_field_names, field)) + + for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':'tags', 'all_series':'series', 'all_publishers':'publisher'}.iteritems(): + setattr(self, func, partial(self.field_id_map, field)) + self.all_tags = lambda : list(self.all_tag_names()) + self.last_update_check = self.last_modified() def close(self): @@ -129,6 +137,12 @@ class LibraryDatabase(object): for book_id in self.data.cache.all_book_ids(): yield book_id + def get_usage_count_by_id(self, field): + return [[k, v] for k, v in self.new_api.get_usage_count_by_id(field).iteritems()] + + def field_id_map(self, field): + return [(k, v) for k, v in self.new_api.get_id_map(field).iteritems()] + def refresh(self, field=None, ascending=True): self.data.cache.refresh() self.data.refresh(field=field, ascending=ascending) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index ced21f479d..b7fd99378a 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -139,9 +139,25 @@ class LegacyTest(BaseTest): 'get_next_series_num_for': [('A Series One',)], 'author_sort_from_authors': [(['Author One', 'Author Two', 'Unknown'],)], 'has_book':[(Metadata('title one'),), (Metadata('xxxx1111'),)], + 'all_author_names':[()], + 'all_tag_names':[()], + 'all_series_names':[()], + 'all_publisher_names':[()], + 'all_authors':[()], + 'all_tags2':[()], + 'all_tags':[()], + 'all_publishers':[()], + 'all_titles':[()], + 'all_series':[()], + 'get_usage_count_by_id':[('authors',), ('tags',), ('series',), ('publisher',), ('#tags',), ('languages',)], }.iteritems(): for a in args: - self.assertEqual(getattr(db, meth)(*a), getattr(ndb, meth)(*a), + fmt = lambda x: x + if meth in {'get_usage_count_by_id', 'all_series', 'all_authors', 'all_tags2', 'all_publishers', 'all_titles'}: + fmt = dict + elif meth in {'all_tags'}: + fmt = frozenset + self.assertEqual(fmt(getattr(db, meth)(*a)), fmt(getattr(ndb, meth)(*a)), 'The method: %s() returned different results for argument %s' % (meth, a)) db.close() # }}} @@ -220,7 +236,7 @@ class LegacyTest(BaseTest): 'get_feeds', 'get_feed', 'update_feed', 'remove_feeds', 'add_feed', 'set_feeds', } SKIP_ARGSPEC = { - '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', + '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', 'all_tags', } missing = [] @@ -238,10 +254,11 @@ class LegacyTest(BaseTest): if attr not in SKIP_ARGSPEC: try: argspec = inspect.getargspec(obj) + nargspec = inspect.getargspec(nobj) except TypeError: pass else: - compare_argspecs(argspec, inspect.getargspec(nobj), attr) + compare_argspecs(argspec, nargspec, attr) finally: for db in (ndb, db): db.close() From c1cca5213a310c0e0b0483fa455b2aeab20002dc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 13:21:08 +0530 Subject: [PATCH 0152/1154] The field API --- src/calibre/db/legacy.py | 13 +++++++++++++ src/calibre/db/tests/legacy.py | 8 ++++++++ src/calibre/library/database2.py | 4 ++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 5f04dd18b2..4c825247e9 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -73,6 +73,14 @@ class LibraryDatabase(object): setattr(self, func, partial(self.field_id_map, field)) self.all_tags = lambda : list(self.all_tag_names()) + for func in ( + 'standard_field_keys', 'custom_field_keys', 'all_field_keys', + 'searchable_fields', 'sortable_field_keys', + 'search_term_to_field_key', 'custom_field_metadata', + 'all_metadata'): + setattr(self, func, getattr(self.field_metadata, func)) + self.metadata_for_field = self.field_metadata.get + self.last_update_check = self.last_modified() def close(self): @@ -273,6 +281,11 @@ class LibraryDatabase(object): return list(self.new_api.get_ids_for_custom_book_data(name)) # }}} + def get_field(self, index, key, default=None, index_is_id=False): + book_id = index if index_is_id else self.data.index_to_id(index) + mi = self.new_api.get_metadata(book_id, get_cover=key == 'cover') + return mi.get(key, default) + # Private interface {{{ def __iter__(self): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index b7fd99378a..6413dc0d8f 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -149,7 +149,15 @@ class LegacyTest(BaseTest): 'all_publishers':[()], 'all_titles':[()], 'all_series':[()], + 'standard_field_keys':[()], + 'all_field_keys':[()], + 'searchable_fields':[()], + 'search_term_to_field_key':[('author',), ('tag',)], + 'metadata_for_field':[('title',), ('tags',)], + 'sortable_field_keys':[()], + 'custom_field_keys':[(True,), (False,)], 'get_usage_count_by_id':[('authors',), ('tags',), ('series',), ('publisher',), ('#tags',), ('languages',)], + 'get_field':[(1, 'title'), (2, 'tags'), (0, 'rating'), (1, 'authors'), (2, 'series'), (1, '#tags')], }.iteritems(): for a in args: fmt = lambda x: x diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 0540e8ede4..bd3f155c9d 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -776,10 +776,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return self.field_metadata.sortable_field_keys() def searchable_fields(self): - return self.field_metadata.searchable_field_keys() + return self.field_metadata.searchable_fields() def search_term_to_field_key(self, term): - return self.field_metadata.search_term_to_key(term) + return self.field_metadata.search_term_to_field_key(term) def custom_field_metadata(self, include_composites=True): return self.field_metadata.custom_field_metadata(include_composites) From 89f712dc53d982c737c5152a2eeec5b06bd83a3a Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Fri, 12 Jul 2013 10:01:42 +0200 Subject: [PATCH 0153/1154] Recover from yet another Amazon EU website change. This time they introduced a new table form with a very different structure. They switch between the form types on a seemingly random basis. --- .../gui2/store/stores/amazon_de_plugin.py | 65 ++++++++++++++----- .../gui2/store/stores/amazon_es_plugin.py | 64 ++++++++++++++---- .../gui2/store/stores/amazon_fr_plugin.py | 64 ++++++++++++++---- .../gui2/store/stores/amazon_it_plugin.py | 65 +++++++++++++++---- .../gui2/store/stores/amazon_uk_plugin.py | 64 ++++++++++++++---- 5 files changed, 251 insertions(+), 71 deletions(-) diff --git a/src/calibre/gui2/store/stores/amazon_de_plugin.py b/src/calibre/gui2/store/stores/amazon_de_plugin.py index 6833bd3710..72c69415bc 100644 --- a/src/calibre/gui2/store/stores/amazon_de_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_de_plugin.py @@ -52,25 +52,61 @@ class AmazonDEKindleStore(StorePlugin): def search(self, query, max_results=10, timeout=60): url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+') + #print(url) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read())#.decode('latin-1', 'replace')) + allText = f.read() + doc = html.fromstring(allText)#.decode('latin-1', 'replace')) + + if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'): + #print('grid form') + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'): + #print('ilo form') + data_xpath = '//li[(@class="ilo")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + # Results can be in a grid (table) or a column + price_xpath = ( + './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'): + #print('list form') + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + else: + # URK -- whats this? + print('unknown result table form for Amazon EU search') + #with open("c:/amazon_search_results.html", "w") as out: + # out.write(allText) + return - data_xpath = '//div[contains(@class, "prod")]' - # Results can be in a grid (table) or a column - format_xpath = ( - './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' - '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') - asin_xpath = '@name' - cover_xpath = './/img[@class="productImage"]/@src' - title_xpath = './/h3[@class="newaps"]/a//text()' - author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' - # Results can be in a grid (table) or a column - price_xpath = ( - './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' - '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') for data in doc.xpath(data_xpath): if counter <= 0: @@ -120,4 +156,3 @@ class AmazonDEKindleStore(StorePlugin): def get_details(self, search_result, timeout): pass - diff --git a/src/calibre/gui2/store/stores/amazon_es_plugin.py b/src/calibre/gui2/store/stores/amazon_es_plugin.py index 0b71ae657b..2caff27477 100644 --- a/src/calibre/gui2/store/stores/amazon_es_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_es_plugin.py @@ -51,25 +51,61 @@ class AmazonESKindleStore(StorePlugin): def search(self, query, max_results=10, timeout=60): url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+') + #print(url) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read())#.decode('latin-1', 'replace')) + allText = f.read() + doc = html.fromstring(allText)#.decode('latin-1', 'replace')) + + if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'): + #print('grid form') + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'): + #print('ilo form') + data_xpath = '//li[(@class="ilo")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + # Results can be in a grid (table) or a column + price_xpath = ( + './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'): + #print('list form') + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + else: + # URK -- whats this? + print('unknown result table form for Amazon EU search') + #with open("c:/amazon_search_results.html", "w") as out: + # out.write(allText) + return - data_xpath = '//div[contains(@class, "prod")]' - # Results can be in a grid (table) or a column - format_xpath = ( - './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' - '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') - asin_xpath = '@name' - cover_xpath = './/img[@class="productImage"]/@src' - title_xpath = './/h3[@class="newaps"]/a//text()' - author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' - # Results can be in a grid (table) or a column - price_xpath = ( - './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' - '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') for data in doc.xpath(data_xpath): if counter <= 0: diff --git a/src/calibre/gui2/store/stores/amazon_fr_plugin.py b/src/calibre/gui2/store/stores/amazon_fr_plugin.py index 4520a3a104..3261e3acb1 100644 --- a/src/calibre/gui2/store/stores/amazon_fr_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_fr_plugin.py @@ -48,25 +48,61 @@ class AmazonFRKindleStore(StorePlugin): def search(self, query, max_results=10, timeout=60): url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+') + #print(url) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read())#.decode('latin-1', 'replace')) + allText = f.read() + doc = html.fromstring(allText)#.decode('latin-1', 'replace')) + + if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'): + #print('grid form') + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'): + #print('ilo form') + data_xpath = '//li[(@class="ilo")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + # Results can be in a grid (table) or a column + price_xpath = ( + './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'): + #print('list form') + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + else: + # URK -- whats this? + print('unknown result table form for Amazon EU search') + #with open("c:/amazon_search_results.html", "w") as out: + # out.write(allText) + return - data_xpath = '//div[contains(@class, "prod")]' - # Results can be in a grid (table) or a column - format_xpath = ( - './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' - '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') - asin_xpath = '@name' - cover_xpath = './/img[@class="productImage"]/@src' - title_xpath = './/h3[@class="newaps"]/a//text()' - author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' - # Results can be in a grid (table) or a column - price_xpath = ( - './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' - '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') for data in doc.xpath(data_xpath): if counter <= 0: diff --git a/src/calibre/gui2/store/stores/amazon_it_plugin.py b/src/calibre/gui2/store/stores/amazon_it_plugin.py index f8a756d1d5..4df20d10f5 100644 --- a/src/calibre/gui2/store/stores/amazon_it_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_it_plugin.py @@ -51,25 +51,61 @@ class AmazonITKindleStore(StorePlugin): def search(self, query, max_results=10, timeout=60): url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+') + #print(url) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read())#.decode('latin-1', 'replace')) + allText = f.read() + doc = html.fromstring(allText)#.decode('latin-1', 'replace')) + + if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'): + #print('grid form') + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'): + #print('ilo form') + data_xpath = '//li[(@class="ilo")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + # Results can be in a grid (table) or a column + price_xpath = ( + './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'): + #print('list form') + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + else: + # URK -- whats this? + print('unknown result table form for Amazon EU search') + #with open("c:/amazon_search_results.html", "w") as out: + # out.write(allText) + return - data_xpath = '//div[contains(@class, "prod")]' - # Results can be in a grid (table) or a column - format_xpath = ( - './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' - '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') - asin_xpath = '@name' - cover_xpath = './/img[@class="productImage"]/@src' - title_xpath = './/h3[@class="newaps"]/a//text()' - author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' - # Results can be in a grid (table) or a column - price_xpath = ( - './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' - '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') for data in doc.xpath(data_xpath): if counter <= 0: @@ -119,3 +155,4 @@ class AmazonITKindleStore(StorePlugin): def get_details(self, search_result, timeout): pass + diff --git a/src/calibre/gui2/store/stores/amazon_uk_plugin.py b/src/calibre/gui2/store/stores/amazon_uk_plugin.py index f6082ac790..836e46f89c 100644 --- a/src/calibre/gui2/store/stores/amazon_uk_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_uk_plugin.py @@ -55,25 +55,61 @@ class AmazonUKKindleStore(StorePlugin): def search(self, query, max_results=10, timeout=60): url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+') + #print(url) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read())#.decode('latin-1', 'replace')) + allText = f.read() + doc = html.fromstring(allText)#.decode('latin-1', 'replace')) + + if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'): + #print('grid form') + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'): + #print('ilo form') + data_xpath = '//li[(@class="ilo")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + # Results can be in a grid (table) or a column + price_xpath = ( + './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'): + #print('list form') + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + else: + # URK -- whats this? + print('unknown result table form for Amazon EU search') + #with open("c:/amazon_search_results.html", "w") as out: + # out.write(allText) + return - data_xpath = '//div[contains(@class, "prod")]' - # Results can be in a grid (table) or a column - format_xpath = ( - './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' - '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') - asin_xpath = '@name' - cover_xpath = './/img[@class="productImage"]/@src' - title_xpath = './/h3[@class="newaps"]/a//text()' - author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' - # Results can be in a grid (table) or a column - price_xpath = ( - './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' - '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') for data in doc.xpath(data_xpath): if counter <= 0: From 998a79075ce9c68f5c0e7860af663dcc5ed2bde2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 13:39:14 +0530 Subject: [PATCH 0154/1154] all_formats() --- src/calibre/db/cache.py | 3 +++ src/calibre/db/legacy.py | 1 + src/calibre/db/tests/legacy.py | 1 + 3 files changed, 5 insertions(+) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index a322c9b6d7..ce921537f2 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -363,6 +363,9 @@ class Cache(object): @read_api def all_field_names(self, field): ''' Frozen set of all fields names (should only be used for many-one and many-many fields) ''' + if field == 'formats': + return frozenset(self.fields[field].table.col_book_map) + try: return frozenset(self.fields[field].table.id_map.itervalues()) except AttributeError: diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 4c825247e9..398d1b1a86 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -68,6 +68,7 @@ class LibraryDatabase(object): for field in ('authors', 'tags', 'publisher', 'series'): name = field[:-1] if field in {'authors', 'tags'} else field setattr(self, 'all_%s_names' % name, partial(self.new_api.all_field_names, field)) + self.all_formats = partial(self.new_api.all_field_names, 'formats') for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':'tags', 'all_series':'series', 'all_publishers':'publisher'}.iteritems(): setattr(self, func, partial(self.field_id_map, field)) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 6413dc0d8f..b816341b5a 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -158,6 +158,7 @@ class LegacyTest(BaseTest): 'custom_field_keys':[(True,), (False,)], 'get_usage_count_by_id':[('authors',), ('tags',), ('series',), ('publisher',), ('#tags',), ('languages',)], 'get_field':[(1, 'title'), (2, 'tags'), (0, 'rating'), (1, 'authors'), (2, 'series'), (1, '#tags')], + 'all_formats':[()], }.iteritems(): for a in args: fmt = lambda x: x From 7ad78fc4475e40d95d7e1ac77614baab9302ea90 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Fri, 12 Jul 2013 10:10:31 +0200 Subject: [PATCH 0155/1154] Forgot to update the version number... --- src/calibre/gui2/store/stores/amazon_de_plugin.py | 2 +- src/calibre/gui2/store/stores/amazon_es_plugin.py | 2 +- src/calibre/gui2/store/stores/amazon_fr_plugin.py | 2 +- src/calibre/gui2/store/stores/amazon_it_plugin.py | 2 +- src/calibre/gui2/store/stores/amazon_uk_plugin.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/store/stores/amazon_de_plugin.py b/src/calibre/gui2/store/stores/amazon_de_plugin.py index 72c69415bc..7bb2f43047 100644 --- a/src/calibre/gui2/store/stores/amazon_de_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_de_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 3 # Needed for dynamic plugin loading +store_version = 4 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' diff --git a/src/calibre/gui2/store/stores/amazon_es_plugin.py b/src/calibre/gui2/store/stores/amazon_es_plugin.py index 2caff27477..1613e3f13a 100644 --- a/src/calibre/gui2/store/stores/amazon_es_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_es_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 3 # Needed for dynamic plugin loading +store_version = 4 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' diff --git a/src/calibre/gui2/store/stores/amazon_fr_plugin.py b/src/calibre/gui2/store/stores/amazon_fr_plugin.py index 3261e3acb1..4d210111e8 100644 --- a/src/calibre/gui2/store/stores/amazon_fr_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_fr_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 3 # Needed for dynamic plugin loading +store_version = 4 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' diff --git a/src/calibre/gui2/store/stores/amazon_it_plugin.py b/src/calibre/gui2/store/stores/amazon_it_plugin.py index 4df20d10f5..1e905d370e 100644 --- a/src/calibre/gui2/store/stores/amazon_it_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_it_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 3 # Needed for dynamic plugin loading +store_version = 4 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' diff --git a/src/calibre/gui2/store/stores/amazon_uk_plugin.py b/src/calibre/gui2/store/stores/amazon_uk_plugin.py index 836e46f89c..0cd5f34dbd 100644 --- a/src/calibre/gui2/store/stores/amazon_uk_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_uk_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 3 # Needed for dynamic plugin loading +store_version = 4 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' From 9249fd8a3e79ebe8a9dc7049e6fad8878d1f8c2d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 14:25:39 +0530 Subject: [PATCH 0156/1154] Refactor the author data api for multiple authors --- src/calibre/db/cache.py | 17 +++++++++-------- src/calibre/db/tests/legacy.py | 2 ++ src/calibre/db/view.py | 6 ++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index ce921537f2..2c74a31438 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -161,7 +161,8 @@ class Cache(object): def _get_metadata(self, book_id, get_user_categories=True): # {{{ mi = Metadata(None, template_cache=self.formatter_template_cache) author_ids = self._field_ids_for('authors', book_id) - aut_list = [self._author_data(i) for i in author_ids] + adata = self._author_data(author_ids) + aut_list = [adata[i] for i in author_ids] aum = [] aus = {} aul = {} @@ -388,17 +389,17 @@ class Cache(object): raise ValueError('%s is not a many-one or many-many field' % field) @read_api - def author_data(self, author_id): + def author_data(self, author_ids): ''' Return author data as a dictionary with keys: name, sort, link - If no author with the specified id is found an empty dictionary is - returned. + If no authors with the specified ids are found an empty dictionary is + returned. If author_ids is None, data for all authors is returned. ''' - try: - return self.fields['authors'].author_data(author_id) - except (KeyError, IndexError): - return {} + af = self.fields['authors'] + if author_ids is None: + author_ids = tuple(af.table.id_map) + return {aid:af.author_data(aid) for aid in author_ids if aid in af.table.id_map} @read_api def format_metadata(self, book_id, fmt, allow_cache=True): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index b816341b5a..b28a5cb93e 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -243,6 +243,8 @@ class LegacyTest(BaseTest): '_set_title', '_set_custom', '_update_author_in_cache', # Feeds are now stored in the config folder 'get_feeds', 'get_feed', 'update_feed', 'remove_feeds', 'add_feed', 'set_feeds', + # Obsolete/broken methods + 'author_id', # replaced by get_author_id } SKIP_ARGSPEC = { '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', 'all_tags', diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 3f135860d9..ecd5182232 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -189,10 +189,8 @@ class View(object): id_ = idx if index_is_id else self.index_to_id(idx) with self.cache.read_lock: ids = self.cache._field_ids_for('authors', id_) - ans = [] - for id_ in ids: - data = self.cache._author_data(id_) - ans.append(':::'.join((data['name'], data['sort'], data['link']))) + adata = self.cache._author_data(ids) + ans = [':::'.join((adata[aid]['name'], adata[aid]['sort'], adata[aid]['link'])) for aid in ids if aid in adata] return ':#:'.join(ans) if ans else default_value def multisort(self, fields=[], subsort=False, only_ids=None): From ca8f29db0bd79392a51a59ef7e57598b54b161c3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 14:43:08 +0530 Subject: [PATCH 0157/1154] get_*_with_ids() API --- src/calibre/db/cache.py | 2 +- src/calibre/db/legacy.py | 6 ++++++ src/calibre/db/tests/legacy.py | 29 ++++++++++++++++++----------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 2c74a31438..eca5a2a395 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -389,7 +389,7 @@ class Cache(object): raise ValueError('%s is not a many-one or many-many field' % field) @read_api - def author_data(self, author_ids): + def author_data(self, author_ids=None): ''' Return author data as a dictionary with keys: name, sort, link diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 398d1b1a86..7f375756e7 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -73,6 +73,12 @@ class LibraryDatabase(object): for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':'tags', 'all_series':'series', 'all_publishers':'publisher'}.iteritems(): setattr(self, func, partial(self.field_id_map, field)) self.all_tags = lambda : list(self.all_tag_names()) + self.get_authors_with_ids = lambda : [[aid, adata['name'], adata['sort'], adata['link']] for aid, adata in self.new_api.author_data().iteritems()] + self.get_tags_with_ids = lambda : [[tid, tag] for tid, tag in self.new_api.get_id_map('tags').iteritems()] + self.get_series_with_ids = lambda : [[tid, tag] for tid, tag in self.new_api.get_id_map('series').iteritems()] + self.get_publishers_with_ids = lambda : [[tid, tag] for tid, tag in self.new_api.get_id_map('publisher').iteritems()] + self.get_ratings_with_ids = lambda : [[tid, tag] for tid, tag in self.new_api.get_id_map('rating').iteritems()] + self.get_languages_with_ids = lambda : [[tid, tag] for tid, tag in self.new_api.get_id_map('languages').iteritems()] for func in ( 'standard_field_keys', 'custom_field_keys', 'all_field_keys', diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index b28a5cb93e..b731dcc819 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -143,12 +143,12 @@ class LegacyTest(BaseTest): 'all_tag_names':[()], 'all_series_names':[()], 'all_publisher_names':[()], - 'all_authors':[()], - 'all_tags2':[()], - 'all_tags':[()], - 'all_publishers':[()], - 'all_titles':[()], - 'all_series':[()], + '!all_authors':[()], + '!all_tags2':[()], + '@all_tags':[()], + '!all_publishers':[()], + '!all_titles':[()], + '!all_series':[()], 'standard_field_keys':[()], 'all_field_keys':[()], 'searchable_fields':[()], @@ -156,16 +156,23 @@ class LegacyTest(BaseTest): 'metadata_for_field':[('title',), ('tags',)], 'sortable_field_keys':[()], 'custom_field_keys':[(True,), (False,)], - 'get_usage_count_by_id':[('authors',), ('tags',), ('series',), ('publisher',), ('#tags',), ('languages',)], + '!get_usage_count_by_id':[('authors',), ('tags',), ('series',), ('publisher',), ('#tags',), ('languages',)], 'get_field':[(1, 'title'), (2, 'tags'), (0, 'rating'), (1, 'authors'), (2, 'series'), (1, '#tags')], 'all_formats':[()], + 'get_authors_with_ids':[()], + '!get_tags_with_ids':[()], + '!get_series_with_ids':[()], + '!get_publishers_with_ids':[()], + '!get_ratings_with_ids':[()], + '!get_languages_with_ids':[()], }.iteritems(): for a in args: fmt = lambda x: x - if meth in {'get_usage_count_by_id', 'all_series', 'all_authors', 'all_tags2', 'all_publishers', 'all_titles'}: - fmt = dict - elif meth in {'all_tags'}: - fmt = frozenset + if meth[0] in {'!', '@'}: + fmt = {'!':dict, '@':frozenset}[meth[0]] + meth = meth[1:] + elif meth == 'get_authors_with_ids': + fmt = lambda val:{x[0]:tuple(x[1:]) for x in val} self.assertEqual(fmt(getattr(db, meth)(*a)), fmt(getattr(ndb, meth)(*a)), 'The method: %s() returned different results for argument %s' % (meth, a)) db.close() From d80f36ef28d93795c5c2fe1d76b086a1e22a131c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 15:16:16 +0530 Subject: [PATCH 0158/1154] Use MethodType for legacy interface to ensure method signatures match --- src/calibre/db/legacy.py | 32 ++++++++++++++++++++++---------- src/calibre/db/tests/legacy.py | 2 +- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 7f375756e7..2c84542d4f 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' -import os, traceback +import os, traceback, types from functools import partial from future_builtins import zip @@ -62,24 +62,36 @@ class LibraryDatabase(object): setattr(self, prop, partial(self.get_property, loc=self.FIELD_MAP[fm])) + MT = lambda func: types.MethodType(func, self, LibraryDatabase) + for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'): setattr(self, meth, getattr(self.new_api, meth)) + # Legacy API to get information about many-(one, many) fields for field in ('authors', 'tags', 'publisher', 'series'): + def getter(field): + def func(self): + return self.new_api.all_field_names(field) + return func name = field[:-1] if field in {'authors', 'tags'} else field - setattr(self, 'all_%s_names' % name, partial(self.new_api.all_field_names, field)) - self.all_formats = partial(self.new_api.all_field_names, 'formats') + setattr(self, 'all_%s_names' % name, MT(getter(field))) + self.all_formats = MT(lambda self:self.new_api.all_field_names('formats')) for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':'tags', 'all_series':'series', 'all_publishers':'publisher'}.iteritems(): setattr(self, func, partial(self.field_id_map, field)) - self.all_tags = lambda : list(self.all_tag_names()) - self.get_authors_with_ids = lambda : [[aid, adata['name'], adata['sort'], adata['link']] for aid, adata in self.new_api.author_data().iteritems()] - self.get_tags_with_ids = lambda : [[tid, tag] for tid, tag in self.new_api.get_id_map('tags').iteritems()] - self.get_series_with_ids = lambda : [[tid, tag] for tid, tag in self.new_api.get_id_map('series').iteritems()] - self.get_publishers_with_ids = lambda : [[tid, tag] for tid, tag in self.new_api.get_id_map('publisher').iteritems()] - self.get_ratings_with_ids = lambda : [[tid, tag] for tid, tag in self.new_api.get_id_map('rating').iteritems()] - self.get_languages_with_ids = lambda : [[tid, tag] for tid, tag in self.new_api.get_id_map('languages').iteritems()] + self.all_tags = MT(lambda self: list(self.all_tag_names())) + self.get_authors_with_ids = MT( + lambda self: [[aid, adata['name'], adata['sort'], adata['link']] for aid, adata in self.new_api.author_data().iteritems()]) + for field in ('tags', 'series', 'publishers', 'ratings', 'languages'): + def getter(field): + fname = field[:-1] if field in {'publishers', 'ratings'} else field + def func(self): + return [[tid, tag] for tid, tag in self.new_api.get_id_map(fname).iteritems()] + return func + setattr(self, 'get_%s_with_ids' % field, + MT(getter(field))) + # Legacy field API for func in ( 'standard_field_keys', 'custom_field_keys', 'all_field_keys', 'searchable_fields', 'sortable_field_keys', diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index b731dcc819..55fac0120d 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -254,7 +254,7 @@ class LegacyTest(BaseTest): 'author_id', # replaced by get_author_id } SKIP_ARGSPEC = { - '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', 'all_tags', + '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', } missing = [] From 71e27433ec30d3f8995d5bade3af0eb3341aac41 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 15:37:07 +0530 Subject: [PATCH 0159/1154] API to get item name from item id --- src/calibre/db/cache.py | 4 ++++ src/calibre/db/legacy.py | 7 +++++++ src/calibre/db/tests/legacy.py | 7 ++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index eca5a2a395..bc8f1024f5 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -388,6 +388,10 @@ class Cache(object): return self.fields[field].table.book_col_map.copy() raise ValueError('%s is not a many-one or many-many field' % field) + @read_api + def get_item_name(self, field, item_id): + return self.fields[field].table.id_map[item_id] + @read_api def author_data(self, author_ids=None): ''' diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 2c84542d4f..f8faed4290 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -90,6 +90,13 @@ class LibraryDatabase(object): return func setattr(self, 'get_%s_with_ids' % field, MT(getter(field))) + for field in ('author', 'tag', 'series'): + def getter(field): + field = field if field == 'series' else (field+'s') + def func(self, item_id): + return self.new_api.get_item_name(field, item_id) + return func + setattr(self, '%s_name' % field, MT(getter(field))) # Legacy field API for func in ( diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 55fac0120d..96fc98d5be 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -32,7 +32,9 @@ class ET(object): return newres def compare_argspecs(old, new, attr): - ok = len(old.args) == len(new.args) and len(old.defaults or ()) == len(new.defaults or ()) and old.args[-len(old.defaults or ()):] == new.args[-len(new.defaults or ()):] # noqa + num = len(old.defaults or ()) + + ok = len(old.args) == len(new.args) and old.defaults == new.defaults and (num == 0 or old.args[-num:] == new.args[-num:]) if not ok: raise AssertionError('The argspec for %s does not match. %r != %r' % (attr, old, new)) @@ -165,6 +167,9 @@ class LegacyTest(BaseTest): '!get_publishers_with_ids':[()], '!get_ratings_with_ids':[()], '!get_languages_with_ids':[()], + 'tag_name':[(3,)], + 'author_name':[(3,)], + 'series_name':[(3,)], }.iteritems(): for a in args: fmt = lambda x: x From 14441ff0524064a05ba2d05c80f82b6ba12edf7b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 16:01:16 +0530 Subject: [PATCH 0160/1154] ... --- src/calibre/db/tests/legacy.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 96fc98d5be..5df09ec065 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -32,6 +32,8 @@ class ET(object): return newres def compare_argspecs(old, new, attr): + # We dont compare the names of the non-keyword arguments as they are often + # different and they dont affect the usage of the API. num = len(old.defaults or ()) ok = len(old.args) == len(new.args) and old.defaults == new.defaults and (num == 0 or old.args[-num:] == new.args[-num:]) From 598a604853a8b0b8a2b435aa5b4fc1c3bf0adb54 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 17:11:13 +0530 Subject: [PATCH 0161/1154] MTP driver: Ignore the zinio folder by default --- src/calibre/devices/mtp/driver.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index f0e532639a..40fa9d900b 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -83,7 +83,7 @@ class MTP_DEVICE(BASE): return name in { 'alarms', 'android', 'dcim', 'movies', 'music', 'notifications', 'pictures', 'ringtones', 'samsung', 'sony', 'htc', 'bluetooth', - 'games', 'lost.dir', 'video', 'whatsapp', 'image'} + 'games', 'lost.dir', 'video', 'whatsapp', 'image', 'com.zinio.mobile.android.reader'} def configure_for_kindle_app(self): proxy = self.prefs @@ -159,16 +159,17 @@ class MTP_DEVICE(BASE): def get_driveinfo(self): if not self.driveinfo: self.driveinfo = {} - for sid, location_code in ( (self._main_id, 'main'), (self._carda_id, + for sid, location_code in ((self._main_id, 'main'), (self._carda_id, 'A'), (self._cardb_id, 'B')): - if sid is None: continue + if sid is None: + continue self._update_drive_info(self.filesystem_cache.storage(sid), location_code) return self.driveinfo def get_device_information(self, end_session=True): self.report_progress(1.0, _('Get device information...')) dinfo = self.get_basic_device_information() - return tuple( list(dinfo) + [self.driveinfo] ) + return tuple(list(dinfo) + [self.driveinfo]) def card_prefix(self, end_session=True): return (self._carda_id, self._cardb_id) @@ -190,7 +191,7 @@ class MTP_DEVICE(BASE): from calibre.devices.mtp.books import JSONCodec from calibre.devices.mtp.books import BookList, Book self.report_progress(0, _('Listing files, this can take a while')) - self.get_driveinfo() # Ensure driveinfo is loaded + self.get_driveinfo() # Ensure driveinfo is loaded sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(oncard, self._main_id) if sid is None: @@ -230,7 +231,7 @@ class MTP_DEVICE(BASE): cached_metadata.path = mtp_file.mtp_id_path debug('Using cached metadata for', '/'.join(mtp_file.full_path)) - continue # No need to update metadata + continue # No need to update metadata book = cached_metadata else: book = Book(sid, '/'.join(relpath)) @@ -352,8 +353,8 @@ class MTP_DEVICE(BASE): def prefix_for_location(self, on_card): if self.location_paths is None: self.location_paths = {} - for sid, loc in ( (self._main_id, None), (self._carda_id, 'carda'), - (self._cardb_id, 'cardb') ): + for sid, loc in ((self._main_id, None), (self._carda_id, 'carda'), + (self._cardb_id, 'cardb')): if sid is not None: storage = self.filesystem_cache.storage(sid) prefixes = self.get_pref('send_to') @@ -470,7 +471,8 @@ class MTP_DEVICE(BASE): def remove_books_from_metadata(self, paths, booklists): self.report_progress(0, _('Removing books from metadata')) - class NextPath(Exception): pass + class NextPath(Exception): + pass for i, path in enumerate(paths): try: @@ -549,3 +551,4 @@ if __name__ == '__main__': dev.shutdown() + From a8033fc610b940a493734624485579bb5c8cc65e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 17:27:13 +0530 Subject: [PATCH 0162/1154] Author data legacy API --- src/calibre/db/legacy.py | 19 +++++++++++++++++++ src/calibre/db/tests/legacy.py | 3 +++ 2 files changed, 22 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index f8faed4290..0ab29d2e3d 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -312,6 +312,25 @@ class LibraryDatabase(object): mi = self.new_api.get_metadata(book_id, get_cover=key == 'cover') return mi.get(key, default) + def authors_sort_strings(self, index, index_is_id=False): + book_id = index if index_is_id else self.data.index_to_id(index) + with self.new_api.read_lock: + af = self.new_api.fields['authors'].table + authors = af.book_col_map.get(book_id, ()) + adata = self.new_api._author_data(authors) + return [adata[aid]['sort'] for aid in authors] + + def author_sort_from_book(self, index, index_is_id=False): + return ' & '.join(self.authors_sort_strings(index, index_is_id=index_is_id)) + + def authors_with_sort_strings(self, index, index_is_id=False): + book_id = index if index_is_id else self.data.index_to_id(index) + with self.new_api.read_lock: + af = self.new_api.fields['authors'].table + authors = af.book_col_map.get(book_id, ()) + adata = self.new_api._author_data(authors) + return [(aid, adata[aid]['name'], adata[aid]['sort'], adata[aid]['link']) for aid in authors] + # Private interface {{{ def __iter__(self): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 5df09ec065..fa76439387 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -172,6 +172,9 @@ class LegacyTest(BaseTest): 'tag_name':[(3,)], 'author_name':[(3,)], 'series_name':[(3,)], + 'authors_sort_strings':[(0,), (1,), (2,)], + 'author_sort_from_book':[(0,), (1,), (2,)], + 'authors_with_sort_strings':[(0,), (1,), (2,)], }.iteritems(): for a in args: fmt = lambda x: x From 7c1421d15a6137a4a5fa90ec060d1ebec6e1433f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 17:38:16 +0530 Subject: [PATCH 0163/1154] ... --- src/calibre/db/legacy.py | 6 ++---- src/calibre/db/tests/legacy.py | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 0ab29d2e3d..ff0c742e9d 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -315,8 +315,7 @@ class LibraryDatabase(object): def authors_sort_strings(self, index, index_is_id=False): book_id = index if index_is_id else self.data.index_to_id(index) with self.new_api.read_lock: - af = self.new_api.fields['authors'].table - authors = af.book_col_map.get(book_id, ()) + authors = self.new_api._field_ids_for('authors', book_id) adata = self.new_api._author_data(authors) return [adata[aid]['sort'] for aid in authors] @@ -326,8 +325,7 @@ class LibraryDatabase(object): def authors_with_sort_strings(self, index, index_is_id=False): book_id = index if index_is_id else self.data.index_to_id(index) with self.new_api.read_lock: - af = self.new_api.fields['authors'].table - authors = af.book_col_map.get(book_id, ()) + authors = self.new_api._field_ids_for('authors', book_id) adata = self.new_api._author_data(authors) return [(aid, adata[aid]['name'], adata[aid]['sort'], adata[aid]['link']) for aid in authors] diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index fa76439387..daa53b7ddd 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -262,6 +262,8 @@ class LegacyTest(BaseTest): 'get_feeds', 'get_feed', 'update_feed', 'remove_feeds', 'add_feed', 'set_feeds', # Obsolete/broken methods 'author_id', # replaced by get_author_id + 'books_for_author', # broken + 'books_in_old_database', # unused } SKIP_ARGSPEC = { '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', @@ -329,3 +331,4 @@ class LegacyTest(BaseTest): T(('n', object())) old.close() # }}} + From 3719dfb4d8a607566b2ac4f069671ec878b4a44a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 17:42:16 +0530 Subject: [PATCH 0164/1154] book_on_device API --- src/calibre/db/legacy.py | 23 +++++++++++++++++++++++ src/calibre/db/tests/legacy.py | 1 + 2 files changed, 24 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index ff0c742e9d..97be03fb19 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -108,6 +108,7 @@ class LibraryDatabase(object): self.metadata_for_field = self.field_metadata.get self.last_update_check = self.last_modified() + self.book_on_device_func = None def close(self): self.backend.close() @@ -329,6 +330,28 @@ class LibraryDatabase(object): adata = self.new_api._author_data(authors) return [(aid, adata[aid]['name'], adata[aid]['sort'], adata[aid]['link']) for aid in authors] + def book_on_device(self, book_id): + if callable(self.book_on_device_func): + return self.book_on_device_func(book_id) + return None + + def book_on_device_string(self, book_id): + loc = [] + count = 0 + on = self.book_on_device(book_id) + if on is not None: + m, a, b, count = on[:4] + if m is not None: + loc.append(_('Main')) + if a is not None: + loc.append(_('Card A')) + if b is not None: + loc.append(_('Card B')) + return ', '.join(loc) + ((_(' (%s books)')%count) if count > 1 else '') + + def set_book_on_device_func(self, func): + self.book_on_device_func = func + # Private interface {{{ def __iter__(self): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index daa53b7ddd..e5a8a8e7b5 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -175,6 +175,7 @@ class LegacyTest(BaseTest): 'authors_sort_strings':[(0,), (1,), (2,)], 'author_sort_from_book':[(0,), (1,), (2,)], 'authors_with_sort_strings':[(0,), (1,), (2,)], + 'book_on_device_string':[(1,), (2,), (3,)], }.iteritems(): for a in args: fmt = lambda x: x From ecbea8b0e22b507d88db5cfab997d38031a46196 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 18:04:27 +0530 Subject: [PATCH 0165/1154] More API --- src/calibre/db/legacy.py | 16 ++++++++++++++++ src/calibre/db/tests/legacy.py | 3 +++ 2 files changed, 19 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 97be03fb19..a4dd30a88c 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -109,6 +109,9 @@ class LibraryDatabase(object): self.last_update_check = self.last_modified() self.book_on_device_func = None + # Cleaning is not required anymore + self.clean = self.clean_custom = MT(lambda self:None) + self.clean_standard_field = MT(lambda self, field, commit=False:None) def close(self): self.backend.close() @@ -352,6 +355,19 @@ class LibraryDatabase(object): def set_book_on_device_func(self, func): self.book_on_device_func = func + def books_in_series(self, series_id): + with self.new_api.read_lock: + book_ids = self.new_api._books_for_field('series', series_id) + ff = self.new_api._field_for + return sorted(book_ids, key=lambda x:ff('series_index', x)) + + def books_in_series_of(self, index, index_is_id=False): + book_id = index if index_is_id else self.data.index_to_id(index) + series_ids = self.new_api.field_ids_for('series', book_id) + if not series_ids: + return [] + return self.books_in_series(series_ids[0]) + # Private interface {{{ def __iter__(self): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index e5a8a8e7b5..5c26e50ae5 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -176,6 +176,7 @@ class LegacyTest(BaseTest): 'author_sort_from_book':[(0,), (1,), (2,)], 'authors_with_sort_strings':[(0,), (1,), (2,)], 'book_on_device_string':[(1,), (2,), (3,)], + 'books_in_series_of':[(0,), (1,), (2,)], }.iteritems(): for a in args: fmt = lambda x: x @@ -265,6 +266,8 @@ class LegacyTest(BaseTest): 'author_id', # replaced by get_author_id 'books_for_author', # broken 'books_in_old_database', # unused + 'clean_user_categories', # internal API + 'cleanup_tags', # internal API } SKIP_ARGSPEC = { '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', From 1e4d11d78fb59a7cb548106ac9b699a158bac559 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 18:21:37 +0530 Subject: [PATCH 0166/1154] books_with_same_title() --- src/calibre/db/legacy.py | 16 +++++++++++++++- src/calibre/db/tests/legacy.py | 6 ++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index a4dd30a88c..8c5fa5bd31 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -10,6 +10,7 @@ import os, traceback, types from functools import partial from future_builtins import zip +from calibre import force_unicode from calibre.db import _get_next_series_num_for_list, _get_series_values from calibre.db.adding import ( find_books_in_directory, import_book_directory_multiple, @@ -112,6 +113,8 @@ class LibraryDatabase(object): # Cleaning is not required anymore self.clean = self.clean_custom = MT(lambda self:None) self.clean_standard_field = MT(lambda self, field, commit=False:None) + # apsw operates in autocommit mode + self.commit = MT(lambda self:None) def close(self): self.backend.close() @@ -368,8 +371,19 @@ class LibraryDatabase(object): return [] return self.books_in_series(series_ids[0]) - # Private interface {{{ + def books_with_same_title(self, mi, all_matches=True): + title = mi.title + ans = set() + if title: + title = icu_lower(force_unicode(title)) + for book_id, x in self.new_api.get_id_map('title').iteritems(): + if icu_lower(x) == title: + ans.add(book_id) + if not all_matches: + break + return ans + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): yield row diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 5c26e50ae5..2f5c879532 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -177,6 +177,7 @@ class LegacyTest(BaseTest): 'authors_with_sort_strings':[(0,), (1,), (2,)], 'book_on_device_string':[(1,), (2,), (3,)], 'books_in_series_of':[(0,), (1,), (2,)], + 'books_with_same_title':[(Metadata(db.title(0)),), (Metadata(db.title(1)),), (Metadata('1234'),)], }.iteritems(): for a in args: fmt = lambda x: x @@ -266,8 +267,9 @@ class LegacyTest(BaseTest): 'author_id', # replaced by get_author_id 'books_for_author', # broken 'books_in_old_database', # unused - 'clean_user_categories', # internal API - 'cleanup_tags', # internal API + + # Internal API + 'clean_user_categories', 'cleanup_tags', 'books_list_filter', } SKIP_ARGSPEC = { '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', From 72c0346874fa2bb96236b76c0a1c7a49752a3400 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 18:23:34 +0530 Subject: [PATCH 0167/1154] Update lamebook --- recipes/lamebook.recipe | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipes/lamebook.recipe b/recipes/lamebook.recipe index e449285d84..0f4fbe9dc0 100644 --- a/recipes/lamebook.recipe +++ b/recipes/lamebook.recipe @@ -13,6 +13,8 @@ class LamebookRecipe(BasicNewsRecipe): language = 'en' use_embedded_content = False publication_type = 'blog' + reverse_article_order = True + encoding = 'utf-8' keep_only_tags = [ dict(name='div', attrs={'class':'entry'}) From 8a3cb9b977abf1b0017fb83694df40d75e032d19 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Fri, 12 Jul 2013 17:22:45 +0200 Subject: [PATCH 0168/1154] Add ability to create a composite column containing the virtual libraries that the book is a member of. Intermediate commit -- still testing --- src/calibre/gui2/search_restriction_mixin.py | 7 ++ src/calibre/library/caches.py | 115 ++++++++++++------- src/calibre/library/cli.py | 2 +- src/calibre/library/database2.py | 6 + src/calibre/library/field_metadata.py | 10 ++ src/calibre/utils/formatter_functions.py | 12 +- src/calibre/utils/search_query_parser.py | 15 ++- 7 files changed, 119 insertions(+), 48 deletions(-) diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index b986a2a78e..a90d607ea9 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -332,6 +332,7 @@ class SearchRestrictionMixin(object): virt_libs = db.prefs.get('virtual_libraries', {}) virt_libs[name] = search db.prefs.set('virtual_libraries', virt_libs) + db.data.invalidate_virtual_libraries_caches(db) def do_create_edit(self, name=None): db = self.library_view.model().db @@ -341,8 +342,11 @@ class SearchRestrictionMixin(object): if name: self._remove_vl(name, reapply=False) self.add_virtual_library(db, cd.library_name, cd.library_search) + db.data.invalidate_virtual_libraries_caches(db) if not name or name == db.data.get_base_restriction_name(): self.apply_virtual_library(cd.library_name) + else: + self.tags_view.recount() def virtual_library_clicked(self): m = self.virtual_library_menu @@ -462,6 +466,9 @@ class SearchRestrictionMixin(object): default_yes=False): return self._remove_vl(name, reapply=True) + db = self.library_view.model().db + db.data.invalidate_virtual_libraries_caches(db) + self.tags_view.recount() def _remove_vl(self, name, reapply=True): db = self.library_view.model().db diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index e552ead591..05d964be71 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -2,7 +2,7 @@ # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai from __future__ import with_statement -__license__ = 'GPL v3' +__license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' @@ -144,7 +144,8 @@ def force_to_bool(val): class CacheRow(list): # {{{ - def __init__(self, db, composites, val, series_col, series_sort_col): + def __init__(self, db, composites, val, series_col, series_sort_col, + virtual_library_col): self.db = db self._composites = composites list.__init__(self, val) @@ -152,6 +153,8 @@ class CacheRow(list): # {{{ self._series_col = series_col self._series_sort_col = series_sort_col self._series_sort = None + self._virt_lib_col = virtual_library_col + self._virt_libs = None def __getitem__(self, col): if self._must_do: @@ -171,7 +174,7 @@ class CacheRow(list): # {{{ mi = self.db.get_metadata(id_, index_is_id=True, get_user_categories=False) for c in self._composites: - self[c] = mi.get(self._composites[c]) + self[c] = mi.get(self._composites[c]) if col == self._series_sort_col and self._series_sort is None: if self[self._series_col]: self._series_sort = title_sort(self[self._series_col]) @@ -179,6 +182,26 @@ class CacheRow(list): # {{{ else: self._series_sort = '' self[self._series_sort_col] = '' + + if col == self._virt_lib_col and self._virt_libs is None: + try: + if not getattr(self.db.data, '_virt_libs_computed', False): + self.db.data._ids_in_virt_libs = {} + for v,s in self.db.prefs.get('virtual_libraries', {}).iteritems(): + self.db.data._ids_in_virt_libs[v] = self.db.data.search_raw(s) + self.db.data._virt_libs_computed = True + r = [] + for v in self.db.prefs.get('virtual_libraries', {}).keys(): + # optimize the lookup of the ID -- it is always zero + if self[0] in self.db.data._ids_in_virt_libs[v]: + r.append(v) + from calibre.utils.icu import sort_key + self._virt_libs = ", ".join(sorted(r, key=sort_key)) + self[self._virt_lib_col] = self._virt_libs + except: + print len(self) + traceback.print_exc() + return list.__getitem__(self, col) def __getslice__(self, i, j): @@ -186,7 +209,7 @@ class CacheRow(list): # {{{ def refresh_composites(self): for c in self._composites: - self[c] = None + self[c] = None self._must_do = True # }}} @@ -206,6 +229,7 @@ class ResultCache(SearchQueryParser): # {{{ self.composites[field_metadata[key]['rec_index']] = key self.series_col = field_metadata['series']['rec_index'] self.series_sort_col = field_metadata['series_sort']['rec_index'] + self.virtual_libraries_col = field_metadata['virtual_libraries']['rec_index'] self._data = [] self._map = self._map_filtered = [] self.first_sort = True @@ -312,12 +336,12 @@ class ResultCache(SearchQueryParser): # {{{ '<=':[2, relop_le] } - local_today = ('_today', icu_lower(_('today'))) - local_yesterday = ('_yesterday', icu_lower(_('yesterday'))) - local_thismonth = ('_thismonth', icu_lower(_('thismonth'))) - local_daysago = icu_lower(_('daysago')) - local_daysago_len = len(local_daysago) - untrans_daysago = '_daysago' + local_today = ('_today', icu_lower(_('today'))) + local_yesterday = ('_yesterday', icu_lower(_('yesterday'))) + local_thismonth = ('_thismonth', icu_lower(_('thismonth'))) + local_daysago = icu_lower(_('daysago')) + local_daysago_len = len(local_daysago) + untrans_daysago = '_daysago' untrans_daysago_len = len('_daysago') def get_dates_matches(self, location, query, candidates): @@ -413,21 +437,21 @@ class ResultCache(SearchQueryParser): # {{{ if val_func is None: loc = self.field_metadata[location]['rec_index'] - val_func = lambda item, loc=loc: item[loc] + val_func = lambda item, loc = loc: item[loc] q = '' cast = adjust = lambda x: x dt = self.field_metadata[location]['datatype'] if query == 'false': if dt == 'rating' or location == 'cover': - relop = lambda x,y: not bool(x) + relop = lambda x, y: not bool(x) else: - relop = lambda x,y: x is None + relop = lambda x, y: x is None elif query == 'true': if dt == 'rating' or location == 'cover': - relop = lambda x,y: bool(x) + relop = lambda x, y: bool(x) else: - relop = lambda x,y: x is not None + relop = lambda x, y: x is not None else: relop = None for k in self.numeric_search_relops.keys(): @@ -441,7 +465,7 @@ class ResultCache(SearchQueryParser): # {{{ cast = lambda x: int(x) elif dt == 'rating': cast = lambda x: 0 if x is None else int(x) - adjust = lambda x: x/2 + adjust = lambda x: x / 2 elif dt in ('float', 'composite'): cast = lambda x : float(x) else: # count operation @@ -449,7 +473,7 @@ class ResultCache(SearchQueryParser): # {{{ if len(query) > 1: mult = query[-1:].lower() - mult = {'k':1024.,'m': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) + mult = {'k':1024., 'm': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) if mult != 1.0: query = query[:-1] else: @@ -568,12 +592,12 @@ class ResultCache(SearchQueryParser): # {{{ query = icu_lower(query) return matchkind, query - local_no = icu_lower(_('no')) - local_yes = icu_lower(_('yes')) + local_no = icu_lower(_('no')) + local_yes = icu_lower(_('yes')) local_unchecked = icu_lower(_('unchecked')) - local_checked = icu_lower(_('checked')) - local_empty = icu_lower(_('empty')) - local_blank = icu_lower(_('blank')) + local_checked = icu_lower(_('checked')) + local_empty = icu_lower(_('empty')) + local_blank = icu_lower(_('blank')) local_bool_values = ( local_no, local_unchecked, '_no', 'false', local_yes, local_checked, '_yes', 'true', @@ -696,8 +720,8 @@ class ResultCache(SearchQueryParser): # {{{ if fm['is_multiple'] and \ len(query) > 1 and query.startswith('#') and \ query[1:1] in '=<>!': - vf = lambda item, loc=fm['rec_index'], \ - ms=fm['is_multiple']['cache_to_list']:\ + vf = lambda item, loc = fm['rec_index'], \ + ms = fm['is_multiple']['cache_to_list']:\ len(item[loc].split(ms)) if item[loc] is not None else 0 return self.get_numeric_matches(location, query[1:], candidates, val_func=vf) @@ -707,7 +731,7 @@ class ResultCache(SearchQueryParser): # {{{ if fm.get('is_csp', False): if location == 'identifiers' and original_location == 'isbn': return self.get_keypair_matches('identifiers', - '=isbn:'+query, candidates) + '=isbn:' + query, candidates) return self.get_keypair_matches(location, query, candidates) # check for user categories @@ -759,7 +783,7 @@ class ResultCache(SearchQueryParser): # {{{ q = canonicalize_lang(query) if q is None: lm = lang_map() - rm = {v.lower():k for k,v in lm.iteritems()} + rm = {v.lower():k for k, v in lm.iteritems()} q = rm.get(query, query) else: q = query @@ -772,7 +796,7 @@ class ResultCache(SearchQueryParser): # {{{ if not item[loc]: if q == 'false' and matchkind == CONTAINS_MATCH: matches.add(item[0]) - continue # item is empty. No possible matches below + continue # item is empty. No possible matches below if q == 'false'and matchkind == CONTAINS_MATCH: # Field has something in it, so a false query does not match continue @@ -816,6 +840,13 @@ class ResultCache(SearchQueryParser): # {{{ current_candidates -= matches return matches + def invalidate_virtual_libraries_caches(self, db): + self.refresh(db) + + def search_raw(self, query): + matches = self.parse(query) + return matches + def search(self, query, return_matches=False): ans = self.search_getting_ids(query, self.search_restriction, set_restriction_count=True) @@ -973,10 +1004,11 @@ class ResultCache(SearchQueryParser): # {{{ try: self._data[id] = CacheRow(db, self.composites, db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0], - self.series_col, self.series_sort_col) + self.series_col, self.series_sort_col, + self.virtual_libraries_col) self._data[id].append(db.book_on_device_string(id)) - self._data[id].append(self.marked_ids_dict.get(id, None)) - self._data[id].append(None) + self._data[id].extend((self.marked_ids_dict.get(id, None), None, None)) + self._virt_libs_computed = False self._uuid_map[self._data[id][self._uuid_column_index]] = id except IndexError: return None @@ -989,14 +1021,15 @@ class ResultCache(SearchQueryParser): # {{{ def books_added(self, ids, db): if not ids: return - self._data.extend(repeat(None, max(ids)-len(self._data)+2)) + self._data.extend(repeat(None, max(ids) - len(self._data) + 2)) for id in ids: self._data[id] = CacheRow(db, self.composites, db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0], - self.series_col, self.series_sort_col) + self.series_col, self.series_sort_col, + self.virtual_libraries_col) self._data[id].append(db.book_on_device_string(id)) - self._data[id].append(self.marked_ids_dict.get(id, None)) - self._data[id].append(None) # Series sort column + self._data[id].extend((self.marked_ids_dict.get(id, None), None, None)) + self._virt_libs_computed = False self._uuid_map[self._data[id][self._uuid_column_index]] = id self._map[0:0] = ids self._map_filtered[0:0] = ids @@ -1020,20 +1053,22 @@ class ResultCache(SearchQueryParser): # {{{ db.initialize_template_cache() temp = db.conn.get('SELECT * FROM meta2') - self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else [] + self._data = list(itertools.repeat(None, temp[-1][0] + 2)) if temp else [] for r in temp: self._data[r[0]] = CacheRow(db, self.composites, r, - self.series_col, self.series_sort_col) + self.series_col, self.series_sort_col, + self.virtual_libraries_col) self._uuid_map[self._data[r[0]][self._uuid_column_index]] = r[0] for item in self._data: if item is not None: item.append(db.book_on_device_string(item[0])) - # Temp mark and series_sort columns - item.extend((None, None)) + # Temp mark, series_sort, virtual_library columns + item.extend((None, None, None)) + self._virt_libs_computed = False marked_col = self.FIELD_MAP['marked'] - for id_,val in self.marked_ids_dict.iteritems(): + for id_, val in self.marked_ids_dict.iteritems(): try: self._data[id_][marked_col] = val except: @@ -1134,7 +1169,7 @@ class SortKeyGenerator(object): for i, candidate in enumerate( ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')): if val.endswith(candidate): - p = 1024**(i) + p = 1024 ** (i) val = val[:-len(candidate)].strip() break val = locale.atof(val) * p diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 547cc5bc08..a86ec7ef6d 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -575,7 +575,7 @@ def command_set_metadata(args, dbpath): for key in sorted(db.field_metadata.all_field_keys()): m = db.field_metadata[key] if (key not in {'formats', 'series_sort', 'ondevice', 'path', - 'last_modified'} and m['is_editable'] and m['name']): + 'virtual_libraries', 'last_modified'} and m['is_editable'] and m['name']): yield key, m if m['datatype'] == 'series': si = m.copy() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index bd3f155c9d..d23633e433 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -449,6 +449,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.field_metadata.set_field_record_index('marked', base, prefer_custom=False) self.FIELD_MAP['series_sort'] = base = base+1 self.field_metadata.set_field_record_index('series_sort', base, prefer_custom=False) + self.FIELD_MAP['virtual_libraries'] = base = base+1 + self.field_metadata.set_field_record_index('virtual_libraries', base, prefer_custom=False) script = ''' DROP VIEW IF EXISTS meta2; @@ -992,6 +994,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.book_size = row[fm['size']] mi.ondevice_col= row[fm['ondevice']] mi.last_modified = row[fm['last_modified']] + + mi._base_db_row = row # So the formatter functions can see the underlying data + mi._virt_lib_column = fm['virtual_libraries'] + formats = row[fm['formats']] mi.format_metadata = {} if not formats: diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 08c26e95a9..a96f819b58 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -387,6 +387,16 @@ class FieldMetadata(dict): 'is_custom':False, 'is_category':False, 'is_csp': False}), + ('virtual_libraries', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':{}, + 'kind':'field', + 'name':_('Virtual Libraries'), + 'search_terms':['virtual_libraries'], + 'is_custom':False, + 'is_category':False, + 'is_csp': False}), ] # }}} diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 73dad7422b..c94467fec0 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -1209,9 +1209,19 @@ class BuiltinFinishFormatting(BuiltinFormatterFunction): return val return prefix + formatter._do_format(val, fmt) + suffix +class BuiltinBookInVirtualLibraries(BuiltinFormatterFunction): + name = 'book_in_virtual_libraries' + arg_count = 0 + category = 'Get values from metadata' + __doc__ = doc = _('book_in_virtual_libraries() -- returns a list of ' + 'virtual libraries that this book is in.') + + def evaluate(self, formatter, kwargs, mi, locals_): + return mi._base_db_row[mi._virt_lib_column ] + _formatter_builtins = [ BuiltinAdd(), BuiltinAnd(), BuiltinApproximateFormats(), - BuiltinAssign(), BuiltinBooksize(), + BuiltinAssign(), BuiltinBookInVirtualLibraries(), BuiltinBooksize(), BuiltinCapitalize(), BuiltinCmp(), BuiltinContains(), BuiltinCount(), BuiltinCurrentLibraryName(), BuiltinCurrentLibraryPath(), BuiltinDaysBetween(), BuiltinDivide(), BuiltinEval(), BuiltinFirstNonEmpty(), diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index 2682088681..08a70a533d 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -294,6 +294,7 @@ class SearchQueryParser(object): def __init__(self, locations, test=False, optimize=False): self.sqp_initialize(locations, test=test, optimize=optimize) + self.sqp_parsed_search_cache = {} self.parser = Parser() def sqp_change_locations(self, locations): @@ -308,8 +309,7 @@ class SearchQueryParser(object): # empty the list of searches used for recursion testing self.recurse_level = 0 self.searches_seen = set([]) - candidates = self.universal_set() - return self._parse(query, candidates) + return self._parse(query) # this parse is used internally because it doesn't clear the # recursive search test list. However, we permit seeing the @@ -317,10 +317,13 @@ class SearchQueryParser(object): # another search. def _parse(self, query, candidates=None): self.recurse_level += 1 - try: - res = self.parser.parse(query, self.locations) - except RuntimeError: - raise ParseException(_('Failed to parse query, recursion limit reached: %s')%repr(query)) + res = self.sqp_parsed_search_cache.get(query, None) + if res is None: + try: + res = self.parser.parse(query, self.locations) + self.sqp_parsed_search_cache[query] = res + except RuntimeError: + raise ParseException(_('Failed to parse query, recursion limit reached: %s')%repr(query)) if candidates is None: candidates = self.universal_set() t = self.evaluate(res, candidates) From 86c314bbc5a618b46e8429ba26da6ef30793a74e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gae=CC=88tan=20Lehmann?= Date: Fri, 12 Jul 2013 18:33:12 +0200 Subject: [PATCH 0169/1154] add monde-diplomatique.fr recipe --- recipes/icons/le_monde_diplomatique_fr.png | Bin 0 -> 446 bytes recipes/le_monde_diplomatique_fr.recipe | 111 +++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 recipes/icons/le_monde_diplomatique_fr.png create mode 100644 recipes/le_monde_diplomatique_fr.recipe diff --git a/recipes/icons/le_monde_diplomatique_fr.png b/recipes/icons/le_monde_diplomatique_fr.png new file mode 100644 index 0000000000000000000000000000000000000000..ae4547977f67c6988ecf696047b47482effa8ddf GIT binary patch literal 446 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE3?yBabR7dyEa{HEjtmSN`?>!lvVtU&J%W50 z7^>757#dm_7=8hT8eT9klo~KFyh>nTu$sZZAYL$MSD+10!Y05c#P!UXGhe=Z{`c?i zOa_J%HJ8W`|Cl(zu^J-YSPTNYul2KxwhLQP$Q#x!4 zr4Nj^*=~OQplTk&#D2-_NexCU3wTo(H-X&ystszz{M@xoaYxD z{krHL^S0l%^Q$@Jn(O`s*IwmZW5-qS@Zi5#pzBmiTq8oyUAEN_sDZ)L)z4*}Q$iB}g`T6{ literal 0 HcmV?d00001 diff --git a/recipes/le_monde_diplomatique_fr.recipe b/recipes/le_monde_diplomatique_fr.recipe new file mode 100644 index 0000000000..b099cf3729 --- /dev/null +++ b/recipes/le_monde_diplomatique_fr.recipe @@ -0,0 +1,111 @@ +# vim:fileencoding=utf-8 +from __future__ import unicode_literals + +__license__ = 'GPL v3' +__copyright__ = '2013' +''' +monde-diplomatique.fr +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe +from calibre.web.feeds import feeds_from_index + +class LeMondeDiplomatiqueSiteWeb(BasicNewsRecipe): + title = u'Le Monde diplomatique.fr' + __author__ = 'Gaëtan Lehmann' + description = "Le Monde diplomatique est un mensuel français d’information et d’opinion à la ligne éditoriale nettement engagée en faveur d'une gauche de rupture avec le capitalisme. Il aborde de nombreux sujets — géopolitique, relations internationales, économie, questions sociales, écologie, culture, médias, …" + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + publisher = 'monde-diplomatique.fr' + category = 'news, France, world' + language = 'fr' + masthead_url = 'http://www.monde-diplomatique.fr/squelettes/images/logotyfa.png' + timefmt = ' [%d %b %Y]' + no_stylesheets = True + + feeds = [(u'Blogs', u'http://blog.mondediplo.net/spip.php?page=backend'), (u'Archives', u'http://www.monde-diplomatique.fr/rss/')] + + preprocess_regexps = [ + (re.compile(r'(.*) - Les blogs du Diplo'), lambda m: '' + m.group(1) + ''), + (re.compile(r'

    (.*) - Les blogs du Diplo

    '), lambda m: '

    ' + m.group(1) + '

    '), + (re.compile(r'(.*) \(Le Monde diplomatique\)'), lambda m: '' + m.group(1) + ''), + (re.compile(r'

    (.*) \(Le Monde diplomatique\)

    '), lambda m: '

    ' + m.group(1) + '

    '), + (re.compile(r'

    Grand format

    '), lambda m: '')] + + remove_tags = [dict(name='div', attrs={'class':'voiraussi liste'}), + dict(name='ul', attrs={'class':'hermetique carto hombre_demi_inverse'}), + dict(name='a', attrs={'class':'tousles'}), + dict(name='h3', attrs={'class':'cat'}), + dict(name='div', attrs={'class':'logodiplo'}), + dict(name='img', attrs={'class':'spip_logos'}), + dict(name='p', attrs={'id':'hierarchie'}), + dict(name='div', attrs={'class':'espace'})] + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + ,'linearize_tables': True + } + + remove_empty_feeds = True + + filterDuplicates = True + + # don't use parse_index - we need it to send an exception so we can mix + # feed and parse_index results in parse_feeds + def parse_index_valise(self): + articles = [] + soup = self.index_to_soup('http://www.monde-diplomatique.fr/carnet/') + cnt = soup.find('ul',attrs={'class':'hermetique liste'}) + for item in cnt.findAll('li'): + description = '' + feed_link = item.find('a') + desc = item.find('div',attrs={'class':'intro'}) + date = item.find('div',attrs={'class':'dates_auteurs'}) + if desc: + description = desc.string + if feed_link and feed_link.has_key('href'): + url = 'http://www.monde-diplomatique.fr' + feed_link['href'] + title = self.tag_to_string(feed_link) + articles.append({ + 'title' :title + ,'date' :date.string.strip() + ,'url' :url + ,'description':description + }) + return [("La valise diplomatique", articles)] + + def parse_index_cartes(self): + articles = [] + soup = self.index_to_soup('http://www.monde-diplomatique.fr/cartes/') + cnt = soup.find('div',attrs={'class':'decale hermetique'}) + for item in cnt.findAll('div',attrs={'class':re.compile('grid_3 filet hombre_demi')}): + feed_link = item.find('a',attrs={'class':'couve'}) + h3 = item.find('h3') + authorAndDate = item.find('div',attrs={'class':'dates_auteurs'}) + author, date = authorAndDate.string.strip().split(', ') + if feed_link and feed_link.has_key('href'): + url = 'http://www.monde-diplomatique.fr' + feed_link['href'] + title = self.tag_to_string(h3) + articles.append({ + 'title' :title + ,'date' :date + ,'url' :url + ,'description': author + }) + return [("Cartes", articles)] + + def parse_feeds(self): + feeds = BasicNewsRecipe.parse_feeds(self) + valise = feeds_from_index(self.parse_index_valise(), oldest_article=self.oldest_article, + max_articles_per_feed=self.max_articles_per_feed, + log=self.log) + cartes = feeds_from_index(self.parse_index_cartes(), oldest_article=self.oldest_article, + max_articles_per_feed=self.max_articles_per_feed, + log=self.log) + feeds = valise + feeds + cartes + return feeds From 452ff0f8a842fd8c93350bf7322c5aba3784cc16 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 12 Jul 2013 22:17:58 +0530 Subject: [PATCH 0170/1154] oops, accidentally unstaged the files in the last commit --- recipes/icons/le_monde_diplomatique_fr.png | Bin 0 -> 446 bytes recipes/le_monde_diplomatique_fr.recipe | 111 +++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 recipes/icons/le_monde_diplomatique_fr.png create mode 100644 recipes/le_monde_diplomatique_fr.recipe diff --git a/recipes/icons/le_monde_diplomatique_fr.png b/recipes/icons/le_monde_diplomatique_fr.png new file mode 100644 index 0000000000000000000000000000000000000000..ae4547977f67c6988ecf696047b47482effa8ddf GIT binary patch literal 446 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE3?yBabR7dyEa{HEjtmSN`?>!lvVtU&J%W50 z7^>757#dm_7=8hT8eT9klo~KFyh>nTu$sZZAYL$MSD+10!Y05c#P!UXGhe=Z{`c?i zOa_J%HJ8W`|Cl(zu^J-YSPTNYul2KxwhLQP$Q#x!4 zr4Nj^*=~OQplTk&#D2-_NexCU3wTo(H-X&ystszz{M@xoaYxD z{krHL^S0l%^Q$@Jn(O`s*IwmZW5-qS@Zi5#pzBmiTq8oyUAEN_sDZ)L)z4*}Q$iB}g`T6{ literal 0 HcmV?d00001 diff --git a/recipes/le_monde_diplomatique_fr.recipe b/recipes/le_monde_diplomatique_fr.recipe new file mode 100644 index 0000000000..f7c3b30fa0 --- /dev/null +++ b/recipes/le_monde_diplomatique_fr.recipe @@ -0,0 +1,111 @@ +# vim:fileencoding=utf-8 +from __future__ import unicode_literals + +__license__ = 'GPL v3' +__copyright__ = '2013' +''' +monde-diplomatique.fr +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe +from calibre.web.feeds import feeds_from_index + +class LeMondeDiplomatiqueSiteWeb(BasicNewsRecipe): + title = u'Le Monde diplomatique.fr' + __author__ = 'Gaëtan Lehmann' + description = "Le Monde diplomatique est un mensuel français d’information et d’opinion à la ligne éditoriale nettement engagée en faveur d'une gauche de rupture avec le capitalisme. Il aborde de nombreux sujets — géopolitique, relations internationales, économie, questions sociales, écologie, culture, médias, …" # noqa + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + publisher = 'monde-diplomatique.fr' + category = 'news, France, world' + language = 'fr' + masthead_url = 'http://www.monde-diplomatique.fr/squelettes/images/logotyfa.png' + timefmt = ' [%d %b %Y]' + no_stylesheets = True + + feeds = [(u'Blogs', u'http://blog.mondediplo.net/spip.php?page=backend'), (u'Archives', u'http://www.monde-diplomatique.fr/rss/')] + + preprocess_regexps = [ + (re.compile(r'(.*) - Les blogs du Diplo'), lambda m: '' + m.group(1) + ''), + (re.compile(r'

    (.*) - Les blogs du Diplo

    '), lambda m: '

    ' + m.group(1) + '

    '), + (re.compile(r'(.*) \(Le Monde diplomatique\)'), lambda m: '' + m.group(1) + ''), + (re.compile(r'

    (.*) \(Le Monde diplomatique\)

    '), lambda m: '

    ' + m.group(1) + '

    '), + (re.compile(r'

    Grand format

    '), lambda m: '')] + + remove_tags = [dict(name='div', attrs={'class':'voiraussi liste'}), + dict(name='ul', attrs={'class':'hermetique carto hombre_demi_inverse'}), + dict(name='a', attrs={'class':'tousles'}), + dict(name='h3', attrs={'class':'cat'}), + dict(name='div', attrs={'class':'logodiplo'}), + dict(name='img', attrs={'class':'spip_logos'}), + dict(name='p', attrs={'id':'hierarchie'}), + dict(name='div', attrs={'class':'espace'})] + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + ,'linearize_tables': True + } + + remove_empty_feeds = True + + filterDuplicates = True + + # don't use parse_index - we need it to send an exception so we can mix + # feed and parse_index results in parse_feeds + def parse_index_valise(self): + articles = [] + soup = self.index_to_soup('http://www.monde-diplomatique.fr/carnet/') + cnt = soup.find('ul',attrs={'class':'hermetique liste'}) + for item in cnt.findAll('li'): + description = '' + feed_link = item.find('a') + desc = item.find('div',attrs={'class':'intro'}) + date = item.find('div',attrs={'class':'dates_auteurs'}) + if desc: + description = desc.string + if feed_link and feed_link.has_key('href'): + url = 'http://www.monde-diplomatique.fr' + feed_link['href'] + title = self.tag_to_string(feed_link) + articles.append({ + 'title' :title + ,'date' :date.string.strip() + ,'url' :url + ,'description':description + }) + return [("La valise diplomatique", articles)] + + def parse_index_cartes(self): + articles = [] + soup = self.index_to_soup('http://www.monde-diplomatique.fr/cartes/') + cnt = soup.find('div',attrs={'class':'decale hermetique'}) + for item in cnt.findAll('div',attrs={'class':re.compile('grid_3 filet hombre_demi')}): + feed_link = item.find('a',attrs={'class':'couve'}) + h3 = item.find('h3') + authorAndDate = item.find('div',attrs={'class':'dates_auteurs'}) + author, date = authorAndDate.string.strip().split(', ') + if feed_link and feed_link.has_key('href'): + url = 'http://www.monde-diplomatique.fr' + feed_link['href'] + title = self.tag_to_string(h3) + articles.append({ + 'title' :title + ,'date' :date + ,'url' :url + ,'description': author + }) + return [("Cartes", articles)] + + def parse_feeds(self): + feeds = BasicNewsRecipe.parse_feeds(self) + valise = feeds_from_index(self.parse_index_valise(), oldest_article=self.oldest_article, + max_articles_per_feed=self.max_articles_per_feed, + log=self.log) + cartes = feeds_from_index(self.parse_index_cartes(), oldest_article=self.oldest_article, + max_articles_per_feed=self.max_articles_per_feed, + log=self.log) + feeds = valise + feeds + cartes + return feeds From 2a5e7e09ae423249c317971fe46a3ef41c1bd634 Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 12 Jul 2013 22:28:11 -0400 Subject: [PATCH 0171/1154] Store: Update Amazon plugin for new site layout. --- .../gui2/store/stores/amazon_plugin.py | 50 +++++++++++++++---- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/store/stores/amazon_plugin.py b/src/calibre/gui2/store/stores/amazon_plugin.py index 33f8f9b048..82e83401e8 100644 --- a/src/calibre/gui2/store/stores/amazon_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 3 # Needed for dynamic plugin loading +store_version = 4 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' @@ -126,15 +126,47 @@ class AmazonKindleStore(StorePlugin): counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read().decode('latin-1', 'replace')) + doc = html.fromstring(f.read()) - data_xpath = '//div[contains(@class, "prod")]' - format_xpath = './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()' - asin_xpath = '@name' - cover_xpath = './/img[@class="productImage"]/@src' - title_xpath = './/h3[@class="newaps"]/a//text()' - author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' - price_xpath = './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]//span[contains(@class, "lrg") and contains(@class, "bld")]/text()' + if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'): + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'): + data_xpath = '//li[(@class="ilo")]' + format_xpath = ( + './/ul[contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + # Results can be in a grid (table) or a column + price_xpath = ( + './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'): + data_xpath = '//div[contains(@class, "prod")]' + format_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()') + asin_xpath = '@name' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/h3[@class="newaps"]/a//text()' + author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()' + price_xpath = ( + './/ul[contains(@class, "rsltL")]' + '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()') + else: + return for data in doc.xpath(data_xpath): if counter <= 0: From 309813c8e0e7141a5ef93fed31df05bf7e9c4f36 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 13 Jul 2013 08:52:18 +0530 Subject: [PATCH 0172/1154] Ignore more unused API --- src/calibre/db/tests/legacy.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 2f5c879532..eabd351bde 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -269,7 +269,9 @@ class LegacyTest(BaseTest): 'books_in_old_database', # unused # Internal API - 'clean_user_categories', 'cleanup_tags', 'books_list_filter', + 'clean_user_categories', 'cleanup_tags', 'books_list_filter', 'conn', 'connect', 'construct_file_name', + 'construct_path_name', 'clear_dirtied', 'commit_dirty_cache', 'initialize_database', 'initialize_dynamic', + 'run_import_plugins', } SKIP_ARGSPEC = { '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', @@ -280,7 +282,7 @@ class LegacyTest(BaseTest): try: total = 0 for attr in dir(db): - if attr in SKIP_ATTRS: + if attr in SKIP_ATTRS or attr.startswith('upgrade_version'): continue total += 1 if not hasattr(ndb, attr): @@ -302,7 +304,7 @@ class LegacyTest(BaseTest): if missing: pc = len(missing)/total - raise AssertionError('{0:.1%} of API ({2} attrs) are missing. For example: {1}'.format(pc, ', '.join(missing[:5]), len(missing))) + raise AssertionError('{0:.1%} of API ({2} attrs) are missing: {1}'.format(pc, ', '.join(missing), len(missing))) # }}} From 9ea6e0d2a94261619ec65b4b2faf8392937fa32a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 13 Jul 2013 10:03:31 +0530 Subject: [PATCH 0173/1154] conversion_options API --- src/calibre/db/backend.py | 24 +++++++++++++++++++++++- src/calibre/db/cache.py | 16 ++++++++++++++++ src/calibre/db/legacy.py | 12 ++++++++++++ src/calibre/db/tests/legacy.py | 24 +++++++++++++++++++++++- src/calibre/db/tests/writing.py | 17 +++++++++++++++++ 5 files changed, 91 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 8ebdc4a154..0ebc9679b7 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -8,7 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' # Imports {{{ -import os, shutil, uuid, json, glob, time +import os, shutil, uuid, json, glob, time, cPickle from functools import partial import apsw @@ -1216,5 +1216,27 @@ class DB(object): def get_ids_for_custom_book_data(self, name): return frozenset(r[0] for r in self.conn.execute('SELECT book FROM books_plugin_data WHERE name=?', (name,))) + def conversion_options(self, book_id, fmt): + for (data,) in self.conn.get('SELECT data FROM conversion_options WHERE book=? AND format=?', (book_id, fmt.upper())): + if data: + return cPickle.loads(bytes(data)) + + def has_conversion_options(self, ids, fmt='PIPE'): + ids = frozenset(ids) + self.conn.execute('DROP TABLE IF EXISTS conversion_options_temp; CREATE TEMP TABLE conversion_options_temp (id INTEGER PRIMARY KEY);') + self.conn.executemany('INSERT INTO conversion_options_temp VALUES (?)', [(x,) for x in ids]) + for (book_id,) in self.conn.get( + 'SELECT book FROM conversion_options WHERE format=? AND book IN (SELECT id FROM conversion_options_temp)', (fmt.upper(),)): + return True + return False + + def delete_conversion_options(self, book_ids, fmt): + self.conn.executemany('DELETE FROM conversion_options WHERE book=? AND format=?', + [(book_id, fmt.upper()) for book_id in book_ids]) + + def set_conversion_options(self, options, fmt): + options = [(book_id, fmt.upper(), buffer(cPickle.dumps(data, -1))) for book_id, data in options.iteritems()] + self.conn.executemany('INSERT OR REPLACE INTO conversion_options(book,format,data) VALUES (?,?,?)', options) + # }}} diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index bc8f1024f5..a36e53c175 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1208,6 +1208,22 @@ class Cache(object): ''' Return the set of book ids for which name has data. ''' return self.backend.get_ids_for_custom_book_data(name) + @read_api + def conversion_options(self, book_id, fmt='PIPE'): + return self.backend.conversion_options(book_id, fmt) + + @read_api + def has_conversion_options(self, ids, fmt='PIPE'): + return self.backend.has_conversion_options(ids, fmt) + + @write_api + def delete_conversion_options(self, book_ids, fmt='PIPE'): + return self.backend.delete_conversion_options(book_ids, fmt) + + @write_api + def set_conversion_options(self, options, fmt='PIPE'): + ''' options must be a map of the form {book_id:conversion_options} ''' + return self.backend.set_conversion_options(options, fmt) # }}} diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 8c5fa5bd31..03146ea1dc 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -383,6 +383,18 @@ class LibraryDatabase(object): break return ans + def set_conversion_options(self, book_id, fmt, options): + self.new_api.set_conversion_options({book_id:options}, fmt=fmt) + + def conversion_options(self, book_id, fmt): + return self.new_api.conversion_options(book_id, fmt=fmt) + + def has_conversion_options(self, ids, format='PIPE'): + return self.new_api.has_conversion_options(ids, fmt=format) + + def delete_conversion_options(self, book_id, fmt, commit=True): + self.new_api.delete_conversion_options((book_id,), fmt=fmt) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index eabd351bde..bda3401107 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -191,6 +191,27 @@ class LegacyTest(BaseTest): db.close() # }}} + def test_legacy_conversion_options(self): # {{{ + 'Test conversion options API' + ndb = self.init_legacy() + db = self.init_old() + all_ids = ndb.new_api.all_book_ids() + op1, op2 = {'xx':'yy'}, {'yy':'zz'} + for x in ( + ('has_conversion_options', all_ids), + ('conversion_options', 1, 'PIPE'), + ('set_conversion_options', 1, 'PIPE', op1), + ('has_conversion_options', all_ids), + ('conversion_options', 1, 'PIPE'), + ('delete_conversion_options', 1, 'PIPE'), + ('has_conversion_options', all_ids), + ): + meth, args = x[0], x[1:] + self.assertEqual((getattr(db, meth)(*args)), (getattr(ndb, meth)(*args)), + 'The method: %s() returned different results for argument %s' % (meth, args)) + db.close() + # }}} + def test_legacy_adding_books(self): # {{{ 'Test various adding books methods' from calibre.ebooks.metadata.book.base import Metadata @@ -271,7 +292,8 @@ class LegacyTest(BaseTest): # Internal API 'clean_user_categories', 'cleanup_tags', 'books_list_filter', 'conn', 'connect', 'construct_file_name', 'construct_path_name', 'clear_dirtied', 'commit_dirty_cache', 'initialize_database', 'initialize_dynamic', - 'run_import_plugins', + 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', + 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', } SKIP_ARGSPEC = { '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index cb525900ee..36b6d3d2a3 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -419,3 +419,20 @@ class WritingTest(BaseTest): # }}} + def test_conversion_options(self): # {{{ + ' Test saving of conversion options ' + cache = self.init_cache() + all_ids = cache.all_book_ids() + self.assertFalse(cache.has_conversion_options(all_ids)) + self.assertIsNone(cache.conversion_options(1)) + op1, op2 = {'xx':'yy'}, {'yy':'zz'} + cache.set_conversion_options({1:op1, 2:op2}) + self.assertTrue(cache.has_conversion_options(all_ids)) + self.assertEqual(cache.conversion_options(1), op1) + self.assertEqual(cache.conversion_options(2), op2) + cache.set_conversion_options({1:op2}) + self.assertEqual(cache.conversion_options(1), op2) + cache.delete_conversion_options(all_ids) + self.assertFalse(cache.has_conversion_options(all_ids)) + # }}} + From b2163e3846058b94cc8000511d6790f19ba5e80e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 13 Jul 2013 12:02:17 +0530 Subject: [PATCH 0174/1154] remove_items() API --- src/calibre/db/cache.py | 14 ++++++++++++ src/calibre/db/tables.py | 38 +++++++++++++++++++++++++++++++++ src/calibre/db/tests/legacy.py | 2 +- src/calibre/db/tests/writing.py | 38 +++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index a36e53c175..119e166c49 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -265,8 +265,10 @@ class Cache(object): for name, field in self.fields.iteritems(): if name[0] == '#' and name.endswith('_index'): field.series_field = self.fields[name[:-len('_index')]] + self.fields[name[:-len('_index')]].index_field = field elif name == 'series_index': field.series_field = self.fields['series'] + self.fields['series'].index_field = field elif name == 'authors': field.author_sort_field = self.fields['author_sort'] elif name == 'title': @@ -1179,6 +1181,18 @@ class Cache(object): else: table.remove_books(book_ids, self.backend) + @write_api + def remove_items(self, field, item_ids): + ''' Delete all items in the specified field with the specified ids. Returns the set of affected book ids. ''' + field = self.fields[field] + affected_books = field.table.remove_items(item_ids, self.backend) + if affected_books: + if hasattr(field, 'index_field'): + self._set_field(field.index_field.name, {bid:1.0 for bid in affected_books}) + else: + self._mark_as_dirty(affected_books) + return affected_books + @write_api def add_custom_book_data(self, name, val_map, delete_first=False): ''' Add data for name where val_map is a map of book_ids to values. If diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 19c4ade10c..7715f6abef 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -204,6 +204,21 @@ class ManyToOneTable(Table): [(x,) for x in clean]) return clean + def remove_items(self, item_ids, db): + affected_books = set() + for item_id in item_ids: + val = self.id_map.pop(item_id, null) + if val is null: + continue + book_ids = self.col_book_map.pop(item_id, set()) + for book_id in book_ids: + self.book_col_map.pop(book_id, None) + affected_books.update(book_ids) + item_ids = tuple((x,) for x in item_ids) + db.conn.executemany('DELETE FROM {0} WHERE {1}=?'.format(self.link_table, self.metadata['link_column']), item_ids) + db.conn.executemany('DELETE FROM {0} WHERE id=?'.format(self.metadata['table']), item_ids) + return affected_books + class ManyToManyTable(ManyToOneTable): ''' @@ -250,6 +265,21 @@ class ManyToManyTable(ManyToOneTable): [(x,) for x in clean]) return clean + def remove_items(self, item_ids, db): + affected_books = set() + for item_id in item_ids: + val = self.id_map.pop(item_id, null) + if val is null: + continue + book_ids = self.col_book_map.pop(item_id, set()) + for book_id in book_ids: + self.book_col_map[book_id] = tuple(x for x in self.book_col_map.get(book_id, ()) if x != item_id) + affected_books.update(book_ids) + item_ids = tuple((x,) for x in item_ids) + db.conn.executemany('DELETE FROM {0} WHERE {1}=?'.format(self.link_table, self.metadata['link_column']), item_ids) + db.conn.executemany('DELETE FROM {0} WHERE id=?'.format(self.metadata['table']), item_ids) + return affected_books + class AuthorsTable(ManyToManyTable): def read_id_maps(self, db): @@ -274,6 +304,9 @@ class AuthorsTable(ManyToManyTable): self.asort_map.pop(item_id, None) return clean + def remove_items(self, item_ids, db): + raise ValueError('Direct removal of authors is not allowed') + class FormatsTable(ManyToManyTable): do_clean_on_remove = False @@ -331,6 +364,9 @@ class FormatsTable(ManyToManyTable): return {book_id:zero_max(book_id) for book_id in formats_map} + def remove_items(self, item_ids, db): + raise NotImplementedError('Cannot delete a format directly') + def update_fmt(self, book_id, fmt, fname, size, db): fmts = list(self.book_col_map.get(book_id, [])) try: @@ -381,4 +417,6 @@ class IdentifiersTable(ManyToManyTable): clean.add(item_id) return clean + def remove_items(self, item_ids, db): + raise NotImplementedError('Direct deletion of identifiers is not implemented') diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index bda3401107..7182a6ef06 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -293,7 +293,7 @@ class LegacyTest(BaseTest): 'clean_user_categories', 'cleanup_tags', 'books_list_filter', 'conn', 'connect', 'construct_file_name', 'construct_path_name', 'clear_dirtied', 'commit_dirty_cache', 'initialize_database', 'initialize_dynamic', 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', - 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', + 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', 'dirty_books_referencing', } SKIP_ARGSPEC = { '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 36b6d3d2a3..c4918b4c4b 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -436,3 +436,41 @@ class WritingTest(BaseTest): self.assertFalse(cache.has_conversion_options(all_ids)) # }}} + def test_remove_items(self): # {{{ + ' Test removal of many-(many,one) items ' + cache = self.init_cache() + tmap = cache.get_id_map('tags') + self.assertEqual(cache.remove_items('tags', tmap), {1, 2}) + tmap = cache.get_id_map('#tags') + t = {v:k for k, v in tmap.iteritems()}['My Tag Two'] + self.assertEqual(cache.remove_items('#tags', (t,)), {1, 2}) + + smap = cache.get_id_map('series') + self.assertEqual(cache.remove_items('series', smap), {1, 2}) + smap = cache.get_id_map('#series') + s = {v:k for k, v in smap.iteritems()}['My Series Two'] + self.assertEqual(cache.remove_items('#series', (s,)), {1}) + + for c in (cache, self.init_cache()): + self.assertFalse(c.get_id_map('tags')) + self.assertFalse(c.all_field_names('tags')) + for bid in c.all_book_ids(): + self.assertFalse(c.field_for('tags', bid)) + + self.assertEqual(len(c.get_id_map('#tags')), 1) + self.assertEqual(c.all_field_names('#tags'), {'My Tag One'}) + for bid in c.all_book_ids(): + self.assertIn(c.field_for('#tags', bid), ((), ('My Tag One',))) + + for bid in (1, 2): + self.assertEqual(c.field_for('series_index', bid), 1.0) + self.assertFalse(c.get_id_map('series')) + self.assertFalse(c.all_field_names('series')) + for bid in c.all_book_ids(): + self.assertFalse(c.field_for('series', bid)) + + self.assertEqual(c.field_for('series_index', 1), 1.0) + self.assertEqual(c.all_field_names('#series'), {'My Series One'}) + for bid in c.all_book_ids(): + self.assertIn(c.field_for('#series', bid), (None, 'My Series One')) + # }}} From ba2618f03d90625d17bf8ff4081ba461c6c10413 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 13 Jul 2013 12:45:35 +0530 Subject: [PATCH 0175/1154] delete_*_using_id() API --- src/calibre/db/legacy.py | 7 +++++++ src/calibre/db/tests/legacy.py | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 03146ea1dc..6128d6a09a 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -98,6 +98,13 @@ class LibraryDatabase(object): return self.new_api.get_item_name(field, item_id) return func setattr(self, '%s_name' % field, MT(getter(field))) + for field in ('publisher', 'series', 'tag'): + def getter(field): + fname = 'tags' if field == 'tag' else field + def func(self, item_id): + self.new_api.remove_items(fname, (item_id,)) + return func + setattr(self, 'delete_%s_using_id' % field, MT(getter(field))) # Legacy field API for func in ( diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 7182a6ef06..76deaea792 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -212,6 +212,31 @@ class LegacyTest(BaseTest): db.close() # }}} + def test_legacy_delete_using(self): # {{{ + 'Test delete_using() API' + ndb = self.init_legacy() + db = self.init_old() + cache = ndb.new_api + tmap = cache.get_id_map('tags') + t = next(tmap.iterkeys()) + pmap = cache.get_id_map('publisher') + p = next(pmap.iterkeys()) + for x in ( + ('delete_tag_using_id', t), + ('delete_publisher_using_id', p), + (db.refresh,), + ('all_tag_names',), ('tags', 0), ('tags', 1), ('tags', 2), + ('all_publisher_names',), ('publisher', 0), ('publisher', 1), ('publisher', 2), + ): + meth, args = x[0], x[1:] + if callable(meth): + meth(*args) + else: + self.assertEqual((getattr(db, meth)(*args)), (getattr(ndb, meth)(*args)), + 'The method: %s() returned different results for argument %s' % (meth, args)) + db.close() + # }}} + def test_legacy_adding_books(self): # {{{ 'Test various adding books methods' from calibre.ebooks.metadata.book.base import Metadata From a87be86958d7ce9a2a7ecd9719ee2ad0328163c2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 13 Jul 2013 14:04:11 +0530 Subject: [PATCH 0176/1154] set() API --- src/calibre/db/legacy.py | 7 +++++ src/calibre/db/tests/legacy.py | 47 ++++++++++++++++++++++++++------ src/calibre/db/write.py | 2 +- src/calibre/library/database2.py | 2 +- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 6128d6a09a..83392d6d16 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -402,6 +402,13 @@ class LibraryDatabase(object): def delete_conversion_options(self, book_id, fmt, commit=True): self.new_api.delete_conversion_options((book_id,), fmt=fmt) + def set(self, index, field, val, allow_case_change=False): + book_id = self.data.index_to_id(index) + try: + return self.new_api.set_field(field, {book_id:val}, allow_case_change=allow_case_change) + finally: + self.notify('metadata', [book_id]) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 76deaea792..037f972010 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -40,6 +40,19 @@ def compare_argspecs(old, new, attr): if not ok: raise AssertionError('The argspec for %s does not match. %r != %r' % (attr, old, new)) +def run_funcs(self, db, ndb, funcs): + for func in funcs: + meth, args = func[0], func[1:] + if callable(meth): + meth(*args) + else: + fmt = lambda x:x + if meth[0] in {'!', '@', '#'}: + fmt = {'!':dict, '@':frozenset, '#':lambda x:set((x or '').split(','))}[meth[0]] + meth = meth[1:] + self.assertEqual(fmt(getattr(db, meth)(*args)), fmt(getattr(ndb, meth)(*args)), + 'The method: %s() returned different results for argument %s' % (meth, args)) + class LegacyTest(BaseTest): ''' Test the emulation of the legacy interface. ''' @@ -137,8 +150,9 @@ class LegacyTest(BaseTest): def test_legacy_direct(self): # {{{ 'Test methods that are directly equivalent in the old and new interface' from calibre.ebooks.metadata.book.base import Metadata - ndb = self.init_legacy() + ndb = self.init_legacy(self.cloned_library) db = self.init_old() + for meth, args in { 'get_next_series_num_for': [('A Series One',)], 'author_sort_from_authors': [(['Author One', 'Author Two', 'Unknown'],)], @@ -221,19 +235,13 @@ class LegacyTest(BaseTest): t = next(tmap.iterkeys()) pmap = cache.get_id_map('publisher') p = next(pmap.iterkeys()) - for x in ( + run_funcs(self, db, ndb, ( ('delete_tag_using_id', t), ('delete_publisher_using_id', p), (db.refresh,), ('all_tag_names',), ('tags', 0), ('tags', 1), ('tags', 2), ('all_publisher_names',), ('publisher', 0), ('publisher', 1), ('publisher', 2), - ): - meth, args = x[0], x[1:] - if callable(meth): - meth(*args) - else: - self.assertEqual((getattr(db, meth)(*args)), (getattr(ndb, meth)(*args)), - 'The method: %s() returned different results for argument %s' % (meth, args)) + )) db.close() # }}} @@ -387,3 +395,24 @@ class LegacyTest(BaseTest): old.close() # }}} + def test_legacy_setters(self): # {{{ + 'Test methods that are directly equivalent in the old and new interface' + ndb = self.init_legacy(self.cloned_library) + db = self.init_old(self.cloned_library) + + run_funcs(self, db, ndb, ( + ('set', 0, 'title', 'newtitle'), + ('set', 0, 'tags', 't1,t2,tag one', True), + ('set', 0, 'authors', 'author one & Author Two', True), + ('set', 0, 'rating', 3.2), + ('set', 0, 'publisher', 'publisher one', True), + (db.refresh,), + ('title', 0), + ('rating', 0), + ('#tags', 0), ('#tags', 1), ('#tags', 2), + ('authors', 0), ('authors', 1), ('authors', 2), + ('publisher', 0), ('publisher', 1), ('publisher', 2), + )) + db.close() + + # }}} diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py index 9bae3e6abb..a257788a60 100644 --- a/src/calibre/db/write.py +++ b/src/calibre/db/write.py @@ -142,7 +142,7 @@ def get_adapter(name, metadata): elif dt == 'comments': ans = single_text elif dt == 'rating': - ans = lambda x: None if x in {None, 0} else min(10., max(0., adapt_number(float, x))) + ans = lambda x: None if x in {None, 0} else min(10, max(0, adapt_number(int, x))) elif dt == 'enumeration': ans = single_text elif dt == 'composite': diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index bd3f155c9d..61c1653cee 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -2206,7 +2206,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): id = self.data[row][0] col = self.FIELD_MAP[column] - books_to_refresh = set() + books_to_refresh = {id} set_args = (row, col, val) if column == 'authors': val = string_to_authors(val) From 3d122c8e7d120358c38fb84be2f51ff8a8df1cd0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 13 Jul 2013 14:10:34 +0530 Subject: [PATCH 0177/1154] Add note about migrating refresh() behavior --- src/calibre/db/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index eded760cde..781e886567 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -114,4 +114,7 @@ Various things that require other things before they can be migrated: 3. From refresh in the legacy interface: Rember to flush the composite column template cache. 4. Replace the metadatabackup thread with the new implementation when using the new backend. + 5. In the new API refresh() does not re-read from disk. That might break a + few things, for example content server reloading on db change as well as + dump/restore of db? ''' From 090eed4153772a7a6cbcdf67ddb321eae324f57c Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sat, 13 Jul 2013 11:09:14 +0200 Subject: [PATCH 0178/1154] For the virtual library column stuff: avoid using refresh when invalidating libraries. --- src/calibre/gui2/search_restriction_mixin.py | 6 +++--- src/calibre/library/caches.py | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index a90d607ea9..63094b45f0 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -332,7 +332,7 @@ class SearchRestrictionMixin(object): virt_libs = db.prefs.get('virtual_libraries', {}) virt_libs[name] = search db.prefs.set('virtual_libraries', virt_libs) - db.data.invalidate_virtual_libraries_caches(db) + db.data.invalidate_virtual_libraries_caches() def do_create_edit(self, name=None): db = self.library_view.model().db @@ -342,7 +342,7 @@ class SearchRestrictionMixin(object): if name: self._remove_vl(name, reapply=False) self.add_virtual_library(db, cd.library_name, cd.library_search) - db.data.invalidate_virtual_libraries_caches(db) + db.data.invalidate_virtual_libraries_caches() if not name or name == db.data.get_base_restriction_name(): self.apply_virtual_library(cd.library_name) else: @@ -467,7 +467,7 @@ class SearchRestrictionMixin(object): return self._remove_vl(name, reapply=True) db = self.library_view.model().db - db.data.invalidate_virtual_libraries_caches(db) + db.data.invalidate_virtual_libraries_caches() self.tags_view.recount() def _remove_vl(self, name, reapply=True): diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 05d964be71..544a8e4b56 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -212,6 +212,8 @@ class CacheRow(list): # {{{ self[c] = None self._must_do = True + def refresh_virtual_libraries(self): + self._virt_libs = None # }}} class ResultCache(SearchQueryParser): # {{{ @@ -247,6 +249,8 @@ class ResultCache(SearchQueryParser): # {{{ pref_use_primary_find_in_search = prefs['use_primary_find_in_search'] self._uuid_column_index = self.FIELD_MAP['uuid'] self._uuid_map = {} + self._virt_libs_computed = False + self._ids_in_virt_libs = {} def break_cycles(self): self._data = self.field_metadata = self.FIELD_MAP = \ @@ -840,8 +844,14 @@ class ResultCache(SearchQueryParser): # {{{ current_candidates -= matches return matches - def invalidate_virtual_libraries_caches(self, db): - self.refresh(db) + def invalidate_virtual_libraries_caches(self): + self._virt_libs_computed = False + self._ids_in_virt_libs = {} + + for row in self._data: + if row is not None: + row.refresh_virtual_libraries() + row.refresh_composites() def search_raw(self, query): matches = self.parse(query) From 642b6732c7170372d75029838a0a3774122b37e8 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sat, 13 Jul 2013 11:38:23 +0200 Subject: [PATCH 0179/1154] [Bug 1200826] [NEW] Matching a library book to device search default --- src/calibre/gui2/dialogs/match_books.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/dialogs/match_books.py b/src/calibre/gui2/dialogs/match_books.py index 6914c8bb84..6ac7aa533c 100644 --- a/src/calibre/gui2/dialogs/match_books.py +++ b/src/calibre/gui2/dialogs/match_books.py @@ -107,6 +107,8 @@ class MatchBooks(QDialog, Ui_MatchBooks): self.buttonBox.rejected.connect(self.reject) self.ignore_next_key = False + self.search_text.setText(self.device_db[self.current_device_book_id].title) + def return_pressed(self): self.ignore_next_key = True self.do_search() From 2dec08b6fbb0f4330bf408d237bd37f25da3707a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 13 Jul 2013 15:25:10 +0530 Subject: [PATCH 0180/1154] Fix #1200826 [Matching a library book to device search default](https://bugs.launchpad.net/calibre/+bug/1200826) (patch taken from cbhaley) --- src/calibre/gui2/dialogs/match_books.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/dialogs/match_books.py b/src/calibre/gui2/dialogs/match_books.py index 6914c8bb84..6ac7aa533c 100644 --- a/src/calibre/gui2/dialogs/match_books.py +++ b/src/calibre/gui2/dialogs/match_books.py @@ -107,6 +107,8 @@ class MatchBooks(QDialog, Ui_MatchBooks): self.buttonBox.rejected.connect(self.reject) self.ignore_next_key = False + self.search_text.setText(self.device_db[self.current_device_book_id].title) + def return_pressed(self): self.ignore_next_key = True self.do_search() From 1c75418d2065b51fd9782206705cbe938ee08c1a Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sat, 13 Jul 2013 12:52:47 +0200 Subject: [PATCH 0181/1154] Back the virtual library column stuff out of this branch. --- src/calibre/gui2/search_restriction_mixin.py | 7 -- src/calibre/library/caches.py | 125 ++++++------------- src/calibre/library/cli.py | 2 +- src/calibre/library/database2.py | 6 - src/calibre/library/field_metadata.py | 10 -- src/calibre/utils/formatter_functions.py | 12 +- src/calibre/utils/search_query_parser.py | 15 +-- 7 files changed, 48 insertions(+), 129 deletions(-) diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index 63094b45f0..b986a2a78e 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -332,7 +332,6 @@ class SearchRestrictionMixin(object): virt_libs = db.prefs.get('virtual_libraries', {}) virt_libs[name] = search db.prefs.set('virtual_libraries', virt_libs) - db.data.invalidate_virtual_libraries_caches() def do_create_edit(self, name=None): db = self.library_view.model().db @@ -342,11 +341,8 @@ class SearchRestrictionMixin(object): if name: self._remove_vl(name, reapply=False) self.add_virtual_library(db, cd.library_name, cd.library_search) - db.data.invalidate_virtual_libraries_caches() if not name or name == db.data.get_base_restriction_name(): self.apply_virtual_library(cd.library_name) - else: - self.tags_view.recount() def virtual_library_clicked(self): m = self.virtual_library_menu @@ -466,9 +462,6 @@ class SearchRestrictionMixin(object): default_yes=False): return self._remove_vl(name, reapply=True) - db = self.library_view.model().db - db.data.invalidate_virtual_libraries_caches() - self.tags_view.recount() def _remove_vl(self, name, reapply=True): db = self.library_view.model().db diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 544a8e4b56..e552ead591 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -2,7 +2,7 @@ # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai from __future__ import with_statement -__license__ = 'GPL v3' +__license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' @@ -144,8 +144,7 @@ def force_to_bool(val): class CacheRow(list): # {{{ - def __init__(self, db, composites, val, series_col, series_sort_col, - virtual_library_col): + def __init__(self, db, composites, val, series_col, series_sort_col): self.db = db self._composites = composites list.__init__(self, val) @@ -153,8 +152,6 @@ class CacheRow(list): # {{{ self._series_col = series_col self._series_sort_col = series_sort_col self._series_sort = None - self._virt_lib_col = virtual_library_col - self._virt_libs = None def __getitem__(self, col): if self._must_do: @@ -174,7 +171,7 @@ class CacheRow(list): # {{{ mi = self.db.get_metadata(id_, index_is_id=True, get_user_categories=False) for c in self._composites: - self[c] = mi.get(self._composites[c]) + self[c] = mi.get(self._composites[c]) if col == self._series_sort_col and self._series_sort is None: if self[self._series_col]: self._series_sort = title_sort(self[self._series_col]) @@ -182,26 +179,6 @@ class CacheRow(list): # {{{ else: self._series_sort = '' self[self._series_sort_col] = '' - - if col == self._virt_lib_col and self._virt_libs is None: - try: - if not getattr(self.db.data, '_virt_libs_computed', False): - self.db.data._ids_in_virt_libs = {} - for v,s in self.db.prefs.get('virtual_libraries', {}).iteritems(): - self.db.data._ids_in_virt_libs[v] = self.db.data.search_raw(s) - self.db.data._virt_libs_computed = True - r = [] - for v in self.db.prefs.get('virtual_libraries', {}).keys(): - # optimize the lookup of the ID -- it is always zero - if self[0] in self.db.data._ids_in_virt_libs[v]: - r.append(v) - from calibre.utils.icu import sort_key - self._virt_libs = ", ".join(sorted(r, key=sort_key)) - self[self._virt_lib_col] = self._virt_libs - except: - print len(self) - traceback.print_exc() - return list.__getitem__(self, col) def __getslice__(self, i, j): @@ -209,11 +186,9 @@ class CacheRow(list): # {{{ def refresh_composites(self): for c in self._composites: - self[c] = None + self[c] = None self._must_do = True - def refresh_virtual_libraries(self): - self._virt_libs = None # }}} class ResultCache(SearchQueryParser): # {{{ @@ -231,7 +206,6 @@ class ResultCache(SearchQueryParser): # {{{ self.composites[field_metadata[key]['rec_index']] = key self.series_col = field_metadata['series']['rec_index'] self.series_sort_col = field_metadata['series_sort']['rec_index'] - self.virtual_libraries_col = field_metadata['virtual_libraries']['rec_index'] self._data = [] self._map = self._map_filtered = [] self.first_sort = True @@ -249,8 +223,6 @@ class ResultCache(SearchQueryParser): # {{{ pref_use_primary_find_in_search = prefs['use_primary_find_in_search'] self._uuid_column_index = self.FIELD_MAP['uuid'] self._uuid_map = {} - self._virt_libs_computed = False - self._ids_in_virt_libs = {} def break_cycles(self): self._data = self.field_metadata = self.FIELD_MAP = \ @@ -340,12 +312,12 @@ class ResultCache(SearchQueryParser): # {{{ '<=':[2, relop_le] } - local_today = ('_today', icu_lower(_('today'))) - local_yesterday = ('_yesterday', icu_lower(_('yesterday'))) - local_thismonth = ('_thismonth', icu_lower(_('thismonth'))) - local_daysago = icu_lower(_('daysago')) - local_daysago_len = len(local_daysago) - untrans_daysago = '_daysago' + local_today = ('_today', icu_lower(_('today'))) + local_yesterday = ('_yesterday', icu_lower(_('yesterday'))) + local_thismonth = ('_thismonth', icu_lower(_('thismonth'))) + local_daysago = icu_lower(_('daysago')) + local_daysago_len = len(local_daysago) + untrans_daysago = '_daysago' untrans_daysago_len = len('_daysago') def get_dates_matches(self, location, query, candidates): @@ -441,21 +413,21 @@ class ResultCache(SearchQueryParser): # {{{ if val_func is None: loc = self.field_metadata[location]['rec_index'] - val_func = lambda item, loc = loc: item[loc] + val_func = lambda item, loc=loc: item[loc] q = '' cast = adjust = lambda x: x dt = self.field_metadata[location]['datatype'] if query == 'false': if dt == 'rating' or location == 'cover': - relop = lambda x, y: not bool(x) + relop = lambda x,y: not bool(x) else: - relop = lambda x, y: x is None + relop = lambda x,y: x is None elif query == 'true': if dt == 'rating' or location == 'cover': - relop = lambda x, y: bool(x) + relop = lambda x,y: bool(x) else: - relop = lambda x, y: x is not None + relop = lambda x,y: x is not None else: relop = None for k in self.numeric_search_relops.keys(): @@ -469,7 +441,7 @@ class ResultCache(SearchQueryParser): # {{{ cast = lambda x: int(x) elif dt == 'rating': cast = lambda x: 0 if x is None else int(x) - adjust = lambda x: x / 2 + adjust = lambda x: x/2 elif dt in ('float', 'composite'): cast = lambda x : float(x) else: # count operation @@ -477,7 +449,7 @@ class ResultCache(SearchQueryParser): # {{{ if len(query) > 1: mult = query[-1:].lower() - mult = {'k':1024., 'm': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) + mult = {'k':1024.,'m': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) if mult != 1.0: query = query[:-1] else: @@ -596,12 +568,12 @@ class ResultCache(SearchQueryParser): # {{{ query = icu_lower(query) return matchkind, query - local_no = icu_lower(_('no')) - local_yes = icu_lower(_('yes')) + local_no = icu_lower(_('no')) + local_yes = icu_lower(_('yes')) local_unchecked = icu_lower(_('unchecked')) - local_checked = icu_lower(_('checked')) - local_empty = icu_lower(_('empty')) - local_blank = icu_lower(_('blank')) + local_checked = icu_lower(_('checked')) + local_empty = icu_lower(_('empty')) + local_blank = icu_lower(_('blank')) local_bool_values = ( local_no, local_unchecked, '_no', 'false', local_yes, local_checked, '_yes', 'true', @@ -724,8 +696,8 @@ class ResultCache(SearchQueryParser): # {{{ if fm['is_multiple'] and \ len(query) > 1 and query.startswith('#') and \ query[1:1] in '=<>!': - vf = lambda item, loc = fm['rec_index'], \ - ms = fm['is_multiple']['cache_to_list']:\ + vf = lambda item, loc=fm['rec_index'], \ + ms=fm['is_multiple']['cache_to_list']:\ len(item[loc].split(ms)) if item[loc] is not None else 0 return self.get_numeric_matches(location, query[1:], candidates, val_func=vf) @@ -735,7 +707,7 @@ class ResultCache(SearchQueryParser): # {{{ if fm.get('is_csp', False): if location == 'identifiers' and original_location == 'isbn': return self.get_keypair_matches('identifiers', - '=isbn:' + query, candidates) + '=isbn:'+query, candidates) return self.get_keypair_matches(location, query, candidates) # check for user categories @@ -787,7 +759,7 @@ class ResultCache(SearchQueryParser): # {{{ q = canonicalize_lang(query) if q is None: lm = lang_map() - rm = {v.lower():k for k, v in lm.iteritems()} + rm = {v.lower():k for k,v in lm.iteritems()} q = rm.get(query, query) else: q = query @@ -800,7 +772,7 @@ class ResultCache(SearchQueryParser): # {{{ if not item[loc]: if q == 'false' and matchkind == CONTAINS_MATCH: matches.add(item[0]) - continue # item is empty. No possible matches below + continue # item is empty. No possible matches below if q == 'false'and matchkind == CONTAINS_MATCH: # Field has something in it, so a false query does not match continue @@ -844,19 +816,6 @@ class ResultCache(SearchQueryParser): # {{{ current_candidates -= matches return matches - def invalidate_virtual_libraries_caches(self): - self._virt_libs_computed = False - self._ids_in_virt_libs = {} - - for row in self._data: - if row is not None: - row.refresh_virtual_libraries() - row.refresh_composites() - - def search_raw(self, query): - matches = self.parse(query) - return matches - def search(self, query, return_matches=False): ans = self.search_getting_ids(query, self.search_restriction, set_restriction_count=True) @@ -1014,11 +973,10 @@ class ResultCache(SearchQueryParser): # {{{ try: self._data[id] = CacheRow(db, self.composites, db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0], - self.series_col, self.series_sort_col, - self.virtual_libraries_col) + self.series_col, self.series_sort_col) self._data[id].append(db.book_on_device_string(id)) - self._data[id].extend((self.marked_ids_dict.get(id, None), None, None)) - self._virt_libs_computed = False + self._data[id].append(self.marked_ids_dict.get(id, None)) + self._data[id].append(None) self._uuid_map[self._data[id][self._uuid_column_index]] = id except IndexError: return None @@ -1031,15 +989,14 @@ class ResultCache(SearchQueryParser): # {{{ def books_added(self, ids, db): if not ids: return - self._data.extend(repeat(None, max(ids) - len(self._data) + 2)) + self._data.extend(repeat(None, max(ids)-len(self._data)+2)) for id in ids: self._data[id] = CacheRow(db, self.composites, db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0], - self.series_col, self.series_sort_col, - self.virtual_libraries_col) + self.series_col, self.series_sort_col) self._data[id].append(db.book_on_device_string(id)) - self._data[id].extend((self.marked_ids_dict.get(id, None), None, None)) - self._virt_libs_computed = False + self._data[id].append(self.marked_ids_dict.get(id, None)) + self._data[id].append(None) # Series sort column self._uuid_map[self._data[id][self._uuid_column_index]] = id self._map[0:0] = ids self._map_filtered[0:0] = ids @@ -1063,22 +1020,20 @@ class ResultCache(SearchQueryParser): # {{{ db.initialize_template_cache() temp = db.conn.get('SELECT * FROM meta2') - self._data = list(itertools.repeat(None, temp[-1][0] + 2)) if temp else [] + self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else [] for r in temp: self._data[r[0]] = CacheRow(db, self.composites, r, - self.series_col, self.series_sort_col, - self.virtual_libraries_col) + self.series_col, self.series_sort_col) self._uuid_map[self._data[r[0]][self._uuid_column_index]] = r[0] for item in self._data: if item is not None: item.append(db.book_on_device_string(item[0])) - # Temp mark, series_sort, virtual_library columns - item.extend((None, None, None)) + # Temp mark and series_sort columns + item.extend((None, None)) - self._virt_libs_computed = False marked_col = self.FIELD_MAP['marked'] - for id_, val in self.marked_ids_dict.iteritems(): + for id_,val in self.marked_ids_dict.iteritems(): try: self._data[id_][marked_col] = val except: @@ -1179,7 +1134,7 @@ class SortKeyGenerator(object): for i, candidate in enumerate( ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')): if val.endswith(candidate): - p = 1024 ** (i) + p = 1024**(i) val = val[:-len(candidate)].strip() break val = locale.atof(val) * p diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index a86ec7ef6d..547cc5bc08 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -575,7 +575,7 @@ def command_set_metadata(args, dbpath): for key in sorted(db.field_metadata.all_field_keys()): m = db.field_metadata[key] if (key not in {'formats', 'series_sort', 'ondevice', 'path', - 'virtual_libraries', 'last_modified'} and m['is_editable'] and m['name']): + 'last_modified'} and m['is_editable'] and m['name']): yield key, m if m['datatype'] == 'series': si = m.copy() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 00c6fb057f..61c1653cee 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -449,8 +449,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.field_metadata.set_field_record_index('marked', base, prefer_custom=False) self.FIELD_MAP['series_sort'] = base = base+1 self.field_metadata.set_field_record_index('series_sort', base, prefer_custom=False) - self.FIELD_MAP['virtual_libraries'] = base = base+1 - self.field_metadata.set_field_record_index('virtual_libraries', base, prefer_custom=False) script = ''' DROP VIEW IF EXISTS meta2; @@ -994,10 +992,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.book_size = row[fm['size']] mi.ondevice_col= row[fm['ondevice']] mi.last_modified = row[fm['last_modified']] - - mi._base_db_row = row # So the formatter functions can see the underlying data - mi._virt_lib_column = fm['virtual_libraries'] - formats = row[fm['formats']] mi.format_metadata = {} if not formats: diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index a96f819b58..08c26e95a9 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -387,16 +387,6 @@ class FieldMetadata(dict): 'is_custom':False, 'is_category':False, 'is_csp': False}), - ('virtual_libraries', {'table':None, - 'column':None, - 'datatype':'text', - 'is_multiple':{}, - 'kind':'field', - 'name':_('Virtual Libraries'), - 'search_terms':['virtual_libraries'], - 'is_custom':False, - 'is_category':False, - 'is_csp': False}), ] # }}} diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index c94467fec0..73dad7422b 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -1209,19 +1209,9 @@ class BuiltinFinishFormatting(BuiltinFormatterFunction): return val return prefix + formatter._do_format(val, fmt) + suffix -class BuiltinBookInVirtualLibraries(BuiltinFormatterFunction): - name = 'book_in_virtual_libraries' - arg_count = 0 - category = 'Get values from metadata' - __doc__ = doc = _('book_in_virtual_libraries() -- returns a list of ' - 'virtual libraries that this book is in.') - - def evaluate(self, formatter, kwargs, mi, locals_): - return mi._base_db_row[mi._virt_lib_column ] - _formatter_builtins = [ BuiltinAdd(), BuiltinAnd(), BuiltinApproximateFormats(), - BuiltinAssign(), BuiltinBookInVirtualLibraries(), BuiltinBooksize(), + BuiltinAssign(), BuiltinBooksize(), BuiltinCapitalize(), BuiltinCmp(), BuiltinContains(), BuiltinCount(), BuiltinCurrentLibraryName(), BuiltinCurrentLibraryPath(), BuiltinDaysBetween(), BuiltinDivide(), BuiltinEval(), BuiltinFirstNonEmpty(), diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index 08a70a533d..2682088681 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -294,7 +294,6 @@ class SearchQueryParser(object): def __init__(self, locations, test=False, optimize=False): self.sqp_initialize(locations, test=test, optimize=optimize) - self.sqp_parsed_search_cache = {} self.parser = Parser() def sqp_change_locations(self, locations): @@ -309,7 +308,8 @@ class SearchQueryParser(object): # empty the list of searches used for recursion testing self.recurse_level = 0 self.searches_seen = set([]) - return self._parse(query) + candidates = self.universal_set() + return self._parse(query, candidates) # this parse is used internally because it doesn't clear the # recursive search test list. However, we permit seeing the @@ -317,13 +317,10 @@ class SearchQueryParser(object): # another search. def _parse(self, query, candidates=None): self.recurse_level += 1 - res = self.sqp_parsed_search_cache.get(query, None) - if res is None: - try: - res = self.parser.parse(query, self.locations) - self.sqp_parsed_search_cache[query] = res - except RuntimeError: - raise ParseException(_('Failed to parse query, recursion limit reached: %s')%repr(query)) + try: + res = self.parser.parse(query, self.locations) + except RuntimeError: + raise ParseException(_('Failed to parse query, recursion limit reached: %s')%repr(query)) if candidates is None: candidates = self.universal_set() t = self.evaluate(res, candidates) From 69c3f37f334f4df4dd4c0fc4935f308200c1e92f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 13 Jul 2013 19:04:42 +0530 Subject: [PATCH 0182/1154] Fix case changing even when allow_case_changes=False when setting series/tags/publisher and the value being set is the only instance in the db. Also rationalize the way books_to_refresh works. --- src/calibre/library/database2.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 61c1653cee..b61544f172 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -2423,7 +2423,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if not authors: authors = [_('Unknown')] self.conn.execute('DELETE FROM books_authors_link WHERE book=?',(id,)) - books_to_refresh = set([]) + books_to_refresh = {id} final_authors = [] for a in authors: case_change = False @@ -2615,10 +2615,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def set_publisher(self, id, publisher, notify=True, commit=True, allow_case_change=False): self.conn.execute('DELETE FROM books_publishers_link WHERE book=?',(id,)) - self.conn.execute('''DELETE FROM publishers WHERE (SELECT COUNT(id) - FROM books_publishers_link - WHERE publisher=publishers.id) < 1''') - books_to_refresh = set([]) + books_to_refresh = {id} if publisher: case_change = False if not isinstance(publisher, unicode): @@ -2634,6 +2631,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): case_change = True else: publisher = cur_name + books_to_refresh = set() else: aid = self.conn.execute('''INSERT INTO publishers(name) VALUES (?)''', (publisher,)).lastrowid @@ -2643,6 +2641,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): bks = self.conn.get('''SELECT book FROM books_publishers_link WHERE publisher=?''', (aid,)) books_to_refresh |= set([bk[0] for bk in bks]) + self.conn.execute('''DELETE FROM publishers WHERE (SELECT COUNT(id) + FROM books_publishers_link + WHERE publisher=publishers.id) < 1''') + self.dirtied(set([id])|books_to_refresh, commit=False) if commit: self.conn.commit() @@ -3054,11 +3056,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tags = [] if not append: self.conn.execute('DELETE FROM books_tags_link WHERE book=?', (id,)) - self.conn.execute('''DELETE FROM tags WHERE (SELECT COUNT(id) - FROM books_tags_link WHERE tag=tags.id) < 1''') otags = self.get_tags(id) tags = self.cleanup_tags(tags) - books_to_refresh = set([]) + books_to_refresh = {id} for tag in (set(tags)-otags): case_changed = False tag = tag.strip() @@ -3089,6 +3089,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): bks = self.conn.get('SELECT book FROM books_tags_link WHERE tag=?', (tid,)) books_to_refresh |= set([bk[0] for bk in bks]) + self.conn.execute('''DELETE FROM tags WHERE (SELECT COUNT(id) + FROM books_tags_link WHERE tag=tags.id) < 1''') self.dirtied(set([id])|books_to_refresh, commit=False) if commit: self.conn.commit() @@ -3139,11 +3141,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def set_series(self, id, series, notify=True, commit=True, allow_case_change=True): self.conn.execute('DELETE FROM books_series_link WHERE book=?',(id,)) - self.conn.execute('''DELETE FROM series - WHERE (SELECT COUNT(id) FROM books_series_link - WHERE series=series.id) < 1''') (series, idx) = self._get_series_values(series) - books_to_refresh = set([]) + books_to_refresh = {id} if series: case_change = False if not isinstance(series, unicode): @@ -3159,6 +3158,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): case_change = True else: series = cur_name + books_to_refresh = set() else: aid = self.conn.execute('INSERT INTO series(name) VALUES (?)', (series,)).lastrowid self.conn.execute('INSERT INTO books_series_link(book, series) VALUES (?,?)', (id, aid)) @@ -3168,6 +3168,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): bks = self.conn.get('SELECT book FROM books_series_link WHERE series=?', (aid,)) books_to_refresh |= set([bk[0] for bk in bks]) + self.conn.execute('''DELETE FROM series + WHERE (SELECT COUNT(id) FROM books_series_link + WHERE series=series.id) < 1''') self.dirtied([id], commit=False) if commit: self.conn.commit() From 9c64054826335e1fa6ae634cce9c0b49b6392d4e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 08:54:59 +0530 Subject: [PATCH 0183/1154] Basic setters API --- src/calibre/db/legacy.py | 44 +++++++++++++++++++++++--- src/calibre/db/tests/legacy.py | 58 +++++++++++++++++++++++++++++----- 2 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 83392d6d16..0f0f35ea87 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -52,18 +52,22 @@ class LibraryDatabase(object): self.get_property = self.data.get_property + MT = lambda func: types.MethodType(func, self, LibraryDatabase) + for prop in ( - 'author_sort', 'authors', 'comment', 'comments', - 'publisher', 'rating', 'series', 'series_index', 'tags', - 'title', 'timestamp', 'uuid', 'pubdate', 'ondevice', - 'metadata_last_modified', 'languages', + 'author_sort', 'authors', 'comment', 'comments', 'publisher', + 'rating', 'series', 'series_index', 'tags', 'title', 'title_sort', + 'timestamp', 'uuid', 'pubdate', 'ondevice', + 'metadata_last_modified', 'languages', ): fm = {'comment':'comments', 'metadata_last_modified': 'last_modified', 'title_sort':'sort'}.get(prop, prop) setattr(self, prop, partial(self.get_property, loc=self.FIELD_MAP[fm])) - MT = lambda func: types.MethodType(func, self, LibraryDatabase) + self.has_cover = MT(lambda self, book_id:self.new_api.field_for('cover', book_id)) + self.get_identifiers = MT( + lambda self, index, index_is_id=False: self.new_api.field_for('identifiers', index if index_is_id else self.data.index_to_id(index))) for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'): setattr(self, meth, getattr(self.new_api, meth)) @@ -115,6 +119,36 @@ class LibraryDatabase(object): setattr(self, func, getattr(self.field_metadata, func)) self.metadata_for_field = self.field_metadata.get + # Legacy setter API + for field in ( + '!authors', 'author_sort', 'comment', 'has_cover', 'identifiers', 'languages', + 'pubdate', '!publisher', 'rating', '!series', 'series_index', 'timestamp', 'uuid', + ): + def setter(field): + has_case_change = field.startswith('!') + field = {'comment':'comments',}.get(field, field) + if has_case_change: + field = field[1:] + acc = field == 'series' + def func(self, book_id, val, notify=True, commit=True, allow_case_change=acc): + ret = self.new_api.set_field(field, {book_id:val}, allow_case_change=allow_case_change) + if notify: + self.notify([book_id]) + return ret + elif field == 'has_cover': + def func(self, book_id, val): + self.new_api.set_field('cover', {book_id:bool(val)}) + else: + def func(self, book_id, val, notify=True, commit=True): + if not val and field == 'uuid': + return + ret = self.new_api.set_field(field, {book_id:val}) + if notify: + self.notify([book_id]) + return ret if field == 'languages' else None + return func + setattr(self, 'set_%s' % field.replace('!', ''), MT(setter(field))) + self.last_update_check = self.last_modified() self.book_on_device_func = None # Cleaning is not required anymore diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 037f972010..f6f0fdd12c 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -47,11 +47,15 @@ def run_funcs(self, db, ndb, funcs): meth(*args) else: fmt = lambda x:x - if meth[0] in {'!', '@', '#'}: - fmt = {'!':dict, '@':frozenset, '#':lambda x:set((x or '').split(','))}[meth[0]] + if meth[0] in {'!', '@', '#', '+'}: + if meth[0] != '+': + fmt = {'!':dict, '@':frozenset, '#':lambda x:set((x or '').split(','))}[meth[0]] + else: + fmt = args[-1] + args = args[:-1] meth = meth[1:] - self.assertEqual(fmt(getattr(db, meth)(*args)), fmt(getattr(ndb, meth)(*args)), - 'The method: %s() returned different results for argument %s' % (meth, args)) + res1, res2 = fmt(getattr(db, meth)(*args)), fmt(getattr(ndb, meth)(*args)) + self.assertEqual(res1, res2, 'The method: %s() returned different results for argument %s' % (meth, args)) class LegacyTest(BaseTest): @@ -129,7 +133,7 @@ class LegacyTest(BaseTest): def test_legacy_getters(self): # {{{ ' Test various functions to get individual bits of metadata ' old = self.init_old() - getters = ('path', 'abspath', 'title', 'authors', 'series', + getters = ('path', 'abspath', 'title', 'title_sort', 'authors', 'series', 'publisher', 'author_sort', 'authors', 'comments', 'comment', 'publisher', 'rating', 'series_index', 'tags', 'timestamp', 'uuid', 'pubdate', 'ondevice', @@ -327,6 +331,7 @@ class LegacyTest(BaseTest): 'construct_path_name', 'clear_dirtied', 'commit_dirty_cache', 'initialize_database', 'initialize_dynamic', 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', 'dirty_books_referencing', + 'windows_check_if_files_in_use', } SKIP_ARGSPEC = { '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', @@ -400,12 +405,51 @@ class LegacyTest(BaseTest): ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) + run_funcs(self, db, ndb, ( + ('set_authors', 1, ('author one',),), ('set_authors', 2, ('author two',), True, True, True), + ('set_author_sort', 3, 'new_aus'), + ('set_comment', 1, ''), ('set_comment', 2, None), ('set_comment', 3, '

    a comment

    '), + ('set_has_cover', 1, True), ('set_has_cover', 2, True), ('set_has_cover', 3, 1), + ('set_identifiers', 2, {'test':'', 'a':'b'}), ('set_identifiers', 3, {'id':'1', 'url':'http://acme.com'}), ('set_identifiers', 1, {}), + ('set_languages', 1, ('en',)), + ('set_languages', 2, ()), + ('set_languages', 3, ('deu', 'spa', 'fra')), + ('set_pubdate', 1, None), ('set_pubdate', 2, '2011-1-7'), + ('set_series', 1, 'a series one'), ('set_series', 2, 'another series [7]'), ('set_series', 3, 'a third series'), + ('set_publisher', 1, 'publisher two'), ('set_publisher', 2, None), ('set_publisher', 3, 'a third puB'), + ('set_rating', 1, 2.3), ('set_rating', 2, 0), ('set_rating', 3, 8), + ('set_timestamp', 1, None), ('set_timestamp', 2, '2011-1-7'), + ('set_uuid', 1, None), ('set_uuid', 2, 'a test uuid'), + + (db.refresh,), + + ('authors', 0), ('authors', 1), ('authors', 2), + ('author_sort', 0), ('author_sort', 1), ('author_sort', 2), + ('has_cover', 3), ('has_cover', 1), ('has_cover', 2), + ('get_identifiers', 0), ('get_identifiers', 1), ('get_identifiers', 2), + ('pubdate', 0), ('pubdate', 1), ('pubdate', 2), + ('timestamp', 0), ('timestamp', 1), ('timestamp', 2), + ('publisher', 0), ('publisher', 1), ('publisher', 2), + ('rating', 0), ('+rating', 1, lambda x: x or 0), ('rating', 2), + ('series', 0), ('series', 1), ('series', 2), + ('series_index', 0), ('series_index', 1), ('series_index', 2), + ('uuid', 0), ('uuid', 1), ('uuid', 2), + + ('set_series_index', 1, 2.3), ('set_series_index', 2, 0), ('set_series_index', 3, 8), + (db.refresh,), + ('series_index', 0), ('series_index', 1), ('series_index', 2), + )) + db.close() + + ndb = self.init_legacy(self.cloned_library) + db = self.init_old(self.cloned_library) + run_funcs(self, db, ndb, ( ('set', 0, 'title', 'newtitle'), ('set', 0, 'tags', 't1,t2,tag one', True), ('set', 0, 'authors', 'author one & Author Two', True), ('set', 0, 'rating', 3.2), - ('set', 0, 'publisher', 'publisher one', True), + ('set', 0, 'publisher', 'publisher one', False), (db.refresh,), ('title', 0), ('rating', 0), @@ -413,6 +457,4 @@ class LegacyTest(BaseTest): ('authors', 0), ('authors', 1), ('authors', 2), ('publisher', 0), ('publisher', 1), ('publisher', 2), )) - db.close() - # }}} From 867f46db2b4593ad88564e8da7c651bcde257021 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 09:43:18 +0530 Subject: [PATCH 0184/1154] Move dynamic method generation to class instead of object level --- src/calibre/db/__init__.py | 1 + src/calibre/db/legacy.py | 239 +++++++++++++++++++-------------- src/calibre/db/tests/legacy.py | 2 +- 3 files changed, 138 insertions(+), 104 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 781e886567..47e44335ea 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -117,4 +117,5 @@ Various things that require other things before they can be migrated: 5. In the new API refresh() does not re-read from disk. That might break a few things, for example content server reloading on db change as well as dump/restore of db? + 6. grep the sources for TODO ''' diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 0f0f35ea87..ec2d6ffec4 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -7,7 +7,6 @@ __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' import os, traceback, types -from functools import partial from future_builtins import zip from calibre import force_unicode @@ -52,110 +51,8 @@ class LibraryDatabase(object): self.get_property = self.data.get_property - MT = lambda func: types.MethodType(func, self, LibraryDatabase) - - for prop in ( - 'author_sort', 'authors', 'comment', 'comments', 'publisher', - 'rating', 'series', 'series_index', 'tags', 'title', 'title_sort', - 'timestamp', 'uuid', 'pubdate', 'ondevice', - 'metadata_last_modified', 'languages', - ): - fm = {'comment':'comments', 'metadata_last_modified': - 'last_modified', 'title_sort':'sort'}.get(prop, prop) - setattr(self, prop, partial(self.get_property, - loc=self.FIELD_MAP[fm])) - - self.has_cover = MT(lambda self, book_id:self.new_api.field_for('cover', book_id)) - self.get_identifiers = MT( - lambda self, index, index_is_id=False: self.new_api.field_for('identifiers', index if index_is_id else self.data.index_to_id(index))) - - for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'): - setattr(self, meth, getattr(self.new_api, meth)) - - # Legacy API to get information about many-(one, many) fields - for field in ('authors', 'tags', 'publisher', 'series'): - def getter(field): - def func(self): - return self.new_api.all_field_names(field) - return func - name = field[:-1] if field in {'authors', 'tags'} else field - setattr(self, 'all_%s_names' % name, MT(getter(field))) - self.all_formats = MT(lambda self:self.new_api.all_field_names('formats')) - - for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':'tags', 'all_series':'series', 'all_publishers':'publisher'}.iteritems(): - setattr(self, func, partial(self.field_id_map, field)) - self.all_tags = MT(lambda self: list(self.all_tag_names())) - self.get_authors_with_ids = MT( - lambda self: [[aid, adata['name'], adata['sort'], adata['link']] for aid, adata in self.new_api.author_data().iteritems()]) - for field in ('tags', 'series', 'publishers', 'ratings', 'languages'): - def getter(field): - fname = field[:-1] if field in {'publishers', 'ratings'} else field - def func(self): - return [[tid, tag] for tid, tag in self.new_api.get_id_map(fname).iteritems()] - return func - setattr(self, 'get_%s_with_ids' % field, - MT(getter(field))) - for field in ('author', 'tag', 'series'): - def getter(field): - field = field if field == 'series' else (field+'s') - def func(self, item_id): - return self.new_api.get_item_name(field, item_id) - return func - setattr(self, '%s_name' % field, MT(getter(field))) - for field in ('publisher', 'series', 'tag'): - def getter(field): - fname = 'tags' if field == 'tag' else field - def func(self, item_id): - self.new_api.remove_items(fname, (item_id,)) - return func - setattr(self, 'delete_%s_using_id' % field, MT(getter(field))) - - # Legacy field API - for func in ( - 'standard_field_keys', 'custom_field_keys', 'all_field_keys', - 'searchable_fields', 'sortable_field_keys', - 'search_term_to_field_key', 'custom_field_metadata', - 'all_metadata'): - setattr(self, func, getattr(self.field_metadata, func)) - self.metadata_for_field = self.field_metadata.get - - # Legacy setter API - for field in ( - '!authors', 'author_sort', 'comment', 'has_cover', 'identifiers', 'languages', - 'pubdate', '!publisher', 'rating', '!series', 'series_index', 'timestamp', 'uuid', - ): - def setter(field): - has_case_change = field.startswith('!') - field = {'comment':'comments',}.get(field, field) - if has_case_change: - field = field[1:] - acc = field == 'series' - def func(self, book_id, val, notify=True, commit=True, allow_case_change=acc): - ret = self.new_api.set_field(field, {book_id:val}, allow_case_change=allow_case_change) - if notify: - self.notify([book_id]) - return ret - elif field == 'has_cover': - def func(self, book_id, val): - self.new_api.set_field('cover', {book_id:bool(val)}) - else: - def func(self, book_id, val, notify=True, commit=True): - if not val and field == 'uuid': - return - ret = self.new_api.set_field(field, {book_id:val}) - if notify: - self.notify([book_id]) - return ret if field == 'languages' else None - return func - setattr(self, 'set_%s' % field.replace('!', ''), MT(setter(field))) - self.last_update_check = self.last_modified() self.book_on_device_func = None - # Cleaning is not required anymore - self.clean = self.clean_custom = MT(lambda self:None) - self.clean_standard_field = MT(lambda self, field, commit=False:None) - # apsw operates in autocommit mode - self.commit = MT(lambda self:None) def close(self): self.backend.close() @@ -456,3 +353,139 @@ class LibraryDatabase(object): # }}} +MT = lambda func: types.MethodType(func, None, LibraryDatabase) + +# Legacy getter API {{{ +for prop in ('author_sort', 'authors', 'comment', 'comments', 'publisher', + 'rating', 'series', 'series_index', 'tags', 'title', 'title_sort', + 'timestamp', 'uuid', 'pubdate', 'ondevice', 'metadata_last_modified', 'languages',): + def getter(prop): + fm = {'comment':'comments', 'metadata_last_modified': + 'last_modified', 'title_sort':'sort'}.get(prop, prop) + def func(self, index, index_is_id=False): + return self.get_property(index, index_is_id=index_is_id, loc=self.FIELD_MAP[fm]) + return func + setattr(LibraryDatabase, prop, MT(getter(prop))) + +LibraryDatabase.has_cover = MT(lambda self, book_id:self.new_api.field_for('cover', book_id)) +LibraryDatabase.get_identifiers = MT( + lambda self, index, index_is_id=False: self.new_api.field_for('identifiers', index if index_is_id else self.data.index_to_id(index))) +# }}} + +# Legacy setter API {{{ +for field in ( + '!authors', 'author_sort', 'comment', 'has_cover', 'identifiers', 'languages', + 'pubdate', '!publisher', 'rating', '!series', 'series_index', 'timestamp', 'uuid', +): + def setter(field): + has_case_change = field.startswith('!') + field = {'comment':'comments',}.get(field, field) + if has_case_change: + field = field[1:] + acc = field == 'series' + def func(self, book_id, val, notify=True, commit=True, allow_case_change=acc): + ret = self.new_api.set_field(field, {book_id:val}, allow_case_change=allow_case_change) + if notify: + self.notify([book_id]) + return ret + elif field == 'has_cover': + def func(self, book_id, val): + self.new_api.set_field('cover', {book_id:bool(val)}) + else: + def func(self, book_id, val, notify=True, commit=True): + if not val and field == 'uuid': + return + ret = self.new_api.set_field(field, {book_id:val}) + if notify: + self.notify([book_id]) + return ret if field == 'languages' else None + return func + setattr(LibraryDatabase, 'set_%s' % field.replace('!', ''), MT(setter(field))) +# }}} + +# Legacy API to get information about many-(one, many) fields {{{ +for field in ('authors', 'tags', 'publisher', 'series'): + def getter(field): + def func(self): + return self.new_api.all_field_names(field) + return func + name = field[:-1] if field in {'authors', 'tags'} else field + setattr(LibraryDatabase, 'all_%s_names' % name, MT(getter(field))) + LibraryDatabase.all_formats = MT(lambda self:self.new_api.all_field_names('formats')) + +for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':'tags', 'all_series':'series', 'all_publishers':'publisher'}.iteritems(): + def getter(field): + def func(self): + return self.field_id_map(field) + return func + setattr(LibraryDatabase, func, MT(getter(field))) + +LibraryDatabase.all_tags = MT(lambda self: list(self.all_tag_names())) +LibraryDatabase.get_authors_with_ids = MT( + lambda self: [[aid, adata['name'], adata['sort'], adata['link']] for aid, adata in self.new_api.author_data().iteritems()]) + +for field in ('tags', 'series', 'publishers', 'ratings', 'languages'): + def getter(field): + fname = field[:-1] if field in {'publishers', 'ratings'} else field + def func(self): + return [[tid, tag] for tid, tag in self.new_api.get_id_map(fname).iteritems()] + return func + setattr(LibraryDatabase, 'get_%s_with_ids' % field, MT(getter(field))) + +for field in ('author', 'tag', 'series'): + def getter(field): + field = field if field == 'series' else (field+'s') + def func(self, item_id): + return self.new_api.get_item_name(field, item_id) + return func + setattr(LibraryDatabase, '%s_name' % field, MT(getter(field))) + +for field in ('publisher', 'series', 'tag'): + def getter(field): + fname = 'tags' if field == 'tag' else field + def func(self, item_id): + self.new_api.remove_items(fname, (item_id,)) + return func + setattr(LibraryDatabase, 'delete_%s_using_id' % field, MT(getter(field))) +# }}} + +# Legacy field API {{{ +for func in ( + 'standard_field_keys', '!custom_field_keys', 'all_field_keys', + 'searchable_fields', 'sortable_field_keys', + 'search_term_to_field_key', '!custom_field_metadata', + 'all_metadata'): + def getter(func): + if func.startswith('!'): + func = func[1:] + def meth(self, include_composites=True): + return getattr(self.field_metadata, func)(include_composites=include_composites) + elif func == 'search_term_to_field_key': + def meth(self, term): + return self.field_metadata.search_term_to_field_key(term) + else: + def meth(self): + return getattr(self.field_metadata, func)() + return meth + setattr(LibraryDatabase, func.replace('!', ''), MT(getter(func))) +LibraryDatabase.metadata_for_field = MT(lambda self, field:self.field_metadata.get(field)) + +# }}} + +# Miscellaneous API {{{ +for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'): + def getter(meth): + def func(self, x): + return getattr(self.new_api, meth)(x) + return func + setattr(LibraryDatabase, meth, MT(getter(meth))) + +# Cleaning is not required anymore +LibraryDatabase.clean = LibraryDatabase.clean_custom = MT(lambda self:None) +LibraryDatabase.clean_standard_field = MT(lambda self, field, commit=False:None) +# apsw operates in autocommit mode +LibraryDatabase.commit = MT(lambda self:None) +# }}} + +del MT + diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index f6f0fdd12c..b3b8d86d59 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -334,7 +334,7 @@ class LegacyTest(BaseTest): 'windows_check_if_files_in_use', } SKIP_ARGSPEC = { - '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', + '__init__', } missing = [] From a3884f22f5fde66cac13b8b6c0b48939f38867eb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 10:25:08 +0530 Subject: [PATCH 0185/1154] title and title_sort setters --- src/calibre/db/legacy.py | 11 +++++++---- src/calibre/db/tests/legacy.py | 5 +++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index ec2d6ffec4..f21d938009 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -376,10 +376,11 @@ LibraryDatabase.get_identifiers = MT( for field in ( '!authors', 'author_sort', 'comment', 'has_cover', 'identifiers', 'languages', 'pubdate', '!publisher', 'rating', '!series', 'series_index', 'timestamp', 'uuid', + 'title', 'title_sort', ): def setter(field): has_case_change = field.startswith('!') - field = {'comment':'comments',}.get(field, field) + field = {'comment':'comments', 'title_sort':'sort'}.get(field, field) if has_case_change: field = field[1:] acc = field == 'series' @@ -392,13 +393,15 @@ for field in ( def func(self, book_id, val): self.new_api.set_field('cover', {book_id:bool(val)}) else: + null_field = field in {'title', 'sort', 'uuid'} + retval = (True if field == 'sort' else None) def func(self, book_id, val, notify=True, commit=True): - if not val and field == 'uuid': - return + if not val and null_field: + return (False if field == 'sort' else None) ret = self.new_api.set_field(field, {book_id:val}) if notify: self.notify([book_id]) - return ret if field == 'languages' else None + return ret if field == 'languages' else retval return func setattr(LibraryDatabase, 'set_%s' % field.replace('!', ''), MT(setter(field))) # }}} diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index b3b8d86d59..6c67fcea36 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -420,9 +420,12 @@ class LegacyTest(BaseTest): ('set_rating', 1, 2.3), ('set_rating', 2, 0), ('set_rating', 3, 8), ('set_timestamp', 1, None), ('set_timestamp', 2, '2011-1-7'), ('set_uuid', 1, None), ('set_uuid', 2, 'a test uuid'), + ('set_title', 1, 'title two'), ('set_title', 2, None), ('set_title', 3, 'The Test Title'), (db.refresh,), + ('title', 0), ('title', 1), ('title', 2), + ('title_sort', 0), ('title_sort', 1), ('title_sort', 2), ('authors', 0), ('authors', 1), ('authors', 2), ('author_sort', 0), ('author_sort', 1), ('author_sort', 2), ('has_cover', 3), ('has_cover', 1), ('has_cover', 2), @@ -435,9 +438,11 @@ class LegacyTest(BaseTest): ('series_index', 0), ('series_index', 1), ('series_index', 2), ('uuid', 0), ('uuid', 1), ('uuid', 2), + ('set_title_sort', 1, 'Title Two'), ('set_title_sort', 2, None), ('set_title_sort', 3, 'The Test Title_sort'), ('set_series_index', 1, 2.3), ('set_series_index', 2, 0), ('set_series_index', 3, 8), (db.refresh,), ('series_index', 0), ('series_index', 1), ('series_index', 2), + ('title_sort', 0), ('title_sort', 1), ('title_sort', 2), )) db.close() From e923fbcffd15e3243b4ca089356bb1400cc47af3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 13:05:15 +0530 Subject: [PATCH 0186/1154] More set_*() API --- src/calibre/db/legacy.py | 32 ++++++++++++++++++++++++++++++++ src/calibre/db/tables.py | 3 +++ src/calibre/db/tests/legacy.py | 26 ++++++++++++++++++++++---- src/calibre/db/write.py | 4 ++-- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index f21d938009..9d9e93c02b 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -18,6 +18,7 @@ from calibre.db.backend import DB from calibre.db.cache import Cache from calibre.db.categories import CATEGORY_SORTS from calibre.db.view import View +from calibre.db.write import clean_identifier from calibre.utils.date import utcnow class LibraryDatabase(object): @@ -340,6 +341,36 @@ class LibraryDatabase(object): finally: self.notify('metadata', [book_id]) + def set_identifier(self, book_id, typ, val, notify=True, commit=True): + with self.new_api.write_lock: + identifiers = self.new_api._field_for('identifiers', book_id) + typ, val = clean_identifier(typ, val) + if typ: + identifiers[typ] = val + self.new_api._set_field('identifiers', {book_id:identifiers}) + self.notify('metadata', [book_id]) + + def set_isbn(self, book_id, isbn, notify=True, commit=True): + self.set_identifier(book_id, 'isbn', isbn, notify=notify, commit=commit) + + def set_tags(self, book_id, tags, append=False, notify=True, commit=True, allow_case_change=False): + tags = tags or [] + with self.new_api.write_lock: + if append: + otags = self.new_api._field_for('tags', book_id) + existing = {icu_lower(x) for x in otags} + tags = list(otags) + [x for x in tags if icu_lower(x) not in existing] + ret = self.new_api._set_field('tags', {book_id:tags}, allow_case_change=allow_case_change) + if notify: + self.notify('metadata', [book_id]) + return ret + + def set_metadata(self, book_id, mi, ignore_errors=False, set_title=True, + set_authors=True, commit=True, force_changes=False, notify=True): + self.new_api.set_metadata(book_id, mi, ignore_errors=ignore_errors, set_title=set_title, set_authors=set_authors, force_changes=force_changes) + if notify: + self.notify('metadata', [book_id]) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): @@ -424,6 +455,7 @@ for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':' setattr(LibraryDatabase, func, MT(getter(field))) LibraryDatabase.all_tags = MT(lambda self: list(self.all_tag_names())) +LibraryDatabase.get_all_identifier_types = MT(lambda self: list(self.new_api.fields['identifiers'].table.all_identifier_types())) LibraryDatabase.get_authors_with_ids = MT( lambda self: [[aid, adata['name'], adata['sort'], adata['link']] for aid, adata in self.new_api.author_data().iteritems()]) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 7715f6abef..46c4554586 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -420,3 +420,6 @@ class IdentifiersTable(ManyToManyTable): def remove_items(self, item_ids, db): raise NotImplementedError('Direct deletion of identifiers is not implemented') + def all_identifier_types(self): + return frozenset(k for k, v in self.col_book_map.iteritems() if v) + diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 6c67fcea36..b0ef9fbe1e 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -49,7 +49,7 @@ def run_funcs(self, db, ndb, funcs): fmt = lambda x:x if meth[0] in {'!', '@', '#', '+'}: if meth[0] != '+': - fmt = {'!':dict, '@':frozenset, '#':lambda x:set((x or '').split(','))}[meth[0]] + fmt = {'!':dict, '@':lambda x:frozenset(x or ()), '#':lambda x:set((x or '').split(','))}[meth[0]] else: fmt = args[-1] args = args[:-1] @@ -168,6 +168,7 @@ class LegacyTest(BaseTest): '!all_authors':[()], '!all_tags2':[()], '@all_tags':[()], + '@get_all_identifier_types':[()], '!all_publishers':[()], '!all_titles':[()], '!all_series':[()], @@ -331,7 +332,7 @@ class LegacyTest(BaseTest): 'construct_path_name', 'clear_dirtied', 'commit_dirty_cache', 'initialize_database', 'initialize_dynamic', 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', 'dirty_books_referencing', - 'windows_check_if_files_in_use', + 'windows_check_if_files_in_use', 'get_metadata_for_dump', 'get_a_dirtied_book', } SKIP_ARGSPEC = { '__init__', @@ -402,6 +403,7 @@ class LegacyTest(BaseTest): def test_legacy_setters(self): # {{{ 'Test methods that are directly equivalent in the old and new interface' + from calibre.ebooks.metadata.book.base import Metadata ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) @@ -421,9 +423,8 @@ class LegacyTest(BaseTest): ('set_timestamp', 1, None), ('set_timestamp', 2, '2011-1-7'), ('set_uuid', 1, None), ('set_uuid', 2, 'a test uuid'), ('set_title', 1, 'title two'), ('set_title', 2, None), ('set_title', 3, 'The Test Title'), - + ('set_tags', 1, ['a1', 'a2'], True), ('set_tags', 2, ['b1', 'tag one'], False, False, False, True), ('set_tags', 3, ['A1']), (db.refresh,), - ('title', 0), ('title', 1), ('title', 2), ('title_sort', 0), ('title_sort', 1), ('title_sort', 2), ('authors', 0), ('authors', 1), ('authors', 2), @@ -437,12 +438,29 @@ class LegacyTest(BaseTest): ('series', 0), ('series', 1), ('series', 2), ('series_index', 0), ('series_index', 1), ('series_index', 2), ('uuid', 0), ('uuid', 1), ('uuid', 2), + ('@tags', 0), ('@tags', 1), ('@tags', 2), + ('@all_tags',), + ('@get_all_identifier_types',), ('set_title_sort', 1, 'Title Two'), ('set_title_sort', 2, None), ('set_title_sort', 3, 'The Test Title_sort'), ('set_series_index', 1, 2.3), ('set_series_index', 2, 0), ('set_series_index', 3, 8), + ('set_identifier', 1, 'moose', 'val'), ('set_identifier', 2, 'test', ''), ('set_identifier', 3, '', ''), (db.refresh,), ('series_index', 0), ('series_index', 1), ('series_index', 2), ('title_sort', 0), ('title_sort', 1), ('title_sort', 2), + ('get_identifiers', 0), ('get_identifiers', 1), ('get_identifiers', 2), + ('@get_all_identifier_types',), + + ('set_metadata', 1, Metadata('title', ('a1',)), False, False, False, True, True), + ('set_metadata', 3, Metadata('title', ('a1',))), + (db.refresh,), + ('title', 0), ('title', 1), ('title', 2), + ('title_sort', 0), ('title_sort', 1), ('title_sort', 2), + ('authors', 0), ('authors', 1), ('authors', 2), + ('author_sort', 0), ('author_sort', 1), ('author_sort', 2), + ('@tags', 0), ('@tags', 1), ('@tags', 2), + ('@all_tags',), + ('@get_all_identifier_types',), )) db.close() diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py index a257788a60..5b35248353 100644 --- a/src/calibre/db/write.py +++ b/src/calibre/db/write.py @@ -107,8 +107,8 @@ def adapt_languages(to_tuple, x): return tuple(ans) def clean_identifier(typ, val): - typ = icu_lower(typ).strip().replace(':', '').replace(',', '') - val = val.strip().replace(',', '|').replace(':', '|') + typ = icu_lower(typ or '').strip().replace(':', '').replace(',', '') + val = (val or '').strip().replace(',', '|').replace(':', '|') return typ, val def adapt_identifiers(to_tuple, x): From e87e6941dc7af81bf9e8c276c9c84127a131312c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 13:51:14 +0530 Subject: [PATCH 0187/1154] Show auto convert format in question dialog --- src/calibre/gui2/email.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index f8c7552437..665b19cc5a 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -257,8 +257,8 @@ class EmailMixin(object): # {{{ else: autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto] if self.auto_convert_question( - _('Auto convert the following books before sending via ' - 'email?'), autos): + _('Auto convert the following books to %s before sending via ' + 'email?') % format.upper(), autos): self.iactions['Convert Books'].auto_convert_mail(to, fmts, delete_from_library, auto, format, subject) if bad: From 96aa24f59f3e27393a541f9d282590ea349b3c8a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 14:03:09 +0530 Subject: [PATCH 0188/1154] Fix auto-conversion regression Fix regression in last release that broke auto-conversion of ebooks when sending to device/sending by email. Fixes #1200864 [False conversion from mobi to epub by using Tolino](https://bugs.launchpad.net/calibre/+bug/1200864) --- src/calibre/gui2/convert/bulk.py | 2 +- src/calibre/gui2/convert/single.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/convert/bulk.py b/src/calibre/gui2/convert/bulk.py index 91efc73ca9..8c1ef8cb96 100644 --- a/src/calibre/gui2/convert/bulk.py +++ b/src/calibre/gui2/convert/bulk.py @@ -119,7 +119,7 @@ class BulkConfig(Config): def setup_output_formats(self, db, preferred_output_format): if preferred_output_format: - preferred_output_format = preferred_output_format.lower() + preferred_output_format = preferred_output_format.upper() output_formats = get_output_formats(preferred_output_format) preferred_output_format = preferred_output_format if \ preferred_output_format and preferred_output_format \ diff --git a/src/calibre/gui2/convert/single.py b/src/calibre/gui2/convert/single.py index e8342610dd..945ed42594 100644 --- a/src/calibre/gui2/convert/single.py +++ b/src/calibre/gui2/convert/single.py @@ -252,7 +252,7 @@ class Config(ResizableDialog, Ui_Dialog): def setup_input_output_formats(self, db, book_id, preferred_input_format, preferred_output_format): if preferred_output_format: - preferred_output_format = preferred_output_format.lower() + preferred_output_format = preferred_output_format.upper() output_formats = get_output_formats(preferred_output_format) input_format, input_formats = get_input_format_for_book(db, book_id, preferred_input_format) From 72d5185347f8e155febe151d7fc8866f82573e7d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 14:53:14 +0530 Subject: [PATCH 0189/1154] tags manipulation API --- src/calibre/db/legacy.py | 56 +++++++++++++++++++++++++++++++++- src/calibre/db/tests/legacy.py | 35 +++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 9d9e93c02b..428e164f06 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -9,7 +9,8 @@ __copyright__ = '2013, Kovid Goyal ' import os, traceback, types from future_builtins import zip -from calibre import force_unicode +from calibre import force_unicode, isbytestring +from calibre.constants import preferred_encoding from calibre.db import _get_next_series_num_for_list, _get_series_values from calibre.db.adding import ( find_books_in_directory, import_book_directory_multiple, @@ -21,6 +22,19 @@ from calibre.db.view import View from calibre.db.write import clean_identifier from calibre.utils.date import utcnow +def cleanup_tags(tags): + tags = [x.strip().replace(',', ';') for x in tags if x.strip()] + tags = [x.decode(preferred_encoding, 'replace') + if isbytestring(x) else x for x in tags] + tags = [u' '.join(x.split()) for x in tags] + ans, seen = [], set([]) + for tag in tags: + if tag.lower() not in seen: + seen.add(tag.lower()) + ans.append(tag) + return ans + + class LibraryDatabase(object): ''' Emulate the old LibraryDatabase2 interface ''' @@ -371,6 +385,44 @@ class LibraryDatabase(object): if notify: self.notify('metadata', [book_id]) + def remove_all_tags(self, ids, notify=False, commit=True): + self.new_api.set_field('tags', {book_id:() for book_id in ids}) + if notify: + self.notify('metadata', ids) + + def bulk_modify_tags(self, ids, add=[], remove=[], notify=False): + add = cleanup_tags(add) + remove = cleanup_tags(remove) + remove = set(remove) - set(add) + if not ids or (not add and not remove): + return + remove = {icu_lower(x) for x in remove} + with self.new_api.write_lock: + val_map = {} + for book_id in ids: + tags = list(self.new_api._field_for('tags', book_id)) + existing = {icu_lower(x) for x in tags} + tags.extend(t for t in add if icu_lower(t) not in existing) + tags = tuple(t for t in tags if icu_lower(t) not in remove) + val_map[book_id] = tags + self.new_api._set_field('tags', val_map, allow_case_change=False) + + if notify: + self.notify('metadata', ids) + + def unapply_tags(self, book_id, tags, notify=True): + self.bulk_modify_tags((book_id,), remove=tags, notify=notify) + + def is_tag_used(self, tag): + return icu_lower(tag) in {icu_lower(x) for x in self.new_api.all_field_names('tags')} + + def delete_tag(self, tag): + with self.new_api.write_lock: + tag_map = {icu_lower(v):k for k, v in self.new_api._get_id_map('tags').iteritems()} + tid = tag_map.get(icu_lower(tag), None) + if tid is not None: + self.new_api._remove_items('tags', (tid,)) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): @@ -399,6 +451,7 @@ for prop in ('author_sort', 'authors', 'comment', 'comments', 'publisher', setattr(LibraryDatabase, prop, MT(getter(prop))) LibraryDatabase.has_cover = MT(lambda self, book_id:self.new_api.field_for('cover', book_id)) +LibraryDatabase.get_tags = MT(lambda self, book_id:set(self.new_api.field_for('tags', book_id))) LibraryDatabase.get_identifiers = MT( lambda self, index, index_is_id=False: self.new_api.field_for('identifiers', index if index_is_id else self.data.index_to_id(index))) # }}} @@ -524,3 +577,4 @@ LibraryDatabase.commit = MT(lambda self:None) del MT + diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index b0ef9fbe1e..0a9aff1c9c 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -404,6 +404,26 @@ class LegacyTest(BaseTest): def test_legacy_setters(self): # {{{ 'Test methods that are directly equivalent in the old and new interface' from calibre.ebooks.metadata.book.base import Metadata + ndb = self.init_legacy(self.cloned_library) + db = self.init_old(self.cloned_library) + run_funcs(self, db, ndb, ( + ('get_tags', 0), ('get_tags', 1), ('get_tags', 2), + ('is_tag_used', 'News'), ('is_tag_used', 'xchkjgfh'), + ('bulk_modify_tags', (1,), ['t1'], ['News']), + ('bulk_modify_tags', (2,), ['t1'], ['Tag One', 'Tag Two']), + ('bulk_modify_tags', (3,), ['t1', 't2', 't3']), + (db.clean,), + ('@all_tags',), + ('@tags', 0), ('@tags', 1), ('@tags', 2), + + ('unapply_tags', 1, ['t1']), + ('unapply_tags', 2, ['xxxx']), + ('unapply_tags', 3, ['t2', 't3']), + (db.clean,), + ('@all_tags',), + ('@tags', 0), ('@tags', 1), ('@tags', 2), + )) + ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) @@ -479,5 +499,20 @@ class LegacyTest(BaseTest): ('#tags', 0), ('#tags', 1), ('#tags', 2), ('authors', 0), ('authors', 1), ('authors', 2), ('publisher', 0), ('publisher', 1), ('publisher', 2), + ('delete_tag', 'T1'), ('delete_tag', 'T2'), ('delete_tag', 'Tag one'), ('delete_tag', 'News'), + (db.clean,), (db.refresh,), + ('@all_tags',), + ('#tags', 0), ('#tags', 1), ('#tags', 2), )) + + ndb = self.init_legacy(self.cloned_library) + db = self.init_old(self.cloned_library) + run_funcs(self, db, ndb, ( + ('remove_all_tags', (1, 2, 3)), + (db.clean,), + ('@all_tags',), + ('@tags', 0), ('@tags', 1), ('@tags', 2), + )) + + # }}} From 73536b642133530158ecc4d0e53a3b62c2ab69bb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 15:12:20 +0530 Subject: [PATCH 0190/1154] ... --- src/calibre/db/legacy.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 428e164f06..05362d98df 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -417,11 +417,15 @@ class LibraryDatabase(object): return icu_lower(tag) in {icu_lower(x) for x in self.new_api.all_field_names('tags')} def delete_tag(self, tag): + self.delete_tags((tag,)) + + def delete_tags(self, tags): with self.new_api.write_lock: tag_map = {icu_lower(v):k for k, v in self.new_api._get_id_map('tags').iteritems()} - tid = tag_map.get(icu_lower(tag), None) - if tid is not None: - self.new_api._remove_items('tags', (tid,)) + tag_ids = (tag_map.get(icu_lower(tag), None) for tag in tags) + tag_ids = tuple(tid for tid in tag_ids if tid is not None) + if tag_ids: + self.new_api._remove_items('tags', tag_ids) # Private interface {{{ def __iter__(self): From 34abccc4f128c5f2fb514adf271df4b650279b26 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 15:48:56 +0530 Subject: [PATCH 0191/1154] Miscellaneous API --- src/calibre/db/legacy.py | 30 +++++++++++++++++++++--------- src/calibre/db/tests/legacy.py | 10 ++++++++-- src/calibre/db/view.py | 3 +++ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 05362d98df..580fcc2ae6 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -63,11 +63,13 @@ class LibraryDatabase(object): cache = self.new_api = Cache(backend) cache.init() self.data = View(cache) + self.id = self.data.index_to_id self.get_property = self.data.get_property self.last_update_check = self.last_modified() self.book_on_device_func = None + self.is_case_sensitive = getattr(backend, 'is_case_sensitive', False) def close(self): self.backend.close() @@ -131,6 +133,10 @@ class LibraryDatabase(object): for book_id in self.data.cache.all_book_ids(): yield book_id + def is_empty(self): + with self.new_api.read_lock: + return not bool(self.new_api.fields['title'].table.book_col_map) + def get_usage_count_by_id(self, field): return [[k, v] for k, v in self.new_api.get_usage_count_by_id(field).iteritems()] @@ -161,7 +167,7 @@ class LibraryDatabase(object): def path(self, index, index_is_id=False): 'Return the relative path to the directory containing this books files as a unicode string.' - book_id = index if index_is_id else self.data.index_to_id(index) + book_id = index if index_is_id else self.id(index) return self.new_api.field_for('path', book_id).replace('/', os.sep) def abspath(self, index, index_is_id=False, create_dirs=True): @@ -224,7 +230,7 @@ class LibraryDatabase(object): def add_format(self, index, fmt, stream, index_is_id=False, path=None, notify=True, replace=True, copy_function=None): ''' path and copy_function are ignored by the new API ''' - book_id = index if index_is_id else self.data.index_to_id(index) + book_id = index if index_is_id else self.id(index) try: return self.new_api.add_format(book_id, fmt, stream, replace=replace, run_hooks=False, dbapi=self) except: @@ -234,7 +240,7 @@ class LibraryDatabase(object): def add_format_with_hooks(self, index, fmt, fpath, index_is_id=False, path=None, notify=True, replace=True): ''' path is ignored by the new API ''' - book_id = index if index_is_id else self.data.index_to_id(index) + book_id = index if index_is_id else self.id(index) try: return self.new_api.add_format(book_id, fmt, fpath, replace=replace, run_hooks=True, dbapi=self) except: @@ -268,12 +274,12 @@ class LibraryDatabase(object): # }}} def get_field(self, index, key, default=None, index_is_id=False): - book_id = index if index_is_id else self.data.index_to_id(index) + book_id = index if index_is_id else self.id(index) mi = self.new_api.get_metadata(book_id, get_cover=key == 'cover') return mi.get(key, default) def authors_sort_strings(self, index, index_is_id=False): - book_id = index if index_is_id else self.data.index_to_id(index) + book_id = index if index_is_id else self.id(index) with self.new_api.read_lock: authors = self.new_api._field_ids_for('authors', book_id) adata = self.new_api._author_data(authors) @@ -283,7 +289,7 @@ class LibraryDatabase(object): return ' & '.join(self.authors_sort_strings(index, index_is_id=index_is_id)) def authors_with_sort_strings(self, index, index_is_id=False): - book_id = index if index_is_id else self.data.index_to_id(index) + book_id = index if index_is_id else self.id(index) with self.new_api.read_lock: authors = self.new_api._field_ids_for('authors', book_id) adata = self.new_api._author_data(authors) @@ -318,7 +324,7 @@ class LibraryDatabase(object): return sorted(book_ids, key=lambda x:ff('series_index', x)) def books_in_series_of(self, index, index_is_id=False): - book_id = index if index_is_id else self.data.index_to_id(index) + book_id = index if index_is_id else self.id(index) series_ids = self.new_api.field_ids_for('series', book_id) if not series_ids: return [] @@ -349,7 +355,7 @@ class LibraryDatabase(object): self.new_api.delete_conversion_options((book_id,), fmt=fmt) def set(self, index, field, val, allow_case_change=False): - book_id = self.data.index_to_id(index) + book_id = self.id(index) try: return self.new_api.set_field(field, {book_id:val}, allow_case_change=allow_case_change) finally: @@ -427,6 +433,9 @@ class LibraryDatabase(object): if tag_ids: self.new_api._remove_items('tags', tag_ids) + def has_id(self, book_id): + return book_id in self.new_api.all_book_ids() + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): @@ -454,10 +463,13 @@ for prop in ('author_sort', 'authors', 'comment', 'comments', 'publisher', return func setattr(LibraryDatabase, prop, MT(getter(prop))) +LibraryDatabase.index = MT(lambda self, book_id, cache=False:self.data.id_to_index(book_id)) LibraryDatabase.has_cover = MT(lambda self, book_id:self.new_api.field_for('cover', book_id)) LibraryDatabase.get_tags = MT(lambda self, book_id:set(self.new_api.field_for('tags', book_id))) LibraryDatabase.get_identifiers = MT( - lambda self, index, index_is_id=False: self.new_api.field_for('identifiers', index if index_is_id else self.data.index_to_id(index))) + lambda self, index, index_is_id=False: self.new_api.field_for('identifiers', index if index_is_id else self.id(index))) +LibraryDatabase.isbn = MT( + lambda self, index, index_is_id=False: self.get_identifiers(index, index_is_id=index_is_id).get('isbn', None)) # }}} # Legacy setter API {{{ diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 0a9aff1c9c..8d07e0cff2 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -161,6 +161,10 @@ class LegacyTest(BaseTest): 'get_next_series_num_for': [('A Series One',)], 'author_sort_from_authors': [(['Author One', 'Author Two', 'Unknown'],)], 'has_book':[(Metadata('title one'),), (Metadata('xxxx1111'),)], + 'has_id':[(1,), (2,), (3,), (9999,)], + 'id':[(1,), (2,), (0,),], + 'index':[(1,), (2,), (3,), ], + 'is_empty':[()], 'all_author_names':[()], 'all_tag_names':[()], 'all_series_names':[()], @@ -332,7 +336,7 @@ class LegacyTest(BaseTest): 'construct_path_name', 'clear_dirtied', 'commit_dirty_cache', 'initialize_database', 'initialize_dynamic', 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', 'dirty_books_referencing', - 'windows_check_if_files_in_use', 'get_metadata_for_dump', 'get_a_dirtied_book', + 'windows_check_if_files_in_use', 'get_metadata_for_dump', 'get_a_dirtied_book', 'dirtied_sequence', } SKIP_ARGSPEC = { '__init__', @@ -432,7 +436,7 @@ class LegacyTest(BaseTest): ('set_author_sort', 3, 'new_aus'), ('set_comment', 1, ''), ('set_comment', 2, None), ('set_comment', 3, '

    a comment

    '), ('set_has_cover', 1, True), ('set_has_cover', 2, True), ('set_has_cover', 3, 1), - ('set_identifiers', 2, {'test':'', 'a':'b'}), ('set_identifiers', 3, {'id':'1', 'url':'http://acme.com'}), ('set_identifiers', 1, {}), + ('set_identifiers', 2, {'test':'', 'a':'b'}), ('set_identifiers', 3, {'id':'1', 'isbn':'9783161484100'}), ('set_identifiers', 1, {}), ('set_languages', 1, ('en',)), ('set_languages', 2, ()), ('set_languages', 3, ('deu', 'spa', 'fra')), @@ -458,6 +462,7 @@ class LegacyTest(BaseTest): ('series', 0), ('series', 1), ('series', 2), ('series_index', 0), ('series_index', 1), ('series_index', 2), ('uuid', 0), ('uuid', 1), ('uuid', 2), + ('isbn', 0), ('isbn', 1), ('isbn', 2), ('@tags', 0), ('@tags', 1), ('@tags', 2), ('@all_tags',), ('@get_all_identifier_types',), @@ -516,3 +521,4 @@ class LegacyTest(BaseTest): # }}} + diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index ecd5182232..43aed74f59 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -161,6 +161,9 @@ class View(object): def index_to_id(self, idx): return self._map_filtered[idx] + def id_to_index(self, book_id): + return self._map.index(book_id) + def _get(self, field, idx, index_is_id=True, default_value=None, fmt=lambda x:x): id_ = idx if index_is_id else self.index_to_id(idx) if index_is_id and id_ not in self.cache.all_book_ids(): From 9ec8aac3f47cdaff374ad99dad5c4ad9b456f663 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 16:10:17 +0530 Subject: [PATCH 0192/1154] Fix img flip regression in PDF PDF Input: Fix a regression that caused some images to be flipped when converting PDF files that use image rotation operators. See #1201083 --- src/calibre/ebooks/pdf/pdftohtml.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/calibre/ebooks/pdf/pdftohtml.py b/src/calibre/ebooks/pdf/pdftohtml.py index ca950b84b2..47bd64c2d0 100644 --- a/src/calibre/ebooks/pdf/pdftohtml.py +++ b/src/calibre/ebooks/pdf/pdftohtml.py @@ -118,7 +118,7 @@ def flip_image(img, flip): im.save(img) def flip_images(raw): - for match in re.finditer(b']+/?>', raw): + for match in re.finditer(b']+/?>', raw, flags=re.I): img = match.group() m = re.search(br'class="(x|y|xy)flip"', img) if m is None: continue @@ -127,7 +127,6 @@ def flip_images(raw): if src is None: continue img = src.group(1) if not os.path.exists(img): continue - print ('Flipping image %s: %s'%(img, flip)) flip_image(img, flip) raw = re.sub(br'\s*', b'', raw, flags=re.I|re.DOTALL) return raw From 3e47e065ee1d1181177f41be6d8b119a0fd03e11 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 14 Jul 2013 17:07:58 +0530 Subject: [PATCH 0193/1154] formats API --- src/calibre/db/backend.py | 17 ++++++++++++- src/calibre/db/cache.py | 45 ++++++++++++++++++++++++++-------- src/calibre/db/legacy.py | 39 +++++++++++++++++++++++++++++ src/calibre/db/tests/legacy.py | 11 +++++++++ 4 files changed, 101 insertions(+), 11 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 0ebc9679b7..2c4dfb8395 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -8,7 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' # Imports {{{ -import os, shutil, uuid, json, glob, time, cPickle +import os, shutil, uuid, json, glob, time, cPickle, hashlib from functools import partial import apsw @@ -17,7 +17,9 @@ from calibre import isbytestring, force_unicode, prints from calibre.constants import (iswindows, filesystem_encoding, preferred_encoding) from calibre.ptempfile import PersistentTemporaryFile +from calibre.db import SPOOL_SIZE from calibre.db.schema_upgrades import SchemaUpgrade +from calibre.db.errors import NoSuchFormat from calibre.library.field_metadata import FieldMetadata from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.utils.icu import sort_key @@ -926,6 +928,19 @@ class DB(object): shutil.copyfile(candidates[0], fmt_path) return fmt_path + def format_hash(self, book_id, fmt, fname, path): + path = self.format_abspath(book_id, fmt, fname, path) + if path is None: + raise NoSuchFormat('Record %d has no fmt: %s'%(book_id, fmt)) + sha = hashlib.sha256() + with lopen(path, 'rb') as f: + while True: + raw = f.read(SPOOL_SIZE) + sha.update(raw) + if len(raw) < SPOOL_SIZE: + break + return sha.hexdigest() + def format_metadata(self, book_id, fmt, fname, path): path = self.format_abspath(book_id, fmt, fname, path) ans = {} diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 119e166c49..c615f62bf7 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -408,7 +408,16 @@ class Cache(object): return {aid:af.author_data(aid) for aid in author_ids if aid in af.table.id_map} @read_api - def format_metadata(self, book_id, fmt, allow_cache=True): + def format_hash(self, book_id, fmt): + try: + name = self.fields['formats'].format_fname(book_id, fmt) + path = self._field_for('path', book_id).replace('/', os.sep) + except: + raise NoSuchFormat('Record %d has no fmt: %s'%(book_id, fmt)) + return self.backend.format_hash(book_id, fmt, name, path) + + @api + def format_metadata(self, book_id, fmt, allow_cache=True, update_db=False): if not fmt: return {} fmt = fmt.upper() @@ -416,18 +425,30 @@ class Cache(object): x = self.format_metadata_cache[book_id].get(fmt, None) if x is not None: return x - try: - name = self.fields['formats'].format_fname(book_id, fmt) - path = self._field_for('path', book_id).replace('/', os.sep) - except: - return {} + with self.read_lock: + try: + name = self.fields['formats'].format_fname(book_id, fmt) + path = self._field_for('path', book_id).replace('/', os.sep) + except: + return {} + + ans = {} + if path and name: + ans = self.backend.format_metadata(book_id, fmt, name, path) + self.format_metadata_cache[book_id][fmt] = ans + if update_db and 'size' in ans: + with self.write_lock: + max_size = self.fields['formats'].table.update_fmt(book_id, fmt, name, ans['size'], self.backend) + self.fields['size'].table.update_sizes({book_id: max_size}) - ans = {} - if path and name: - ans = self.backend.format_metadata(book_id, fmt, name, path) - self.format_metadata_cache[book_id][fmt] = ans return ans + @read_api + def format_files(self, book_id): + field = self.fields['formats'] + fmts = field.table.book_col_map.get(book_id, ()) + return {fmt:field.format_fname(book_id, fmt) for fmt in fmts} + @read_api def pref(self, name, default=None): return self.backend.prefs.get(name, default) @@ -524,6 +545,7 @@ class Cache(object): the path is different from the current path (taking case sensitivity into account). ''' + fmt = (fmt or '').upper() try: name = self.fields['formats'].format_fname(book_id, fmt) path = self._field_for('path', book_id).replace('/', os.sep) @@ -544,6 +566,7 @@ class Cache(object): Apart from the viewer, I don't believe any of the others do any file I/O with the results of this call. ''' + fmt = (fmt or '').upper() try: name = self.fields['formats'].format_fname(book_id, fmt) path = self._field_for('path', book_id).replace('/', os.sep) @@ -555,6 +578,7 @@ class Cache(object): @read_api def has_format(self, book_id, fmt): 'Return True iff the format exists on disk' + fmt = (fmt or '').upper() try: name = self.fields['formats'].format_fname(book_id, fmt) path = self._field_for('path', book_id).replace('/', os.sep) @@ -601,6 +625,7 @@ class Cache(object): this means that repeated calls yield the same temp file (which is re-created each time) ''' + fmt = (fmt or '').upper() ext = ('.'+fmt.lower()) if fmt else '' if as_path: if preserve_filename: diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 580fcc2ae6..0ec44e9670 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -17,6 +17,7 @@ from calibre.db.adding import ( import_book_directory, recursive_import, add_catalog, add_news) from calibre.db.backend import DB from calibre.db.cache import Cache +from calibre.db.errors import NoSuchFormat from calibre.db.categories import CATEGORY_SORTS from calibre.db.view import View from calibre.db.write import clean_identifier @@ -436,6 +437,43 @@ class LibraryDatabase(object): def has_id(self, book_id): return book_id in self.new_api.all_book_ids() + def format(self, index, fmt, index_is_id=False, as_file=False, mode='r+b', as_path=False, preserve_filename=False): + book_id = index if index_is_id else self.id(index) + return self.new_api.format(book_id, fmt, as_file=as_file, as_path=as_path, preserve_filename=preserve_filename) + + def format_abspath(self, index, fmt, index_is_id=False): + book_id = index if index_is_id else self.id(index) + return self.new_api.format_abspath(book_id, fmt) + + def format_path(self, index, fmt, index_is_id=False): + book_id = index if index_is_id else self.id(index) + ans = self.new_api.format_abspath(book_id, fmt) + if ans is None: + raise NoSuchFormat('Record %d has no format: %s'%(book_id, fmt)) + return ans + + def format_files(self, index, index_is_id=False): + book_id = index if index_is_id else self.id(index) + return [(v, k) for k, v in self.new_api.format_files(book_id).iteritems()] + + def format_metadata(self, book_id, fmt, allow_cache=True, update_db=False, commit=False): + return self.new_api.format_metadata(book_id, fmt, allow_cache=allow_cache, update_db=update_db) + + def format_last_modified(self, book_id, fmt): + m = self.format_metadata(book_id, fmt) + if m: + return m['mtime'] + + def formats(self, index, index_is_id=False, verify_formats=True): + book_id = index if index_is_id else self.id(index) + ans = self.new_api.formats(book_id, verify_formats=verify_formats) + if ans: + return ','.join(ans) + + def has_format(self, index, fmt, index_is_id=False): + book_id = index if index_is_id else self.id(index) + return self.new_api.has_format(book_id, fmt) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): @@ -463,6 +501,7 @@ for prop in ('author_sort', 'authors', 'comment', 'comments', 'publisher', return func setattr(LibraryDatabase, prop, MT(getter(prop))) +LibraryDatabase.format_hash = MT(lambda self, book_id, fmt:self.new_api.format_hash(book_id, fmt)) LibraryDatabase.index = MT(lambda self, book_id, cache=False:self.data.id_to_index(book_id)) LibraryDatabase.has_cover = MT(lambda self, book_id:self.new_api.field_for('cover', book_id)) LibraryDatabase.get_tags = MT(lambda self, book_id:set(self.new_api.field_for('tags', book_id))) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 8d07e0cff2..1cae34fd04 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -11,6 +11,7 @@ from io import BytesIO from repr import repr from functools import partial from tempfile import NamedTemporaryFile +from operator import itemgetter from calibre.db.tests.base import BaseTest @@ -159,6 +160,11 @@ class LegacyTest(BaseTest): for meth, args in { 'get_next_series_num_for': [('A Series One',)], + 'format':[(1, 'FMT1', True), (2, 'FMT1', True), (0, 'xxxxxx')], + 'has_format':[(1, 'FMT1', True), (2, 'FMT1', True), (0, 'xxxxxx')], + '@format_files':[(0,),(1,),(2,)], + 'formats':[(0,),(1,),(2,)], + 'format_hash':[(1, 'FMT1'),(1, 'FMT2'), (2, 'FMT1')], 'author_sort_from_authors': [(['Author One', 'Author Two', 'Unknown'],)], 'has_book':[(Metadata('title one'),), (Metadata('xxxx1111'),)], 'has_id':[(1,), (2,), (3,), (9999,)], @@ -330,6 +336,7 @@ class LegacyTest(BaseTest): 'author_id', # replaced by get_author_id 'books_for_author', # broken 'books_in_old_database', # unused + 'migrate_old', # no longer supported # Internal API 'clean_user_categories', 'cleanup_tags', 'books_list_filter', 'conn', 'connect', 'construct_file_name', @@ -337,6 +344,7 @@ class LegacyTest(BaseTest): 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', 'dirty_books_referencing', 'windows_check_if_files_in_use', 'get_metadata_for_dump', 'get_a_dirtied_book', 'dirtied_sequence', + 'format_filename_cache', 'format_metadata_cache', 'filter', 'create_version1', } SKIP_ARGSPEC = { '__init__', @@ -411,6 +419,9 @@ class LegacyTest(BaseTest): ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) run_funcs(self, db, ndb, ( + ('+format_metadata', 1, 'FMT1', itemgetter('size')), + ('+format_metadata', 1, 'FMT2', itemgetter('size')), + ('+format_metadata', 2, 'FMT1', itemgetter('size')), ('get_tags', 0), ('get_tags', 1), ('get_tags', 2), ('is_tag_used', 'News'), ('is_tag_used', 'xchkjgfh'), ('bulk_modify_tags', (1,), ['t1'], ['News']), From 055bee6610a2704926921af7c6b050a48cb6e0e7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Jul 2013 10:47:54 +0530 Subject: [PATCH 0194/1154] Implement refresh/reload semantics --- src/calibre/db/__init__.py | 4 +++- src/calibre/db/cache.py | 26 +++++++++++++++++++++----- src/calibre/db/fields.py | 35 ++++++++++++++++++++++++----------- src/calibre/db/legacy.py | 35 ++++++++++++++++------------------- src/calibre/db/view.py | 11 +++++++++++ 5 files changed, 75 insertions(+), 36 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 47e44335ea..8f83a6bb81 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -116,6 +116,8 @@ Various things that require other things before they can be migrated: 4. Replace the metadatabackup thread with the new implementation when using the new backend. 5. In the new API refresh() does not re-read from disk. That might break a few things, for example content server reloading on db change as well as - dump/restore of db? + dump/restore of db and the refreshdb: action in gui2/ui.py. Probaly you'll have to create a dedicated API for + refreshing the db from disk and change the code to use it instead of the overloaded refresh (which is often used + to reread data from the db after writing to it). See reload_from_db() in cache.py 6. grep the sources for TODO ''' diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index c615f62bf7..d278e880b7 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -146,11 +146,18 @@ class Cache(object): self.formatter_template_cache = {} @write_api - def refresh(self): - self._initialize_template_cache() + def clear_caches(self, book_ids=None): + self._initialize_template_cache() # Clear the formatter template cache + for field in self.fields.itervalues(): + if hasattr(field, 'clear_caches'): + field.clear_caches(book_ids=book_ids) # Clear the composite cache and ondevice caches + self.format_metadata_cache.clear() + + @write_api + def reload_from_db(self, clear_caches=True): + if clear_caches: + self._clear_caches() for field in self.fields.itervalues(): - if hasattr(field, 'clear_cache'): - field.clear_cache() # Clear the composite cache if hasattr(field, 'table'): field.table.read(self.backend) # Reread data from metadata.db @@ -786,7 +793,7 @@ class Cache(object): if dirtied and self.composites: for name in self.composites: - self.fields[name].pop_cache(dirtied) + self.fields[name].clear_caches(book_ids=dirtied) if dirtied and update_path and do_path_update: self._update_path(dirtied, mark_as_dirtied=False) @@ -1264,6 +1271,15 @@ class Cache(object): ''' options must be a map of the form {book_id:conversion_options} ''' return self.backend.set_conversion_options(options, fmt) + @write_api + def refresh_format_cache(self): + self.fields['formats'].table.read(self.backend) + self.format_metadata_cache.clear() + + @write_api + def refresh_ondevice(self): + self.fields['ondevice'].clear_caches() + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 20d0d75ff4..e18179e4b1 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -11,7 +11,7 @@ __docformat__ = 'restructuredtext en' from threading import Lock from collections import defaultdict, Counter -from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY +from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY, null from calibre.db.write import Writer from calibre.ebooks.metadata import title_sort from calibre.utils.config_base import tweaks @@ -163,14 +163,13 @@ class CompositeField(OneToOneField): self._render_cache[book_id] = ans return ans - def clear_cache(self): + def clear_caches(self, book_ids=None): with self._lock: - self._render_cache = {} - - def pop_cache(self, book_ids): - with self._lock: - for book_id in book_ids: - self._render_cache.pop(book_id, None) + if book_ids is None: + self._render_cache.clear() + else: + for book_id in book_ids: + self._render_cache.pop(book_id, None) def get_value_with_cache(self, book_id, get_metadata): with self._lock: @@ -218,11 +217,25 @@ class OnDeviceField(OneToOneField): self.name = name self.book_on_device_func = None self.is_multiple = False + self.cache = {} + self._lock = Lock() + + def clear_caches(self, book_ids=None): + with self._lock: + if book_ids is None: + self.cache.clear() + else: + for book_id in book_ids: + self.cache.pop(book_id, None) def book_on_device(self, book_id): - if callable(self.book_on_device_func): - return self.book_on_device_func(book_id) - return None + with self._lock: + ans = self.cache.get(book_id, null) + if ans is null and callable(self.book_on_device_func): + ans = self.book_on_device_func(book_id) + with self._lock: + self.cache[book_id] = ans + return None if ans is null else ans def set_book_on_device_func(self, func): self.book_on_device_func = func diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 0ec44e9670..f9e4b52896 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -69,7 +69,7 @@ class LibraryDatabase(object): self.get_property = self.data.get_property self.last_update_check = self.last_modified() - self.book_on_device_func = None + self.refresh_ids = self.data.refresh_ids self.is_case_sensitive = getattr(backend, 'is_case_sensitive', False) def close(self): @@ -108,7 +108,7 @@ class LibraryDatabase(object): def check_if_modified(self): if self.last_modified() > self.last_update_check: - self.refresh() + self.new_api.reload_from_db() self.last_update_check = utcnow() @property @@ -145,7 +145,6 @@ class LibraryDatabase(object): return [(k, v) for k, v in self.new_api.get_id_map(field).iteritems()] def refresh(self, field=None, ascending=True): - self.data.cache.refresh() self.data.refresh(field=field, ascending=ascending) def add_listener(self, listener): @@ -297,26 +296,18 @@ class LibraryDatabase(object): return [(aid, adata[aid]['name'], adata[aid]['sort'], adata[aid]['link']) for aid in authors] def book_on_device(self, book_id): - if callable(self.book_on_device_func): - return self.book_on_device_func(book_id) - return None + with self.new_api.read_lock: + return self.new_api.fields['ondevice'].book_on_device(book_id) def book_on_device_string(self, book_id): - loc = [] - count = 0 - on = self.book_on_device(book_id) - if on is not None: - m, a, b, count = on[:4] - if m is not None: - loc.append(_('Main')) - if a is not None: - loc.append(_('Card A')) - if b is not None: - loc.append(_('Card B')) - return ', '.join(loc) + ((_(' (%s books)')%count) if count > 1 else '') + return self.new_api.field_for('ondevice', book_id) def set_book_on_device_func(self, func): - self.book_on_device_func = func + self.new_api.fields['ondevice'].set_book_on_device_func(func) + + @property + def book_on_device_func(self): + return self.new_api.fields['ondevice'].book_on_device_func def books_in_series(self, series_id): with self.new_api.read_lock: @@ -474,6 +465,12 @@ class LibraryDatabase(object): book_id = index if index_is_id else self.id(index) return self.new_api.has_format(book_id, fmt) + def refresh_format_cache(self): + self.new_api.refresh_format_cache() + + def refresh_ondevice(self): + self.new_api.refresh_ondevice() + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 43aed74f59..bc341698e2 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en' import weakref from functools import partial from itertools import izip, imap +from future_builtins import map from calibre.ebooks.metadata import title_sort from calibre.utils.config_base import tweaks @@ -163,6 +164,7 @@ class View(object): def id_to_index(self, book_id): return self._map.index(book_id) + row = index_to_id def _get(self, field, idx, index_is_id=True, default_value=None, fmt=lambda x:x): id_ = idx if index_is_id else self.index_to_id(idx) @@ -307,8 +309,17 @@ class View(object): def refresh(self, field=None, ascending=True): self._map = tuple(self.cache.all_book_ids()) self._map_filtered = tuple(self._map) + self.cache.clear_caches() if field is not None: self.sort(field, ascending) if self.search_restriction or self.base_restriction: self.search('', return_matches=False) + def refresh_ids(self, db, ids): + self.cache.clear_caches(book_ids=ids) + try: + return list(map(self.id_to_index, ids)) + except ValueError: + pass + return None + From e8a912267d3ced92ec82ca709ff284d3713b4e2f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Jul 2013 11:17:03 +0530 Subject: [PATCH 0195/1154] update_last_modified() and set_marked_ids() --- src/calibre/db/legacy.py | 5 +++++ src/calibre/db/tests/legacy.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index f9e4b52896..b2ad232f24 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -70,6 +70,7 @@ class LibraryDatabase(object): self.last_update_check = self.last_modified() self.refresh_ids = self.data.refresh_ids + self.set_marked_ids = self.data.set_marked_ids self.is_case_sensitive = getattr(backend, 'is_case_sensitive', False) def close(self): @@ -540,6 +541,10 @@ for field in ( return ret if field == 'languages' else retval return func setattr(LibraryDatabase, 'set_%s' % field.replace('!', ''), MT(setter(field))) + +LibraryDatabase.update_last_modified = MT( + lambda self, book_ids, commit=False, now=None: self.new_api.update_last_modified(book_ids, now=now)) + # }}} # Legacy API to get information about many-(one, many) fields {{{ diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 1cae34fd04..3f2c8729b8 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -416,6 +416,8 @@ class LegacyTest(BaseTest): def test_legacy_setters(self): # {{{ 'Test methods that are directly equivalent in the old and new interface' from calibre.ebooks.metadata.book.base import Metadata + from calibre.utils.date import now + n = now() ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) run_funcs(self, db, ndb, ( @@ -437,6 +439,9 @@ class LegacyTest(BaseTest): (db.clean,), ('@all_tags',), ('@tags', 0), ('@tags', 1), ('@tags', 2), + + ('update_last_modified', (1,), True, n), ('update_last_modified', (3,), True, n), + ('metadata_last_modified', 1, True), ('metadata_last_modified', 3, True), )) ndb = self.init_legacy(self.cloned_library) From 51018ff76f9f9345dd12a92fdbbf767e4303dc2e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Jul 2013 13:34:43 +0530 Subject: [PATCH 0196/1154] tags_older_than() --- src/calibre/db/__init__.py | 6 ++--- src/calibre/db/cache.py | 45 ++++++++++++++++++++++++++++++++++ src/calibre/db/legacy.py | 4 +++ src/calibre/db/tests/legacy.py | 20 +++++++++------ 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 8f83a6bb81..3a14552281 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -107,12 +107,10 @@ Various things that require other things before they can be migrated: 1. From initialize_dynamic(): set_saved_searches, load_user_template_functions. Also add custom columns/categories/searches info into - self.field_metadata. Finally, implement metadata dirtied - functionality. + self.field_metadata. 2. Catching DatabaseException and sqlite.Error when creating new libraries/switching/on calibre startup. - 3. From refresh in the legacy interface: Rember to flush the composite - column template cache. + 3. Port library/restore.py 4. Replace the metadatabackup thread with the new implementation when using the new backend. 5. In the new API refresh() does not re-read from disk. That might break a few things, for example content server reloading on db change as well as diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index d278e880b7..28fff9268b 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1280,6 +1280,51 @@ class Cache(object): def refresh_ondevice(self): self.fields['ondevice'].clear_caches() + @read_api + def tags_older_than(self, tag, delta=None, must_have_tag=None, must_have_authors=None): + ''' + Return the ids of all books having the tag ``tag`` that are older than + than the specified time. tag comparison is case insensitive. + + :param delta: A timedelta object or None. If None, then all ids with + the tag are returned. + :param must_have_tag: If not None the list of matches will be + restricted to books that have this tag + :param must_have_authors: A list of authors. If not None the list of + matches will be restricted to books that have these authors (case + insensitive). + ''' + tag_map = {icu_lower(v):k for k, v in self._get_id_map('tags').iteritems()} + tag = icu_lower(tag.strip()) + mht = icu_lower(must_have_tag.strip()) if must_have_tag else None + tag_id, mht_id = tag_map.get(tag, None), tag_map.get(mht, None) + ans = set() + if mht_id is None and mht: + return ans + if tag_id is not None: + tagged_books = self._books_for_field('tags', tag_id) + if mht_id is not None and tagged_books: + tagged_books = tagged_books.intersection(self._books_for_field('tags', mht_id)) + if tagged_books: + if must_have_authors is not None: + amap = {icu_lower(v):k for k, v in self._get_id_map('authors').iteritems()} + books = None + for author in must_have_authors: + abooks = self._books_for_field('authors', amap.get(icu_lower(author), None)) + books = abooks if books is None else books.intersection(abooks) + if not books: + break + tagged_books = tagged_books.intersection(books or set()) + if delta is None: + ans = tagged_books + else: + now = nowf() + for book_id in tagged_books: + ts = self._field_for('timestamp', book_id) + if (now - ts) > delta: + ans.add(book_id) + return ans + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index b2ad232f24..62cfe08838 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -472,6 +472,10 @@ class LibraryDatabase(object): def refresh_ondevice(self): self.new_api.refresh_ondevice() + def tags_older_than(self, tag, delta, must_have_tag=None, must_have_authors=None): + for book_id in sorted(self.new_api.tags_older_than(tag, delta=delta, must_have_tag=must_have_tag, must_have_authors=must_have_authors)): + yield book_id + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 3f2c8729b8..4d25c8798a 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -153,13 +153,19 @@ class LegacyTest(BaseTest): # }}} def test_legacy_direct(self): # {{{ - 'Test methods that are directly equivalent in the old and new interface' + 'Test read-only methods that are directly equivalent in the old and new interface' from calibre.ebooks.metadata.book.base import Metadata + from datetime import timedelta ndb = self.init_legacy(self.cloned_library) db = self.init_old() for meth, args in { 'get_next_series_num_for': [('A Series One',)], + '@tags_older_than': [ + ('News', None), ('Tag One', None), ('xxxx', None), ('Tag One', None, 'News'), ('News', None, 'xxxx'), + ('News', None, None, ['xxxxxxx']), ('News', None, 'Tag One', ['Author Two', 'Author One']), + ('News', timedelta(0), None, None), ('News', timedelta(100000)), + ], 'format':[(1, 'FMT1', True), (2, 'FMT1', True), (0, 'xxxxxx')], 'has_format':[(1, 'FMT1', True), (2, 'FMT1', True), (0, 'xxxxxx')], '@format_files':[(0,),(1,),(2,)], @@ -208,13 +214,13 @@ class LegacyTest(BaseTest): 'books_in_series_of':[(0,), (1,), (2,)], 'books_with_same_title':[(Metadata(db.title(0)),), (Metadata(db.title(1)),), (Metadata('1234'),)], }.iteritems(): + fmt = lambda x: x + if meth[0] in {'!', '@'}: + fmt = {'!':dict, '@':frozenset}[meth[0]] + meth = meth[1:] + elif meth == 'get_authors_with_ids': + fmt = lambda val:{x[0]:tuple(x[1:]) for x in val} for a in args: - fmt = lambda x: x - if meth[0] in {'!', '@'}: - fmt = {'!':dict, '@':frozenset}[meth[0]] - meth = meth[1:] - elif meth == 'get_authors_with_ids': - fmt = lambda val:{x[0]:tuple(x[1:]) for x in val} self.assertEqual(fmt(getattr(db, meth)(*a)), fmt(getattr(ndb, meth)(*a)), 'The method: %s() returned different results for argument %s' % (meth, a)) db.close() From 3e184968f25462361fbe349eabf6a758e94eaf9f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Jul 2013 15:35:55 +0530 Subject: [PATCH 0197/1154] More API --- src/calibre/db/cache.py | 26 ++++++++++++++++++++++++++ src/calibre/db/legacy.py | 18 ++++++++++++++++++ src/calibre/db/tables.py | 7 +++++++ src/calibre/db/tests/legacy.py | 17 ++++++++++++++++- 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 28fff9268b..42e53a6f47 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1325,6 +1325,32 @@ class Cache(object): ans.add(book_id) return ans + @write_api + def set_sort_for_authors(self, author_id_to_sort_map, update_books=True): + self.fields['authors'].table.set_sort_names(author_id_to_sort_map, self.backend) + changed_books = set() + if update_books: + val_map = {} + for author_id in author_id_to_sort_map: + books = self._books_for_field('authors', author_id) + changed_books |= books + for book_id in books: + authors = self._field_ids_for('authors', book_id) + adata = self._author_data(authors) + sorts = [adata[x]['sort'] for x in authors] + val_map[book_id] = ' & '.join(sorts) + if val_map: + self._set_field('author_sort', val_map) + return changed_books + + @write_api + def set_link_for_authors(self, author_id_to_link_map): + self.fields['authors'].table.set_links(author_id_to_link_map, self.backend) + changed_books = set() + for author_id in author_id_to_link_map: + changed_books |= self._books_for_field('authors', author_id) + return changed_books + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 62cfe08838..fef440dd6d 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -296,6 +296,16 @@ class LibraryDatabase(object): adata = self.new_api._author_data(authors) return [(aid, adata[aid]['name'], adata[aid]['sort'], adata[aid]['link']) for aid in authors] + def set_sort_field_for_author(self, old_id, new_sort, commit=True, notify=False): + changed_books = self.new_api.set_sort_for_authors({old_id:new_sort}) + if notify: + self.notify('metadata', list(changed_books)) + + def set_link_field_for_author(self, aid, link, commit=True, notify=False): + changed_books = self.new_api.set_link_for_authors({aid:link}) + if notify: + self.notify('metadata', list(changed_books)) + def book_on_device(self, book_id): with self.new_api.read_lock: return self.new_api.fields['ondevice'].book_on_device(book_id) @@ -476,6 +486,14 @@ class LibraryDatabase(object): for book_id in sorted(self.new_api.tags_older_than(tag, delta=delta, must_have_tag=must_have_tag, must_have_authors=must_have_authors)): yield book_id + def sizeof_format(self, index, fmt, index_is_id=False): + book_id = index if index_is_id else self.id(index) + return self.new_api.format_metadata(book_id, fmt).get('size', None) + + def get_metadata(self, index, index_is_id=False, get_cover=False, get_user_categories=True, cover_as_data=False): + book_id = index if index_is_id else self.id(index) + return self.new_api.get_metadata(book_id, get_cover=get_cover, get_user_categories=get_user_categories, cover_as_data=cover_as_data) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 46c4554586..81c66a0ca5 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -293,10 +293,17 @@ class AuthorsTable(ManyToManyTable): self.alink_map[row[0]] = row[3] def set_sort_names(self, aus_map, db): + aus_map = {aid:(a or '').strip() for aid, a in aus_map.iteritems()} self.asort_map.update(aus_map) db.conn.executemany('UPDATE authors SET sort=? WHERE id=?', [(v, k) for k, v in aus_map.iteritems()]) + def set_links(self, link_map, db): + link_map = {aid:(l or '').strip() for aid, l in link_map.iteritems()} + self.alink_map.update(link_map) + db.conn.executemany('UPDATE authors SET link=? WHERE id=?', + [(v, k) for k, v in link_map.iteritems()]) + def remove_books(self, book_ids, db): clean = ManyToManyTable.remove_books(self, book_ids, db) for item_id in clean: diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 4d25c8798a..0707022674 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -168,6 +168,7 @@ class LegacyTest(BaseTest): ], 'format':[(1, 'FMT1', True), (2, 'FMT1', True), (0, 'xxxxxx')], 'has_format':[(1, 'FMT1', True), (2, 'FMT1', True), (0, 'xxxxxx')], + 'sizeof_format':[(1, 'FMT1', True), (2, 'FMT1', True), (0, 'xxxxxx')], '@format_files':[(0,),(1,),(2,)], 'formats':[(0,),(1,),(2,)], 'format_hash':[(1, 'FMT1'),(1, 'FMT2'), (2, 'FMT1')], @@ -341,7 +342,7 @@ class LegacyTest(BaseTest): # Obsolete/broken methods 'author_id', # replaced by get_author_id 'books_for_author', # broken - 'books_in_old_database', # unused + 'books_in_old_database', 'sizeof_old_database', # unused 'migrate_old', # no longer supported # Internal API @@ -425,6 +426,8 @@ class LegacyTest(BaseTest): from calibre.utils.date import now n = now() ndb = self.init_legacy(self.cloned_library) + amap = ndb.new_api.get_id_map('authors') + sorts = [(aid, 's%d' % aid) for aid in amap] db = self.init_old(self.cloned_library) run_funcs(self, db, ndb, ( ('+format_metadata', 1, 'FMT1', itemgetter('size')), @@ -448,7 +451,19 @@ class LegacyTest(BaseTest): ('update_last_modified', (1,), True, n), ('update_last_modified', (3,), True, n), ('metadata_last_modified', 1, True), ('metadata_last_modified', 3, True), + ('set_sort_field_for_author', sorts[0][0], sorts[0][1]), + ('set_sort_field_for_author', sorts[1][0], sorts[1][1]), + ('set_sort_field_for_author', sorts[2][0], sorts[2][1]), + ('set_link_field_for_author', sorts[0][0], sorts[0][1]), + ('set_link_field_for_author', sorts[1][0], sorts[1][1]), + ('set_link_field_for_author', sorts[2][0], sorts[2][1]), + (db.refresh,), + ('author_sort', 0), ('author_sort', 1), ('author_sort', 2), )) + omi = [db.get_metadata(x) for x in (0, 1, 2)] + nmi = [ndb.get_metadata(x) for x in (0, 1, 2)] + self.assertEqual([x.author_sort_map for x in omi], [x.author_sort_map for x in nmi]) + self.assertEqual([x.author_link_map for x in omi], [x.author_link_map for x in nmi]) ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) From dc97d6ad695cd3b010a81024827e63ba69e85db4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Jul 2013 15:41:19 +0530 Subject: [PATCH 0198/1154] ... --- src/calibre/db/tests/legacy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 0707022674..b609cb478d 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -344,6 +344,7 @@ class LegacyTest(BaseTest): 'books_for_author', # broken 'books_in_old_database', 'sizeof_old_database', # unused 'migrate_old', # no longer supported + 'remove_unused_series', # superseded by clean API # Internal API 'clean_user_categories', 'cleanup_tags', 'books_list_filter', 'conn', 'connect', 'construct_file_name', From 12cded043fb1796f564346a412072ded8519b60b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Jul 2013 16:47:54 +0530 Subject: [PATCH 0199/1154] EPUB Output: Generate inline ToC EPUB Output: Add an option to insert an inline Table of COntents into the main text. Fixes #1201006 [Epub output: not possible to insert inline toc](https://bugs.launchpad.net/calibre/+bug/1201006) --- .../ebooks/conversion/plugins/epub_output.py | 44 +++++++---- src/calibre/ebooks/mobi/writer8/toc.py | 40 ++++++++-- src/calibre/gui2/convert/epub_output.py | 1 + src/calibre/gui2/convert/epub_output.ui | 77 +++++++++++++------ 4 files changed, 114 insertions(+), 48 deletions(-) diff --git a/src/calibre/ebooks/conversion/plugins/epub_output.py b/src/calibre/ebooks/conversion/plugins/epub_output.py index f09f2560b0..5569be4304 100644 --- a/src/calibre/ebooks/conversion/plugins/epub_output.py +++ b/src/calibre/ebooks/conversion/plugins/epub_output.py @@ -105,14 +105,23 @@ class EPUBOutput(OutputFormatPlugin): ' EPUB, putting all files into the top level.') ), + OptionRecommendation(name='epub_inline_toc', recommended_value=False, + help=_('Insert an inline Table of Contents that will appear as part of the main book content.') + ), + + OptionRecommendation(name='epub_toc_at_end', recommended_value=False, + help=_('Put the inserted inline Table of Contents at the end of the book instead of the start.') + ), + + OptionRecommendation(name='toc_title', recommended_value=None, + help=_('Title for any generated in-line table of contents.') + ), ]) recommendations = set([('pretty_print', True, OptionRecommendation.HIGH)]) - - - def workaround_webkit_quirks(self): # {{{ + def workaround_webkit_quirks(self): # {{{ from calibre.ebooks.oeb.base import XPath for x in self.oeb.spine: root = x.data @@ -128,13 +137,13 @@ class EPUBOutput(OutputFormatPlugin): pre.tag = 'div' # }}} - def upshift_markup(self): # {{{ + def upshift_markup(self): # {{{ 'Upgrade markup to comply with XHTML 1.1 where possible' from calibre.ebooks.oeb.base import XPath, XML for x in self.oeb.spine: root = x.data if (not root.get(XML('lang'))) and (root.get('lang')): - root.set(XML('lang'), root.get('lang')) + root.set(XML('lang'), root.get('lang')) body = XPath('//h:body')(root) if body: body = body[0] @@ -159,12 +168,17 @@ class EPUBOutput(OutputFormatPlugin): else: seen_names.add(name) - # }}} - def convert(self, oeb, output_path, input_plugin, opts, log): self.log, self.opts, self.oeb = log, opts, oeb + if self.opts.epub_inline_toc: + from calibre.ebooks.mobi.writer8.toc import TOCAdder + opts.mobi_toc_at_start = not opts.epub_toc_at_end + opts.mobi_passthrough = False + opts.no_inline_toc = False + TOCAdder(oeb, opts, replace_previous_inline_toc=True, ignore_existing_toc=True) + if self.opts.epub_flatten: from calibre.ebooks.oeb.transforms.filenames import FlatFilenames FlatFilenames()(oeb, opts) @@ -234,7 +248,7 @@ class EPUBOutput(OutputFormatPlugin): oeb_output = plugin_for_output_format('oeb') oeb_output.convert(oeb, tdir, input_plugin, opts, log) opf = [x for x in os.listdir(tdir) if x.endswith('.opf')][0] - self.condense_ncx([os.path.join(tdir, x) for x in os.listdir(tdir)\ + self.condense_ncx([os.path.join(tdir, x) for x in os.listdir(tdir) if x.endswith('.ncx')][0]) encryption = None if encrypted_fonts: @@ -261,7 +275,7 @@ class EPUBOutput(OutputFormatPlugin): zf.extractall(path=opts.extract_to) self.log.info('EPUB extracted to', opts.extract_to) - def encrypt_fonts(self, uris, tdir, uuid): # {{{ + def encrypt_fonts(self, uris, tdir, uuid): # {{{ from binascii import unhexlify key = re.sub(r'[^a-fA-F0-9]', '', uuid) @@ -301,14 +315,14 @@ class EPUBOutput(OutputFormatPlugin): '''%(uri.replace('"', '\\"'))) if fonts: - ans = ''' ''' - ans += (u'\n'.join(fonts)).encode('utf-8') - ans += '\n' - return ans + ans += (u'\n'.join(fonts)).encode('utf-8') + ans += '\n' + return ans # }}} def condense_ncx(self, ncx_path): @@ -323,7 +337,7 @@ class EPUBOutput(OutputFormatPlugin): compressed = etree.tostring(tree.getroot(), encoding='utf-8') open(ncx_path, 'wb').write(compressed) - def workaround_ade_quirks(self): # {{{ + def workaround_ade_quirks(self): # {{{ ''' Perform various markup transforms to get the output to render correctly in the quirky ADE. @@ -462,7 +476,7 @@ class EPUBOutput(OutputFormatPlugin): # }}} - def workaround_sony_quirks(self): # {{{ + def workaround_sony_quirks(self): # {{{ ''' Perform toc link transforms to alleviate slow loading. ''' diff --git a/src/calibre/ebooks/mobi/writer8/toc.py b/src/calibre/ebooks/mobi/writer8/toc.py index 7bae35ae98..640e8bec5f 100644 --- a/src/calibre/ebooks/mobi/writer8/toc.py +++ b/src/calibre/ebooks/mobi/writer8/toc.py @@ -34,9 +34,17 @@ TEMPLATE = ''' ''' +def find_previous_calibre_inline_toc(oeb): + if 'toc' in oeb.guide: + href = urlnormalize(oeb.guide['toc'].href.partition('#')[0]) + if href in oeb.manifest.hrefs: + item = oeb.manifest.hrefs[href] + if (hasattr(item.data, 'xpath') and XPath('//h:body[@id="calibre_generated_inline_toc"]')(item.data)): + return item + class TOCAdder(object): - def __init__(self, oeb, opts): + def __init__(self, oeb, opts, replace_previous_inline_toc=False, ignore_existing_toc=False): self.oeb, self.opts, self.log = oeb, opts, oeb.log self.title = opts.toc_title or DEFAULT_TITLE self.at_start = opts.mobi_toc_at_start @@ -44,6 +52,12 @@ class TOCAdder(object): self.added_toc_guide_entry = False self.has_toc = oeb.toc and oeb.toc.count() > 1 + self.tocitem = tocitem = None + if find_previous_calibre_inline_toc: + tocitem = self.tocitem = find_previous_calibre_inline_toc(oeb) + if ignore_existing_toc and 'toc' in oeb.guide: + oeb.guide.remove('toc') + if 'toc' in oeb.guide: # Remove spurious toc entry from guide if it is not in spine or it # does not have any hyperlinks @@ -81,13 +95,19 @@ class TOCAdder(object): for child in self.oeb.toc: self.process_toc_node(child, parent) - id, href = oeb.manifest.generate('contents', 'contents.xhtml') - item = self.generated_item = oeb.manifest.add(id, href, XHTML_MIME, - data=root) - if self.at_start: - oeb.spine.insert(0, item, linear=True) + if tocitem is not None: + href = tocitem.href + if oeb.spine.index(tocitem) > -1: + oeb.spine.remove(tocitem) + tocitem.data = root else: - oeb.spine.add(item, linear=False) + id, href = oeb.manifest.generate('contents', 'contents.xhtml') + tocitem = self.generated_item = oeb.manifest.add(id, href, XHTML_MIME, + data=root) + if self.at_start: + oeb.spine.insert(0, tocitem, linear=True) + else: + oeb.spine.add(tocitem, linear=False) oeb.guide.add('toc', 'Table of Contents', href) @@ -95,7 +115,10 @@ class TOCAdder(object): li = parent.makeelement(XHTML('li')) li.tail = '\n'+ ('\t'*level) parent.append(li) - a = parent.makeelement(XHTML('a'), href=toc.href or '#') + href = toc.href + if self.tocitem is not None and href: + href = self.tocitem.relhref(toc.href) + a = parent.makeelement(XHTML('a'), href=href or '#') a.text = toc.title li.append(a) if toc.count() > 0: @@ -115,3 +138,4 @@ class TOCAdder(object): self.oeb.guide.remove('toc') self.added_toc_guide_entry = False + diff --git a/src/calibre/gui2/convert/epub_output.py b/src/calibre/gui2/convert/epub_output.py index 2fcbd751fe..5fbfd74d05 100644 --- a/src/calibre/gui2/convert/epub_output.py +++ b/src/calibre/gui2/convert/epub_output.py @@ -21,6 +21,7 @@ class PluginWidget(Widget, Ui_Form): Widget.__init__(self, parent, ['dont_split_on_page_breaks', 'flow_size', 'no_default_epub_cover', 'no_svg_cover', + 'epub_inline_toc', 'epub_toc_at_end', 'toc_title', 'preserve_cover_aspect_ratio', 'epub_flatten'] ) for i in range(2): diff --git a/src/calibre/gui2/convert/epub_output.ui b/src/calibre/gui2/convert/epub_output.ui index 606ed62065..fa939d289f 100644 --- a/src/calibre/gui2/convert/epub_output.ui +++ b/src/calibre/gui2/convert/epub_output.ui @@ -6,7 +6,7 @@ 0 0 - 400 + 644 300 @@ -14,27 +14,6 @@ Form - - - - Do not &split on page breaks - - - - - - - No default &cover - - - - - - - No &SVG cover - - - @@ -42,7 +21,7 @@ - + Split files &larger than: @@ -52,7 +31,7 @@ - + KB @@ -68,7 +47,7 @@ - + Qt::Vertical @@ -81,6 +60,41 @@ + + + + No default &cover + + + + + + + No &SVG cover + + + + + + + Insert inline &Table of Contents + + + + + + + Do not &split on page breaks + + + + + + + Put inserted Table of Contents at the &end of the book + + + @@ -88,6 +102,19 @@ + + + + &Title for inserted ToC: + + + opt_toc_title + + + + + + From 67025df8ef24cc788a68f78a488b21d22b102ab3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Jul 2013 18:31:55 +0530 Subject: [PATCH 0200/1154] Implement refreshdb: --- src/calibre/db/__init__.py | 8 ++------ src/calibre/gui2/ui.py | 7 ++++++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 3a14552281..99154ad618 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -112,10 +112,6 @@ Various things that require other things before they can be migrated: libraries/switching/on calibre startup. 3. Port library/restore.py 4. Replace the metadatabackup thread with the new implementation when using the new backend. - 5. In the new API refresh() does not re-read from disk. That might break a - few things, for example content server reloading on db change as well as - dump/restore of db and the refreshdb: action in gui2/ui.py. Probaly you'll have to create a dedicated API for - refreshing the db from disk and change the code to use it instead of the overloaded refresh (which is often used - to reread data from the db after writing to it). See reload_from_db() in cache.py - 6. grep the sources for TODO + 5. grep the sources for TODO + 6. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add all work ''' diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 06cb4d904f..229ed0933d 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -532,7 +532,12 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.raise_() self.activateWindow() elif msg.startswith('refreshdb:'): - self.library_view.model().refresh() + db = self.library_view.model().db + if hasattr(db, 'new_api'): + db.new_api.reload_from_db() + self.library_view.model().resort() + else: + self.library_view.model().refresh() self.library_view.model().research() self.tags_view.recount() self.library_view.model().db.refresh_format_cache() From eac44f83b8d80e5820d567f44027bd7f6b676e62 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Jul 2013 18:53:55 +0530 Subject: [PATCH 0201/1154] More API --- src/calibre/db/legacy.py | 19 +++++++++++++++++++ src/calibre/db/tests/legacy.py | 8 +++++++- src/calibre/db/view.py | 3 +++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index fef440dd6d..ec480c3d2d 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -65,6 +65,7 @@ class LibraryDatabase(object): cache.init() self.data = View(cache) self.id = self.data.index_to_id + self.count = self.data.count self.get_property = self.data.get_property @@ -84,6 +85,10 @@ class LibraryDatabase(object): delattr(self, x) # Library wide properties {{{ + @property + def prefs(self): + return self.new_api.backend.prefs + @property def field_metadata(self): return self.backend.field_metadata @@ -521,6 +526,18 @@ for prop in ('author_sort', 'authors', 'comment', 'comments', 'publisher', return func setattr(LibraryDatabase, prop, MT(getter(prop))) +for prop in ('series', 'publisher'): + def getter(field): + def func(self, index, index_is_id=False): + book_id = index if index_is_id else self.id(index) + ans = self.new_api.field_ids_for(field, book_id) + try: + return ans[0] + except IndexError: + pass + return func + setattr(LibraryDatabase, prop + '_id', MT(getter(prop))) + LibraryDatabase.format_hash = MT(lambda self, book_id, fmt:self.new_api.format_hash(book_id, fmt)) LibraryDatabase.index = MT(lambda self, book_id, cache=False:self.data.id_to_index(book_id)) LibraryDatabase.has_cover = MT(lambda self, book_id:self.new_api.field_for('cover', book_id)) @@ -590,6 +607,8 @@ LibraryDatabase.all_tags = MT(lambda self: list(self.all_tag_names())) LibraryDatabase.get_all_identifier_types = MT(lambda self: list(self.new_api.fields['identifiers'].table.all_identifier_types())) LibraryDatabase.get_authors_with_ids = MT( lambda self: [[aid, adata['name'], adata['sort'], adata['link']] for aid, adata in self.new_api.author_data().iteritems()]) +LibraryDatabase.get_author_id = MT( + lambda self, author: {icu_lower(v):k for k, v in self.new_api.get_id_map('authors').iteritems()}.get(icu_lower(author), None)) for field in ('tags', 'series', 'publishers', 'ratings', 'languages'): def getter(field): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index b609cb478d..5f1aa2ff41 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -159,8 +159,13 @@ class LegacyTest(BaseTest): ndb = self.init_legacy(self.cloned_library) db = self.init_old() + self.assertEqual(dict(db.prefs), dict(ndb.prefs)) + for meth, args in { 'get_next_series_num_for': [('A Series One',)], + 'get_author_id': [('author one',), ('unknown',), ('xxxxx',)], + 'series_id': [(0,), (1,), (2,)], + 'publisher_id': [(0,), (1,), (2,)], '@tags_older_than': [ ('News', None), ('Tag One', None), ('xxxx', None), ('Tag One', None, 'News'), ('News', None, 'xxxx'), ('News', None, None, ['xxxxxxx']), ('News', None, 'Tag One', ['Author Two', 'Author One']), @@ -178,6 +183,7 @@ class LegacyTest(BaseTest): 'id':[(1,), (2,), (0,),], 'index':[(1,), (2,), (3,), ], 'is_empty':[()], + 'count':[()], 'all_author_names':[()], 'all_tag_names':[()], 'all_series_names':[()], @@ -352,7 +358,7 @@ class LegacyTest(BaseTest): 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', 'dirty_books_referencing', 'windows_check_if_files_in_use', 'get_metadata_for_dump', 'get_a_dirtied_book', 'dirtied_sequence', - 'format_filename_cache', 'format_metadata_cache', 'filter', 'create_version1', + 'format_filename_cache', 'format_metadata_cache', 'filter', 'create_version1', 'normpath', } SKIP_ARGSPEC = { '__init__', diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index bc341698e2..bb9131e212 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -120,6 +120,9 @@ class View(object): self._map = tuple(sorted(self.cache.all_book_ids())) self._map_filtered = tuple(self._map) + def count(self): + return len(self._map) + def get_property(self, id_or_index, index_is_id=False, loc=-1): book_id = id_or_index if index_is_id else self._map_filtered[id_or_index] return self._field_getters[loc](book_id) From abc9232016ee33a518f99dee2970ad88e46a4a8c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Jul 2013 19:07:33 +0530 Subject: [PATCH 0202/1154] ... --- src/calibre/db/cache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 42e53a6f47..330e8dfba5 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1341,6 +1341,7 @@ class Cache(object): val_map[book_id] = ' & '.join(sorts) if val_map: self._set_field('author_sort', val_map) + self._mark_as_dirty(changed_books) return changed_books @write_api @@ -1349,6 +1350,7 @@ class Cache(object): changed_books = set() for author_id in author_id_to_link_map: changed_books |= self._books_for_field('authors', author_id) + self._mark_as_dirty(changed_books) return changed_books # }}} From cdf50da0342729728efc99d90eb519c5400c7387 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 15 Jul 2013 21:16:07 +0530 Subject: [PATCH 0203/1154] get_id_from_uuid() --- src/calibre/db/cache.py | 5 +++++ src/calibre/db/legacy.py | 4 ++++ src/calibre/db/tables.py | 3 +++ src/calibre/db/tests/legacy.py | 1 + 4 files changed, 13 insertions(+) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 330e8dfba5..ea62f3b90a 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1353,6 +1353,11 @@ class Cache(object): self._mark_as_dirty(changed_books) return changed_books + @read_api + def lookup_by_uuid(self, uuid): + return self.fields['uuid'].table.lookup_by_uuid(uuid) + + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index ec480c3d2d..1faf9a2d1a 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -153,6 +153,10 @@ class LibraryDatabase(object): def refresh(self, field=None, ascending=True): self.data.refresh(field=field, ascending=ascending) + def get_id_from_uuid(self, uuid): + if uuid: + return self.new_api.lookup_by_uuid(uuid) + def add_listener(self, listener): ''' Add a listener. Will be called on change events with two arguments. diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 81c66a0ca5..e76423d971 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -137,6 +137,9 @@ class UUIDTable(OneToOneTable): clean.add(val) return clean + def lookup_by_uuid(self, uuid): + return self.uuid_to_id_map.get(uuid, None) + class CompositeTable(OneToOneTable): def read(self, db): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 5f1aa2ff41..725f6c3ad9 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -163,6 +163,7 @@ class LegacyTest(BaseTest): for meth, args in { 'get_next_series_num_for': [('A Series One',)], + 'get_id_from_uuid':[('ddddd',), (db.uuid(1, True),)], 'get_author_id': [('author one',), ('unknown',), ('xxxxx',)], 'series_id': [(0,), (1,), (2,)], 'publisher_id': [(0,), (1,), (2,)], From 22f2aca3ebee11ac3a7938946035925402880201 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 09:09:19 +0530 Subject: [PATCH 0204/1154] Dont chdir() when doing bulk metadata Works around broken windows systems with temp folder permission issues. --- src/calibre/ebooks/metadata/sources/worker.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/worker.py b/src/calibre/ebooks/metadata/sources/worker.py index 1c83f965e1..ebe764c68f 100644 --- a/src/calibre/ebooks/metadata/sources/worker.py +++ b/src/calibre/ebooks/metadata/sources/worker.py @@ -50,14 +50,13 @@ def merge_result(oldmi, newmi, ensure_fields=None): return newmi def main(do_identify, covers, metadata, ensure_fields, tdir): - os.chdir(tdir) failed_ids = set() failed_covers = set() all_failed = True log = GUILog() for book_id, mi in metadata.iteritems(): - mi = OPF(BytesIO(mi), basedir=os.getcwdu(), + mi = OPF(BytesIO(mi), basedir=tdir, populate_spine=False).to_book_metadata() title, authors, identifiers = mi.title, mi.authors, mi.identifiers cdata = None @@ -77,7 +76,7 @@ def main(do_identify, covers, metadata, ensure_fields, tdir): if not mi.is_null('rating'): # set_metadata expects a rating out of 10 mi.rating *= 2 - with open('%d.mi'%book_id, 'wb') as f: + with open(os.path.join(tdir, '%d.mi'%book_id), 'wb') as f: f.write(metadata_to_opf(mi, default_lang='und')) else: log.error('Failed to download metadata for', title) @@ -89,11 +88,11 @@ def main(do_identify, covers, metadata, ensure_fields, tdir): if cdata is None: failed_covers.add(book_id) else: - with open('%d.cover'%book_id, 'wb') as f: + with open(os.path.join(tdir, '%d.cover'%book_id), 'wb') as f: f.write(cdata[-1]) all_failed = False - with open('%d.log'%book_id, 'wb') as f: + with open(os.path.join(tdir, '%d.log'%book_id), 'wb') as f: f.write(log.plain_text.encode('utf-8')) return failed_ids, failed_covers, all_failed From 0c6d820f2b60914b5a19cb5e04bee3424a251ca9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 15:00:02 +0530 Subject: [PATCH 0205/1154] Renaming of many-(one,many) items --- src/calibre/db/cache.py | 27 +++++++++++++ src/calibre/db/legacy.py | 5 +-- src/calibre/db/tables.py | 66 +++++++++++++++++++++++++++++++ src/calibre/db/tests/writing.py | 69 +++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 4 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index ea62f3b90a..c662b2a951 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1213,6 +1213,33 @@ class Cache(object): else: table.remove_books(book_ids, self.backend) + @read_api + def author_sort_strings_for_books(self, book_ids): + val_map = {} + for book_id in book_ids: + authors = self._field_ids_for('authors', book_id) + adata = self._author_data(authors) + val_map[book_id] = tuple(adata[aid]['sort'] for aid in authors) + return val_map + + @write_api + def rename_items(self, field, item_id_to_new_name_map): + try: + func = self.fields[field].table.rename_item + except AttributeError: + raise ValueError('Cannot rename items for one-one fields: %s' % field) + affected_books = set() + for item_id, new_name in item_id_to_new_name_map.iteritems(): + affected_books.update(func(item_id, new_name, self.backend)) + if affected_books: + if field == 'authors': + self._set_field('author_sort', # also marks as dirty + {k:' & '.join(v) for k, v in self._author_sort_strings_for_books(affected_books).iteritems()}) + self._update_path(affected_books, mark_as_dirtied=False) + else: + self._mark_as_dirty(affected_books) + return affected_books + @write_api def remove_items(self, field, item_ids): ''' Delete all items in the specified field with the specified ids. Returns the set of affected book ids. ''' diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 1faf9a2d1a..5df491eca3 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -290,10 +290,7 @@ class LibraryDatabase(object): def authors_sort_strings(self, index, index_is_id=False): book_id = index if index_is_id else self.id(index) - with self.new_api.read_lock: - authors = self.new_api._field_ids_for('authors', book_id) - adata = self.new_api._author_data(authors) - return [adata[aid]['sort'] for aid in authors] + return list(self.author_sort_strings_for_books.canonical_author_sort_for_books((book_id,))[book_id]) def author_sort_from_book(self, index, index_is_id=False): return ' & '.join(self.authors_sort_strings(index, index_is_id=index_is_id)) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index e76423d971..274cffd6d5 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -222,6 +222,29 @@ class ManyToOneTable(Table): db.conn.executemany('DELETE FROM {0} WHERE id=?'.format(self.metadata['table']), item_ids) return affected_books + def rename_item(self, item_id, new_name, db): + rmap = {icu_lower(v):k for k, v in self.id_map.iteritems()} + existing_item = rmap.get(icu_lower(new_name), None) + table, col, lcol = self.metadata['table'], self.metadata['column'], self.metadata['link_column'] + affected_books = self.col_book_map.get(item_id, set()) + if existing_item is None or existing_item == item_id: + # A simple rename will do the trick + self.id_map[item_id] = new_name + db.conn.execute('UPDATE {0} SET {1}=? WHERE id=?'.format(table, col), (new_name, item_id)) + else: + # We have to replace + self.id_map.pop(item_id, None) + books = self.col_book_map.pop(item_id, set()) + for book_id in books: + self.book_col_map[book_id] = existing_item + self.col_book_map[existing_item].update(books) + # For custom series this means that the series index can + # potentially have duplicates/be incorrect, but there is no way to + # handle that in this context. + db.conn.execute('UPDATE {0} SET {1}=? WHERE {1}=?; DELETE FROM {2} WHERE id=?'.format( + self.link_table, lcol, table), (existing_item, item_id, item_id)) + return affected_books + class ManyToManyTable(ManyToOneTable): ''' @@ -283,6 +306,32 @@ class ManyToManyTable(ManyToOneTable): db.conn.executemany('DELETE FROM {0} WHERE id=?'.format(self.metadata['table']), item_ids) return affected_books + def rename_item(self, item_id, new_name, db): + rmap = {icu_lower(v):k for k, v in self.id_map.iteritems()} + existing_item = rmap.get(icu_lower(new_name), None) + table, col, lcol = self.metadata['table'], self.metadata['column'], self.metadata['link_column'] + affected_books = self.col_book_map.get(item_id, set()) + if existing_item is None or existing_item == item_id: + # A simple rename will do the trick + self.id_map[item_id] = new_name + db.conn.execute('UPDATE {0} SET {1}=? WHERE id=?'.format(table, col), (new_name, item_id)) + else: + # We have to replace + self.id_map.pop(item_id, None) + books = self.col_book_map.pop(item_id, set()) + # Replacing item_id with existing_item could cause the same id to + # appear twice in the book list. Handle that by removing existing + # item from the book list before replacing. + for book_id in books: + self.book_col_map[book_id] = tuple((existing_item if x == item_id else x) for x in self.book_col_map.get(book_id, ()) if x != existing_item) + self.col_book_map[existing_item].update(books) + db.conn.executemany('DELETE FROM {0} WHERE book=? AND {1}=?'.format(self.link_table, lcol), [ + (book_id, existing_item) for book_id in books]) + db.conn.execute('UPDATE {0} SET {1}=? WHERE {1}=?; DELETE FROM {2} WHERE id=?'.format( + self.link_table, lcol, table), (existing_item, item_id, item_id)) + return affected_books + + class AuthorsTable(ManyToManyTable): def read_id_maps(self, db): @@ -314,6 +363,17 @@ class AuthorsTable(ManyToManyTable): self.asort_map.pop(item_id, None) return clean + def rename_item(self, item_id, new_name, db): + ret = ManyToManyTable.rename_item(self, item_id, new_name, db) + if item_id not in self.id_map: + self.alink_map.pop(item_id, None) + self.asort_map.pop(item_id, None) + else: + # Was a simple rename, update the author sort value + self.set_sort_names({item_id:author_to_author_sort(new_name)}, db) + + return ret + def remove_items(self, item_ids, db): raise ValueError('Direct removal of authors is not allowed') @@ -377,6 +437,9 @@ class FormatsTable(ManyToManyTable): def remove_items(self, item_ids, db): raise NotImplementedError('Cannot delete a format directly') + def rename_item(self, item_id, new_name, db): + raise NotImplementedError('Cannot rename formats') + def update_fmt(self, book_id, fmt, fname, size, db): fmts = list(self.book_col_map.get(book_id, [])) try: @@ -430,6 +493,9 @@ class IdentifiersTable(ManyToManyTable): def remove_items(self, item_ids, db): raise NotImplementedError('Direct deletion of identifiers is not implemented') + def rename_item(self, item_id, new_name, db): + raise NotImplementedError('Cannot rename identifiers') + def all_identifier_types(self): return frozenset(k for k, v in self.col_book_map.iteritems() if v) diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index c4918b4c4b..a2a36ec340 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -474,3 +474,72 @@ class WritingTest(BaseTest): for bid in c.all_book_ids(): self.assertIn(c.field_for('#series', bid), (None, 'My Series One')) # }}} + + def test_rename_items(self): # {{{ + ' Test renaming of many-(many,one) items ' + cl = self.cloned_library + cache = self.init_cache(cl) + # Check that renaming authors updates author sort and path + a = {v:k for k, v in cache.get_id_map('authors').iteritems()}['Unknown'] + self.assertEqual(cache.rename_items('authors', {a:'New Author'}), {3}) + a = {v:k for k, v in cache.get_id_map('authors').iteritems()}['Author One'] + self.assertEqual(cache.rename_items('authors', {a:'Author Two'}), {1, 2}) + for c in (cache, self.init_cache(cl)): + self.assertEqual(c.all_field_names('authors'), {'New Author', 'Author Two'}) + self.assertEqual(c.field_for('author_sort', 3), 'Author, New') + self.assertIn('New Author/', c.field_for('path', 3)) + self.assertEqual(c.field_for('authors', 1), ('Author Two',)) + self.assertEqual(c.field_for('author_sort', 1), 'Two, Author') + + t = {v:k for k, v in cache.get_id_map('tags').iteritems()}['Tag One'] + # Test case change + self.assertEqual(cache.rename_items('tags', {t:'tag one'}), {1, 2}) + for c in (cache, self.init_cache(cl)): + self.assertEqual(c.all_field_names('tags'), {'tag one', 'Tag Two', 'News'}) + self.assertEqual(set(c.field_for('tags', 1)), {'tag one', 'News'}) + self.assertEqual(set(c.field_for('tags', 2)), {'tag one', 'Tag Two'}) + # Test new name + self.assertEqual(cache.rename_items('tags', {t:'t1'}), {1,2}) + for c in (cache, self.init_cache(cl)): + self.assertEqual(c.all_field_names('tags'), {'t1', 'Tag Two', 'News'}) + self.assertEqual(set(c.field_for('tags', 1)), {'t1', 'News'}) + self.assertEqual(set(c.field_for('tags', 2)), {'t1', 'Tag Two'}) + # Test rename to existing + self.assertEqual(cache.rename_items('tags', {t:'Tag Two'}), {1,2}) + for c in (cache, self.init_cache(cl)): + self.assertEqual(c.all_field_names('tags'), {'Tag Two', 'News'}) + self.assertEqual(set(c.field_for('tags', 1)), {'Tag Two', 'News'}) + self.assertEqual(set(c.field_for('tags', 2)), {'Tag Two'}) + # Test on a custom column + t = {v:k for k, v in cache.get_id_map('#tags').iteritems()}['My Tag One'] + self.assertEqual(cache.rename_items('#tags', {t:'My Tag Two'}), {2}) + for c in (cache, self.init_cache(cl)): + self.assertEqual(c.all_field_names('#tags'), {'My Tag Two'}) + self.assertEqual(set(c.field_for('#tags', 2)), {'My Tag Two'}) + + # Test a Many-one field + s = {v:k for k, v in cache.get_id_map('series').iteritems()}['A Series One'] + # Test case change + self.assertEqual(cache.rename_items('series', {s:'a series one'}), {1, 2}) + for c in (cache, self.init_cache(cl)): + self.assertEqual(c.all_field_names('series'), {'a series one'}) + self.assertEqual(c.field_for('series', 1), 'a series one') + self.assertEqual(c.field_for('series_index', 1), 2.0) + + # Test new name + self.assertEqual(cache.rename_items('series', {s:'series'}), {1, 2}) + for c in (cache, self.init_cache(cl)): + self.assertEqual(c.all_field_names('series'), {'series'}) + self.assertEqual(c.field_for('series', 1), 'series') + self.assertEqual(c.field_for('series', 2), 'series') + self.assertEqual(c.field_for('series_index', 1), 2.0) + + s = {v:k for k, v in cache.get_id_map('#series').iteritems()}['My Series One'] + # Test custom column with rename to existing + self.assertEqual(cache.rename_items('#series', {s:'My Series Two'}), {2}) + for c in (cache, self.init_cache(cl)): + self.assertEqual(c.all_field_names('#series'), {'My Series Two'}) + self.assertEqual(c.field_for('#series', 2), 'My Series Two') + self.assertEqual(c.field_for('#series_index', 1), 3.0) + self.assertEqual(c.field_for('#series_index', 2), 1.0) + # }}} From 7819f37a45a802185842d6d5b18d5fab5a110a90 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 15:09:03 +0530 Subject: [PATCH 0206/1154] When renaming return id map as well --- src/calibre/db/cache.py | 7 +++++-- src/calibre/db/tables.py | 8 ++++++-- src/calibre/db/tests/writing.py | 18 +++++++++--------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index c662b2a951..178e4d3285 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1229,8 +1229,11 @@ class Cache(object): except AttributeError: raise ValueError('Cannot rename items for one-one fields: %s' % field) affected_books = set() + id_map = {} for item_id, new_name in item_id_to_new_name_map.iteritems(): - affected_books.update(func(item_id, new_name, self.backend)) + books, new_id = func(item_id, new_name, self.backend) + affected_books.update(books) + id_map[item_id] = new_id if affected_books: if field == 'authors': self._set_field('author_sort', # also marks as dirty @@ -1238,7 +1241,7 @@ class Cache(object): self._update_path(affected_books, mark_as_dirtied=False) else: self._mark_as_dirty(affected_books) - return affected_books + return affected_books, id_map @write_api def remove_items(self, field, item_ids): diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 274cffd6d5..4ab3269ef0 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -227,12 +227,14 @@ class ManyToOneTable(Table): existing_item = rmap.get(icu_lower(new_name), None) table, col, lcol = self.metadata['table'], self.metadata['column'], self.metadata['link_column'] affected_books = self.col_book_map.get(item_id, set()) + new_id = item_id if existing_item is None or existing_item == item_id: # A simple rename will do the trick self.id_map[item_id] = new_name db.conn.execute('UPDATE {0} SET {1}=? WHERE id=?'.format(table, col), (new_name, item_id)) else: # We have to replace + new_id = existing_item self.id_map.pop(item_id, None) books = self.col_book_map.pop(item_id, set()) for book_id in books: @@ -243,7 +245,7 @@ class ManyToOneTable(Table): # handle that in this context. db.conn.execute('UPDATE {0} SET {1}=? WHERE {1}=?; DELETE FROM {2} WHERE id=?'.format( self.link_table, lcol, table), (existing_item, item_id, item_id)) - return affected_books + return affected_books, new_id class ManyToManyTable(ManyToOneTable): @@ -311,12 +313,14 @@ class ManyToManyTable(ManyToOneTable): existing_item = rmap.get(icu_lower(new_name), None) table, col, lcol = self.metadata['table'], self.metadata['column'], self.metadata['link_column'] affected_books = self.col_book_map.get(item_id, set()) + new_id = item_id if existing_item is None or existing_item == item_id: # A simple rename will do the trick self.id_map[item_id] = new_name db.conn.execute('UPDATE {0} SET {1}=? WHERE id=?'.format(table, col), (new_name, item_id)) else: # We have to replace + new_id = existing_item self.id_map.pop(item_id, None) books = self.col_book_map.pop(item_id, set()) # Replacing item_id with existing_item could cause the same id to @@ -329,7 +333,7 @@ class ManyToManyTable(ManyToOneTable): (book_id, existing_item) for book_id in books]) db.conn.execute('UPDATE {0} SET {1}=? WHERE {1}=?; DELETE FROM {2} WHERE id=?'.format( self.link_table, lcol, table), (existing_item, item_id, item_id)) - return affected_books + return affected_books, new_id class AuthorsTable(ManyToManyTable): diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index a2a36ec340..9a882bbd03 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -481,9 +481,9 @@ class WritingTest(BaseTest): cache = self.init_cache(cl) # Check that renaming authors updates author sort and path a = {v:k for k, v in cache.get_id_map('authors').iteritems()}['Unknown'] - self.assertEqual(cache.rename_items('authors', {a:'New Author'}), {3}) + self.assertEqual(cache.rename_items('authors', {a:'New Author'})[0], {3}) a = {v:k for k, v in cache.get_id_map('authors').iteritems()}['Author One'] - self.assertEqual(cache.rename_items('authors', {a:'Author Two'}), {1, 2}) + self.assertEqual(cache.rename_items('authors', {a:'Author Two'})[0], {1, 2}) for c in (cache, self.init_cache(cl)): self.assertEqual(c.all_field_names('authors'), {'New Author', 'Author Two'}) self.assertEqual(c.field_for('author_sort', 3), 'Author, New') @@ -493,26 +493,26 @@ class WritingTest(BaseTest): t = {v:k for k, v in cache.get_id_map('tags').iteritems()}['Tag One'] # Test case change - self.assertEqual(cache.rename_items('tags', {t:'tag one'}), {1, 2}) + self.assertEqual(cache.rename_items('tags', {t:'tag one'}), ({1, 2}, {t:t})) for c in (cache, self.init_cache(cl)): self.assertEqual(c.all_field_names('tags'), {'tag one', 'Tag Two', 'News'}) self.assertEqual(set(c.field_for('tags', 1)), {'tag one', 'News'}) self.assertEqual(set(c.field_for('tags', 2)), {'tag one', 'Tag Two'}) # Test new name - self.assertEqual(cache.rename_items('tags', {t:'t1'}), {1,2}) + self.assertEqual(cache.rename_items('tags', {t:'t1'})[0], {1,2}) for c in (cache, self.init_cache(cl)): self.assertEqual(c.all_field_names('tags'), {'t1', 'Tag Two', 'News'}) self.assertEqual(set(c.field_for('tags', 1)), {'t1', 'News'}) self.assertEqual(set(c.field_for('tags', 2)), {'t1', 'Tag Two'}) # Test rename to existing - self.assertEqual(cache.rename_items('tags', {t:'Tag Two'}), {1,2}) + self.assertEqual(cache.rename_items('tags', {t:'Tag Two'})[0], {1,2}) for c in (cache, self.init_cache(cl)): self.assertEqual(c.all_field_names('tags'), {'Tag Two', 'News'}) self.assertEqual(set(c.field_for('tags', 1)), {'Tag Two', 'News'}) self.assertEqual(set(c.field_for('tags', 2)), {'Tag Two'}) # Test on a custom column t = {v:k for k, v in cache.get_id_map('#tags').iteritems()}['My Tag One'] - self.assertEqual(cache.rename_items('#tags', {t:'My Tag Two'}), {2}) + self.assertEqual(cache.rename_items('#tags', {t:'My Tag Two'})[0], {2}) for c in (cache, self.init_cache(cl)): self.assertEqual(c.all_field_names('#tags'), {'My Tag Two'}) self.assertEqual(set(c.field_for('#tags', 2)), {'My Tag Two'}) @@ -520,14 +520,14 @@ class WritingTest(BaseTest): # Test a Many-one field s = {v:k for k, v in cache.get_id_map('series').iteritems()}['A Series One'] # Test case change - self.assertEqual(cache.rename_items('series', {s:'a series one'}), {1, 2}) + self.assertEqual(cache.rename_items('series', {s:'a series one'}), ({1, 2}, {s:s})) for c in (cache, self.init_cache(cl)): self.assertEqual(c.all_field_names('series'), {'a series one'}) self.assertEqual(c.field_for('series', 1), 'a series one') self.assertEqual(c.field_for('series_index', 1), 2.0) # Test new name - self.assertEqual(cache.rename_items('series', {s:'series'}), {1, 2}) + self.assertEqual(cache.rename_items('series', {s:'series'})[0], {1, 2}) for c in (cache, self.init_cache(cl)): self.assertEqual(c.all_field_names('series'), {'series'}) self.assertEqual(c.field_for('series', 1), 'series') @@ -536,7 +536,7 @@ class WritingTest(BaseTest): s = {v:k for k, v in cache.get_id_map('#series').iteritems()}['My Series One'] # Test custom column with rename to existing - self.assertEqual(cache.rename_items('#series', {s:'My Series Two'}), {2}) + self.assertEqual(cache.rename_items('#series', {s:'My Series Two'})[0], {2}) for c in (cache, self.init_cache(cl)): self.assertEqual(c.all_field_names('#series'), {'My Series Two'}) self.assertEqual(c.field_for('#series', 2), 'My Series Two') From 064294fa3213a1666ea8c5128418865836a09254 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 15:56:57 +0530 Subject: [PATCH 0207/1154] Ensure tweaks are set to default when running tests --- src/calibre/db/tests/base.py | 5 +++++ src/calibre/utils/config_base.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index b94faf6b28..dd87ab1583 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -21,6 +21,11 @@ class BaseTest(unittest.TestCase): longMessage = True maxDiff = None + @classmethod + def setUpClass(cls): + from calibre.utils.config_base import reset_tweaks_to_default + reset_tweaks_to_default() + def setUp(self): self.library_path = self.mkdtemp() self.create_db(self.library_path) diff --git a/src/calibre/utils/config_base.py b/src/calibre/utils/config_base.py index a31b7052b1..a9860b60fa 100644 --- a/src/calibre/utils/config_base.py +++ b/src/calibre/utils/config_base.py @@ -474,4 +474,11 @@ def write_tweaks(raw): tweaks = read_tweaks() +def reset_tweaks_to_default(): + global tweaks + default_tweaks = P('default_tweaks.py', data=True, + allow_user_override=False) + dl, dg = {}, {} + exec default_tweaks in dg, dl + tweaks = dl From 17c520d1d911eb66a7cfa7e2b4820b2979061cb0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 16:11:04 +0530 Subject: [PATCH 0208/1154] Legacy rename API --- src/calibre/db/cache.py | 26 ++++++++++++++++---------- src/calibre/db/legacy.py | 15 ++++++++++++++- src/calibre/db/tests/legacy.py | 27 +++++++++++++++++++++++++++ src/calibre/db/tests/writing.py | 2 +- 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 178e4d3285..6564a9a9a2 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -30,7 +30,7 @@ from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ptempfile import (base_dir, PersistentTemporaryFile, SpooledTemporaryFile) -from calibre.utils.config import prefs +from calibre.utils.config import prefs, tweaks from calibre.utils.date import now as nowf, utcnow, UNDEFINED_DATE from calibre.utils.icu import sort_key @@ -1100,16 +1100,16 @@ class Cache(object): self._update_last_modified(tuple(formats_map.iterkeys())) @read_api - def get_next_series_num_for(self, series): + def get_next_series_num_for(self, series, field='series'): books = () - sf = self.fields['series'] + sf = self.fields[field] if series: q = icu_lower(series) - for val, book_ids in sf.iter_searchable_values(self._get_metadata, frozenset(self.all_book_ids())): + for val, book_ids in sf.iter_searchable_values(self._get_metadata, frozenset(self._all_book_ids())): if q == icu_lower(val): books = book_ids break - series_indices = sorted(self._field_for('series_index', book_id) for book_id in books) + series_indices = sorted(self._field_for(sf.index_field.name, book_id) for book_id in books) return _get_next_series_num_for_list(tuple(series_indices), unwrap=False) @read_api @@ -1223,24 +1223,30 @@ class Cache(object): return val_map @write_api - def rename_items(self, field, item_id_to_new_name_map): + def rename_items(self, field, item_id_to_new_name_map, change_index=True): + f = self.fields[field] try: - func = self.fields[field].table.rename_item + func = f.table.rename_item except AttributeError: raise ValueError('Cannot rename items for one-one fields: %s' % field) affected_books = set() + moved_books = set() id_map = {} for item_id, new_name in item_id_to_new_name_map.iteritems(): books, new_id = func(item_id, new_name, self.backend) affected_books.update(books) id_map[item_id] = new_id + if new_id != item_id: + moved_books.update(books) if affected_books: if field == 'authors': - self._set_field('author_sort', # also marks as dirty + self._set_field('author_sort', {k:' & '.join(v) for k, v in self._author_sort_strings_for_books(affected_books).iteritems()}) self._update_path(affected_books, mark_as_dirtied=False) - else: - self._mark_as_dirty(affected_books) + elif change_index and hasattr(f, 'index_field') and tweaks['series_index_auto_increment'] != 'no_change': + for book_id in moved_books: + self._set_field(f.index_field.name, {book_id:self._get_next_series_num_for(self._field_for(field, book_id), field=field)}) + self._mark_as_dirty(affected_books) return affected_books, id_map @write_api diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 5df491eca3..a6e6bd9fa7 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -290,7 +290,7 @@ class LibraryDatabase(object): def authors_sort_strings(self, index, index_is_id=False): book_id = index if index_is_id else self.id(index) - return list(self.author_sort_strings_for_books.canonical_author_sort_for_books((book_id,))[book_id]) + return list(self.new_api.author_sort_strings_for_books((book_id,))[book_id]) def author_sort_from_book(self, index, index_is_id=False): return ' & '.join(self.authors_sort_strings(index, index_is_id=index_is_id)) @@ -500,6 +500,9 @@ class LibraryDatabase(object): book_id = index if index_is_id else self.id(index) return self.new_api.get_metadata(book_id, get_cover=get_cover, get_user_categories=get_user_categories, cover_as_data=cover_as_data) + def rename_series(self, old_id, new_name, change_index=True): + self.new_api.rename_items('series', {old_id:new_name}, change_index=change_index) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): @@ -582,6 +585,16 @@ for field in ( return func setattr(LibraryDatabase, 'set_%s' % field.replace('!', ''), MT(setter(field))) +for field in ('authors', 'tags', 'publisher'): + def renamer(field): + def func(self, old_id, new_name): + id_map = self.new_api.rename_items(field, {old_id:new_name})[1] + if field == 'authors': + return id_map[old_id] + return func + fname = field[:-1] if field in {'tags', 'authors'} else field + setattr(LibraryDatabase, 'rename_%s' % fname, MT(renamer(field))) + LibraryDatabase.update_last_modified = MT( lambda self, book_ids, commit=False, now=None: self.new_api.update_last_modified(book_ids, now=now)) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 725f6c3ad9..09ae756d10 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -472,6 +472,7 @@ class LegacyTest(BaseTest): nmi = [ndb.get_metadata(x) for x in (0, 1, 2)] self.assertEqual([x.author_sort_map for x in omi], [x.author_sort_map for x in nmi]) self.assertEqual([x.author_link_map for x in omi], [x.author_link_map for x in nmi]) + db.close() ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) @@ -554,6 +555,7 @@ class LegacyTest(BaseTest): ('@all_tags',), ('#tags', 0), ('#tags', 1), ('#tags', 2), )) + db.close() ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) @@ -563,6 +565,31 @@ class LegacyTest(BaseTest): ('@all_tags',), ('@tags', 0), ('@tags', 1), ('@tags', 2), )) + db.close() + + ndb = self.init_legacy(self.cloned_library) + db = self.init_old(self.cloned_library) + a = {v:k for k, v in ndb.new_api.get_id_map('authors').iteritems()}['Author One'] + t = {v:k for k, v in ndb.new_api.get_id_map('tags').iteritems()}['Tag One'] + s = {v:k for k, v in ndb.new_api.get_id_map('series').iteritems()}['A Series One'] + p = {v:k for k, v in ndb.new_api.get_id_map('publisher').iteritems()}['Publisher One'] + run_funcs(self, db, ndb, ( + ('rename_author', a, 'Author Two'), + ('rename_tag', t, 'News'), + ('rename_series', s, 'ss'), + ('rename_publisher', p, 'publisher one'), + (db.clean,), + (db.refresh,), + ('@all_tags',), + ('tags', 0), ('tags', 1), ('tags', 2), + ('series', 0), ('series', 1), ('series', 2), + ('publisher', 0), ('publisher', 1), ('publisher', 2), + ('series_index', 0), ('series_index', 1), ('series_index', 2), + ('authors', 0), ('authors', 1), ('authors', 2), + ('author_sort', 0), ('author_sort', 1), ('author_sort', 2), + )) + db.close() + # }}} diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 9a882bbd03..26f73964df 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -541,5 +541,5 @@ class WritingTest(BaseTest): self.assertEqual(c.all_field_names('#series'), {'My Series Two'}) self.assertEqual(c.field_for('#series', 2), 'My Series Two') self.assertEqual(c.field_for('#series_index', 1), 3.0) - self.assertEqual(c.field_for('#series_index', 2), 1.0) + self.assertEqual(c.field_for('#series_index', 2), 4.0) # }}} From 522ddc9c91d9b6c5b54335c873864e3217308a85 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 16:25:52 +0530 Subject: [PATCH 0209/1154] ... --- setup/file_hosting_servers.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst index c72998958e..5b494cf066 100644 --- a/setup/file_hosting_servers.rst +++ b/setup/file_hosting_servers.rst @@ -22,6 +22,7 @@ mkdir -p /root/staging /root/work/vim /srv/download /srv/manual scp .zshrc .vimrc server: scp -r ~/work/vim/zsh-syntax-highlighting server:work/vim +scp -r ~/work/vim/zsh-history-substring-search server:work/vim If the server has a backup hard-disk, mount it at /mnt/backup and edit /etc/fstab so that it is auto-mounted. Then, add the following to crontab:: From a824208ea2fcfb9fd36c13d129f01cfa02ac7ef6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 17:26:18 +0530 Subject: [PATCH 0210/1154] Il Foglio by faber1971 --- recipes/il_foglio.recipe | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 recipes/il_foglio.recipe diff --git a/recipes/il_foglio.recipe b/recipes/il_foglio.recipe new file mode 100644 index 0000000000..9d5e8aa2e6 --- /dev/null +++ b/recipes/il_foglio.recipe @@ -0,0 +1,16 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1373969939(BasicNewsRecipe): + title = u'Il Foglio - Editoriali' + oldest_article = 1 + max_articles_per_feed = 10 + auto_cleanup = False + keep_only_tags = [ + dict(name='div', attrs={'class':'sec_item'}) + ] + feeds = [(u'Il Foglio - Editoriali', u'http://feed43.com/8814237344800115.xml')] + no_stylesheets = True + __author__ = 'faber1971' + description = 'Leading articles from an Italian newspaper - v1.00 (16 July, 2013)' + language = 'it' + masthead_url = 'http://www.ilfoglio.it/media/img/interface/logo_testata_small.gif' From a65f8042434dccb4da3f178133ba980213f48a53 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 17:49:19 +0530 Subject: [PATCH 0211/1154] Some misc legacy API --- src/calibre/db/legacy.py | 12 +++++++----- src/calibre/db/tests/legacy.py | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index a6e6bd9fa7..f13e8107df 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -65,14 +65,13 @@ class LibraryDatabase(object): cache.init() self.data = View(cache) self.id = self.data.index_to_id - self.count = self.data.count + for x in ('get_property', 'count', 'refresh_ids', 'set_marked_ids', + 'multisort', 'search', 'search_getting_ids'): + setattr(self, x, getattr(self.data, x)) - self.get_property = self.data.get_property + self.is_case_sensitive = getattr(backend, 'is_case_sensitive', False) self.last_update_check = self.last_modified() - self.refresh_ids = self.data.refresh_ids - self.set_marked_ids = self.data.set_marked_ids - self.is_case_sensitive = getattr(backend, 'is_case_sensitive', False) def close(self): self.backend.close() @@ -283,6 +282,9 @@ class LibraryDatabase(object): return list(self.new_api.get_ids_for_custom_book_data(name)) # }}} + def sort(self, field, ascending, subsort=False): + self.multisort([(field, ascending)]) + def get_field(self, index, key, default=None, index_is_id=False): book_id = index if index_is_id else self.id(index) mi = self.new_api.get_metadata(book_id, get_cover=key == 'cover') diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 09ae756d10..137acff8d5 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -359,7 +359,8 @@ class LegacyTest(BaseTest): 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', 'dirty_books_referencing', 'windows_check_if_files_in_use', 'get_metadata_for_dump', 'get_a_dirtied_book', 'dirtied_sequence', - 'format_filename_cache', 'format_metadata_cache', 'filter', 'create_version1', 'normpath', + 'format_filename_cache', 'format_metadata_cache', 'filter', 'create_version1', 'normpath', 'custom_data_adapters', + 'custom_table_names', 'custom_columns_in_meta', 'custom_tables', } SKIP_ARGSPEC = { '__init__', From a49b518cde782a8efc43c39aed998d1d5f6fafde Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 17:58:22 +0530 Subject: [PATCH 0212/1154] cover_last_modified() --- src/calibre/db/backend.py | 7 +++++++ src/calibre/db/cache.py | 8 ++++++++ src/calibre/db/legacy.py | 4 ++++ 3 files changed, 19 insertions(+) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 2c4dfb8395..1f561980bd 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -963,6 +963,13 @@ class DB(object): import traceback traceback.print_exc() + def cover_last_modified(self, path): + path = os.path.abspath(os.path.join(self.library_path, path, 'cover.jpg')) + try: + return utcfromtimestamp(os.stat(path).st_mtime) + except EnvironmentError: + pass # Cover doesn't exist + def copy_cover_to(self, path, dest, windows_atomic_move=None, use_hardlink=False): path = os.path.abspath(os.path.join(self.library_path, path, 'cover.jpg')) if windows_atomic_move is not None: diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 6564a9a9a2..e7c3114f0d 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -526,6 +526,14 @@ class Cache(object): ret = i return ret + @read_api + def cover_last_modified(self, book_id): + try: + path = self._field_for('path', book_id).replace('/', os.sep) + except AttributeError: + return + return self.backend.cover_last_modified(path) + @read_api def copy_cover_to(self, book_id, dest, use_hardlink=False): ''' diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index f13e8107df..a6d7f22989 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -290,6 +290,10 @@ class LibraryDatabase(object): mi = self.new_api.get_metadata(book_id, get_cover=key == 'cover') return mi.get(key, default) + def cover_last_modified(self, index, index_is_id=False): + book_id = index if index_is_id else self.id(index) + return self.new_api.cover_last_modified(book_id) or self.last_modified() + def authors_sort_strings(self, index, index_is_id=False): book_id = index if index_is_id else self.id(index) return list(self.new_api.author_sort_strings_for_books((book_id,))[book_id]) From a57a35557263dde8496de90e5888f94e749c1bdc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 18:15:34 +0530 Subject: [PATCH 0213/1154] More API --- src/calibre/db/legacy.py | 17 +++++++++++++++++ src/calibre/db/tests/legacy.py | 14 +++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index a6d7f22989..3301bebdf3 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -294,6 +294,23 @@ class LibraryDatabase(object): book_id = index if index_is_id else self.id(index) return self.new_api.cover_last_modified(book_id) or self.last_modified() + def cover(self, index, index_is_id=False, as_file=False, as_image=False, as_path=False): + book_id = index if index_is_id else self.id(index) + return self.new_api.cover(book_id, as_file=as_file, as_image=as_image, as_path=as_path) + + def copy_cover_to(self, index, dest, index_is_id=False, windows_atomic_move=None, use_hardlink=False): + book_id = index if index_is_id else self.id(index) + return self.new_api.copy_cover_to(book_id, dest, use_hardlink=use_hardlink) + + def copy_format_to(self, index, fmt, dest, index_is_id=False, windows_atomic_move=None, use_hardlink=False): + book_id = index if index_is_id else self.id(index) + return self.new_api.copy_format_to(book_id, fmt, dest, use_hardlink=use_hardlink) + + def delete_book(self, book_id, notify=True, commit=True, permanent=False, do_clean=True): + self.new_api.remove_books((book_id,), permanent=permanent) + if notify: + self.notify('delete', [id]) + def authors_sort_strings(self, index, index_is_id=False): book_id = index if index_is_id else self.id(index) return list(self.new_api.author_sort_strings_for_books((book_id,))[book_id]) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 137acff8d5..4986b1b5dd 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -164,6 +164,7 @@ class LegacyTest(BaseTest): for meth, args in { 'get_next_series_num_for': [('A Series One',)], 'get_id_from_uuid':[('ddddd',), (db.uuid(1, True),)], + 'cover':[(0,), (1,), (2,)], 'get_author_id': [('author one',), ('unknown',), ('xxxxx',)], 'series_id': [(0,), (1,), (2,)], 'publisher_id': [(0,), (1,), (2,)], @@ -231,6 +232,14 @@ class LegacyTest(BaseTest): for a in args: self.assertEqual(fmt(getattr(db, meth)(*a)), fmt(getattr(ndb, meth)(*a)), 'The method: %s() returned different results for argument %s' % (meth, a)) + d1, d2 = BytesIO(), BytesIO() + db.copy_cover_to(1, d1, True) + ndb.copy_cover_to(1, d2, True) + self.assertTrue(d1.getvalue() == d2.getvalue()) + d1, d2 = BytesIO(), BytesIO() + db.copy_format_to(1, 'FMT1', d1, True) + ndb.copy_format_to(1, 'FMT1', d2, True) + self.assertTrue(d1.getvalue() == d2.getvalue()) db.close() # }}} @@ -275,7 +284,7 @@ class LegacyTest(BaseTest): # }}} def test_legacy_adding_books(self): # {{{ - 'Test various adding books methods' + 'Test various adding/deleting books methods' from calibre.ebooks.metadata.book.base import Metadata legacy, old = self.init_legacy(self.cloned_library), self.init_old(self.cloned_library) mi = Metadata('Added Book0', authors=('Added Author',)) @@ -332,6 +341,9 @@ class LegacyTest(BaseTest): self.assertEqual(cache.field_for('authors', bid), ('calibre',)) self.assertEqual(cache.field_for('tags', bid), (_('News'), 'Events', 'one', 'two')) + legacy.delete_book(1) + old.delete_book(1) + self.assertNotIn(1, legacy.all_ids()) old.close() # }}} From 70f338b047a730cc0961d50c4b663bfaaa4cb15b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 18:22:09 +0530 Subject: [PATCH 0214/1154] More API --- src/calibre/db/legacy.py | 6 ++++++ src/calibre/db/tests/legacy.py | 1 + 2 files changed, 7 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 3301bebdf3..be979ebed3 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -311,6 +311,12 @@ class LibraryDatabase(object): if notify: self.notify('delete', [id]) + def dirtied(self, book_ids, commit=True): + self.new_api.mark_as_dirty(book_ids) + + def dump_metadata(self, book_ids=None, remove_from_dirtied=True, commit=True, callback=None): + self.new_api.dump_metadata(book_ids=book_ids, remove_from_dirtied=remove_from_dirtied, callback=callback) + def authors_sort_strings(self, index, index_is_id=False): book_id = index if index_is_id else self.id(index) return list(self.new_api.author_sort_strings_for_books((book_id,))[book_id]) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 4986b1b5dd..216a0499ec 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -344,6 +344,7 @@ class LegacyTest(BaseTest): legacy.delete_book(1) old.delete_book(1) self.assertNotIn(1, legacy.all_ids()) + legacy.dump_metadata((2,3)) old.close() # }}} From 62d1dfbeef792a805f46e66cbdf7e0448eb5ed7c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 22:19:58 +0530 Subject: [PATCH 0215/1154] Start work on legacy custom column API --- src/calibre/db/backend.py | 7 +++++++ src/calibre/db/legacy.py | 4 +++- src/calibre/db/tests/legacy.py | 14 ++++++++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 1f561980bd..20ca7eda3f 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -549,6 +549,7 @@ class DB(object): # Load metadata for custom columns self.custom_column_label_map, self.custom_column_num_map = {}, {} + self.custom_column_num_to_label_map = {} triggers = [] remove = [] custom_tables = self.custom_tables @@ -586,6 +587,7 @@ class DB(object): self.custom_column_num_map[data['num']] = \ self.custom_column_label_map[data['label']] = data + self.custom_column_num_to_label_map[data['num']] = data['label'] # Create Foreign Key triggers if data['normalized']: @@ -785,6 +787,11 @@ class DB(object): self._conn = Connection(self.dbpath) return self._conn + def custom_field_name(self, label=None, num=None): + if label is not None: + return self.field_metadata.custom_field_prefix + label + return self.field_metadata.custom_field_prefix + self.custom_column_num_to_label_map[num] + def close(self): if self._conn is not None: self._conn.close() diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index be979ebed3..9073ed05ae 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -70,6 +70,7 @@ class LibraryDatabase(object): setattr(self, x, getattr(self.data, x)) self.is_case_sensitive = getattr(backend, 'is_case_sensitive', False) + self.custom_field_name = backend.custom_field_name self.last_update_check = self.last_modified() @@ -637,7 +638,8 @@ for field in ('authors', 'tags', 'publisher', 'series'): return func name = field[:-1] if field in {'authors', 'tags'} else field setattr(LibraryDatabase, 'all_%s_names' % name, MT(getter(field))) - LibraryDatabase.all_formats = MT(lambda self:self.new_api.all_field_names('formats')) +LibraryDatabase.all_formats = MT(lambda self:self.new_api.all_field_names('formats')) +LibraryDatabase.all_custom = MT(lambda self, label=None, num=None:self.new_api.all_field_names(self.custom_field_name(label, num))) for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':'tags', 'all_series':'series', 'all_publishers':'publisher'}.iteritems(): def getter(field): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 216a0499ec..3e08274561 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -604,7 +604,17 @@ class LegacyTest(BaseTest): )) db.close() - - + # }}} + + def test_legacy_custom(self): # {{{ + 'Test the legacy API for custom columns' + ndb = self.init_legacy(self.cloned_library) + db = self.init_old(self.cloned_library) + run_funcs(self, db, ndb, ( + ('all_custom', 'series'), + ('all_custom', 'tags'), + ('all_custom', 'rating'), + ('all_custom', 'authors'), + )) # }}} From 7ceedb4e3ce3356e893dc20d8b4b4a137c4ea5f3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 16 Jul 2013 22:35:23 +0530 Subject: [PATCH 0216/1154] get_custome_items_with_ids() --- src/calibre/db/legacy.py | 6 ++++++ src/calibre/db/tests/legacy.py | 12 ++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 9073ed05ae..4d29a6b18f 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -150,6 +150,12 @@ class LibraryDatabase(object): def field_id_map(self, field): return [(k, v) for k, v in self.new_api.get_id_map(field).iteritems()] + def get_custom_items_with_ids(self, label=None, num=None): + try: + return [[k, v] for k, v in self.new_api.get_id_map(self.custom_field_name(label, num)).iteritems()] + except ValueError: + return [] + def refresh(self, field=None, ascending=True): self.data.refresh(field=field, ascending=ascending) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 3e08274561..45d1367b4e 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -48,9 +48,10 @@ def run_funcs(self, db, ndb, funcs): meth(*args) else: fmt = lambda x:x - if meth[0] in {'!', '@', '#', '+'}: + if meth[0] in {'!', '@', '#', '+', '$'}: if meth[0] != '+': - fmt = {'!':dict, '@':lambda x:frozenset(x or ()), '#':lambda x:set((x or '').split(','))}[meth[0]] + fmt = {'!':dict, '@':lambda x:frozenset(x or ()), '#':lambda x:set((x or '').split(',')), + '$':lambda x:set(tuple(y) for y in x)}[meth[0]] else: fmt = args[-1] args = args[:-1] @@ -611,10 +612,9 @@ class LegacyTest(BaseTest): ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) run_funcs(self, db, ndb, ( - ('all_custom', 'series'), - ('all_custom', 'tags'), - ('all_custom', 'rating'), - ('all_custom', 'authors'), + ('all_custom', 'series'), ('all_custom', 'tags'), ('all_custom', 'rating'), ('all_custom', 'authors'), + ('$get_custom_items_with_ids', 'series'), ('$get_custom_items_with_ids', 'tags'), ('$get_custom_items_with_ids', 'float'), + ('$get_custom_items_with_ids', 'rating'), ('$get_custom_items_with_ids', 'authors'), )) # }}} From 2e6813ce02b7d3ae37014d67a892e9bc68ddb25a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 07:59:44 +0530 Subject: [PATCH 0217/1154] Fix memory card not being detected for Elonex 621 on Windows --- src/calibre/devices/eb600/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/eb600/driver.py b/src/calibre/devices/eb600/driver.py index f647c28a75..e51633f3a1 100644 --- a/src/calibre/devices/eb600/driver.py +++ b/src/calibre/devices/eb600/driver.py @@ -86,7 +86,7 @@ class COOL_ER(EB600): FORMATS = ['epub', 'mobi', 'prc', 'pdf', 'txt'] VENDOR_NAME = 'COOL-ER' - WINDOWS_MAIN_MEM = 'EREADER' + WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EREADER' OSX_MAIN_MEM = 'COOL-ER eReader Media' From 1dc169db4cba678a28baae4e001dcd7974135339 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 08:23:15 +0530 Subject: [PATCH 0218/1154] Driver for LG Android phone Fixes #1202013 [Calibre does not recognize my device](https://bugs.launchpad.net/calibre/+bug/1202013) --- src/calibre/devices/android/driver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 1880324fdc..a0eb021289 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -151,6 +151,7 @@ class ANDROID(USBMS): 0x61ce : [0x226, 0x227, 0x9999, 0x100], 0x618e : [0x226, 0x227, 0x9999, 0x100], 0x6205 : [0x226, 0x227, 0x9999, 0x100], + 0x6234 : [0x231], }, # Archos @@ -254,7 +255,7 @@ class ANDROID(USBMS): 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E', 'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD', 'ST80208-1', 'XT894', - '_USB', 'PROD_TAB13-201', 'URFPAD2', 'MID1126', + '_USB', 'PROD_TAB13-201', 'URFPAD2', 'MID1126', 'ANDROID_PLATFORM', ] OSX_MAIN_MEM = 'Android Device Main Memory' From 6c1bcc65033b5b489b4c503b6165d221be55ca4c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 09:36:12 +0530 Subject: [PATCH 0219/1154] CC get API --- src/calibre/db/backend.py | 5 +++++ src/calibre/db/fields.py | 6 +++++- src/calibre/db/legacy.py | 35 ++++++++++++++++++++++++++++++++++ src/calibre/db/tables.py | 1 + src/calibre/db/tests/legacy.py | 11 +++++++++-- 5 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 20ca7eda3f..13e8b80ff5 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -792,6 +792,11 @@ class DB(object): return self.field_metadata.custom_field_prefix + label return self.field_metadata.custom_field_prefix + self.custom_column_num_to_label_map[num] + def custom_field_metadata(self, label=None, num=None): + if label is not None: + return self.custom_column_label_map[label] + return self.custom_column_num_map[num] + def close(self): if self._conn is not None: self._conn.close() diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index e18179e4b1..dd0165b44e 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -325,7 +325,11 @@ class ManyToManyField(Field): def for_book(self, book_id, default_value=None): ids = self.table.book_col_map.get(book_id, ()) if ids: - ans = tuple(self.table.id_map[i] for i in ids) + ans = (self.table.id_map[i] for i in ids) + if self.table.sort_alpha: + ans = tuple(sorted(ans, key=sort_key)) + else: + ans = tuple(ans) else: ans = default_value return ans diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 4d29a6b18f..44b26f5f43 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -539,6 +539,41 @@ class LibraryDatabase(object): def rename_series(self, old_id, new_name, change_index=True): self.new_api.rename_items('series', {old_id:new_name}, change_index=change_index) + def get_custom(self, index, label=None, num=None, index_is_id=False): + book_id = index if index_is_id else self.id(index) + ans = self.new_api.field_for(self.custom_field_name(label, num), book_id) + if isinstance(ans, tuple): + ans = list(ans) + return ans + + def get_custom_extra(self, index, label=None, num=None, index_is_id=False): + data = self.backend.custom_field_metadata(label, num) + # add future datatypes with an extra column here + if data['datatype'] != 'series': + return None + book_id = index if index_is_id else self.id(index) + return self.new_api.field_for(self.custom_field_name(label, num) + '_index', book_id) + + def get_custom_and_extra(self, index, label=None, num=None, index_is_id=False): + book_id = index if index_is_id else self.id(index) + data = self.backend.custom_field_metadata(label, num) + ans = self.new_api.field_for(self.custom_field_name(label, num), book_id) + if isinstance(ans, tuple): + ans = list(ans) + if data['datatype'] != 'series': + return (ans, None) + return (ans, self.new_api.field_for(self.custom_field_name(label, num) + '_index', book_id)) + + def get_next_cc_series_num_for(self, series, label=None, num=None): + data = self.backend.custom_field_metadata(label, num) + if data['datatype'] != 'series': + return None + return self.new_api.get_next_series_num_for(series, field=self.custom_field_name(label, num)) + + def is_item_used_in_multiple(self, item, label=None, num=None): + existing_tags = self.all_custom(label=label, num=num) + return icu_lower(item) in {icu_lower(t) for t in existing_tags} + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 4ab3269ef0..9b9ff4e9e0 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -44,6 +44,7 @@ class Table(object): def __init__(self, name, metadata, link_table=None): self.name, self.metadata = name, metadata + self.sort_alpha = metadata.get('is_multiple', False) and metadata.get('display', {}).get('sort_alpha', False) # self.unserialize() maps values from the db to python objects self.unserialize = \ diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 45d1367b4e..2db0394d20 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -612,9 +612,16 @@ class LegacyTest(BaseTest): ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) run_funcs(self, db, ndb, ( - ('all_custom', 'series'), ('all_custom', 'tags'), ('all_custom', 'rating'), ('all_custom', 'authors'), + ('all_custom', 'series'), ('all_custom', 'tags'), ('all_custom', 'rating'), ('all_custom', 'authors'), ('all_custom', None, 7), + ('get_next_cc_series_num_for', 'My Series One', 'series'), ('get_next_cc_series_num_for', 'My Series Two', 'series'), + ('is_item_used_in_multiple', 'My Tag One', 'tags'), + ('is_item_used_in_multiple', 'My Series One', 'series'), ('$get_custom_items_with_ids', 'series'), ('$get_custom_items_with_ids', 'tags'), ('$get_custom_items_with_ids', 'float'), - ('$get_custom_items_with_ids', 'rating'), ('$get_custom_items_with_ids', 'authors'), + ('$get_custom_items_with_ids', 'rating'), ('$get_custom_items_with_ids', 'authors'), ('$get_custom_items_with_ids', None, 7), )) + for label in ('tags', 'series', 'authors', 'comments', 'rating', 'date', 'yesno', 'isbn', 'enum', 'formats', 'float', 'comp_tags'): + for func in ('get_custom', 'get_custom_extra', 'get_custom_and_extra'): + run_funcs(self, db, ndb, [(func, idx, label) for idx in range(3)]) + db.close() # }}} From dd5ccfd75ce830165ec2d8ceacfb244e1ffbabcc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 10:45:51 +0530 Subject: [PATCH 0220/1154] More CC API --- src/calibre/db/legacy.py | 15 +++++++++++++++ src/calibre/db/tests/legacy.py | 21 +++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 44b26f5f43..eb0debe758 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -574,6 +574,21 @@ class LibraryDatabase(object): existing_tags = self.all_custom(label=label, num=num) return icu_lower(item) in {icu_lower(t) for t in existing_tags} + def delete_custom_item_using_id(self, item_id, label=None, num=None): + self.new_api.remove_items(self.custom_field_name(label, num), (item_id,)) + + def rename_custom_item(self, old_id, new_name, label=None, num=None): + self.new_api.rename_items(self.custom_field_name(label, num), {old_id:new_name}, change_index=False) + + def delete_item_from_multiple(self, item, label=None, num=None): + field = self.custom_field_name(label, num) + existing = self.new_api.get_id_map(field) + rmap = {icu_lower(v):k for k, v in existing.iteritems()} + item_id = rmap.get(icu_lower(item), None) + if item_id is None: + return [] + return list(self.new_api.remove_items(field, (item_id,))) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 2db0394d20..83325c1615 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -611,6 +611,7 @@ class LegacyTest(BaseTest): 'Test the legacy API for custom columns' ndb = self.init_legacy(self.cloned_library) db = self.init_old(self.cloned_library) + # Test getting run_funcs(self, db, ndb, ( ('all_custom', 'series'), ('all_custom', 'tags'), ('all_custom', 'rating'), ('all_custom', 'authors'), ('all_custom', None, 7), ('get_next_cc_series_num_for', 'My Series One', 'series'), ('get_next_cc_series_num_for', 'My Series Two', 'series'), @@ -622,6 +623,26 @@ class LegacyTest(BaseTest): for label in ('tags', 'series', 'authors', 'comments', 'rating', 'date', 'yesno', 'isbn', 'enum', 'formats', 'float', 'comp_tags'): for func in ('get_custom', 'get_custom_extra', 'get_custom_and_extra'): run_funcs(self, db, ndb, [(func, idx, label) for idx in range(3)]) + + # Test renaming/deleting + t = {v:k for k, v in ndb.new_api.get_id_map('#tags').iteritems()}['My Tag One'] + t2 = {v:k for k, v in ndb.new_api.get_id_map('#tags').iteritems()}['My Tag Two'] + a = {v:k for k, v in ndb.new_api.get_id_map('#authors').iteritems()}['My Author Two'] + a2 = {v:k for k, v in ndb.new_api.get_id_map('#authors').iteritems()}['Custom One'] + s = {v:k for k, v in ndb.new_api.get_id_map('#series').iteritems()}['My Series One'] + run_funcs(self, db, ndb, ( + ('delete_custom_item_using_id', t, 'tags'), + ('delete_custom_item_using_id', a, 'authors'), + ('rename_custom_item', t2, 't2', 'tags'), + ('rename_custom_item', a2, 'custom one', 'authors'), + ('rename_custom_item', s, 'My Series Two', 'series'), + ('delete_item_from_multiple', 'custom two', 'authors'), + (db.clean,), + (db.refresh,), + ('all_custom', 'series'), ('all_custom', 'tags'), ('all_custom', 'authors'), + )) + for label in ('tags', 'authors', 'series'): + run_funcs(self, db, ndb, [('get_custom_and_extra', idx, label) for idx in range(3)]) db.close() # }}} From fb38f7d5653e164232c53f9dcb42914f2a9add7c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 15:52:31 +0530 Subject: [PATCH 0221/1154] Fucking McAfee --- manual/faq.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/manual/faq.rst b/manual/faq.rst index e5a6342cf8..71246acdcd 100644 --- a/manual/faq.rst +++ b/manual/faq.rst @@ -840,6 +840,18 @@ If you still cannot get the installer to work and you are on windows, you can us My antivirus program claims |app| is a virus/trojan? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. note :: + As of July, 2013 McAfee Site Advisor has started warning that + http://calibre-ebook.com is unsafe, with no stated reason or justification. + McAfee is wrong, the mistake has been reported to them, by several people, + but they have not corrected it. McAfee SiteAdvisor is a notoriously + unreliable service, see for example + http://www.naturalnews.com/041170_McAfee_Site_Advisor_false_information.html or + http://www.snapfiles.com/siteadvisor.html or + http://en.wikipedia.org/wiki/McAfee_SiteAdvisor#Criticism We strongly urge + you to stop using McAfee products, find a more competent security provider + to give your business to. + The first thing to check is that you are downloading |app| from the official website: ``_. |app| is a very popular program and unscrupulous people try to setup websites offering it for download to fool From de5237c4d5c3e26cec9de2cfcd38de0e02d483e8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 16:24:42 +0530 Subject: [PATCH 0222/1154] CC set_* API --- src/calibre/db/legacy.py | 72 ++++++++++++++++++++++++++++++++-- src/calibre/db/tests/legacy.py | 49 ++++++++++++++++++++++- 2 files changed, 116 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index eb0debe758..b814b1e23e 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -441,26 +441,39 @@ class LibraryDatabase(object): if notify: self.notify('metadata', ids) - def bulk_modify_tags(self, ids, add=[], remove=[], notify=False): + def _do_bulk_modify(self, field, ids, add, remove, notify): add = cleanup_tags(add) remove = cleanup_tags(remove) remove = set(remove) - set(add) if not ids or (not add and not remove): return + remove = {icu_lower(x) for x in remove} with self.new_api.write_lock: val_map = {} for book_id in ids: - tags = list(self.new_api._field_for('tags', book_id)) + tags = list(self.new_api._field_for(field, book_id)) existing = {icu_lower(x) for x in tags} tags.extend(t for t in add if icu_lower(t) not in existing) tags = tuple(t for t in tags if icu_lower(t) not in remove) val_map[book_id] = tags - self.new_api._set_field('tags', val_map, allow_case_change=False) + self.new_api._set_field(field, val_map, allow_case_change=False) if notify: self.notify('metadata', ids) + def bulk_modify_tags(self, ids, add=[], remove=[], notify=False): + self._do_bulk_modify('tags', ids, add, remove, notify) + + def set_custom_bulk_multiple(self, ids, add=[], remove=[], label=None, num=None, notify=False): + data = self.backend.custom_field_metadata(label, num) + if not data['editable']: + raise ValueError('Column %r is not editable'%data['label']) + if data['datatype'] != 'text' or not data['is_multiple']: + raise ValueError('Column %r is not text/multiple'%data['label']) + field = self.custom_field_name(label, num) + self._do_bulk_modify(field, ids, add, remove, notify) + def unapply_tags(self, book_id, tags, notify=True): self.bulk_modify_tags((book_id,), remove=tags, notify=notify) @@ -589,6 +602,57 @@ class LibraryDatabase(object): return [] return list(self.new_api.remove_items(field, (item_id,))) + def set_custom(self, book_id, val, label=None, num=None, append=False, + notify=True, extra=None, commit=True, allow_case_change=False): + field = self.custom_field_name(label, num) + data = self.backend.custom_field_metadata(label, num) + if data['datatype'] == 'composite': + return set() + if not data['editable']: + raise ValueError('Column %r is not editable'%data['label']) + if data['datatype'] == 'enumeration' and ( + val and val not in data['display']['enum_values']): + return set() + with self.new_api.write_lock: + if append and data['is_multiple']: + current = self.new_api._field_for(field, book_id) + existing = {icu_lower(x) for x in current} + val = current + tuple(x for x in self.new_api.fields[field].writer.adapter(val) if icu_lower(x) not in existing) + affected_books = self.new_api._set_field(field, {book_id:val}, allow_case_change=allow_case_change) + else: + affected_books = self.new_api._set_field(field, {book_id:val}, allow_case_change=allow_case_change) + if data['datatype'] == 'series': + extra = 1.0 if extra is None else extra + self.new_api._set_field(field + '_index', {book_id:extra}) + if notify and affected_books: + self.notify('metadata', list(affected_books)) + return affected_books + + def set_custom_bulk(self, ids, val, label=None, num=None, + append=False, notify=True, extras=None): + if extras is not None and len(extras) != len(ids): + raise ValueError('Length of ids and extras is not the same') + field = self.custom_field_name(label, num) + data = self.backend.custom_field_metadata(label, num) + if data['datatype'] == 'composite': + return set() + if data['datatype'] == 'enumeration' and ( + val and val not in data['display']['enum_values']): + return + if not data['editable']: + raise ValueError('Column %r is not editable'%data['label']) + + if append: + for book_id in ids: + self.set_custom(book_id, val, label=label, num=num, append=True, notify=False) + else: + with self.new_api.write_lock: + self.new_api._set_field(field, {book_id:val for book_id in ids}, allow_case_change=False) + if extras is not None: + self.new_api._set_field(field + '_index', {book_id:val for book_id, val in zip(ids, extras)}) + if notify: + self.notify('metadata', list(ids)) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): @@ -777,3 +841,5 @@ LibraryDatabase.commit = MT(lambda self:None) del MT + + diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 83325c1615..6765392638 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -48,10 +48,10 @@ def run_funcs(self, db, ndb, funcs): meth(*args) else: fmt = lambda x:x - if meth[0] in {'!', '@', '#', '+', '$'}: + if meth[0] in {'!', '@', '#', '+', '$', '-'}: if meth[0] != '+': fmt = {'!':dict, '@':lambda x:frozenset(x or ()), '#':lambda x:set((x or '').split(',')), - '$':lambda x:set(tuple(y) for y in x)}[meth[0]] + '$':lambda x:set(tuple(y) for y in x), '-':lambda x:None}[meth[0]] else: fmt = args[-1] args = args[:-1] @@ -644,5 +644,50 @@ class LegacyTest(BaseTest): for label in ('tags', 'authors', 'series'): run_funcs(self, db, ndb, [('get_custom_and_extra', idx, label) for idx in range(3)]) db.close() + + ndb = self.init_legacy(self.cloned_library) + db = self.init_old(self.cloned_library) + # Test setting + run_funcs(self, db, ndb, ( + ('-set_custom', 1, 't1 & t2', 'authors'), + ('-set_custom', 1, 't3 & t4', 'authors', None, True), + ('-set_custom', 3, 'test one & test Two', 'authors'), + ('-set_custom', 1, 'ijfkghkjdf', 'enum'), + ('-set_custom', 3, 'One', 'enum'), + ('-set_custom', 3, 'xxx', 'formats'), + ('-set_custom', 1, 'my tag two', 'tags', None, False, False, None, True, True), + (db.clean,), (db.refresh,), + ('all_custom', 'series'), ('all_custom', 'tags'), ('all_custom', 'authors'), + )) + for label in ('tags', 'series', 'authors', 'comments', 'rating', 'date', 'yesno', 'isbn', 'enum', 'formats', 'float', 'comp_tags'): + for func in ('get_custom', 'get_custom_extra', 'get_custom_and_extra'): + run_funcs(self, db, ndb, [(func, idx, label) for idx in range(3)]) + db.close() + + ndb = self.init_legacy(self.cloned_library) + db = self.init_old(self.cloned_library) + # Test setting bulk + run_funcs(self, db, ndb, ( + ('set_custom_bulk', (1,2,3), 't1 & t2', 'authors'), + ('set_custom_bulk', (1,2,3), 'a series', 'series', None, False, False, (9, 10, 11)), + ('set_custom_bulk', (1,2,3), 't1', 'tags', None, True), + (db.clean,), (db.refresh,), + ('all_custom', 'series'), ('all_custom', 'tags'), ('all_custom', 'authors'), + )) + for label in ('tags', 'series', 'authors', 'comments', 'rating', 'date', 'yesno', 'isbn', 'enum', 'formats', 'float', 'comp_tags'): + for func in ('get_custom', 'get_custom_extra', 'get_custom_and_extra'): + run_funcs(self, db, ndb, [(func, idx, label) for idx in range(3)]) + db.close() + + ndb = self.init_legacy(self.cloned_library) + db = self.init_old(self.cloned_library) + # Test bulk multiple + run_funcs(self, db, ndb, ( + ('set_custom_bulk_multiple', (1,2,3), ['t1'], ['My Tag One'], 'tags'), + (db.clean,), (db.refresh,), + ('all_custom', 'tags'), + ('get_custom', 0, 'tags'), ('get_custom', 1, 'tags'), ('get_custom', 2, 'tags'), + )) + db.close() # }}} From 3cc7a7374d639bf0a5d202e68b603b4b244d60d1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 17:40:45 +0530 Subject: [PATCH 0223/1154] API for creating, modifying and deleting custom columns --- src/calibre/db/backend.py | 181 +++++++++++++++++++++++++++++++++ src/calibre/db/cache.py | 11 ++ src/calibre/db/legacy.py | 12 +++ src/calibre/db/tests/legacy.py | 20 ++++ 4 files changed, 224 insertions(+) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 13e8b80ff5..fc7d556dc0 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -42,6 +42,8 @@ Differences in semantics from pysqlite: 3. There is no executescript ''' +CUSTOM_DATA_TYPES = frozenset(['rating', 'text', 'comments', 'datetime', + 'int', 'float', 'bool', 'series', 'composite', 'enumeration']) class DynamicFilter(object): # {{{ @@ -797,6 +799,184 @@ class DB(object): return self.custom_column_label_map[label] return self.custom_column_num_map[num] + def set_custom_column_metadata(self, num, name=None, label=None, is_editable=None, display=None): + changed = False + if name is not None: + self.conn.execute('UPDATE custom_columns SET name=? WHERE id=?', (name, num)) + changed = True + if label is not None: + self.conn.execute('UPDATE custom_columns SET label=? WHERE id=?', (label, num)) + changed = True + if is_editable is not None: + self.conn.execute('UPDATE custom_columns SET editable=? WHERE id=?', (bool(is_editable), num)) + self.custom_column_num_map[num]['is_editable'] = bool(is_editable) + changed = True + if display is not None: + self.conn.execute('UPDATE custom_columns SET display=? WHERE id=?', (json.dumps(display), num)) + changed = True + return changed + + def create_custom_column(self, label, name, datatype, is_multiple, editable=True, display={}): # {{{ + import re + if not label: + raise ValueError(_('No label was provided')) + if re.match('^\w*$', label) is None or not label[0].isalpha() or label.lower() != label: + raise ValueError(_('The label must contain only lower case letters, digits and underscores, and start with a letter')) + if datatype not in CUSTOM_DATA_TYPES: + raise ValueError('%r is not a supported data type'%datatype) + normalized = datatype not in ('datetime', 'comments', 'int', 'bool', + 'float', 'composite') + is_multiple = is_multiple and datatype in ('text', 'composite') + self.conn.execute( + ('INSERT INTO ' + 'custom_columns(label,name,datatype,is_multiple,editable,display,normalized)' + 'VALUES (?,?,?,?,?,?,?)'), + (label, name, datatype, is_multiple, editable, json.dumps(display), normalized)) + num = self.conn.last_insert_rowid() + + if datatype in ('rating', 'int'): + dt = 'INT' + elif datatype in ('text', 'comments', 'series', 'composite', 'enumeration'): + dt = 'TEXT' + elif datatype in ('float',): + dt = 'REAL' + elif datatype == 'datetime': + dt = 'timestamp' + elif datatype == 'bool': + dt = 'BOOL' + collate = 'COLLATE NOCASE' if dt == 'TEXT' else '' + table, lt = self.custom_table_names(num) + if normalized: + if datatype == 'series': + s_index = 'extra REAL,' + else: + s_index = '' + lines = [ + '''\ + CREATE TABLE %s( + id INTEGER PRIMARY KEY AUTOINCREMENT, + value %s NOT NULL %s, + UNIQUE(value)); + '''%(table, dt, collate), + + 'CREATE INDEX %s_idx ON %s (value %s);'%(table, table, collate), + + '''\ + CREATE TABLE %s( + id INTEGER PRIMARY KEY AUTOINCREMENT, + book INTEGER NOT NULL, + value INTEGER NOT NULL, + %s + UNIQUE(book, value) + );'''%(lt, s_index), + + 'CREATE INDEX %s_aidx ON %s (value);'%(lt,lt), + 'CREATE INDEX %s_bidx ON %s (book);'%(lt,lt), + + '''\ + CREATE TRIGGER fkc_update_{lt}_a + BEFORE UPDATE OF book ON {lt} + BEGIN + SELECT CASE + WHEN (SELECT id from books WHERE id=NEW.book) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: book not in books') + END; + END; + CREATE TRIGGER fkc_update_{lt}_b + BEFORE UPDATE OF author ON {lt} + BEGIN + SELECT CASE + WHEN (SELECT id from {table} WHERE id=NEW.value) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: value not in {table}') + END; + END; + CREATE TRIGGER fkc_insert_{lt} + BEFORE INSERT ON {lt} + BEGIN + SELECT CASE + WHEN (SELECT id from books WHERE id=NEW.book) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: book not in books') + WHEN (SELECT id from {table} WHERE id=NEW.value) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: value not in {table}') + END; + END; + CREATE TRIGGER fkc_delete_{lt} + AFTER DELETE ON {table} + BEGIN + DELETE FROM {lt} WHERE value=OLD.id; + END; + + CREATE VIEW tag_browser_{table} AS SELECT + id, + value, + (SELECT COUNT(id) FROM {lt} WHERE value={table}.id) count, + (SELECT AVG(r.rating) + FROM {lt}, + books_ratings_link as bl, + ratings as r + WHERE {lt}.value={table}.id and bl.book={lt}.book and + r.id = bl.rating and r.rating <> 0) avg_rating, + value AS sort + FROM {table}; + + CREATE VIEW tag_browser_filtered_{table} AS SELECT + id, + value, + (SELECT COUNT({lt}.id) FROM {lt} WHERE value={table}.id AND + books_list_filter(book)) count, + (SELECT AVG(r.rating) + FROM {lt}, + books_ratings_link as bl, + ratings as r + WHERE {lt}.value={table}.id AND bl.book={lt}.book AND + r.id = bl.rating AND r.rating <> 0 AND + books_list_filter(bl.book)) avg_rating, + value AS sort + FROM {table}; + + '''.format(lt=lt, table=table), + + ] + else: + lines = [ + '''\ + CREATE TABLE %s( + id INTEGER PRIMARY KEY AUTOINCREMENT, + book INTEGER, + value %s NOT NULL %s, + UNIQUE(book)); + '''%(table, dt, collate), + + 'CREATE INDEX %s_idx ON %s (book);'%(table, table), + + '''\ + CREATE TRIGGER fkc_insert_{table} + BEFORE INSERT ON {table} + BEGIN + SELECT CASE + WHEN (SELECT id from books WHERE id=NEW.book) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: book not in books') + END; + END; + CREATE TRIGGER fkc_update_{table} + BEFORE UPDATE OF book ON {table} + BEGIN + SELECT CASE + WHEN (SELECT id from books WHERE id=NEW.book) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: book not in books') + END; + END; + '''.format(table=table), + ] + script = ' \n'.join(lines) + self.conn.execute(script) + return num + # }}} + + def delete_custom_column(self, label=None, num=None): + data = self.custom_field_metadata(label, num) + self.conn.execute('UPDATE custom_columns SET mark_for_delete=1 WHERE id=?', (data['num'],)) + def close(self): if self._conn is not None: self._conn.close() @@ -1274,3 +1454,4 @@ class DB(object): # }}} + diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index e7c3114f0d..d9dda41aa3 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1401,6 +1401,17 @@ class Cache(object): def lookup_by_uuid(self, uuid): return self.fields['uuid'].table.lookup_by_uuid(uuid) + @write_api + def delete_custom_column(self, label=None, num=None): + self.backend.delete_custom_column(label, num) + + @write_api + def create_custom_column(self, label, name, datatype, is_multiple, editable=True, display={}): + self.backend.create_custom_column(label, name, datatype, is_multiple, editable=editable, display=display) + + @write_api + def set_custom_column_metadata(self, num, name=None, label=None, is_editable=None, display=None): + return self.backend.set_custom_column_metadata(num, name=name, label=label, is_editable=is_editable, display=display) # }}} diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index b814b1e23e..8f7a1c577e 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -653,6 +653,17 @@ class LibraryDatabase(object): if notify: self.notify('metadata', list(ids)) + def delete_custom_column(self, label=None, num=None): + self.new_api.delete_custom_column(label, num) + + def create_custom_column(self, label, name, datatype, is_multiple, editable=True, display={}): + self.new_api.create_custom_column(label, name, datatype, is_multiple, editable=editable, display=display) + + def set_custom_column_metadata(self, num, name=None, label=None, is_editable=None, display=None, notify=True): + changed = self.new_api.set_custom_column_metadata(num, name=name, label=label, is_editable=is_editable, display=display) + if changed and notify: + self.notify('metadata', []) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): @@ -843,3 +854,4 @@ del MT + diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 6765392638..6e50c164e3 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -689,5 +689,25 @@ class LegacyTest(BaseTest): ('get_custom', 0, 'tags'), ('get_custom', 1, 'tags'), ('get_custom', 2, 'tags'), )) db.close() + + o = self.cloned_library + n = self.cloned_library + ndb, db = self.init_legacy(n), self.init_old(o) + ndb.create_custom_column('created', 'Created', 'text', True, True, {'moose':'cat'}) + db.create_custom_column('created', 'Created', 'text', True, True, {'moose':'cat'}) + db.close() + ndb, db = self.init_legacy(n), self.init_old(o) + self.assertEqual(db.custom_column_label_map['created'], ndb.backend.custom_field_metadata('created')) + num = db.custom_column_label_map['created']['num'] + ndb.set_custom_column_metadata(num, is_editable=False, name='Crikey', display={}) + db.set_custom_column_metadata(num, is_editable=False, name='Crikey', display={}) + db.close() + ndb, db = self.init_legacy(n), self.init_old(o) + self.assertEqual(db.custom_column_label_map['created'], ndb.backend.custom_field_metadata('created')) + db.close() + ndb = self.init_legacy(n) + ndb.delete_custom_column('created') + ndb = self.init_legacy(n) + self.assertRaises(KeyError, ndb.custom_field_name, num=num) # }}} From a7ca60b0e91c142db6b3ed70db566a0e2bd8f31a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 17:44:50 +0530 Subject: [PATCH 0224/1154] Instructions on how to uninstall McAfee SiteAdvisor --- manual/faq.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/manual/faq.rst b/manual/faq.rst index 71246acdcd..46d675da13 100644 --- a/manual/faq.rst +++ b/manual/faq.rst @@ -846,11 +846,12 @@ My antivirus program claims |app| is a virus/trojan? McAfee is wrong, the mistake has been reported to them, by several people, but they have not corrected it. McAfee SiteAdvisor is a notoriously unreliable service, see for example - http://www.naturalnews.com/041170_McAfee_Site_Advisor_false_information.html or - http://www.snapfiles.com/siteadvisor.html or - http://en.wikipedia.org/wiki/McAfee_SiteAdvisor#Criticism We strongly urge - you to stop using McAfee products, find a more competent security provider + `this page `_ or + `this page `_ or + `this Wikipedia entry `_. + We strongly urge you to stop using McAfee products, find a more competent security provider to give your business to. + Instructions on how to `uninstall McAfee SiteAdvisor `_. The first thing to check is that you are downloading |app| from the official website: ``_. |app| is a very popular program From 0ab4ebbfbd4f38780cf7a8266a3fcba4e18f7ff5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 18:11:06 +0530 Subject: [PATCH 0225/1154] max_size() --- src/calibre/db/legacy.py | 4 ++-- src/calibre/db/tests/legacy.py | 3 +++ src/calibre/library/database.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 8f7a1c577e..fff77948f8 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -680,12 +680,12 @@ class LibraryDatabase(object): MT = lambda func: types.MethodType(func, None, LibraryDatabase) # Legacy getter API {{{ -for prop in ('author_sort', 'authors', 'comment', 'comments', 'publisher', +for prop in ('author_sort', 'authors', 'comment', 'comments', 'publisher', 'max_size', 'rating', 'series', 'series_index', 'tags', 'title', 'title_sort', 'timestamp', 'uuid', 'pubdate', 'ondevice', 'metadata_last_modified', 'languages',): def getter(prop): fm = {'comment':'comments', 'metadata_last_modified': - 'last_modified', 'title_sort':'sort'}.get(prop, prop) + 'last_modified', 'title_sort':'sort', 'max_size':'size'}.get(prop, prop) def func(self, index, index_is_id=False): return self.get_property(index, index_is_id=index_is_id, loc=self.FIELD_MAP[fm]) return func diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 6e50c164e3..aa70a9bb1d 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -15,6 +15,7 @@ from operator import itemgetter from calibre.db.tests.base import BaseTest +# Utils {{{ class ET(object): def __init__(self, func_name, args, kwargs={}, old=None, legacy=None): @@ -58,6 +59,7 @@ def run_funcs(self, db, ndb, funcs): meth = meth[1:] res1, res2 = fmt(getattr(db, meth)(*args)), fmt(getattr(ndb, meth)(*args)) self.assertEqual(res1, res2, 'The method: %s() returned different results for argument %s' % (meth, args)) +# }}} class LegacyTest(BaseTest): @@ -179,6 +181,7 @@ class LegacyTest(BaseTest): 'sizeof_format':[(1, 'FMT1', True), (2, 'FMT1', True), (0, 'xxxxxx')], '@format_files':[(0,),(1,),(2,)], 'formats':[(0,),(1,),(2,)], + 'max_size':[(0,),(1,),(2,)], 'format_hash':[(1, 'FMT1'),(1, 'FMT2'), (2, 'FMT1')], 'author_sort_from_authors': [(['Author One', 'Author Two', 'Unknown'],)], 'has_book':[(Metadata('title one'),), (Metadata('xxxx1111'),)], diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index 0e2bd5876b..6d4f55a535 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -959,7 +959,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE; def max_size(self, index, index_is_id=False): if index_is_id: return self.conn.get('SELECT size FROM meta WHERE id=?', (index,), all=False) - return self.data[index][6] + return self.data[index][4] def cover(self, index, index_is_id=False): '''Cover as a data string or None''' From f75458224bfdc311b4322478c979acab002b117e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 22:11:44 +0530 Subject: [PATCH 0226/1154] E-book viewer: Fix page position incorrect after startup bookmark E-book viewer: Fix a bug that could cause the reported position to be incorrect immediately after opening a previously opened book. This also fixes the Back button not working if a link is clicked on the page immediately after opening the book. --- resources/compiled_coffeescript.zip | Bin 71881 -> 71983 bytes .../ebooks/oeb/display/indexing.coffee | 4 ++++ src/calibre/gui2/viewer/main.py | 10 ++++++++++ 3 files changed, 14 insertions(+) diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index e092b53157f3c30f95712db0b597358b744dc08a..3c5f0e80c6f57fd0b309fa8ea039fcfcd3f1e310 100644 GIT binary patch delta 328 zcmX@Pk!AfR7Ty4FW)=|!5LmwU8ZZ_gK z;NZzjQ_xUM%uCM5FDjmFs3km^U4nOV-3N}zp{yd4A4utJUL>x?jHzm#9OvYFja(pA zn;%Hs6W(msdA(<{-eiuATtE4E8JR?w;f|QjCC(^4-Byf|e|oJrBgb?baYhjZOgDku z3?&+(z%HD=P>j)(=|BO5^Hz*eZhD9~qZHHBDv*dL)5I2t%mr~qBPNGQlfN&JovtXs zsKwMe6C#l$!DzrV^%sP*Qi73Zx~LeV26F{F<8%ixMkNteHjss!K)8U3fuT~25y%4o D7KK^O delta 278 zcmZ3#iRI)*mhBbnjLiI!%q$`dARy?u-)T!=LB~8%1_lt8o4lb)Z1ct_1CGg_3q&?+ zCWkSD7?U-VbvA!Y+a$b^uO@2q{HohM)2qc8xu)NcV3cD98Z-T;1f!A`Ba;X-4r9TV z!3akF$s4Ogr!$K)nljy10ki$Z8RcXSnuxl%M=>yfury3DnBJfVR<%ovQDk~1P$`o^ zCs06s`U-JIEhe4GV9A%_j7Chi7fhFxV033vSTT9wOzG*35{z2R2Aqu3*GVuch_JGO R4C4gC1xySK!eWd-9suzoMJxaS diff --git a/src/calibre/ebooks/oeb/display/indexing.coffee b/src/calibre/ebooks/oeb/display/indexing.coffee index efe42199e9..357128fce9 100644 --- a/src/calibre/ebooks/oeb/display/indexing.coffee +++ b/src/calibre/ebooks/oeb/display/indexing.coffee @@ -50,6 +50,8 @@ class BookIndexing this.last_check = [null, null] cache_valid: (anchors) -> + if not anchors + return false for a in anchors if not Object.prototype.hasOwnProperty.call(this.cache, a) return false @@ -65,6 +67,8 @@ class BookIndexing return this.cache ans = {} + if not anchors + return ans for anchor in anchors elem = document.getElementById(anchor) if elem == null diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index 113e1201e2..8681a0fe21 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -772,12 +772,14 @@ class EbookViewer(MainWindow, Ui_EbookViewer): self.scrolled(self.view.scroll_fraction) def internal_link_clicked(self, frac): + self.update_page_number() # Ensure page number is accurate as it is used for history self.history.add(self.pos.value()) def link_clicked(self, url): path = os.path.abspath(unicode(url.toLocalFile())) frag = None if path in self.iterator.spine: + self.update_page_number() # Ensure page number is accurate as it is used for history self.history.add(self.pos.value()) path = self.iterator.spine[self.iterator.spine.index(path)] if url.hasFragment(): @@ -913,6 +915,14 @@ class EbookViewer(MainWindow, Ui_EbookViewer): else: self.view.document.page_position.restore() self.view.document.after_resize() + # For some reason scroll_fraction returns incorrect results in paged + # mode for some time after a resize is finished. No way of knowing + # exactly how long, so we update it in a second, in the hopes that it + # will be enough *most* of the time. + QTimer.singleShot(1000, self.update_page_number) + + def update_page_number(self): + self.set_page_number(self.view.document.scroll_fraction) def close_progress_indicator(self): self.pi.stop() From 93e68b13986ca19d0b817152b7e04dac47e3f552 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Jul 2013 22:16:45 +0530 Subject: [PATCH 0227/1154] remove_cover() --- src/calibre/db/legacy.py | 5 +++++ src/calibre/db/tests/legacy.py | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index fff77948f8..a4d5a332e5 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -664,6 +664,11 @@ class LibraryDatabase(object): if changed and notify: self.notify('metadata', []) + def remove_cover(self, book_id, notify=True, commit=True): + self.new_api.set_cover({book_id:None}) + if notify: + self.notify('cover', [id]) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index aa70a9bb1d..4f448b5113 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -345,6 +345,12 @@ class LegacyTest(BaseTest): self.assertEqual(cache.field_for('authors', bid), ('calibre',)) self.assertEqual(cache.field_for('tags', bid), (_('News'), 'Events', 'one', 'two')) + self.assertTrue(legacy.cover(1, index_is_id=True)) + self.assertTrue(legacy.has_cover(1)) + legacy.remove_cover(1) + self.assertFalse(legacy.has_cover(1)) + self.assertFalse(legacy.cover(1, index_is_id=True)) + legacy.delete_book(1) old.delete_book(1) self.assertNotIn(1, legacy.all_ids()) From ce4d12711ab8674fad84bb65f26f17fcc7749010 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 18 Jul 2013 08:16:29 +0530 Subject: [PATCH 0228/1154] Correctly identify JPEG files that have no headers --- src/calibre/utils/imghdr.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/utils/imghdr.py b/src/calibre/utils/imghdr.py index c899a5be95..c4070f072e 100644 --- a/src/calibre/utils/imghdr.py +++ b/src/calibre/utils/imghdr.py @@ -26,6 +26,10 @@ def what(file, h=None): finally: if f: f.close() + # There exist some jpeg files with no headers, only the starting two bits + # If we cannot identify as anything else, identify as jpeg. + if h[:2] == b'\xff\xd8': + return 'jpeg' return None From 0c5959f2983cd863ee87dd956b4881c1ba077003 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 18 Jul 2013 10:03:30 +0530 Subject: [PATCH 0229/1154] PDF Output: Fix anchors sometimes located incorrectly PDF Output: Workaround bug in WebKit's getBoundingClientRect() method that could cause links to occasionally point to incorrect locations. Fixes #1202390 [Private bug](https://bugs.launchpad.net/calibre/+bug/1202390) --- resources/compiled_coffeescript.zip | Bin 71983 -> 72211 bytes src/calibre/ebooks/oeb/display/paged.coffee | 16 ++++++++++++++-- src/calibre/ebooks/pdf/render/from_html.py | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index 3c5f0e80c6f57fd0b309fa8ea039fcfcd3f1e310..ddf0db08e77d28f55c7eb8344a7899e938a18244 100644 GIT binary patch delta 260 zcmZ3#iDmK@7U=+QW)=|!5XkcSSSJZ^~onK1zF>ZQqv44_v`CVw#}3V zit=(AMjOQHD3s(EOpbXVy}8f&j~w$#*H4>!qkin>L$!DMWHCl-kmU4>VvN#EDbbTR zR*6n$7H2eNT2csR`-?MLFjZCq+49p@h%;(2m9~P#Uy3stF{wc9=AsQIgS} psdXk;eY_;20n^l9(#B%>@_r5Gd79stY?N45X} diff --git a/src/calibre/ebooks/oeb/display/paged.coffee b/src/calibre/ebooks/oeb/display/paged.coffee index f97f1b3cf8..39c4b12fe9 100644 --- a/src/calibre/ebooks/oeb/display/paged.coffee +++ b/src/calibre/ebooks/oeb/display/paged.coffee @@ -294,9 +294,21 @@ class PagedDisplay return Math.floor(xpos/this.page_width) column_location: (elem) -> - # Return the location of elem relative to its containing column + # Return the location of elem relative to its containing column. + # WARNING: This method may cause the viewport to scroll (to workaround + # a bug in WebKit). br = elem.getBoundingClientRect() - [left, top] = calibre_utils.viewport_to_document(br.left, br.top, elem.ownerDocument) + # Because of a bug in WebKit's getBoundingClientRect() in column + # mode, this position can be inaccurate, see + # https://bugs.launchpad.net/calibre/+bug/1202390 for a test case. + # The usual symptom of the inaccuracy is br.top is highly negative. + if br.top < -100 + # We have to actually scroll the element into view to get its + # position + elem.scrollIntoView() + [left, top] = calibre_utils.viewport_to_document(elem.scrollLeft, elem.scrollTop, elem.ownerDocument) + else + [left, top] = calibre_utils.viewport_to_document(br.left, br.top, elem.ownerDocument) c = this.column_at(left) width = Math.min(br.right, (c+1)*this.page_width) - br.left if br.bottom < br.top diff --git a/src/calibre/ebooks/pdf/render/from_html.py b/src/calibre/ebooks/pdf/render/from_html.py index 8ea1d8203e..771cc33ede 100644 --- a/src/calibre/ebooks/pdf/render/from_html.py +++ b/src/calibre/ebooks/pdf/render/from_html.py @@ -353,6 +353,7 @@ class PDFWriter(QObject): paged_display.layout(); paged_display.fit_images(); py_bridge.value = book_indexing.all_links_and_anchors(); + window.scrollTo(0, 0); // This is needed as getting anchor positions could have caused the viewport to scroll '''%(self.margin_top, 0, self.margin_bottom)) amap = self.bridge_value From 5542dcfbb3e6c303f3e1fd30300452904d3152fb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 18 Jul 2013 11:34:39 +0530 Subject: [PATCH 0230/1154] Dont add page breaks for chapters at the start of the file PDF Output: Fix extra blank page being inserted at the start of the chapter when converting some epub files from feedbooks --- .../ebooks/oeb/transforms/structure.py | 47 +++++++++++++++---- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/structure.py b/src/calibre/ebooks/oeb/transforms/structure.py index cd376b4ec4..50ee4d011d 100644 --- a/src/calibre/ebooks/oeb/transforms/structure.py +++ b/src/calibre/ebooks/oeb/transforms/structure.py @@ -10,7 +10,7 @@ import re, uuid from lxml import etree from urlparse import urlparse -from collections import OrderedDict +from collections import OrderedDict, Counter from calibre.ebooks.oeb.base import XPNSMAP, TOC, XHTML, xml2text, barename from calibre.ebooks import ConversionError @@ -22,6 +22,26 @@ def XPath(x): raise ConversionError( 'The syntax of the XPath expression %s is invalid.' % repr(x)) +def isspace(x): + return not x or x.replace(u'\xa0', u'').isspace() + +def at_start(elem): + ' Return True if there is no content before elem ' + body = XPath('ancestor-or-self::h:body')(elem) + if not body: + return True + body = body[0] + ancestors = frozenset(XPath('ancestor::*')(elem)) + for x in body.iter(): + if x is elem: + return True + if getattr(x, 'tag', None) and x.tag.rpartition('}')[-1] in {'img', 'svg'}: + return False + if isspace(getattr(x, 'text', None)) and (x in ancestors or isspace(getattr(x, 'tail', None))): + continue + return False + return False + class DetectStructure(object): def __call__(self, oeb, opts): @@ -51,7 +71,7 @@ class DetectStructure(object): regexp = re.compile(opts.toc_filter) for node in list(self.oeb.toc.iter()): if not node.title or regexp.search(node.title) is not None: - self.log('Filtering', node.title if node.title else\ + self.log('Filtering', node.title if node.title else 'empty node', 'from TOC') self.oeb.toc.remove(node) @@ -92,7 +112,8 @@ class DetectStructure(object): 'Invalid start reading at XPath expression, ignoring: %s'%expr) return for item in self.oeb.spine: - if not hasattr(item.data, 'xpath'): continue + if not hasattr(item.data, 'xpath'): + continue matches = expr(item.data) if matches: elem = matches[0] @@ -129,17 +150,27 @@ class DetectStructure(object): chapter_mark = self.opts.chapter_mark page_break_before = 'display: block; page-break-before: always' page_break_after = 'display: block; page-break-after: always' + c = Counter() for item, elem in self.detected_chapters: + c[item] += 1 text = xml2text(elem).strip() text = re.sub(r'\s+', ' ', text.strip()) self.log('\tDetected chapter:', text[:50]) if chapter_mark == 'none': continue - elif chapter_mark == 'rule': + if chapter_mark == 'rule': mark = etree.Element(XHTML('hr')) elif chapter_mark == 'pagebreak': + if c[item] < 3 and at_start(elem): + # For the first two elements in this item, check if they + # are at the start of the file, in which case inserting a + # page break in unnecessary and can lead to extra blank + # pages in the PDF Output plugin. We need to use two as + # feedbooks epubs match both a heading tag and its + # containing div with the default chapter expression. + continue mark = etree.Element(XHTML('div'), style=page_break_after) - else: # chapter_mark == 'both': + else: # chapter_mark == 'both': mark = etree.Element(XHTML('hr'), style=page_break_before) try: elem.addprevious(mark) @@ -182,8 +213,6 @@ class DetectStructure(object): self.log('Maximum TOC links reached, stopping.') return - - def elem_to_link(self, item, elem, counter): text = xml2text(elem).strip() if not text: @@ -197,7 +226,6 @@ class DetectStructure(object): href = '#'.join((item.href, id)) return text, href - def add_leveled_toc_items(self): added = OrderedDict() added2 = OrderedDict() @@ -223,7 +251,7 @@ class DetectStructure(object): node = self.oeb.toc.add(text, _href, play_order=self.oeb.toc.next_play_order()) added[elem] = node - #node.add(_('Top'), _href) + # node.add(_('Top'), _href) if self.opts.level2_toc is not None and added: for elem in find_matches(self.opts.level2_toc, document.data): @@ -263,3 +291,4 @@ class DetectStructure(object): play_order=self.oeb.toc.next_play_order()) break + From b982bcc3f2d0f3340764db04c8fd6c4f7c8360ed Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 18 Jul 2013 12:45:35 +0530 Subject: [PATCH 0231/1154] Get Books: Fix title author search handling Get Books: Fix searching for title and author returning some extra matches, if the title starts with an article like the, a or an. Fixes #1200012 ["Get books" search is too fuzzy](https://bugs.launchpad.net/calibre/+bug/1200012) --- src/calibre/gui2/store/search/models.py | 22 ++++++++++++---------- src/calibre/gui2/store/search/search.py | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/store/search/models.py b/src/calibre/gui2/store/search/models.py index 59292b54d7..af2c274bd7 100644 --- a/src/calibre/gui2/store/search/models.py +++ b/src/calibre/gui2/store/search/models.py @@ -229,14 +229,14 @@ class Matches(QAbstractItemModel): if col == 1: return QVariant('

    %s

    ' % result.title) elif col == 2: - return QVariant('

    ' + _('Detected price as: %s. Check with the store before making a purchase to verify this price is correct. This price often does not include promotions the store may be running.') % result.price + '

    ') + return QVariant('

    ' + _('Detected price as: %s. Check with the store before making a purchase to verify this price is correct. This price often does not include promotions the store may be running.') % result.price + '

    ') # noqa elif col == 3: if result.drm == SearchResult.DRM_LOCKED: - return QVariant('

    ' + _('This book as been detected as having DRM restrictions. This book may not work with your reader and you will have limitations placed upon you as to what you can do with this book. Check with the store before making any purchases to ensure you can actually read this book.') + '

    ') + return QVariant('

    ' + _('This book as been detected as having DRM restrictions. This book may not work with your reader and you will have limitations placed upon you as to what you can do with this book. Check with the store before making any purchases to ensure you can actually read this book.') + '

    ') # noqa elif result.drm == SearchResult.DRM_UNLOCKED: - return QVariant('

    ' + _('This book has been detected as being DRM Free. You should be able to use this book on any device provided it is in a format calibre supports for conversion. However, before making a purchase double check the DRM status with the store. The store may not be disclosing the use of DRM.') + '

    ') + return QVariant('

    ' + _('This book has been detected as being DRM Free. You should be able to use this book on any device provided it is in a format calibre supports for conversion. However, before making a purchase double check the DRM status with the store. The store may not be disclosing the use of DRM.') + '

    ') # noqa else: - return QVariant('

    ' + _('The DRM status of this book could not be determined. There is a very high likelihood that this book is actually DRM restricted.') + '

    ') + return QVariant('

    ' + _('The DRM status of this book could not be determined. There is a very high likelihood that this book is actually DRM restricted.') + '

    ') # noqa elif col == 4: return QVariant('

    %s

    ' % result.formats) elif col == 5: @@ -337,7 +337,7 @@ class SearchFilter(SearchQueryParser): def _match(self, query, value, matchkind): for t in value: - try: ### ignore regexp exceptions, required because search-ahead tries before typing is finished + try: # ignore regexp exceptions, required because search-ahead tries before typing is finished t = icu_lower(t) if matchkind == self.EQUALS_MATCH: if query == t: @@ -375,7 +375,7 @@ class SearchFilter(SearchQueryParser): elif query.startswith('~'): matchkind = self.REGEXP_MATCH query = query[1:] - if matchkind != self.REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D + if matchkind != self.REGEXP_MATCH: # leave case in regexps because it can be significant e.g. \S \W \D query = query.lower() if location not in self.USABLE_LOCATIONS: @@ -439,9 +439,9 @@ class SearchFilter(SearchQueryParser): if locvalue in ('affiliate', 'drm', 'download', 'downloads'): continue try: - ### Can't separate authors because comma is used for name sep and author sep - ### Exact match might not get what you want. For that reason, turn author - ### exactmatch searches into contains searches. + # Can't separate authors because comma is used for name sep and author sep + # Exact match might not get what you want. For that reason, turn author + # exactmatch searches into contains searches. if locvalue == 'author' and matchkind == self.EQUALS_MATCH: m = self.CONTAINS_MATCH else: @@ -452,13 +452,15 @@ class SearchFilter(SearchQueryParser): elif locvalue in ('author2', 'title2'): m = self.IN_MATCH vals = re.sub(r'(^|\s)(and|not|or|a|the|is|of|,)(\s|$)', ' ', accessor(sr)).split(' ') + vals = [x for x in vals if x] final_query = query.lower() else: vals = [accessor(sr)] if self._match(final_query, vals, m): matches.add(sr) break - except ValueError: # Unicode errors + except ValueError: # Unicode errors import traceback traceback.print_exc() return matches + diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index 117bc0dca3..7e39e5d513 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -236,11 +236,11 @@ class SearchDialog(QDialog, Ui_Dialog): query = re.sub(r'%s:"[^"]"' % loc, '', query) query = re.sub(r'%s:[^\s]*' % loc, '', query) # Remove logic. - query = re.sub(r'(^|\s)(and|not|or|a|the|is|of)(\s|$)', ' ', query) + query = re.sub(r'(^|\s|")(and|not|or|a|the|is|of)(\s|$|")', r' ', query) # Remove " query = query.replace('"', '') # Remove excess whitespace. - query = re.sub(r'\s{2,}', ' ', query) + query = re.sub(r'\s+', ' ', query) query = query.strip() return query.encode('utf-8') From 781191c21b4059d83efdfb56ac92900f28f5590a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 18 Jul 2013 12:49:29 +0530 Subject: [PATCH 0232/1154] More trekstor 4ink device ids --- src/calibre/devices/misc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index e35db8f03d..13ae870fd1 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -229,7 +229,8 @@ class TREKSTOR(USBMS): 0x0067, # This is for the Pyrus Mini 0x006f, # This is for the Pyrus Maxi 0x003e, # This is for the EBOOK_PLAYER_5M https://bugs.launchpad.net/bugs/792091 - 0x5cL, # This is for the 4ink http://www.mobileread.com/forums/showthread.php?t=191318 + 0x05cL, # This is for the 4ink http://www.mobileread.com/forums/showthread.php?t=191318 + 0x006c, # This is for the 4ink http://www.mobileread.com/forums/showthread.php?t=218273 ] BCD = [0x0002, 0x100, 0x0222] From aefab3b47abfb4077e7c58cd83a4f452ffd99b7f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 18 Jul 2013 13:30:44 +0530 Subject: [PATCH 0233/1154] Fix regression that caused rescaling of ont size in dropcaps generated by the DOCX input plugin --- src/calibre/ebooks/oeb/transforms/flatcss.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 9c08934938..1b678a3fe5 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -371,11 +371,13 @@ class CSSFlattener(object): is_drop_cap = (cssdict.get('float', None) == 'left' and 'font-size' in cssdict and len(node) == 0 and node.text and len(node.text) == 1) - is_drop_cap = is_drop_cap or ( - # The docx input plugin generates drop caps that look like this - len(node) == 1 and not node.text and len(node[0]) == 0 and - node[0].text and not node[0].tail and len(node[0].text) == 1 and - 'line-height' in cssdict and 'font-size' in cssdict) + # Detect drop caps generated by the docx input plugin + if (node.tag and node.tag.endswith('}p') and len(node) == 0 and node.text and len(node.text.strip()) == 1 and + not node.tail and 'line-height' in cssdict and 'font-size' in cssdict): + dp = node.getparent() + if dp.tag and dp.tag.endswith('}div') and len(dp) == 1 and not dp.text: + if stylizer.style(dp).cssdict().get('float', None) == 'left': + is_drop_cap = True if not self.context.disable_font_rescaling and not is_drop_cap: _sbase = self.sbase if self.sbase is not None else \ self.context.source.fbase From 1ee4cad632c83830d216f79d0f1ffba475495d7f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 18 Jul 2013 14:52:09 +0530 Subject: [PATCH 0234/1154] set_cover() --- src/calibre/db/legacy.py | 7 ++++++- src/calibre/db/tests/legacy.py | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index a4d5a332e5..b079c776bb 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -667,7 +667,12 @@ class LibraryDatabase(object): def remove_cover(self, book_id, notify=True, commit=True): self.new_api.set_cover({book_id:None}) if notify: - self.notify('cover', [id]) + self.notify('cover', [book_id]) + + def set_cover(self, book_id, data, notify=True, commit=True): + self.new_api.set_cover({book_id:data}) + if notify: + self.notify('cover', [book_id]) # Private interface {{{ def __iter__(self): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 4f448b5113..091d344e2d 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -346,10 +346,14 @@ class LegacyTest(BaseTest): self.assertEqual(cache.field_for('tags', bid), (_('News'), 'Events', 'one', 'two')) self.assertTrue(legacy.cover(1, index_is_id=True)) + origcov = legacy.cover(1, index_is_id=True) self.assertTrue(legacy.has_cover(1)) legacy.remove_cover(1) self.assertFalse(legacy.has_cover(1)) self.assertFalse(legacy.cover(1, index_is_id=True)) + legacy.set_cover(3, origcov) + self.assertEqual(legacy.cover(3, index_is_id=True), origcov) + self.assertTrue(legacy.has_cover(3)) legacy.delete_book(1) old.delete_book(1) From 76045d2a5490addb8d0e7e79a2af326e91864d82 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 18 Jul 2013 15:25:30 +0530 Subject: [PATCH 0235/1154] categories API --- src/calibre/db/cache.py | 14 ++++++++++++++ src/calibre/db/fields.py | 10 ++++++++++ src/calibre/db/legacy.py | 3 +++ src/calibre/db/tests/legacy.py | 2 ++ 4 files changed, 29 insertions(+) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index d9dda41aa3..82dd762077 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -401,6 +401,12 @@ class Cache(object): def get_item_name(self, field, item_id): return self.fields[field].table.id_map[item_id] + @read_api + def get_item_id(self, field, item_name): + ' Return the item id for item_name (case-insensitive) ' + rmap = {icu_lower(v) if isinstance(v, unicode) else v:k for k, v in self.fields[field].table.id_map.iteritems()} + return rmap.get(icu_lower(item_name) if isinstance(item_name, unicode) else item_name, None) + @read_api def author_data(self, author_ids=None): ''' @@ -1413,6 +1419,14 @@ class Cache(object): def set_custom_column_metadata(self, num, name=None, label=None, is_editable=None, display=None): return self.backend.set_custom_column_metadata(num, name=name, label=label, is_editable=is_editable, display=display) + @read_api + def get_books_for_category(self, category, item_id_or_composite_value): + f = self.fields[category] + if hasattr(f, 'get_books_for_val'): + # Composite field + return f.get_books_for_val(item_id_or_composite_value, self._get_metadata, self._all_book_ids()) + return self._books_for_field(f.name, item_id_or_composite_value) + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index dd0165b44e..e028ff5d99 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -211,6 +211,16 @@ class CompositeField(OneToOneField): ans.append(c) return ans + def get_books_for_val(self, value, get_metadata, book_ids): + is_multiple = self.table.metadata['is_multiple'].get('cache_to_list', None) + ans = set() + for book_id in book_ids: + val = self.get_value_with_cache(book_id, get_metadata) + vals = {x.strip() for x in val.split(is_multiple)} if is_multiple else [val] + if value in vals: + ans.add(book_id) + return ans + class OnDeviceField(OneToOneField): def __init__(self, name, table): diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index b079c776bb..f7cab21baf 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -717,10 +717,13 @@ LibraryDatabase.format_hash = MT(lambda self, book_id, fmt:self.new_api.format_h LibraryDatabase.index = MT(lambda self, book_id, cache=False:self.data.id_to_index(book_id)) LibraryDatabase.has_cover = MT(lambda self, book_id:self.new_api.field_for('cover', book_id)) LibraryDatabase.get_tags = MT(lambda self, book_id:set(self.new_api.field_for('tags', book_id))) +LibraryDatabase.get_categories = MT(lambda self, sort='name', ids=None, icon_map=None:self.new_api.get_categories(sort=sort, book_ids=ids, icon_map=icon_map)) LibraryDatabase.get_identifiers = MT( lambda self, index, index_is_id=False: self.new_api.field_for('identifiers', index if index_is_id else self.id(index))) LibraryDatabase.isbn = MT( lambda self, index, index_is_id=False: self.get_identifiers(index, index_is_id=index_is_id).get('isbn', None)) +LibraryDatabase.get_books_for_category = MT( + lambda self, category, id_:self.new_api.get_books_for_category(category, id_)) # }}} # Legacy setter API {{{ diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 091d344e2d..936eb12e44 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -161,10 +161,12 @@ class LegacyTest(BaseTest): from datetime import timedelta ndb = self.init_legacy(self.cloned_library) db = self.init_old() + newstag = ndb.new_api.get_item_id('tags', 'news') self.assertEqual(dict(db.prefs), dict(ndb.prefs)) for meth, args in { + 'get_books_for_category': [('tags', newstag), ('#formats', 'FMT1')], 'get_next_series_num_for': [('A Series One',)], 'get_id_from_uuid':[('ddddd',), (db.uuid(1, True),)], 'cover':[(0,), (1,), (2,)], From 51620932a80ad8f5cd19245542efb8e396423ad2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 18 Jul 2013 16:33:24 +0530 Subject: [PATCH 0236/1154] Format manipulation API --- src/calibre/db/cache.py | 25 +++++++++++++++++++++++++ src/calibre/db/legacy.py | 22 ++++++++++++++++++++++ src/calibre/db/tests/add_remove.py | 17 +++++++++++++++++ src/calibre/db/tests/legacy.py | 21 +++++++++++++++++++-- src/calibre/library/database2.py | 2 ++ 5 files changed, 85 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 82dd762077..46dc5df6d7 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -607,6 +607,31 @@ class Cache(object): return False return self.backend.has_format(book_id, fmt, name, path) + @api + def save_original_format(self, book_id, fmt): + fmt = fmt.upper() + if 'ORIGINAL' in fmt: + raise ValueError('Cannot save original of an original fmt') + fmtfile = self.format(book_id, fmt, as_file=True) + if fmtfile is None: + return False + with fmtfile: + nfmt = 'ORIGINAL_'+fmt + return self.add_format(book_id, nfmt, fmtfile, run_hooks=False) + + @api + def restore_original_format(self, book_id, original_fmt): + original_fmt = original_fmt.upper() + fmtfile = self.format(book_id, original_fmt, as_file=True) + if fmtfile is not None: + fmt = original_fmt.partition('_')[2] + with self.write_lock: + with fmtfile: + self._add_format(book_id, fmt, fmtfile, run_hooks=False) + self._remove_formats({book_id:(original_fmt,)}) + return True + return False + @read_api def formats(self, book_id, verify_formats=True): ''' diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index f7cab21baf..aeab6e3de0 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -674,6 +674,28 @@ class LibraryDatabase(object): if notify: self.notify('cover', [book_id]) + def original_fmt(self, book_id, fmt): + nfmt = ('ORIGINAL_%s'%fmt).upper() + return nfmt if self.new_api.has_format(book_id, nfmt) else fmt + + def save_original_format(self, book_id, fmt, notify=True): + ret = self.new_api.save_original_format(book_id, fmt) + if ret and notify: + self.notify('metadata', [book_id]) + return ret + + def restore_original_format(self, book_id, original_fmt, notify=True): + ret = self.new_api.restore_original_format(book_id, original_fmt) + if ret and notify: + self.notify('metadata', [book_id]) + return ret + + def remove_format(self, index, fmt, index_is_id=False, notify=True, commit=True, db_only=False): + book_id = index if index_is_id else self.id(index) + self.new_api.remove_formats({book_id:(fmt,)}, db_only=db_only) + if notify: + self.notify('metadata', [book_id]) + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index 76349df1c5..0047a0ec4f 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -251,4 +251,21 @@ class AddRemoveTest(BaseTest): # }}} + def test_original_fmt(self): # {{{ + ' Test management of original fmt ' + af, ae, at = self.assertFalse, self.assertEqual, self.assertTrue + db = self.init_cache() + fmts = db.formats(1) + af(db.has_format(1, 'ORIGINAL_FMT1')) + at(db.save_original_format(1, 'FMT1')) + at(db.has_format(1, 'ORIGINAL_FMT1')) + raw = db.format(1, 'FMT1') + ae(raw, db.format(1, 'ORIGINAL_FMT1')) + db.add_format(1, 'FMT1', BytesIO(b'replacedfmt')) + self.assertNotEqual(db.format(1, 'FMT1'), db.format(1, 'ORIGINAL_FMT1')) + at(db.restore_original_format(1, 'ORIGINAL_FMT1')) + ae(raw, db.format(1, 'FMT1')) + af(db.has_format(1, 'ORIGINAL_FMT1')) + ae(set(fmts), set(db.formats(1, verify_formats=False))) + # }}} diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 936eb12e44..e37d954225 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -49,10 +49,10 @@ def run_funcs(self, db, ndb, funcs): meth(*args) else: fmt = lambda x:x - if meth[0] in {'!', '@', '#', '+', '$', '-'}: + if meth[0] in {'!', '@', '#', '+', '$', '-', '%'}: if meth[0] != '+': fmt = {'!':dict, '@':lambda x:frozenset(x or ()), '#':lambda x:set((x or '').split(',')), - '$':lambda x:set(tuple(y) for y in x), '-':lambda x:None}[meth[0]] + '$':lambda x:set(tuple(y) for y in x), '-':lambda x:None, '%':lambda x: set((x or '').split(','))}[meth[0]] else: fmt = args[-1] args = args[:-1] @@ -357,6 +357,10 @@ class LegacyTest(BaseTest): self.assertEqual(legacy.cover(3, index_is_id=True), origcov) self.assertTrue(legacy.has_cover(3)) + self.assertTrue(legacy.format(1, 'FMT1', index_is_id=True)) + legacy.remove_format(1, 'FMT1', index_is_id=True) + self.assertIsNone(legacy.format(1, 'FMT1', index_is_id=True)) + legacy.delete_book(1) old.delete_book(1) self.assertNotIn(1, legacy.all_ids()) @@ -726,3 +730,16 @@ class LegacyTest(BaseTest): self.assertRaises(KeyError, ndb.custom_field_name, num=num) # }}} + def test_legacy_original_fmt(self): # {{{ + db, ndb = self.init_old(), self.init_legacy() + run_funcs(self, db, ndb, ( + ('original_fmt', 1, 'FMT1'), + ('save_original_format', 1, 'FMT1'), + ('original_fmt', 1, 'FMT1'), + ('restore_original_format', 1, 'ORIGINAL_FMT1'), + ('original_fmt', 1, 'FMT1'), + ('%formats', 1, True), + )) + db.close() + + # }}} diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index b61544f172..4ccba4d9fc 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1570,6 +1570,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): with lopen(opath, 'rb') as f: self.add_format(book_id, fmt, f, index_is_id=True, notify=False) self.remove_format(book_id, original_fmt, index_is_id=True, notify=notify) + return True + return False def delete_book(self, id, notify=True, commit=True, permanent=False, do_clean=True): From 6298b1d0591cd12cb86d9e3214430bdacb3f442f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 18 Jul 2013 16:35:21 +0530 Subject: [PATCH 0237/1154] ... --- src/calibre/db/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 99154ad618..47ceb996ea 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -113,5 +113,7 @@ Various things that require other things before they can be migrated: 3. Port library/restore.py 4. Replace the metadatabackup thread with the new implementation when using the new backend. 5. grep the sources for TODO - 6. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add all work + 6. Check that content server reloading on metadata,db change, metadata + backup, refresh gui on calibredb add and moving libraries all work (check + them on windows as well for file locking issues) ''' From 54054c1c9fde8bd14f80dc2cd564bbc258a85341 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 09:07:04 +0530 Subject: [PATCH 0238/1154] Allow using non-ascii chars in email passwords Fixes #1202825 [SMTP password with special characters not stored correctly](https://bugs.launchpad.net/calibre/+bug/1202825) --- src/calibre/gui2/email.py | 2 +- src/calibre/gui2/wizard/send_email.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 665b19cc5a..6645441158 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -113,7 +113,7 @@ class Sendmail(object): verbose=1, relay=opts.relay_host, username=opts.relay_username, - password=unhexlify(opts.relay_password), port=opts.relay_port, + password=unhexlify(opts.relay_password).decode('utf-8'), port=opts.relay_port, encryption=opts.encryption, debug_output=log.debug) finally: diff --git a/src/calibre/gui2/wizard/send_email.py b/src/calibre/gui2/wizard/send_email.py index 0dc6861116..b183af1744 100644 --- a/src/calibre/gui2/wizard/send_email.py +++ b/src/calibre/gui2/wizard/send_email.py @@ -32,7 +32,7 @@ class TestEmail(QDialog, TE_Dialog): self.to.setText(pa) if opts.relay_host: self.label.setText(_('Using: %(un)s:%(pw)s@%(host)s:%(port)s and %(enc)s encryption')% - dict(un=opts.relay_username, pw=unhexlify(opts.relay_password), + dict(un=opts.relay_username, pw=unhexlify(opts.relay_password).decode('utf-8'), host=opts.relay_host, port=opts.relay_port, enc=opts.encryption)) def test(self, *args): @@ -129,7 +129,7 @@ class SendEmail(QWidget, Ui_Form): self.relay_username.setText(opts.relay_username) self.relay_username.textChanged.connect(self.changed) if opts.relay_password: - self.relay_password.setText(unhexlify(opts.relay_password)) + self.relay_password.setText(unhexlify(opts.relay_password).decode('utf-8')) self.relay_password.textChanged.connect(self.changed) getattr(self, 'relay_'+opts.encryption.lower()).setChecked(True) self.relay_tls.toggled.connect(self.changed) @@ -169,7 +169,7 @@ class SendEmail(QWidget, Ui_Form): sendmail(msg, from_=opts.from_, to=[to], verbose=3, timeout=30, relay=opts.relay_host, username=opts.relay_username, - password=unhexlify(opts.relay_password), + password=unhexlify(opts.relay_password).decode('utf-8'), encryption=opts.encryption, port=opts.relay_port) except: import traceback @@ -248,7 +248,7 @@ class SendEmail(QWidget, Ui_Form): conf.set('relay_host', host if host else None) conf.set('relay_port', self.relay_port.value()) conf.set('relay_username', username if username else None) - conf.set('relay_password', hexlify(password)) + conf.set('relay_password', hexlify(password.encode('utf-8'))) conf.set('encryption', enc_method) return True From dc3dde863a7607ef74471e5113b0092e32fc239a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 09:21:33 +0530 Subject: [PATCH 0239/1154] Update Galaxy's Edge --- recipes/galaxys_edge.recipe | 47 +++---------------------------------- 1 file changed, 3 insertions(+), 44 deletions(-) diff --git a/recipes/galaxys_edge.recipe b/recipes/galaxys_edge.recipe index e6e1dd7475..4406db4620 100644 --- a/recipes/galaxys_edge.recipe +++ b/recipes/galaxys_edge.recipe @@ -14,19 +14,12 @@ class GalaxyEdge(BasicNewsRecipe): auto_cleanup = True - #keep_only_tags = [dict(id='content')] - #remove_tags = [dict(attrs={'class':['article-links', 'breadcr']}), - #dict(id=['email-section', 'right-column', 'printfooter', 'topover', - #'slidebox', 'th_footer'])] - extra_css = '.photo-caption { font-size: smaller }' def parse_index(self): soup = self.index_to_soup('http://www.galaxysedge.com/') - main = soup.find('table', attrs={'width':'911'}) - toc = main.find('td', attrs={'width':'225'}) - - + main = soup.find('table', attrs={'width':'944'}) + toc = main.find('td', attrs={'width':'204'}) current_section = None current_articles = [] @@ -68,41 +61,7 @@ class GalaxyEdge(BasicNewsRecipe): current_articles.append({'title': title, 'url':url, 'description':'', 'date':''}) if current_articles and current_section: - feeds.append((current_section, current_articles)) + feeds.append((current_section, current_articles)) return feeds - - - - #def preprocess_raw_html(self, raw, url): - #return raw.replace('

    ', '

    ').replace('

    ', '

    ') - - #def postprocess_html(self, soup, first_fetch): - #for t in soup.findAll(['table', 'tr', 'td','center']): - #t.name = 'div' - #return soup - - #def parse_index(self): - #today = time.strftime('%Y-%m-%d') - #soup = self.index_to_soup( - #'http://www.thehindu.com/todays-paper/tp-index/?date=' + today) - #div = soup.find(id='left-column') - #feeds = [] - #current_section = None - #current_articles = [] - #for x in div.findAll(['h3', 'div']): - #if current_section and x.get('class', '') == 'tpaper': - #a = x.find('a', href=True) - #if a is not None: - #current_articles.append({'url':a['href']+'?css=print', - #'title':self.tag_to_string(a), 'date': '', - #'description':''}) - #if x.name == 'h3': - #if current_section and current_articles: - #feeds.append((current_section, current_articles)) - #current_section = self.tag_to_string(x) - #current_articles = [] - #return feeds - - From 161233430bc1186483fa54ef14f711eb3e1bb7a4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 09:29:26 +0530 Subject: [PATCH 0240/1154] version 0.9.40 --- Changelog.yaml | 49 ++++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index db25f77a8d..2cbe422226 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,55 @@ # new recipes: # - title: +- version: 0.9.40 + date: 2013-07-19 + + new features: + - title: "EPUB Output: Add an option to insert an inline Table of Contents into the main text." + tickets: [1201006] + + - title: "Driver for LG Android phone" + tickets: [1202013] + + - title: "When matching books in the library against the device manually, pre-fill the search field with the book title" + tickets: [1200826] + + bug fixes: + - title: "PDF Input: Fix a regression that caused some images to be flipped when converting PDF files that use image rotation operators." + tickets: [1201083] + + - title: "Fix regression that caused incorrect font size in dropcaps generated by the DOCX input plugin" + + - title: "Get Books: Fix searching for title and author returning some extra matches, if the title starts with an article like the, a or an." + tickets: [1200012] + + - title: "PDF Output: Fix extra blank page being inserted at the start of the chapter when converting some epub files from feedbooks" + + - title: "PDF Output: Workaround bug in WebKit's getBoundingClientRect() method that could cause links to occasionally point to incorrect locations." + tickets: [1202390] + + - title: "E-book viewer: Fix a bug that could cause the reported position to be incorrect immediately after opening a previously opened book. This also fixes the Back button not working if a link is clicked on the page immediately after opening the book." + + - title: "Fix memory card not being detected for Elonex 621 on Windows" + + - title: "Fix regression in last release that broke auto-conversion of ebooks when sending to device/sending by email." + tickets: [1200864] + + - title: "Get Books: Update amazon plugins for website changes" + + - title: "Allow using non-ascii chars in email passwords." + tickets: [1202825] + + improved recipes: + - Galaxy's Edge + + new recipes: + - title: Il Foglio + author: faber1971 + + - title: Le Monde Diplomatique and Acrimed + author: Gaetan Lehmann + - version: 0.9.39 date: 2013-07-12 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 18b4e3d238..1e0b2a1a83 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 9, 39) +numeric_version = (0, 9, 40) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " From 669efdd6f69e775f3e386071032b54eb9a9ed2ae Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 10:44:46 +0530 Subject: [PATCH 0241/1154] get_data_as_dict() --- src/calibre/db/__init__.py | 64 ++++++++++++++++++++++++++++++ src/calibre/db/legacy.py | 5 ++- src/calibre/db/tests/legacy.py | 6 +++ src/calibre/library/database2.py | 68 ++------------------------------ 4 files changed, 77 insertions(+), 66 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 47ceb996ea..a07fb8b5a3 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -54,6 +54,70 @@ def _get_series_values(val): pass return (val, None) +def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None): + ''' + Return all metadata stored in the database as a dict. Includes paths to + the cover and each format. + + :param prefix: The prefix for all paths. By default, the prefix is the absolute path + to the library folder. + :param ids: Set of ids to return the data for. If None return data for + all entries in database. + ''' + import os + from calibre.ebooks.metadata import authors_to_string + backend = getattr(self, 'backend', self) # Works with both old and legacy interfaces + if prefix is None: + prefix = backend.library_path + fdata = backend.custom_column_num_map + + FIELDS = set(['title', 'sort', 'authors', 'author_sort', 'publisher', + 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', + 'series_index', 'uuid', 'pubdate', 'last_modified', 'identifiers', + 'languages']).union(set(fdata)) + for x, data in fdata.iteritems(): + if data['datatype'] == 'series': + FIELDS.add('%d_index'%x) + data = [] + for record in self.data: + if record is None: + continue + db_id = record[self.FIELD_MAP['id']] + if ids is not None and db_id not in ids: + continue + x = {} + for field in FIELDS: + x[field] = record[self.FIELD_MAP[field]] + data.append(x) + x['id'] = db_id + x['formats'] = [] + isbn = self.isbn(db_id, index_is_id=True) + x['isbn'] = isbn if isbn else '' + if not x['authors']: + x['authors'] = _('Unknown') + x['authors'] = [i.replace('|', ',') for i in x['authors'].split(',')] + if authors_as_string: + x['authors'] = authors_to_string(x['authors']) + x['tags'] = [i.replace('|', ',').strip() for i in x['tags'].split(',')] if x['tags'] else [] + path = os.path.join(prefix, self.path(record[self.FIELD_MAP['id']], index_is_id=True)) + x['cover'] = os.path.join(path, 'cover.jpg') + if not record[self.FIELD_MAP['cover']]: + x['cover'] = None + formats = self.formats(record[self.FIELD_MAP['id']], index_is_id=True) + if formats: + for fmt in formats.split(','): + path = self.format_abspath(x['id'], fmt, index_is_id=True) + if path is None: + continue + if prefix != self.library_path: + path = os.path.relpath(path, self.library_path) + path = os.path.join(prefix, path) + x['formats'].append(path) + x['fmt_'+fmt.lower()] = path + x['available_formats'] = [i.upper() for i in formats.split(',')] + + return data + ''' Rewrite of the calibre database backend. diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index aeab6e3de0..9c6071283e 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -11,7 +11,7 @@ from future_builtins import zip from calibre import force_unicode, isbytestring from calibre.constants import preferred_encoding -from calibre.db import _get_next_series_num_for_list, _get_series_values +from calibre.db import _get_next_series_num_for_list, _get_series_values, get_data_as_dict from calibre.db.adding import ( find_books_in_directory, import_book_directory_multiple, import_book_directory, recursive_import, add_catalog, add_news) @@ -35,7 +35,6 @@ def cleanup_tags(tags): ans.append(tag) return ans - class LibraryDatabase(object): ''' Emulate the old LibraryDatabase2 interface ''' @@ -746,6 +745,7 @@ LibraryDatabase.isbn = MT( lambda self, index, index_is_id=False: self.get_identifiers(index, index_is_id=index_is_id).get('isbn', None)) LibraryDatabase.get_books_for_category = MT( lambda self, category, id_:self.new_api.get_books_for_category(category, id_)) +LibraryDatabase.get_data_as_dict = MT(get_data_as_dict) # }}} # Legacy setter API {{{ @@ -890,3 +890,4 @@ del MT + diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index e37d954225..3e1165e3f2 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -246,6 +246,12 @@ class LegacyTest(BaseTest): db.copy_format_to(1, 'FMT1', d1, True) ndb.copy_format_to(1, 'FMT1', d2, True) self.assertTrue(d1.getvalue() == d2.getvalue()) + old = db.get_data_as_dict(prefix='test-prefix') + new = ndb.get_data_as_dict(prefix='test-prefix') + for o, n in zip(old, new): + o = {type('')(k) if isinstance(k, bytes) else k:set(v) if isinstance(v, list) else v for k, v in o.iteritems()} + n = {k:set(v) if isinstance(v, list) else v for k, v in n.iteritems()} + self.assertEqual(o, n) db.close() # }}} diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 4ccba4d9fc..e0cd9c613e 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -7,14 +7,14 @@ __docformat__ = 'restructuredtext en' The database used to store ebook metadata ''' import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \ - json, uuid, hashlib, copy + json, uuid, hashlib, copy, types from collections import defaultdict import threading, random from itertools import repeat from calibre import prints, force_unicode from calibre.ebooks.metadata import (title_sort, author_to_author_sort, - string_to_authors, authors_to_string, get_title_sort_pat) + string_to_authors, get_title_sort_pat) from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.library.database import LibraryDatabase from calibre.library.field_metadata import FieldMetadata, TagsIcons @@ -41,7 +41,7 @@ from calibre.ebooks import check_ebook_format from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.formatter_functions import load_user_template_functions -from calibre.db import _get_next_series_num_for_list, _get_series_values +from calibre.db import _get_next_series_num_for_list, _get_series_values, get_data_as_dict from calibre.db.adding import find_books_in_directory, import_book_directory_multiple, import_book_directory, recursive_import from calibre.db.errors import NoSuchFormat from calibre.db.lazy import FormatMetadata, FormatsList @@ -135,6 +135,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): read_only=False, is_second_db=False, progress_callback=None, restore_all_prefs=False): self.is_second_db = is_second_db + self.get_data_as_dict = types.MethodType(get_data_as_dict, self, LibraryDatabase2) try: if isbytestring(library_path): library_path = library_path.decode(filesystem_encoding) @@ -3619,67 +3620,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for i in iter(self): yield i[x] - def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None): - ''' - Return all metadata stored in the database as a dict. Includes paths to - the cover and each format. - - :param prefix: The prefix for all paths. By default, the prefix is the absolute path - to the library folder. - :param ids: Set of ids to return the data for. If None return data for - all entries in database. - ''' - if prefix is None: - prefix = self.library_path - fdata = self.custom_column_num_map - - FIELDS = set(['title', 'sort', 'authors', 'author_sort', 'publisher', - 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', - 'series_index', 'uuid', 'pubdate', 'last_modified', 'identifiers', - 'languages']).union(set(fdata)) - for x, data in fdata.iteritems(): - if data['datatype'] == 'series': - FIELDS.add('%d_index'%x) - data = [] - for record in self.data: - if record is None: - continue - db_id = record[self.FIELD_MAP['id']] - if ids is not None and db_id not in ids: - continue - x = {} - for field in FIELDS: - x[field] = record[self.FIELD_MAP[field]] - data.append(x) - x['id'] = db_id - x['formats'] = [] - isbn = self.isbn(db_id, index_is_id=True) - x['isbn'] = isbn if isbn else '' - if not x['authors']: - x['authors'] = _('Unknown') - x['authors'] = [i.replace('|', ',') for i in x['authors'].split(',')] - if authors_as_string: - x['authors'] = authors_to_string(x['authors']) - x['tags'] = [i.replace('|', ',').strip() for i in x['tags'].split(',')] if x['tags'] else [] - path = os.path.join(prefix, self.path(record[self.FIELD_MAP['id']], index_is_id=True)) - x['cover'] = os.path.join(path, 'cover.jpg') - if not record[self.FIELD_MAP['cover']]: - x['cover'] = None - formats = self.formats(record[self.FIELD_MAP['id']], index_is_id=True) - if formats: - for fmt in formats.split(','): - path = self.format_abspath(x['id'], fmt, index_is_id=True) - if path is None: - continue - if prefix != self.library_path: - path = os.path.relpath(path, self.library_path) - path = os.path.join(prefix, path) - x['formats'].append(path) - x['fmt_'+fmt.lower()] = path - x['available_formats'] = [i.upper() for i in formats.split(',')] - - return data - def migrate_old(self, db, progress): from PyQt4.QtCore import QCoreApplication header = _(u'

    Migrating old database to ebook library in %s

    ')%self.library_path From 34704c9735c1ce8a1d29ea580e205d19b28807e7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 11:37:46 +0530 Subject: [PATCH 0242/1154] find_identical_books() --- src/calibre/db/cache.py | 62 +++++++++++++++++++++++++++++++--- src/calibre/db/legacy.py | 1 + src/calibre/db/search.py | 10 +++--- src/calibre/db/tests/legacy.py | 1 + 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 46dc5df6d7..19e8d52134 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback, random, shutil +import os, traceback, random, shutil, re from io import BytesIO from collections import defaultdict from functools import wraps, partial @@ -25,7 +25,7 @@ from calibre.db.tables import VirtualTable from calibre.db.write import get_series_values from calibre.db.lazy import FormatMetadata, FormatsList from calibre.ebooks import check_ebook_format -from calibre.ebooks.metadata import string_to_authors, author_to_author_sort +from calibre.ebooks.metadata import string_to_authors, author_to_author_sort, get_title_sort_pat from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ptempfile import (base_dir, PersistentTemporaryFile, @@ -767,9 +767,8 @@ class Cache(object): return sorted(all_book_ids, key=partial(SortKey, fields, sort_keys)) @read_api - def search(self, query, restriction, virtual_fields=None): - return self._search_api(self, query, restriction, - virtual_fields=virtual_fields) + def search(self, query, restriction='', virtual_fields=None, book_ids=None): + return self._search_api(self, query, restriction, virtual_fields=virtual_fields, book_ids=book_ids) @read_api def get_categories(self, sort='name', book_ids=None, icon_map=None): @@ -1452,6 +1451,59 @@ class Cache(object): return f.get_books_for_val(item_id_or_composite_value, self._get_metadata, self._all_book_ids()) return self._books_for_field(f.name, item_id_or_composite_value) + @read_api + def find_identical_books(self, mi, search_restriction='', book_ids=None): + ''' Finds books that have a superset of the authors in mi and the same + title (title is fuzzy matched) ''' + fuzzy_title_patterns = [(re.compile(pat, re.IGNORECASE) if + isinstance(pat, basestring) else pat, repl) for pat, repl in + [ + (r'[\[\](){}<>\'";,:#]', ''), + (get_title_sort_pat(), ''), + (r'[-._]', ' '), + (r'\s+', ' ') + ] + ] + + def fuzzy_title(title): + title = icu_lower(title.strip()) + for pat, repl in fuzzy_title_patterns: + title = pat.sub(repl, title) + return title + + identical_book_ids = set() + if mi.authors: + try: + quathors = mi.authors[:20] # Too many authors causes parsing of + # the search expression to fail + query = ' and '.join('authors:"=%s"'%(a.replace('"', '')) for a in quathors) + qauthors = mi.authors[20:] + except ValueError: + return identical_book_ids + try: + book_ids = self._search(query, restriction=search_restriction, book_ids=book_ids) + except: + traceback.print_exc() + return identical_book_ids + if qauthors and book_ids: + matches = set() + qauthors = {icu_lower(x) for x in qauthors} + for book_id in book_ids: + aut = self._field_for('authors', book_id) + if aut: + aut = {icu_lower(x) for x in aut} + if aut.issuperset(qauthors): + matches.add(book_id) + book_ids = matches + + for book_id in book_ids: + fbook_title = self._field_for('title', book_id) + fbook_title = fuzzy_title(fbook_title) + mbook_title = fuzzy_title(mi.title) + if fbook_title == mbook_title: + identical_book_ids.add(book_id) + return identical_book_ids + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 9c6071283e..48d1c8b78c 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -746,6 +746,7 @@ LibraryDatabase.isbn = MT( LibraryDatabase.get_books_for_category = MT( lambda self, category, id_:self.new_api.get_books_for_category(category, id_)) LibraryDatabase.get_data_as_dict = MT(get_data_as_dict) +LibraryDatabase.find_identical_books = MT(lambda self, mi:self.new_api.find_identical_books(mi)) # }}} # Legacy setter API {{{ diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 7b4ad90bc3..fbe1515920 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -661,7 +661,7 @@ class Search(object): def change_locations(self, newlocs): self.all_search_locations = newlocs - def __call__(self, dbcache, query, search_restriction, virtual_fields=None): + def __call__(self, dbcache, query, search_restriction, virtual_fields=None, book_ids=None): ''' Return the set of ids of all records that match the specified query and restriction @@ -674,17 +674,15 @@ class Search(object): if search_restriction: q = u'(%s) and (%s)' % (search_restriction, query) - all_book_ids = dbcache._all_book_ids(type=set) + all_book_ids = dbcache._all_book_ids(type=set) if book_ids is None else set(book_ids) if not q: return all_book_ids if not isinstance(q, type(u'')): q = q.decode('utf-8') - # We construct a new parser instance per search as pyparsing is not - # thread safe. On my desktop, constructing a SearchQueryParser instance - # takes 0.000975 seconds and restoring it from a pickle takes - # 0.000974 seconds. + # We construct a new parser instance per search as the parse is not + # thread safe. sqp = Parser( dbcache, all_book_ids, dbcache._pref('grouped_search_terms'), self.date_search, self.num_search, self.bool_search, diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 3e1165e3f2..d3a672bee5 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -166,6 +166,7 @@ class LegacyTest(BaseTest): self.assertEqual(dict(db.prefs), dict(ndb.prefs)) for meth, args in { + 'find_identical_books': [(Metadata('title one', ['author one']),), (Metadata('unknown'),), (Metadata('xxxx'),)], 'get_books_for_category': [('tags', newstag), ('#formats', 'FMT1')], 'get_next_series_num_for': [('A Series One',)], 'get_id_from_uuid':[('ddddd',), (db.uuid(1, True),)], From 467dcf1a8711969358a7758c7c9a0e620a3d5b99 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 12:11:07 +0530 Subject: [PATCH 0243/1154] API coverage is now 100% --- src/calibre/db/backend.py | 53 ++++++++++++++++++++++++++++++ src/calibre/db/cache.py | 12 +++++++ src/calibre/db/legacy.py | 2 ++ src/calibre/db/tests/filesystem.py | 17 ++++++++++ src/calibre/db/tests/legacy.py | 1 + src/calibre/library/database2.py | 6 ++-- 6 files changed, 89 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index fc7d556dc0..4f33a917fa 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1452,6 +1452,59 @@ class DB(object): options = [(book_id, fmt.upper(), buffer(cPickle.dumps(data, -1))) for book_id, data in options.iteritems()] self.conn.executemany('INSERT OR REPLACE INTO conversion_options(book,format,data) VALUES (?,?,?)', options) + def get_top_level_move_items(self, all_paths): + items = set(os.listdir(self.library_path)) + paths = set(all_paths) + paths.update({'metadata.db', 'metadata_db_prefs_backup.json'}) + path_map = {x:x for x in paths} + if not self.is_case_sensitive: + for x in items: + path_map[x.lower()] = x + items = set(path_map) + paths = {x.lower() for x in paths} + items = items.intersection(paths) + return items, path_map + + def move_library_to(self, all_paths, newloc, progress=lambda x: x): + if not os.path.exists(newloc): + os.makedirs(newloc) + old_dirs = set() + items, path_map = self.get_top_level_move_items(all_paths) + for x in items: + src = os.path.join(self.library_path, x) + dest = os.path.join(newloc, path_map[x]) + if os.path.isdir(src): + if os.path.exists(dest): + shutil.rmtree(dest) + shutil.copytree(src, dest) + old_dirs.add(src) + else: + if os.path.exists(dest): + os.remove(dest) + shutil.copyfile(src, dest) + x = path_map[x] + if not isinstance(x, unicode): + x = x.decode(filesystem_encoding, 'replace') + progress(x) + + dbpath = os.path.join(newloc, os.path.basename(self.dbpath)) + opath = self.dbpath + self.conn.close() + self.library_path, self.dbpath = newloc, dbpath + if self._conn is not None: + self._conn.close() + self._conn = None + self.conn + try: + os.unlink(opath) + except: + pass + for loc in old_dirs: + try: + shutil.rmtree(loc) + except: + pass + # }}} diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 19e8d52134..4faa819b42 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1504,6 +1504,18 @@ class Cache(object): identical_book_ids.add(book_id) return identical_book_ids + @read_api + def get_top_level_move_items(self): + all_paths = {self._field_for('path', book_id).partition('/')[0] for book_id in self._all_book_ids()} + return self.backend.get_top_level_move_items(all_paths) + + @write_api + def move_library_to(self, newloc, progress=None): + if progress is None: + progress = lambda x:x + all_paths = {self._field_for('path', book_id).partition('/')[0] for book_id in self._all_book_ids()} + self.backend.move_library_to(all_paths, newloc, progress=progress) + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 48d1c8b78c..97babb3b60 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -747,6 +747,7 @@ LibraryDatabase.get_books_for_category = MT( lambda self, category, id_:self.new_api.get_books_for_category(category, id_)) LibraryDatabase.get_data_as_dict = MT(get_data_as_dict) LibraryDatabase.find_identical_books = MT(lambda self, mi:self.new_api.find_identical_books(mi)) +LibraryDatabase.get_top_level_move_items = MT(lambda self:self.new_api.get_top_level_move_items()) # }}} # Legacy setter API {{{ @@ -878,6 +879,7 @@ for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'): return func setattr(LibraryDatabase, meth, MT(getter(meth))) +LibraryDatabase.move_library_to = MT(lambda self, newloc, progress=None:self.new_api.move_library_to(newloc, progress=progress)) # Cleaning is not required anymore LibraryDatabase.clean = LibraryDatabase.clean_custom = MT(lambda self:None) LibraryDatabase.clean_standard_field = MT(lambda self, field, commit=False:None) diff --git a/src/calibre/db/tests/filesystem.py b/src/calibre/db/tests/filesystem.py index 168eec53a4..c99f5ad512 100644 --- a/src/calibre/db/tests/filesystem.py +++ b/src/calibre/db/tests/filesystem.py @@ -79,4 +79,21 @@ class FilesystemTest(BaseTest): f.close() self.assertNotEqual(cache.field_for('title', 1), 'Moved', 'Title was changed despite file lock') + def test_library_move(self): + ' Test moving of library ' + from calibre.ptempfile import TemporaryDirectory + cache = self.init_cache() + self.assertIn('metadata.db', cache.get_top_level_move_items()[0]) + all_ids = cache.all_book_ids() + fmt1 = cache.format(1, 'FMT1') + cov = cache.cover(1) + with TemporaryDirectory('moved_lib') as tdir: + cache.move_library_to(tdir) + self.assertIn('moved_lib', cache.backend.library_path) + self.assertIn('moved_lib', cache.backend.dbpath) + cache.reload_from_db() + self.assertEqual(all_ids, cache.all_book_ids()) + self.assertEqual(fmt1, cache.format(1, 'FMT1')) + self.assertEqual(cov, cache.cover(1)) + cache.backend.close() diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index d3a672bee5..3e1a1457b3 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -167,6 +167,7 @@ class LegacyTest(BaseTest): for meth, args in { 'find_identical_books': [(Metadata('title one', ['author one']),), (Metadata('unknown'),), (Metadata('xxxx'),)], + 'get_top_level_move_items': [()], 'get_books_for_category': [('tags', newstag), ('#formats', 'FMT1')], 'get_next_series_num_for': [('A Series One',)], 'get_id_from_uuid':[('ddddd',), (db.uuid(1, True),)], diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index e0cd9c613e..9022605024 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -3561,7 +3561,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): path = self.path(x, index_is_id=True) path = path.split(os.sep)[0] paths.add(path) - paths.add('metadata.db') + paths.update({'metadata.db', 'metadata_db_prefs_backup.json'}) path_map = {} for x in paths: path_map[x] = x @@ -3573,7 +3573,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): items = items.intersection(paths) return items, path_map - def move_library_to(self, newloc, progress=lambda x: x): + def move_library_to(self, newloc, progress=None): + if progress is None: + progress = lambda x:x if not os.path.exists(newloc): os.makedirs(newloc) old_dirs = set([]) From 72849dfee67665328ffab092382dd8a62369a304 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 12:12:14 +0530 Subject: [PATCH 0244/1154] ... --- src/calibre/db/tests/filesystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/tests/filesystem.py b/src/calibre/db/tests/filesystem.py index c99f5ad512..5367f62235 100644 --- a/src/calibre/db/tests/filesystem.py +++ b/src/calibre/db/tests/filesystem.py @@ -91,9 +91,9 @@ class FilesystemTest(BaseTest): cache.move_library_to(tdir) self.assertIn('moved_lib', cache.backend.library_path) self.assertIn('moved_lib', cache.backend.dbpath) - cache.reload_from_db() - self.assertEqual(all_ids, cache.all_book_ids()) self.assertEqual(fmt1, cache.format(1, 'FMT1')) self.assertEqual(cov, cache.cover(1)) + cache.reload_from_db() + self.assertEqual(all_ids, cache.all_book_ids()) cache.backend.close() From b60530fafef280db26467e1576c87186bd3b1078 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 13:28:27 +0530 Subject: [PATCH 0245/1154] Implement saved searches API --- src/calibre/db/cache.py | 12 ++++++++---- src/calibre/db/legacy.py | 4 ++++ src/calibre/db/search.py | 16 ++++++++++------ src/calibre/db/tests/reading.py | 2 -- src/calibre/utils/search_query_parser.py | 7 ++++--- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 4faa819b42..c2d094eef0 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -89,7 +89,7 @@ class Cache(object): self.formatter_template_cache = {} self.dirtied_cache = {} self.dirtied_sequence = 0 - self._search_api = Search(self.field_metadata.get_search_terms()) + self._search_api = Search(self, 'saved_searches', self.field_metadata.get_search_terms()) # Implement locking for all simple read/write API methods # An unlocked version of the method is stored with the name starting @@ -127,9 +127,8 @@ class Cache(object): except: traceback.print_exc() - # TODO: Saved searches - # if len(saved_searches().names()): - # self.field_metadata.add_search_category(label='search', name=_('Searches')) + if len(self._search_api.get_saved_searches().names()): + self.field_metadata.add_search_category(label='search', name=_('Searches')) self.field_metadata.add_grouped_search_terms( self._pref('grouped_search_terms', {})) @@ -141,6 +140,11 @@ class Cache(object): if self.dirtied_cache: self.dirtied_sequence = max(self.dirtied_cache.itervalues())+1 + @property + def prefs(self): + 'For internal use only (used by SavedSearchQueries). For thread-safe access to the preferences, use the pref() and set_pref() methods.' + return self.backend.prefs + @write_api def initialize_template_cache(self): self.formatter_template_cache = {} diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 97babb3b60..b938e7dddf 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -22,6 +22,7 @@ from calibre.db.categories import CATEGORY_SORTS from calibre.db.view import View from calibre.db.write import clean_identifier from calibre.utils.date import utcnow +from calibre.utils.search_query_parser import set_saved_searches def cleanup_tags(tags): tags = [x.strip().replace(',', ';') for x in tags if x.strip()] @@ -73,6 +74,9 @@ class LibraryDatabase(object): self.last_update_check = self.last_modified() + if not self.is_second_db: + set_saved_searches(self, 'saved_searches') + def close(self): self.backend.close() diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index fbe1515920..013678e3b3 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -15,7 +15,7 @@ from calibre.utils.config_base import prefs from calibre.utils.date import parse_date, UNDEFINED_DATE, now from calibre.utils.icu import primary_find from calibre.utils.localization import lang_map, canonicalize_lang -from calibre.utils.search_query_parser import SearchQueryParser, ParseException +from calibre.utils.search_query_parser import SearchQueryParser, ParseException, SavedSearchQueries CONTAINS_MATCH = 0 EQUALS_MATCH = 1 @@ -392,7 +392,7 @@ class Parser(SearchQueryParser): def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, bool_search, keypair_search, limit_search_columns, limit_search_columns_to, - locations, virtual_fields): + locations, virtual_fields, get_saved_searches): self.dbcache, self.all_book_ids = dbcache, all_book_ids self.all_search_locations = frozenset(locations) self.grouped_search_terms = gst @@ -403,7 +403,7 @@ class Parser(SearchQueryParser): self.virtual_fields = virtual_fields or {} if 'marked' not in self.virtual_fields: self.virtual_fields['marked'] = self - super(Parser, self).__init__(locations, optimize=True) + super(Parser, self).__init__(locations, optimize=True, get_saved_searches=get_saved_searches) @property def field_metadata(self): @@ -651,12 +651,16 @@ class Parser(SearchQueryParser): class Search(object): - def __init__(self, all_search_locations=()): + def __init__(self, db, opt_name, all_search_locations=()): self.all_search_locations = all_search_locations self.date_search = DateSearch() self.num_search = NumericSearch() self.bool_search = BooleanSearch() self.keypair_search = KeyPairSearch() + self.saved_searches = SavedSearchQueries(db, opt_name) + + def get_saved_searches(self): + return self.saved_searches def change_locations(self, newlocs): self.all_search_locations = newlocs @@ -689,11 +693,11 @@ class Search(object): self.keypair_search, prefs['limit_search_columns'], prefs['limit_search_columns_to'], self.all_search_locations, - virtual_fields) + virtual_fields, self.get_saved_searches) try: ret = sqp.parse(q) finally: - sqp.dbcache = None + sqp.dbcache = sqp.get_saved_searches = None return ret diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 24d80d33c7..fcf309ea66 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -149,8 +149,6 @@ class ReadingTest(BaseTest): '#tags':[3, 2, 1], '#yesno':[3, 1, 2], '#comments':[3, 2, 1], - # TODO: Add an empty book to the db and ensure that empty - # fields sort the same as they do in db2 }.iteritems(): x = list(reversed(order)) self.assertEqual(order, cache.multisort([(field, True)], diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index 2682088681..dc2a7b51b8 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -40,7 +40,7 @@ class SavedSearchQueries(object): self.queries = {} try: self._db = weakref.ref(db) - except: + except TypeError: # db could be None self._db = lambda : None @@ -292,9 +292,10 @@ class SearchQueryParser(object): failed.append(test[0]) return failed - def __init__(self, locations, test=False, optimize=False): + def __init__(self, locations, test=False, optimize=False, get_saved_searches=None): self.sqp_initialize(locations, test=test, optimize=optimize) self.parser = Parser() + self.get_saved_searches = saved_searches if get_saved_searches is None else get_saved_searches def sqp_change_locations(self, locations): self.sqp_initialize(locations, optimize=self.optimize) @@ -367,7 +368,7 @@ class SearchQueryParser(object): raise ParseException(_('Recursive saved search: {0}').format(query)) if self.recurse_level > 5: self.searches_seen.add(query) - return self._parse(saved_searches().lookup(query), candidates) + return self._parse(self.get_saved_searches().lookup(query), candidates) except ParseException as e: raise e except: # convert all exceptions (e.g., missing key) to a parse error From 437d5413062920b21e57a69399a93476515b3681 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 13:56:55 +0530 Subject: [PATCH 0246/1154] Migrate the rest of calibre to use the new saved searches API --- src/calibre/db/categories.py | 3 +-- src/calibre/db/legacy.py | 3 +++ src/calibre/gui2/dialogs/saved_search_editor.py | 3 ++- src/calibre/gui2/search_box.py | 6 +++++- src/calibre/gui2/search_restriction_mixin.py | 4 +++- src/calibre/gui2/tag_browser/model.py | 7 +++---- src/calibre/gui2/tag_browser/view.py | 2 +- src/calibre/gui2/ui.py | 10 ++++++++++ src/calibre/library/cli.py | 4 +--- src/calibre/library/database2.py | 3 +++ src/calibre/library/server/base.py | 3 +-- 11 files changed, 33 insertions(+), 15 deletions(-) diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py index 3f7bbb9e61..df6c1402d2 100644 --- a/src/calibre/db/categories.py +++ b/src/calibre/db/categories.py @@ -16,7 +16,6 @@ from calibre.ebooks.metadata import author_to_author_sort from calibre.library.field_metadata import TagsIcons from calibre.utils.config_base import tweaks from calibre.utils.icu import sort_key -from calibre.utils.search_query_parser import saved_searches CATEGORY_SORTS = ('name', 'popularity', 'rating') # This has to be a tuple not a set @@ -229,7 +228,7 @@ def get_categories(dbcache, sort='name', book_ids=None, icon_map=None): icon = None if icon_map and 'search' in icon_map: icon = icon_map['search'] - ss = saved_searches() + ss = dbcache._search_api.get_saved_searches() for srch in ss.names(): items.append(Tag(srch, tooltip=ss.lookup(srch), sort=srch, icon=icon, category='search', diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index b938e7dddf..838ccdfe21 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -120,6 +120,9 @@ class LibraryDatabase(object): self.new_api.reload_from_db() self.last_update_check = utcnow() + def get_saved_searches(self): + return self.new_api._search_api.get_saved_searches() + @property def custom_column_num_map(self): return self.backend.custom_column_num_map diff --git a/src/calibre/gui2/dialogs/saved_search_editor.py b/src/calibre/gui2/dialogs/saved_search_editor.py index c9f843109a..669771e46e 100644 --- a/src/calibre/gui2/dialogs/saved_search_editor.py +++ b/src/calibre/gui2/dialogs/saved_search_editor.py @@ -7,7 +7,6 @@ from PyQt4.QtCore import SIGNAL from PyQt4.QtGui import QDialog from calibre.gui2.dialogs.saved_search_editor_ui import Ui_SavedSearchEditor -from calibre.utils.search_query_parser import saved_searches from calibre.utils.icu import sort_key from calibre.gui2 import error_dialog from calibre.gui2.dialogs.confirm_delete import confirm @@ -15,6 +14,7 @@ from calibre.gui2.dialogs.confirm_delete import confirm class SavedSearchEditor(QDialog, Ui_SavedSearchEditor): def __init__(self, parent, initial_search=None): + from calibre.gui2.ui import saved_searches QDialog.__init__(self, parent) Ui_SavedSearchEditor.__init__(self) self.setupUi(self) @@ -98,6 +98,7 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor): self.search_text.setPlainText('') def accept(self): + from calibre.gui2.ui import saved_searches if self.current_search_name: self.searches[self.current_search_name] = unicode(self.search_text.toPlainText()) for name in saved_searches().names(): diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index 85ddf533c4..b10fdd40ef 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -18,7 +18,6 @@ from calibre.gui2 import config, error_dialog from calibre.gui2.dialogs.confirm_delete import confirm from calibre.gui2.dialogs.saved_search_editor import SavedSearchEditor from calibre.gui2.dialogs.search import SearchDialog -from calibre.utils.search_query_parser import saved_searches class SearchLineEdit(QLineEdit): # {{{ key_pressed = pyqtSignal(object) @@ -309,6 +308,7 @@ class SavedSearchBox(QComboBox): # {{{ self.saved_search_selected(self.currentText()) def saved_search_selected(self, qname): + from calibre.gui2.ui import saved_searches qname = unicode(qname) if qname is None or not qname.strip(): self.search_box.clear() @@ -322,12 +322,14 @@ class SavedSearchBox(QComboBox): # {{{ self.setToolTip(saved_searches().lookup(qname)) def initialize_saved_search_names(self): + from calibre.gui2.ui import saved_searches qnames = saved_searches().names() self.addItems(qnames) self.setCurrentIndex(-1) # SIGNALed from the main UI def save_search_button_clicked(self): + from calibre.gui2.ui import saved_searches name = unicode(self.currentText()) if not name.strip(): name = unicode(self.search_box.text()).replace('"', '') @@ -346,6 +348,7 @@ class SavedSearchBox(QComboBox): # {{{ self.changed.emit() def delete_current_search(self): + from calibre.gui2.ui import saved_searches idx = self.currentIndex() if idx <= 0: error_dialog(self, _('Delete current search'), @@ -365,6 +368,7 @@ class SavedSearchBox(QComboBox): # {{{ # SIGNALed from the main UI def copy_search_button_clicked(self): + from calibre.gui2.ui import saved_searches idx = self.currentIndex() if idx < 0: return diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index b986a2a78e..71e0f2f392 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -17,7 +17,6 @@ from calibre.gui2.widgets import ComboBoxWithHelp from calibre.utils.config_base import tweaks from calibre.utils.icu import sort_key from calibre.utils.search_query_parser import ParseException -from calibre.utils.search_query_parser import saved_searches class SelectNames(QDialog): # {{{ @@ -179,6 +178,7 @@ class CreateVirtualLibrary(QDialog): # {{{ self.resize(self.sizeHint()+QSize(150, 25)) def search_text_changed(self, txt): + from calibre.gui2.ui import saved_searches searches = [_('Saved searches recognized in the expression:')] txt = unicode(txt) while txt: @@ -234,6 +234,7 @@ class CreateVirtualLibrary(QDialog): # {{{ self.vl_text.setText(self.original_search) def link_activated(self, url): + from calibre.gui2.ui import saved_searches db = self.gui.current_db f, txt = unicode(url).partition('.')[0::2] if f == 'search': @@ -475,6 +476,7 @@ class SearchRestrictionMixin(object): return name[0:MAX_VIRTUAL_LIBRARY_NAME_LENGTH].strip() def build_search_restriction_list(self): + from calibre.gui2.ui import saved_searches m = self.ar_menu m.clear() diff --git a/src/calibre/gui2/tag_browser/model.py b/src/calibre/gui2/tag_browser/model.py index 33d1235f8b..7dba7cfe7d 100644 --- a/src/calibre/gui2/tag_browser/model.py +++ b/src/calibre/gui2/tag_browser/model.py @@ -21,7 +21,6 @@ from calibre.utils.icu import sort_key, lower, strcmp, collation_order from calibre.library.field_metadata import TagsIcons, category_icon_map from calibre.gui2.dialogs.confirm_delete import confirm from calibre.utils.formatter import EvalFormatter -from calibre.utils.search_query_parser import saved_searches TAG_SEARCH_STATES = {'clear': 0, 'mark_plus': 1, 'mark_plusplus': 2, 'mark_minus': 3, 'mark_minusminus': 4} @@ -879,7 +878,7 @@ class TagsModel(QAbstractItemModel): # {{{ traceback.print_exc() self.db.data.change_search_locations(self.db.field_metadata.get_search_terms()) - if len(saved_searches().names()): + if len(self.db.get_saved_searches().names()): tb_cats.add_search_category(label='search', name=_('Searches')) if self.filter_categories_by: @@ -1005,11 +1004,11 @@ class TagsModel(QAbstractItemModel): # {{{ _('Author names cannot contain & characters.')).exec_() return False if key == 'search': - if val in saved_searches().names(): + if val in self.db.get_saved_searches().names(): error_dialog(self.gui_parent, _('Duplicate search name'), _('The saved search name %s is already used.')%val).exec_() return False - saved_searches().rename(unicode(item.data(role).toString()), val) + self.db.get_saved_searches().rename(unicode(item.data(role).toString()), val) item.tag.name = val self.search_item_renamed.emit() # Does a refresh else: diff --git a/src/calibre/gui2/tag_browser/view.py b/src/calibre/gui2/tag_browser/view.py index cefa0f8975..d28b7ca848 100644 --- a/src/calibre/gui2/tag_browser/view.py +++ b/src/calibre/gui2/tag_browser/view.py @@ -20,7 +20,6 @@ from calibre.constants import config_dir from calibre.gui2.tag_browser.model import (TagTreeItem, TAG_SEARCH_STATES, TagsModel) from calibre.gui2 import config, gprefs, choose_files, pixmap_to_data -from calibre.utils.search_query_parser import saved_searches from calibre.utils.icu import sort_key class TagDelegate(QStyledItemDelegate): # {{{ @@ -355,6 +354,7 @@ class TagsView(QTreeView): # {{{ self.delete_user_category.emit(key) return if action == 'delete_search': + from calibre.gui2.ui import saved_searches saved_searches().delete(key) self.rebuild_saved_searches.emit() return diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 229ed0933d..e54d06e671 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -98,6 +98,16 @@ _gui = None def get_gui(): return _gui +def saved_searches(): + 'Return the saved searches defined in the currently open library' + try: + return _gui.library_view.model().db.get_saved_searches() + except AttributeError: + # Happens during initialization of the gui + from calibre.utils.search_query_parser import saved_searches + return saved_searches() + + class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin, SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin, diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 547cc5bc08..7e7a234724 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -1028,10 +1028,8 @@ def command_saved_searches(args, dbpath): print prints(_('Error: You must specify an action (add|remove|list)'), file=sys.stderr) return 1 - from calibre.utils.search_query_parser import saved_searches db = get_db(dbpath, opts) - db - ss = saved_searches() + ss = db.get_saved_searches() if args[0] == 'list': for name in ss.names(): prints(_('Name:'), name) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 9022605024..8ea7e75b59 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -537,6 +537,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if self.user_version == 0: self.user_version = 1 + def get_saved_searches(self): + return saved_searches() + def last_modified(self): ''' Return last modified time as a UTC datetime object''' return utcfromtimestamp(os.stat(self.dbpath).st_mtime) diff --git a/src/calibre/library/server/base.py b/src/calibre/library/server/base.py index bbd5239b42..a677b991f9 100644 --- a/src/calibre/library/server/base.py +++ b/src/calibre/library/server/base.py @@ -25,7 +25,6 @@ from calibre.library.server.opds import OPDSServer from calibre.library.server.cache import Cache from calibre.library.server.browse import BrowseServer from calibre.library.server.ajax import AjaxServer -from calibre.utils.search_query_parser import saved_searches from calibre import prints, as_unicode @@ -210,7 +209,7 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache, if sr: if sr in virt_libs: sr = virt_libs[sr] - elif sr not in saved_searches().names(): + elif sr not in self.db.get_saved_searches().names(): prints('WARNING: Content server: search restriction ', sr, ' does not exist') sr = '' From 96d3c4d2325cf730a2aae954cc8a2a6312fa4d28 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 17:07:40 +0530 Subject: [PATCH 0247/1154] Refactor saved search API to make it thread safe --- src/calibre/db/backend.py | 30 ++------ src/calibre/db/cache.py | 35 +++++++-- src/calibre/db/categories.py | 9 +-- src/calibre/db/legacy.py | 9 ++- src/calibre/db/search.py | 76 +++++++++++++++++-- src/calibre/db/tests/legacy.py | 21 +++++ .../gui2/dialogs/saved_search_editor.py | 18 ++--- src/calibre/gui2/search_box.py | 37 +++++---- src/calibre/gui2/search_restriction_mixin.py | 15 ++-- src/calibre/gui2/tag_browser/model.py | 6 +- src/calibre/gui2/tag_browser/view.py | 3 +- src/calibre/gui2/ui.py | 12 +-- src/calibre/library/cli.py | 9 +-- src/calibre/library/database2.py | 19 ++++- src/calibre/library/server/base.py | 2 +- src/calibre/utils/search_query_parser.py | 14 +++- 16 files changed, 213 insertions(+), 102 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 4f33a917fa..3241493af1 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -70,6 +70,10 @@ class DBPrefs(dict): # {{{ self.db = db self.defaults = {} self.disable_setting = False + self.load_from_db() + + def load_from_db(self): + self.clear() for key, val in self.db.conn.get('SELECT key,val FROM preferences'): try: val = self.raw_to_object(val) @@ -136,28 +140,10 @@ class DBPrefs(dict): # {{{ @classmethod def read_serialized(cls, library_path, recreate_prefs=False): - try: - from_filename = os.path.join(library_path, - 'metadata_db_prefs_backup.json') - with open(from_filename, "rb") as f: - d = json.load(f, object_hook=from_json) - if not recreate_prefs: - return d - cls.clear() - cls.db.conn.execute('DELETE FROM preferences') - for k,v in d.iteritems(): - raw = cls.to_raw(v) - cls.db.conn.execute( - 'INSERT INTO preferences (key,val) VALUES (?,?)', (k, raw)) - cls.db.conn.commit() - cls.clear() - cls.update(d) - return d - except: - import traceback - traceback.print_exc() - raise - return None + from_filename = os.path.join(library_path, + 'metadata_db_prefs_backup.json') + with open(from_filename, "rb") as f: + return json.load(f, object_hook=from_json) # }}} # Extra collators {{{ diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index c2d094eef0..ce8ce288ea 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -89,7 +89,6 @@ class Cache(object): self.formatter_template_cache = {} self.dirtied_cache = {} self.dirtied_sequence = 0 - self._search_api = Search(self, 'saved_searches', self.field_metadata.get_search_terms()) # Implement locking for all simple read/write API methods # An unlocked version of the method is stored with the name starting @@ -105,6 +104,7 @@ class Cache(object): lock = self.read_lock if ira else self.write_lock setattr(self, name, wrap_simple(lock, func)) + self._search_api = Search(self, 'saved_searches', self.field_metadata.get_search_terms()) self.initialize_dynamic() @write_api @@ -127,7 +127,7 @@ class Cache(object): except: traceback.print_exc() - if len(self._search_api.get_saved_searches().names()): + if len(self._search_api.saved_searches.names()) > 0: self.field_metadata.add_search_category(label='search', name=_('Searches')) self.field_metadata.add_grouped_search_terms( @@ -140,11 +140,6 @@ class Cache(object): if self.dirtied_cache: self.dirtied_sequence = max(self.dirtied_cache.itervalues())+1 - @property - def prefs(self): - 'For internal use only (used by SavedSearchQueries). For thread-safe access to the preferences, use the pref() and set_pref() methods.' - return self.backend.prefs - @write_api def initialize_template_cache(self): self.formatter_template_cache = {} @@ -161,6 +156,8 @@ class Cache(object): def reload_from_db(self, clear_caches=True): if clear_caches: self._clear_caches() + self.backend.prefs.load_from_db() + self._search_api.saved_searches.load_from_db() for field in self.fields.itervalues(): if hasattr(field, 'table'): field.table.read(self.backend) # Reread data from metadata.db @@ -1520,6 +1517,30 @@ class Cache(object): all_paths = {self._field_for('path', book_id).partition('/')[0] for book_id in self._all_book_ids()} self.backend.move_library_to(all_paths, newloc, progress=progress) + @read_api + def saved_search_names(self): + return self._search_api.saved_searches.names() + + @read_api + def saved_search_lookup(self, name): + return self._search_api.saved_searches.lookup(name) + + @write_api + def saved_search_set_all(self, smap): + self._search_api.saved_searches.set_all(smap) + + @write_api + def saved_search_delete(self, name): + self._search_api.saved_searches.delete(name) + + @write_api + def saved_search_add(self, name, val): + self._search_api.saved_searches.add(name, val) + + @write_api + def saved_search_rename(self, old_name, new_name): + self._search_api.saved_searches.rename(old_name, new_name) + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py index df6c1402d2..f80bb52c08 100644 --- a/src/calibre/db/categories.py +++ b/src/calibre/db/categories.py @@ -228,11 +228,10 @@ def get_categories(dbcache, sort='name', book_ids=None, icon_map=None): icon = None if icon_map and 'search' in icon_map: icon = icon_map['search'] - ss = dbcache._search_api.get_saved_searches() - for srch in ss.names(): - items.append(Tag(srch, tooltip=ss.lookup(srch), - sort=srch, icon=icon, category='search', - is_editable=False)) + queries = dbcache._search_api.saved_searches.queries + for srch in sorted(queries, key=sort_key): + items.append(Tag(srch, tooltip=queries[srch], sort=srch, icon=icon, + category='search', is_editable=False)) if len(items): categories['search'] = items diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 838ccdfe21..9de6d5ce1f 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -120,9 +120,6 @@ class LibraryDatabase(object): self.new_api.reload_from_db() self.last_update_check = utcnow() - def get_saved_searches(self): - return self.new_api._search_api.get_saved_searches() - @property def custom_column_num_map(self): return self.backend.custom_column_num_map @@ -887,6 +884,12 @@ for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'): setattr(LibraryDatabase, meth, MT(getter(meth))) LibraryDatabase.move_library_to = MT(lambda self, newloc, progress=None:self.new_api.move_library_to(newloc, progress=progress)) +LibraryDatabase.saved_search_names = MT(lambda self:self.new_api.saved_search_names()) +LibraryDatabase.saved_search_lookup = MT(lambda self, x:self.new_api.saved_search_lookup(x)) +LibraryDatabase.saved_search_set_all = MT(lambda self, smap:self.new_api.saved_search_set_all(smap)) +LibraryDatabase.saved_search_delete = MT(lambda self, x:self.new_api.saved_search_delete(x)) +LibraryDatabase.saved_search_add = MT(lambda self, x, y:self.new_api.saved_search_add(x, y)) +LibraryDatabase.saved_search_rename = MT(lambda self, x, y:self.new_api.saved_search_rename(x, y)) # Cleaning is not required anymore LibraryDatabase.clean = LibraryDatabase.clean_custom = MT(lambda self:None) LibraryDatabase.clean_standard_field = MT(lambda self, field, commit=False:None) diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 013678e3b3..f00b7102e7 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -7,15 +7,16 @@ __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re +import re, weakref from functools import partial from datetime import timedelta +from calibre.constants import preferred_encoding from calibre.utils.config_base import prefs from calibre.utils.date import parse_date, UNDEFINED_DATE, now -from calibre.utils.icu import primary_find +from calibre.utils.icu import primary_find, sort_key from calibre.utils.localization import lang_map, canonicalize_lang -from calibre.utils.search_query_parser import SearchQueryParser, ParseException, SavedSearchQueries +from calibre.utils.search_query_parser import SearchQueryParser, ParseException CONTAINS_MATCH = 0 EQUALS_MATCH = 1 @@ -388,11 +389,72 @@ class KeyPairSearch(object): # {{{ # }}} +class SavedSearchQueries(object): # {{{ + queries = {} + opt_name = '' + + def __init__(self, db, _opt_name): + self.opt_name = _opt_name + try: + self._db = weakref.ref(db) + except TypeError: + # db could be None + self._db = lambda : None + self.load_from_db() + + def load_from_db(self): + db = self.db + if db is not None: + self.queries = db._pref(self.opt_name, default={}) + else: + self.queries = {} + + @property + def db(self): + return self._db() + + def force_unicode(self, x): + if not isinstance(x, unicode): + x = x.decode(preferred_encoding, 'replace') + return x + + def add(self, name, value): + db = self.db + if db is not None: + self.queries[self.force_unicode(name)] = self.force_unicode(value).strip() + db._set_pref(self.opt_name, self.queries) + + def lookup(self, name): + return self.queries.get(self.force_unicode(name), None) + + def delete(self, name): + db = self.db + if db is not None: + self.queries.pop(self.force_unicode(name), False) + db._set_pref(self.opt_name, self.queries) + + def rename(self, old_name, new_name): + db = self.db + if db is not None: + self.queries[self.force_unicode(new_name)] = self.queries.get(self.force_unicode(old_name), None) + self.queries.pop(self.force_unicode(old_name), False) + db._set_pref(self.opt_name, self.queries) + + def set_all(self, smap): + db = self.db + if db is not None: + self.queries = smap + db._set_pref(self.opt_name, smap) + + def names(self): + return sorted(self.queries.iterkeys(), key=sort_key) +# }}} + class Parser(SearchQueryParser): def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, bool_search, keypair_search, limit_search_columns, limit_search_columns_to, - locations, virtual_fields, get_saved_searches): + locations, virtual_fields, lookup_saved_search): self.dbcache, self.all_book_ids = dbcache, all_book_ids self.all_search_locations = frozenset(locations) self.grouped_search_terms = gst @@ -403,7 +465,7 @@ class Parser(SearchQueryParser): self.virtual_fields = virtual_fields or {} if 'marked' not in self.virtual_fields: self.virtual_fields['marked'] = self - super(Parser, self).__init__(locations, optimize=True, get_saved_searches=get_saved_searches) + super(Parser, self).__init__(locations, optimize=True, lookup_saved_search=lookup_saved_search) @property def field_metadata(self): @@ -693,11 +755,11 @@ class Search(object): self.keypair_search, prefs['limit_search_columns'], prefs['limit_search_columns_to'], self.all_search_locations, - virtual_fields, self.get_saved_searches) + virtual_fields, self.saved_searches.lookup) try: ret = sqp.parse(q) finally: - sqp.dbcache = sqp.get_saved_searches = None + sqp.dbcache = sqp.lookup_saved_search = None return ret diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 3e1a1457b3..0c4efb9f6e 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -751,3 +751,24 @@ class LegacyTest(BaseTest): db.close() # }}} + + def test_legacy_saved_search(self): # {{{ + ' Test legacy saved search API ' + db, ndb = self.init_old(), self.init_legacy() + run_funcs(self, db, ndb, ( + ('saved_search_set_all', {'one':'a', 'two':'b'}), + ('saved_search_names',), + ('saved_search_lookup', 'one'), + ('saved_search_lookup', 'two'), + ('saved_search_lookup', 'xxx'), + ('saved_search_rename', 'one', '1'), + ('saved_search_names',), + ('saved_search_lookup', '1'), + ('saved_search_delete', '1'), + ('saved_search_names',), + ('saved_search_add', 'n', 'm'), + ('saved_search_names',), + ('saved_search_lookup', 'n'), + )) + # }}} + diff --git a/src/calibre/gui2/dialogs/saved_search_editor.py b/src/calibre/gui2/dialogs/saved_search_editor.py index 669771e46e..40cd16c41d 100644 --- a/src/calibre/gui2/dialogs/saved_search_editor.py +++ b/src/calibre/gui2/dialogs/saved_search_editor.py @@ -14,7 +14,8 @@ from calibre.gui2.dialogs.confirm_delete import confirm class SavedSearchEditor(QDialog, Ui_SavedSearchEditor): def __init__(self, parent, initial_search=None): - from calibre.gui2.ui import saved_searches + from calibre.gui2.ui import get_gui + db = get_gui().current_db QDialog.__init__(self, parent) Ui_SavedSearchEditor.__init__(self) self.setupUi(self) @@ -27,9 +28,9 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor): self.current_search_name = None self.searches = {} - for name in saved_searches().names(): - self.searches[name] = saved_searches().lookup(name) - self.search_names = set([icu_lower(n) for n in saved_searches().names()]) + for name in db.saved_search_names(): + self.searches[name] = db.saved_search_lookup(name) + self.search_names = set([icu_lower(n) for n in db.saved_search_names()]) self.populate_search_list() if initial_search is not None and initial_search in self.searches: @@ -98,11 +99,10 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor): self.search_text.setPlainText('') def accept(self): - from calibre.gui2.ui import saved_searches + from calibre.gui2.ui import get_gui + db = get_gui().current_db if self.current_search_name: self.searches[self.current_search_name] = unicode(self.search_text.toPlainText()) - for name in saved_searches().names(): - saved_searches().delete(name) - for name in self.searches: - saved_searches().add(name, self.searches[name]) + ss = {name:self.searches[name] for name in self.searches} + db.saved_search_set_all(ss) QDialog.accept(self) diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index b10fdd40ef..edd05168f4 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -308,28 +308,35 @@ class SavedSearchBox(QComboBox): # {{{ self.saved_search_selected(self.currentText()) def saved_search_selected(self, qname): - from calibre.gui2.ui import saved_searches + from calibre.gui2.ui import get_gui + db = get_gui().current_db qname = unicode(qname) if qname is None or not qname.strip(): self.search_box.clear() return - if not saved_searches().lookup(qname): + if not db.saved_search_lookup(qname): self.search_box.clear() self.setEditText(qname) return self.search_box.set_search_string(u'search:"%s"' % qname, emit_changed=False) self.setEditText(qname) - self.setToolTip(saved_searches().lookup(qname)) + self.setToolTip(db.saved_search_lookup(qname)) def initialize_saved_search_names(self): - from calibre.gui2.ui import saved_searches - qnames = saved_searches().names() - self.addItems(qnames) + from calibre.gui2.ui import get_gui + gui = get_gui() + try: + names = gui.current_db.saved_search_names() + except AttributeError: + # Happens during gui initialization + names = [] + self.addItems(names) self.setCurrentIndex(-1) # SIGNALed from the main UI def save_search_button_clicked(self): - from calibre.gui2.ui import saved_searches + from calibre.gui2.ui import get_gui + db = get_gui().current_db name = unicode(self.currentText()) if not name.strip(): name = unicode(self.search_box.text()).replace('"', '') @@ -337,8 +344,8 @@ class SavedSearchBox(QComboBox): # {{{ error_dialog(self, _('Create saved search'), _('There is no search to save'), show=True) return - saved_searches().delete(name) - saved_searches().add(name, unicode(self.search_box.text())) + db.saved_search_delete(name) + db.saved_search_add(name, unicode(self.search_box.text())) # now go through an initialization cycle to ensure that the combobox has # the new search in it, that it is selected, and that the search box # references the new search instead of the text in the search. @@ -348,7 +355,8 @@ class SavedSearchBox(QComboBox): # {{{ self.changed.emit() def delete_current_search(self): - from calibre.gui2.ui import saved_searches + from calibre.gui2.ui import get_gui + db = get_gui().current_db idx = self.currentIndex() if idx <= 0: error_dialog(self, _('Delete current search'), @@ -358,21 +366,22 @@ class SavedSearchBox(QComboBox): # {{{ 'permanently deleted. Are you sure?') +'

    ', 'saved_search_delete', self): return - ss = saved_searches().lookup(unicode(self.currentText())) + ss = db.saved_search_lookup(unicode(self.currentText())) if ss is None: return - saved_searches().delete(unicode(self.currentText())) + db.saved_search_delete(unicode(self.currentText())) self.clear() self.search_box.clear() self.changed.emit() # SIGNALed from the main UI def copy_search_button_clicked(self): - from calibre.gui2.ui import saved_searches + from calibre.gui2.ui import get_gui + db = get_gui().current_db idx = self.currentIndex() if idx < 0: return - self.search_box.set_search_string(saved_searches().lookup(unicode(self.currentText()))) + self.search_box.set_search_string(db.saved_search_lookup(unicode(self.currentText()))) # }}} diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index 71e0f2f392..ceed4f1928 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -178,7 +178,7 @@ class CreateVirtualLibrary(QDialog): # {{{ self.resize(self.sizeHint()+QSize(150, 25)) def search_text_changed(self, txt): - from calibre.gui2.ui import saved_searches + db = self.gui.current_db searches = [_('Saved searches recognized in the expression:')] txt = unicode(txt) while txt: @@ -201,9 +201,9 @@ class CreateVirtualLibrary(QDialog): # {{{ search_name = possible_search[0] if search_name.startswith('='): search_name = search_name[1:] - if search_name in saved_searches().names(): + if search_name in db.saved_search_names(): searches.append(search_name + '=' + - saved_searches().lookup(search_name)) + db.saved_search_lookup(search_name)) else: txt = '' else: @@ -234,18 +234,17 @@ class CreateVirtualLibrary(QDialog): # {{{ self.vl_text.setText(self.original_search) def link_activated(self, url): - from calibre.gui2.ui import saved_searches db = self.gui.current_db f, txt = unicode(url).partition('.')[0::2] if f == 'search': - names = saved_searches().names() + names = db.saved_search_names() else: names = getattr(db, 'all_%s_names'%f)() d = SelectNames(names, txt, parent=self) if d.exec_() == d.Accepted: prefix = f+'s' if f in {'tag', 'author'} else f if f == 'search': - search = ['(%s)'%(saved_searches().lookup(x)) for x in d.names] + search = ['(%s)'%(db.saved_search_lookup(x)) for x in d.names] else: search = ['%s:"=%s"'%(prefix, x.replace('"', '\\"')) for x in d.names] if search: @@ -476,7 +475,7 @@ class SearchRestrictionMixin(object): return name[0:MAX_VIRTUAL_LIBRARY_NAME_LENGTH].strip() def build_search_restriction_list(self): - from calibre.gui2.ui import saved_searches + from calibre.gui2.ui import get_gui m = self.ar_menu m.clear() @@ -508,7 +507,7 @@ class SearchRestrictionMixin(object): add_action(current_restriction_text, 2) dex += 1 - for n in sorted(saved_searches().names(), key=sort_key): + for n in sorted(get_gui().current_db.saved_search_names(), key=sort_key): add_action(n, dex) dex += 1 diff --git a/src/calibre/gui2/tag_browser/model.py b/src/calibre/gui2/tag_browser/model.py index 7dba7cfe7d..1b291930fd 100644 --- a/src/calibre/gui2/tag_browser/model.py +++ b/src/calibre/gui2/tag_browser/model.py @@ -878,7 +878,7 @@ class TagsModel(QAbstractItemModel): # {{{ traceback.print_exc() self.db.data.change_search_locations(self.db.field_metadata.get_search_terms()) - if len(self.db.get_saved_searches().names()): + if len(self.db.saved_search_names()): tb_cats.add_search_category(label='search', name=_('Searches')) if self.filter_categories_by: @@ -1004,11 +1004,11 @@ class TagsModel(QAbstractItemModel): # {{{ _('Author names cannot contain & characters.')).exec_() return False if key == 'search': - if val in self.db.get_saved_searches().names(): + if val in self.db.saved_search_names(): error_dialog(self.gui_parent, _('Duplicate search name'), _('The saved search name %s is already used.')%val).exec_() return False - self.db.get_saved_searches().rename(unicode(item.data(role).toString()), val) + self.db.saved_search_rename(unicode(item.data(role).toString()), val) item.tag.name = val self.search_item_renamed.emit() # Does a refresh else: diff --git a/src/calibre/gui2/tag_browser/view.py b/src/calibre/gui2/tag_browser/view.py index d28b7ca848..a48cb502e4 100644 --- a/src/calibre/gui2/tag_browser/view.py +++ b/src/calibre/gui2/tag_browser/view.py @@ -354,8 +354,7 @@ class TagsView(QTreeView): # {{{ self.delete_user_category.emit(key) return if action == 'delete_search': - from calibre.gui2.ui import saved_searches - saved_searches().delete(key) + self.model().db.saved_search_delete(key) self.rebuild_saved_searches.emit() return if action == 'delete_item_from_user_category': diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index e54d06e671..b4d103ca64 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -98,16 +98,6 @@ _gui = None def get_gui(): return _gui -def saved_searches(): - 'Return the saved searches defined in the currently open library' - try: - return _gui.library_view.model().db.get_saved_searches() - except AttributeError: - # Happens during initialization of the gui - from calibre.utils.search_query_parser import saved_searches - return saved_searches() - - class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin, SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin, @@ -312,10 +302,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ ####################### Search boxes ######################## SearchRestrictionMixin.__init__(self) SavedSearchBoxMixin.__init__(self) - SearchBoxMixin.__init__(self) ####################### Library view ######################## LibraryViewMixin.__init__(self, db) + SearchBoxMixin.__init__(self) # Requires current_db if show_gui: self.show() diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 7e7a234724..2978a4e169 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -1029,11 +1029,10 @@ def command_saved_searches(args, dbpath): prints(_('Error: You must specify an action (add|remove|list)'), file=sys.stderr) return 1 db = get_db(dbpath, opts) - ss = db.get_saved_searches() if args[0] == 'list': - for name in ss.names(): + for name in db.saved_search_names(): prints(_('Name:'), name) - prints(_('Search string:'), ss.lookup(name)) + prints(_('Search string:'), db.saved_search_lookup(name)) print elif args[0] == 'add': if len(args) < 3: @@ -1041,7 +1040,7 @@ def command_saved_searches(args, dbpath): print prints(_('Error: You must specify a name and a search string'), file=sys.stderr) return 1 - ss.add(args[1], args[2]) + db.saved_search_add(args[1], args[2]) prints(args[1], _('added')) elif args[0] == 'remove': if len(args) < 2: @@ -1049,7 +1048,7 @@ def command_saved_searches(args, dbpath): print prints(_('Error: You must specify a name'), file=sys.stderr) return 1 - ss.delete(args[1]) + db.saved_search_delete(args[1]) prints(args[1], _('removed')) else: parser.print_help() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 8ea7e75b59..677729a9e9 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -537,8 +537,23 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if self.user_version == 0: self.user_version = 1 - def get_saved_searches(self): - return saved_searches() + def saved_search_names(self): + return saved_searches().names() + + def saved_search_rename(self, old_name, new_name): + saved_searches().rename(old_name, new_name) + + def saved_search_lookup(self, name): + return saved_searches().lookup(name) + + def saved_search_add(self, name, val): + saved_searches().add(name, val) + + def saved_search_delete(self, name): + saved_searches().delete(name) + + def saved_search_set_all(self, smap): + saved_searches().set_all(smap) def last_modified(self): ''' Return last modified time as a UTC datetime object''' diff --git a/src/calibre/library/server/base.py b/src/calibre/library/server/base.py index a677b991f9..f8608bd2d0 100644 --- a/src/calibre/library/server/base.py +++ b/src/calibre/library/server/base.py @@ -209,7 +209,7 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache, if sr: if sr in virt_libs: sr = virt_libs[sr] - elif sr not in self.db.get_saved_searches().names(): + elif sr not in self.db.saved_search_names(): prints('WARNING: Content server: search restriction ', sr, ' does not exist') sr = '' diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index dc2a7b51b8..4406785bb1 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -76,6 +76,11 @@ class SavedSearchQueries(object): self.queries.pop(self.force_unicode(old_name), False) db.prefs[self.opt_name] = self.queries + def set_all(self, smap): + db = self.db + if db is not None: + self.queries = db.prefs[self.opt_name] = smap + def names(self): return sorted(self.queries.keys(),key=sort_key) @@ -93,6 +98,9 @@ def saved_searches(): global ss return ss +def global_lookup_saved_search(name): + return ss.lookup(name) + ''' Parse a search expression into a series of potentially recursive operations. @@ -292,10 +300,10 @@ class SearchQueryParser(object): failed.append(test[0]) return failed - def __init__(self, locations, test=False, optimize=False, get_saved_searches=None): + def __init__(self, locations, test=False, optimize=False, lookup_saved_search=None): self.sqp_initialize(locations, test=test, optimize=optimize) self.parser = Parser() - self.get_saved_searches = saved_searches if get_saved_searches is None else get_saved_searches + self.lookup_saved_search = global_lookup_saved_search if lookup_saved_search is None else lookup_saved_search def sqp_change_locations(self, locations): self.sqp_initialize(locations, optimize=self.optimize) @@ -368,7 +376,7 @@ class SearchQueryParser(object): raise ParseException(_('Recursive saved search: {0}').format(query)) if self.recurse_level > 5: self.searches_seen.add(query) - return self._parse(self.get_saved_searches().lookup(query), candidates) + return self._parse(self.lookup_saved_search(query), candidates) except ParseException as e: raise e except: # convert all exceptions (e.g., missing key) to a parse error From e336d8a2b980d78c8b0e2d943b57dd10f9f77993 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Fri, 19 Jul 2013 14:00:35 +0200 Subject: [PATCH 0248/1154] Strip spaces off of function names before looking them up. They shouldn't ever be significant. --- src/calibre/utils/formatter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index eff38203a0..2fa5901dde 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -137,6 +137,7 @@ class _Parser(object): # We have a function. # Check if it is a known one. We do this here so error reporting is # better, as it can identify the tokens near the problem. + id = id.strip() if id not in funcs: self.error(_('unknown function {0}').format(id)) @@ -246,6 +247,7 @@ class _CompileParser(_Parser): # We have a function. # Check if it is a known one. We do this here so error reporting is # better, as it can identify the tokens near the problem. + id = id.strip() if id not in funcs: self.error(_('unknown function {0}').format(id)) @@ -457,7 +459,7 @@ class TemplateFormatter(string.Formatter): colon += 1 funcs = formatter_functions().get_functions() - fname = fmt[colon:p] + fname = fmt[colon:p].strip() if fname in funcs: func = funcs[fname] if func.arg_count == 2: From 29e2886267825443616a2310bb5b944c0a72cd6a Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Fri, 19 Jul 2013 14:01:47 +0200 Subject: [PATCH 0249/1154] Implement enforcement of unique custom template function names. --- src/calibre/ebooks/metadata/worker.py | 2 +- .../gui2/preferences/template_functions.py | 2 +- src/calibre/gui2/ui.py | 3 + src/calibre/library/database2.py | 3 +- src/calibre/utils/formatter_functions.py | 56 +++++++++++++++++-- 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index 660240571b..e8147958d8 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -318,7 +318,7 @@ def save_book(ids, dpath, plugboards, template_functions, path, recs, from calibre.library.save_to_disk import config, save_serialized_to_disk from calibre.customize.ui import apply_null_metadata from calibre.utils.formatter_functions import load_user_template_functions - load_user_template_functions(template_functions) + load_user_template_functions('', template_functions) opts = config().parse() for name in recs: setattr(opts, name, recs[name]) diff --git a/src/calibre/gui2/preferences/template_functions.py b/src/calibre/gui2/preferences/template_functions.py index f5203e9cdd..4cc07fee3a 100644 --- a/src/calibre/gui2/preferences/template_functions.py +++ b/src/calibre/gui2/preferences/template_functions.py @@ -223,7 +223,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if f in self.builtins: continue func = self.funcs[f] - formatter_functions().register_function(func) + formatter_functions().register_function(self.db.library_id, func) pref_value.append((func.name, func.doc, func.arg_count, func.program_text)) self.db.prefs.set('user_template_functions', pref_value) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index e54d06e671..05f36c2709 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -580,6 +580,9 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ olddb = self.library_view.model().db if copy_structure: default_prefs = olddb.prefs + + from calibre.utils.formatter_functions import unload_user_template_functions + unload_user_template_functions(olddb.library_id ) except: olddb = None try: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 8ea7e75b59..7942c69574 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -325,7 +325,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.prefs.set('user_categories', user_cats) if not self.is_second_db: - load_user_template_functions(self.prefs.get('user_template_functions', [])) + load_user_template_functions(self.library_id, + self.prefs.get('user_template_functions', [])) # Load the format filename cache self.refresh_format_cache() diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 73dad7422b..f8d62c367a 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en' import inspect, re, traceback from math import trunc +from collections import defaultdict from calibre import human_readable from calibre.constants import DEBUG @@ -25,6 +26,7 @@ class FormatterFunctions(object): def __init__(self): self._builtins = {} self._functions = {} + self._functions_from_library = defaultdict(list) def register_builtin(self, func_class): if not isinstance(func_class, FormatterFunction): @@ -38,14 +40,24 @@ class FormatterFunctions(object): for a in func_class.aliases: self._functions[a] = func_class - def register_function(self, func_class): + def register_function(self, library_uuid, func_class, replace=False): if not isinstance(func_class, FormatterFunction): raise ValueError('Class %s is not an instance of FormatterFunction'%( func_class.__class__.__name__)) name = func_class.name - if name in self._functions: + if not replace and name in self._functions: raise ValueError('Name %s already used'%name) self._functions[name] = func_class + self._functions_from_library[library_uuid].append(name) + + def function_exists(self, name): + return self._functions.get(name, None) + + def unregister_functions(self, library_uuid): + if library_uuid in self._functions_from_library: + for name in self._functions_from_library[library_uuid]: + self._functions.pop(name) + self._functions_from_library.pop(library_uuid) def get_builtins(self): return self._builtins @@ -1259,11 +1271,45 @@ class UserFunction(FormatterUserFunction): cls = locals_['UserFunction'](name, doc, arg_count, eval_func) return cls -def load_user_template_functions(funcs): - formatter_functions().reset_to_builtins() +error_function_body = ('def evaluate(self, formatter, kwargs, mi, locals):\n' + '\treturn "' + + _('Duplicate user function name {0}. ' + 'Change the name or ensure that the functions are identical') + + '"') + +def load_user_template_functions(library_uuid, funcs): + unload_user_template_functions(library_uuid) + for func in funcs: try: + # Force a name conflict to test the logic + # if func[0] == 'myFunc2': + # func[0] = 'myFunc3' + + # Compile the function so that the tab processing is done on the + # source. This helps ensure that if the function already is defined + # then white space differences don't cause them to compare differently + cls = compile_user_function(*func) - formatter_functions().register_function(cls) + f = formatter_functions().function_exists(cls.name) + replace = False + if f is not None: + existing_body = f.program_text + new_body = cls.program_text + if new_body != existing_body: + # Change the body of the template function to one that will + # return an error message. Also change the arg count to + # -1 (variable) to avoid template compilation errors + replace = True + func[3] = error_function_body.format(func[0]) + func[2] = -1 + cls = compile_user_function(*func) + else: + continue + + formatter_functions().register_function(library_uuid, cls, replace=replace) except: traceback.print_exc() + +def unload_user_template_functions(library_uuid): + formatter_functions().unregister_functions(library_uuid) \ No newline at end of file From 97d0eed4520efd61e87f57be50d35aca228559e3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 18:25:01 +0530 Subject: [PATCH 0250/1154] Add loading of user template functions to the new backend --- src/calibre/db/__init__.py | 3 +-- src/calibre/db/backend.py | 3 +++ src/calibre/db/legacy.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index a07fb8b5a3..5bc0437fb5 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -168,8 +168,7 @@ How this will proceed: work. Various things that require other things before they can be migrated: - 1. From initialize_dynamic(): set_saved_searches, - load_user_template_functions. Also add custom + 1. From initialize_dynamic(): Also add custom columns/categories/searches info into self.field_metadata. 2. Catching DatabaseException and sqlite.Error when creating new diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 3241493af1..914248fedc 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -29,6 +29,7 @@ from calibre.utils.filenames import (is_case_sensitive, samefile, hardlink_file, WindowsAtomicFolderMove) from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_tree, delete_file +from calibre.utils.formatter_functions import load_user_template_functions from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable, CompositeTable, UUIDTable) @@ -372,6 +373,8 @@ class DB(object): self.initialize_prefs(default_prefs, restore_all_prefs, progress_callback) self.initialize_custom_columns() self.initialize_tables() + load_user_template_functions(self.library_id, + self.prefs.get('user_template_functions', [])) def initialize_prefs(self, default_prefs, restore_all_prefs, progress_callback): # {{{ self.prefs = DBPrefs(self) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 9de6d5ce1f..54f1fe5c1d 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -55,7 +55,7 @@ class LibraryDatabase(object): default_prefs=None, read_only=False, is_second_db=False, progress_callback=lambda x, y:True, restore_all_prefs=False): - self.is_second_db = is_second_db # TODO: Use is_second_db + self.is_second_db = is_second_db self.listeners = set() backend = self.backend = DB(library_path, default_prefs=default_prefs, From faa351c2087b8e588933fb9edffbd4b787469c08 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Fri, 19 Jul 2013 15:42:43 +0200 Subject: [PATCH 0251/1154] Fix problems creating template functions caused by the changes for the new single-name rule --- src/calibre/gui2/preferences/template_functions.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/preferences/template_functions.py b/src/calibre/gui2/preferences/template_functions.py index 4cc07fee3a..6842e0798e 100644 --- a/src/calibre/gui2/preferences/template_functions.py +++ b/src/calibre/gui2/preferences/template_functions.py @@ -13,7 +13,8 @@ from calibre.gui2 import error_dialog, warning_dialog from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.template_functions_ui import Ui_Form from calibre.gui2.widgets import PythonHighlighter -from calibre.utils.formatter_functions import formatter_functions, compile_user_function +from calibre.utils.formatter_functions import (formatter_functions, + compile_user_function, load_user_template_functions) class ConfigWidget(ConfigWidgetBase, Ui_Form): @@ -21,7 +22,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): def genesis(self, gui): self.gui = gui self.db = gui.library_view.model().db - self.current_plugboards = self.db.prefs.get('plugboards',{}) + help_text = _('''

    Here you can add and remove functions used in template processing. A template function is written in python. It takes information from the @@ -217,15 +218,15 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): pass def commit(self): - formatter_functions().reset_to_builtins() + # formatter_functions().reset_to_builtins() pref_value = [] for f in self.funcs: - if f in self.builtins: - continue func = self.funcs[f] - formatter_functions().register_function(self.db.library_id, func) pref_value.append((func.name, func.doc, func.arg_count, func.program_text)) self.db.prefs.set('user_template_functions', pref_value) + formatter_functions().unregister_functions(self.db.library_id) + load_user_template_functions(self.db.library_id, pref_value) + return False if __name__ == '__main__': From d0b826c39ab0bcbb32ee7e33ef275166500a7c75 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 21:11:08 +0530 Subject: [PATCH 0252/1154] Allow using a tweak to switch db backends --- src/calibre/db/__init__.py | 69 +++++---------------- src/calibre/gui2/actions/choose_library.py | 17 +++-- src/calibre/gui2/actions/copy_to_library.py | 4 +- src/calibre/gui2/dialogs/choose_library.py | 7 ++- src/calibre/gui2/main.py | 27 ++++---- src/calibre/gui2/ui.py | 12 ++-- src/calibre/gui2/wizard/__init__.py | 17 ++--- src/calibre/library/__init__.py | 7 ++- src/calibre/library/cli.py | 13 ++-- src/calibre/library/move.py | 6 +- src/calibre/library/server/main.py | 4 +- 11 files changed, 80 insertions(+), 103 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 5bc0437fb5..b8e46b4ed2 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -118,65 +118,26 @@ def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None): return data +def get_db_loader(): + from calibre.utils.config_base import tweaks + if tweaks.get('use_new_db', False): + from calibre.db.legacy import LibraryDatabase as cls + import apsw + errs = (apsw.Error,) + else: + from calibre.library.database2 import LibraryDatabase2 as cls + from calibre.library.sqlite import sqlite, DatabaseException + errs = (sqlite.Error, DatabaseException) + return cls, errs + ''' -Rewrite of the calibre database backend. - -Broad Objectives: - - * Use the sqlite db only as a datastore. i.e. do not do - sorting/searching/concatenation or anything else in sqlite. Instead - mirror the sqlite tables in memory, create caches and lookup maps from - them and create a set_* API that updates the memory caches and the sqlite - correctly. - - * Move from keeping a list of books in memory as a cache to a per table - cache. This allows much faster search and sort operations at the expense - of slightly slower lookup operations. That slowdown can be mitigated by - keeping lots of maps and updating them in the set_* API. Also - get_categories becomes blazingly fast. - - * Separate the database layer from the cache layer more cleanly. Rather - than having the db layer refer to the cache layer and vice versa, the - cache layer will refer to the db layer only and the new API will be - defined on the cache layer. - - * Get rid of index_is_id and other poor design decisions - - * Minimize the API as much as possible and define it cleanly - - * Do not change the on disk format of metadata.db at all (this is for - backwards compatibility) - - * Get rid of the need for a separate db access thread by switching to apsw - to access sqlite, which is thread safe - - * The new API will have methods to efficiently do bulk operations and will - use shared/exclusive/pending locks to serialize access to the in-mem data - structures. Use the same locking scheme as sqlite itself does. - -How this will proceed: - - 1. Create the new API - 2. Create a test suite for it - 3. Write a replacement for LibraryDatabase2 that uses the new API - internally - 4. Lots of testing of calibre with the new LibraryDatabase2 - 5. Gradually migrate code to use the (much faster) new api wherever possible (the new api - will be exposed via db.new_api) - - I plan to work on this slowly, in parallel to normal calibre development - work. - Various things that require other things before they can be migrated: 1. From initialize_dynamic(): Also add custom columns/categories/searches info into self.field_metadata. - 2. Catching DatabaseException and sqlite.Error when creating new - libraries/switching/on calibre startup. - 3. Port library/restore.py - 4. Replace the metadatabackup thread with the new implementation when using the new backend. - 5. grep the sources for TODO - 6. Check that content server reloading on metadata,db change, metadata + 2. Port library/restore.py + 3. Replace the metadatabackup thread with the new implementation when using the new backend. + 4. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) ''' diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index 347b6862a0..6857454ae4 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -19,9 +19,12 @@ from calibre.utils.config import prefs, tweaks from calibre.utils.icu import sort_key from calibre.gui2 import (gprefs, warning_dialog, Dispatcher, error_dialog, question_dialog, info_dialog, open_local_file, choose_dir) -from calibre.library.database2 import LibraryDatabase2 from calibre.gui2.actions import InterfaceAction +def db_class(): + from calibre.db import get_db_loader + return get_db_loader()[0] + class LibraryUsageStats(object): # {{{ def __init__(self): @@ -139,7 +142,7 @@ class MovedDialog(QDialog): # {{{ def accept(self): newloc = unicode(self.loc.text()) - if not LibraryDatabase2.exists_at(newloc): + if not db_class.exists_at(newloc): error_dialog(self, _('No library found'), _('No existing calibre library found at %s')%newloc, show=True) @@ -313,6 +316,7 @@ class ChooseLibraryAction(InterfaceAction): self.qaction.setEnabled(enabled) def rename_requested(self, name, location): + LibraryDatabase = db_class() loc = location.replace('/', os.sep) base = os.path.dirname(loc) newname, ok = QInputDialog.getText(self.gui, _('Rename') + ' ' + name, @@ -328,10 +332,10 @@ class ChooseLibraryAction(InterfaceAction): _('The folder %s already exists. Delete it first.') % newloc, show=True) if (iswindows and len(newloc) > - LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT): + LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT): return error_dialog(self.gui, _('Too long'), _('Path to library too long. Must be less than' - ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT, + ' %d characters.')%LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT, show=True) if not os.path.exists(loc): error_dialog(self.gui, _('Not found'), @@ -387,16 +391,17 @@ class ChooseLibraryAction(InterfaceAction): 'rate of approximately 1 book every three seconds.'), show=True) def restore_database(self): + LibraryDatabase = db_class() m = self.gui.library_view.model() db = m.db if (iswindows and len(db.library_path) > - LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT): + LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT): return error_dialog(self.gui, _('Too long'), _('Path to library too long. Must be less than' ' %d characters. Move your library to a location with' ' a shorter path using Windows Explorer, then point' ' calibre to the new location and try again.')% - LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT, + LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT, show=True) from calibre.gui2.dialogs.restore_library import restore_database diff --git a/src/calibre/gui2/actions/copy_to_library.py b/src/calibre/gui2/actions/copy_to_library.py index 8465cb98f0..f32f060de7 100644 --- a/src/calibre/gui2/actions/copy_to_library.py +++ b/src/calibre/gui2/actions/copy_to_library.py @@ -55,8 +55,8 @@ class Worker(Thread): # {{{ notify=False, replace=replace) def doit(self): - from calibre.library.database2 import LibraryDatabase2 - newdb = LibraryDatabase2(self.loc, is_second_db=True) + from calibre.db import get_db_loader + newdb = get_db_loader()[0](self.loc, is_second_db=True) with closing(newdb): self._doit(newdb) newdb.break_cycles() diff --git a/src/calibre/gui2/dialogs/choose_library.py b/src/calibre/gui2/dialogs/choose_library.py index 91048e8ff1..52e8ef644a 100644 --- a/src/calibre/gui2/dialogs/choose_library.py +++ b/src/calibre/gui2/dialogs/choose_library.py @@ -15,7 +15,6 @@ from calibre.constants import (filesystem_encoding, iswindows, get_portable_base) from calibre import isbytestring, patheq, force_unicode from calibre.gui2.wizard import move_library -from calibre.library.database2 import LibraryDatabase2 class ChooseLibrary(QDialog, Ui_Dialog): @@ -86,6 +85,8 @@ class ChooseLibrary(QDialog, Ui_Dialog): show=True) return False if ac in ('new', 'move'): + from calibre.db import get_db_loader + LibraryDatabase = get_db_loader()[0] if not empty: error_dialog(self, _('Not empty'), _('The folder %s is not empty. Please choose an empty' @@ -93,10 +94,10 @@ class ChooseLibrary(QDialog, Ui_Dialog): show=True) return False if (iswindows and len(loc) > - LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT): + LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT): error_dialog(self, _('Too long'), _('Path to library too long. Must be less than' - ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT, + ' %d characters.')%LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT, show=True) return False diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index f35a9ca083..bcfc88d239 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -15,8 +15,6 @@ from calibre.gui2 import (ORG_NAME, APP_UID, initialize_file_icon_provider, Application, choose_dir, error_dialog, question_dialog, gprefs) from calibre.gui2.main_window import option_parser as _option_parser from calibre.utils.config import prefs, dynamic -from calibre.library.database2 import LibraryDatabase2 -from calibre.library.sqlite import sqlite, DatabaseException if iswindows: winutil = plugins['winutil'][0] @@ -51,7 +49,8 @@ path_to_ebook to the database. def find_portable_library(): base = get_portable_base() - if base is None: return + if base is None: + return import glob candidates = [os.path.basename(os.path.dirname(x)) for x in glob.glob( os.path.join(base, u'*%smetadata.db'%os.sep))] @@ -123,7 +122,7 @@ def get_default_library_path(): def get_library_path(parent=None): library_path = prefs['library_path'] - if library_path is None: # Need to migrate to new database layout + if library_path is None: # Need to migrate to new database layout base = os.path.expanduser('~') if iswindows: base = winutil.special_folder_path(winutil.CSIDL_PERSONAL) @@ -181,7 +180,7 @@ class GuiRunner(QObject): main = Main(self.opts, gui_debug=self.gui_debug) if self.splash_screen is not None: self.splash_screen.showMessage(_('Initializing user interface...')) - with gprefs: # Only write gui.json after initialization is complete + with gprefs: # Only write gui.json after initialization is complete main.initialize(self.library_path, db, self.listener, self.actions) if self.splash_screen is not None: self.splash_screen.finish(main) @@ -224,7 +223,7 @@ class GuiRunner(QObject): try: self.library_path = candidate - db = LibraryDatabase2(candidate) + db = self.db_class(candidate) except: error_dialog(self.splash_screen, _('Bad database location'), _('Bad database location %r. calibre will now quit.' @@ -235,10 +234,12 @@ class GuiRunner(QObject): self.start_gui(db) def initialize_db(self): + from calibre.db import get_db_loader db = None + self.db_class, errs = get_db_loader() try: - db = LibraryDatabase2(self.library_path) - except (sqlite.Error, DatabaseException): + db = self.db_class(self.library_path) + except errs: repair = question_dialog(self.splash_screen, _('Corrupted database'), _('The library database at %s appears to be corrupted. Do ' 'you want calibre to try and rebuild it automatically? ' @@ -249,7 +250,7 @@ class GuiRunner(QObject): ) if repair: if repair_library(self.library_path): - db = LibraryDatabase2(self.library_path) + db = self.db_class(self.library_path) except: error_dialog(self.splash_screen, _('Bad database location'), _('Bad database location %r. Will start with ' @@ -383,7 +384,7 @@ def shutdown_other(rc=None): rc = build_pipe(print_error=False) if rc.conn is None: prints(_('No running calibre found')) - return # No running instance found + return # No running instance found from calibre.utils.lock import singleinstance rc.conn.send('shutdown:') prints(_('Shutdown command sent, waiting for shutdown...')) @@ -441,7 +442,7 @@ def main(args=sys.argv): otherinstance = False try: listener = Listener(address=gui_socket_address()) - except socket.error: # Good si is correct (on UNIX) + except socket.error: # Good si is correct (on UNIX) otherinstance = True else: # On windows only singleinstance can be trusted @@ -458,7 +459,8 @@ if __name__ == '__main__': try: sys.exit(main()) except Exception as err: - if not iswindows: raise + if not iswindows: + raise tb = traceback.format_exc() from PyQt4.QtGui import QErrorMessage logfile = os.path.join(os.path.expanduser('~'), 'calibre.log') @@ -470,3 +472,4 @@ if __name__ == '__main__': unicode(tb).replace('\n', '
    '), log.replace('\n', '
    '))) + diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 80aa66601b..5e4f75895a 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -22,7 +22,7 @@ from calibre import prints, force_unicode from calibre.constants import __appname__, isosx, filesystem_encoding from calibre.utils.config import prefs, dynamic from calibre.utils.ipc.server import Server -from calibre.library.database2 import LibraryDatabase2 +from calibre.db import get_db_loader from calibre.customize.ui import interface_actions, available_store_plugins from calibre.gui2 import (error_dialog, GetMetadata, open_url, gprefs, max_available_height, config, info_dialog, Dispatcher, @@ -42,7 +42,6 @@ from calibre.gui2.search_restriction_mixin import SearchRestrictionMixin from calibre.gui2.tag_browser.ui import TagBrowserMixin from calibre.gui2.keyboard import Manager from calibre.gui2.auto_add import AutoAdder -from calibre.library.sqlite import sqlite, DatabaseException from calibre.gui2.proceed import ProceedQuestion from calibre.gui2.dialogs.message_box import JobError from calibre.gui2.job_indicator import Pointer @@ -572,12 +571,13 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ default_prefs = olddb.prefs from calibre.utils.formatter_functions import unload_user_template_functions - unload_user_template_functions(olddb.library_id ) + unload_user_template_functions(olddb.library_id) except: olddb = None + db_class, errs = get_db_loader() try: - db = LibraryDatabase2(newloc, default_prefs=default_prefs) - except (DatabaseException, sqlite.Error): + db = db_class(newloc, default_prefs=default_prefs) + except errs: if not allow_rebuild: raise import traceback @@ -591,7 +591,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ if repair: from calibre.gui2.dialogs.restore_library import repair_library_at if repair_library_at(newloc, parent=self): - db = LibraryDatabase2(newloc, default_prefs=default_prefs) + db = db_class(newloc, default_prefs=default_prefs) else: return else: diff --git a/src/calibre/gui2/wizard/__init__.py b/src/calibre/gui2/wizard/__init__.py index f813eed892..00fd3bdb29 100644 --- a/src/calibre/gui2/wizard/__init__.py +++ b/src/calibre/gui2/wizard/__init__.py @@ -14,7 +14,6 @@ from contextlib import closing from PyQt4.Qt import (QWizard, QWizardPage, QPixmap, Qt, QAbstractListModel, QVariant, QItemSelectionModel, SIGNAL, QObject, QTimer, pyqtSignal) from calibre import __appname__, patheq -from calibre.library.database2 import LibraryDatabase2 from calibre.library.move import MoveLibrary from calibre.constants import (filesystem_encoding, iswindows, plugins, isportable) @@ -34,6 +33,10 @@ from calibre.customize.ui import device_plugins if iswindows: winutil = plugins['winutil'][0] +def db_class(): + from calibre.db import get_db_loader + return get_db_loader()[0] + # Devices {{{ class Device(object): @@ -623,7 +626,7 @@ def move_library(oldloc, newloc, parent, callback_on_complete): if oldloc and os.access(os.path.join(oldloc, 'metadata.db'), os.R_OK): # Move old library to new location try: - db = LibraryDatabase2(oldloc) + db = db_class()(oldloc) except: return move_library(None, newloc, parent, callback) @@ -636,13 +639,13 @@ def move_library(oldloc, newloc, parent, callback_on_complete): return else: # Create new library at new location - db = LibraryDatabase2(newloc) + db = db_class()(newloc) callback(newloc) return # Try to load existing library at new location try: - LibraryDatabase2(newloc) + db_class()(newloc) except Exception as err: det = traceback.format_exc() error_dialog(parent, _('Invalid database'), @@ -729,7 +732,7 @@ class LibraryPage(QWizardPage, LibraryUI): def is_library_dir_suitable(self, x): try: - return LibraryDatabase2.exists_at(x) or not os.listdir(x) + return db_class().exists_at(x) or not os.listdir(x) except: return False @@ -745,10 +748,10 @@ class LibraryPage(QWizardPage, LibraryUI): _('Select location for books')) if x: if (iswindows and len(x) > - LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT): + db_class().WINDOWS_LIBRARY_PATH_LIMIT): return error_dialog(self, _('Too long'), _('Path to library too long. Must be less than' - ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT, + ' %d characters.')%(db_class().WINDOWS_LIBRARY_PATH_LIMIT), show=True) if not os.path.exists(x): try: diff --git a/src/calibre/library/__init__.py b/src/calibre/library/__init__.py index 3ae237c919..e219d764d5 100644 --- a/src/calibre/library/__init__.py +++ b/src/calibre/library/__init__.py @@ -3,13 +3,13 @@ __copyright__ = '2008, Kovid Goyal ' ''' Code to manage ebook library''' def db(path=None, read_only=False): - from calibre.library.database2 import LibraryDatabase2 + from calibre.db import get_db_loader from calibre.utils.config import prefs - return LibraryDatabase2(path if path else prefs['library_path'], + return get_db_loader()[0](path if path else prefs['library_path'], read_only=read_only) -def generate_test_db(library_path, # {{{ +def generate_test_db(library_path, # {{{ num_of_records=20000, num_of_authors=6000, num_of_tags=10000, @@ -76,3 +76,4 @@ def current_library_name(): if path: return posixpath.basename(path) + diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 2978a4e169..2c6e7cd777 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -15,15 +15,18 @@ from calibre import preferred_encoding, prints, isbytestring from calibre.utils.config import OptionParser, prefs, tweaks from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.metadata.book.base import field_from_string -from calibre.library.database2 import LibraryDatabase2 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF from calibre.utils.date import isoformat +from calibre.db import get_db_loader FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'formats', 'isbn', 'uuid', 'pubdate', 'cover', 'last_modified', 'identifiers']) +def db_class(): + return get_db_loader()[0] + do_notify = True def send_message(msg=''): global do_notify @@ -62,7 +65,7 @@ def get_db(dbpath, options): dbpath = os.path.abspath(dbpath) if options.dont_notify_gui: do_notify = False - return LibraryDatabase2(dbpath) + return db_class()(dbpath) def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, separator, prefix, limit, subtitle='Books in the calibre database'): @@ -1101,7 +1104,7 @@ def command_backup_metadata(args, dbpath): dbpath = opts.library_path if isbytestring(dbpath): dbpath = dbpath.decode(preferred_encoding) - db = LibraryDatabase2(dbpath) + db = db_class()(dbpath) book_ids = None if opts.all: book_ids = db.all_ids() @@ -1183,7 +1186,7 @@ def command_check_library(args, dbpath): for i in list: print ' %-40.40s - %-40.40s'%(i[0], i[1]) - db = LibraryDatabase2(dbpath) + db = db_class()(dbpath) checker = CheckLibrary(dbpath, db) checker.scan_library(names, exts) for check in checks: @@ -1296,7 +1299,7 @@ def command_list_categories(args, dbpath): if isbytestring(dbpath): dbpath = dbpath.decode(preferred_encoding) - db = LibraryDatabase2(dbpath) + db = db_class()(dbpath) category_data = db.get_categories() data = [] report_on = [c.strip() for c in opts.report.split(',') if c.strip()] diff --git a/src/calibre/library/move.py b/src/calibre/library/move.py index d162d962fe..ccebaa0302 100644 --- a/src/calibre/library/move.py +++ b/src/calibre/library/move.py @@ -10,14 +10,14 @@ import time, os from threading import Thread from Queue import Empty -from calibre.library.database2 import LibraryDatabase2 from calibre.utils.ipc.server import Server from calibre.utils.ipc.job import ParallelJob -def move_library(from_, to, notification = lambda x:x): +def move_library(from_, to, notification=lambda x:x): + from calibre.db import get_db_loader time.sleep(1) - old = LibraryDatabase2(from_) + old = get_db_loader()[0](from_) old.move_library_to(to, notification) return True diff --git a/src/calibre/library/server/main.py b/src/calibre/library/server/main.py index 486c0dbb68..a97eb53618 100644 --- a/src/calibre/library/server/main.py +++ b/src/calibre/library/server/main.py @@ -100,7 +100,7 @@ def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): def main(args=sys.argv): - from calibre.library.database2 import LibraryDatabase2 + from calibre.db import get_db_loader parser = option_parser() opts, args = parser.parse_args(args) if opts.daemonize and not iswindows: @@ -116,7 +116,7 @@ def main(args=sys.argv): print('No saved library path. Use the --with-library option' ' to specify the path to the library you want to use.') return 1 - db = LibraryDatabase2(opts.with_library) + db = get_db_loader()[0](opts.with_library) server = LibraryServer(db, opts, show_tracebacks=opts.develop) server.start() return 0 From 934d4ba258fa71a23b6d3fe3b0748d11d5237e35 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 21:15:02 +0530 Subject: [PATCH 0253/1154] Use the correct metadata backup implementation --- src/calibre/db/__init__.py | 3 +-- src/calibre/gui2/library/models.py | 8 ++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index b8e46b4ed2..434d9ae317 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -136,8 +136,7 @@ Various things that require other things before they can be migrated: columns/categories/searches info into self.field_metadata. 2. Port library/restore.py - 3. Replace the metadatabackup thread with the new implementation when using the new backend. - 4. Check that content server reloading on metadata,db change, metadata + 3. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) ''' diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index af293b864d..738b42a669 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -21,7 +21,7 @@ from calibre.utils.date import dt_factory, qt_to_dt, as_local_time from calibre.utils.icu import sort_key from calibre.utils.search_query_parser import SearchQueryParser from calibre.db.search import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH -from calibre.library.caches import (MetadataBackup, force_to_bool) +from calibre.library.caches import force_to_bool from calibre.library.save_to_disk import find_plugboard from calibre import strftime, isbytestring from calibre.constants import filesystem_encoding, DEBUG, config_dir @@ -234,6 +234,10 @@ class BooksModel(QAbstractTableModel): # {{{ self.start_metadata_backup() def start_metadata_backup(self): + if hasattr(self.db, 'new_api'): + from calibre.db.backup import MetadataBackup + else: + from calibre.library.caches import MetadataBackup self.metadata_backup = MetadataBackup(self.db) self.metadata_backup.start() @@ -1209,7 +1213,6 @@ class DeviceBooksModel(BooksModel): # {{{ self.book_in_library = None self.sync_icon = QIcon(I('sync.png')) - def counts(self): return Counts(len(self.db), len(self.db), len(self.map)) @@ -1613,3 +1616,4 @@ class DeviceBooksModel(BooksModel): # {{{ # }}} + From e01a3a263daf17c0ff3da0f3f8d135899060b3c5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 21:24:39 +0530 Subject: [PATCH 0254/1154] Missing legacy API --- src/calibre/db/cache.py | 4 ++++ src/calibre/db/view.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index ce8ce288ea..7a17f3468b 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1541,6 +1541,10 @@ class Cache(object): def saved_search_rename(self, old_name, new_name): self._search_api.saved_searches.rename(old_name, new_name) + @write_api + def change_search_locations(self, newlocs): + self._search_api.change_locations(newlocs) + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index bb9131e212..ffaa0ce746 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -290,6 +290,9 @@ class View(object): def get_search_restriction_book_count(self): return self.search_restriction_book_count + def change_search_locations(self, newlocs): + self.cache.change_search_locations(newlocs) + def set_marked_ids(self, id_dict): ''' ids in id_dict are "marked". They can be searched for by From 1770f6b1e9accd9264cacce45f7f6fa956dd6a26 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 21:41:36 +0530 Subject: [PATCH 0255/1154] Replace use of db.conn.close() as it is not supported in the new backend --- src/calibre/db/__init__.py | 2 +- src/calibre/gui2/dialogs/check_library.py | 4 ++-- src/calibre/gui2/dialogs/restore_library.py | 2 +- src/calibre/gui2/ui.py | 2 +- src/calibre/library/restore.py | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 434d9ae317..930b99f8bf 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -135,7 +135,7 @@ Various things that require other things before they can be migrated: 1. From initialize_dynamic(): Also add custom columns/categories/searches info into self.field_metadata. - 2. Port library/restore.py + 2. Port library/restore.py, check_library.py and reinit_db() from debug.py 3. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) diff --git a/src/calibre/gui2/dialogs/check_library.py b/src/calibre/gui2/dialogs/check_library.py index f3c1022ba7..2fc90e5c94 100644 --- a/src/calibre/gui2/dialogs/check_library.py +++ b/src/calibre/gui2/dialogs/check_library.py @@ -45,7 +45,7 @@ class DBCheck(QDialog): # {{{ self.user_version = self.db.user_version self.rejected = False self.db.clean() - self.db.conn.close() + self.db.close() self.closed_orig_conn = True t = DBThread(self.db.dbpath, False) t.connect() @@ -80,7 +80,7 @@ class DBCheck(QDialog): # {{{ self.pb.setMaximum(self.count) self.pb.setValue(0) self.msg.setText(_('Loading database from SQL')) - self.db.conn.close() + self.db.close() self.ndbpath = PersistentTemporaryFile('.db') self.ndbpath.close() self.ndbpath = self.ndbpath.name diff --git a/src/calibre/gui2/dialogs/restore_library.py b/src/calibre/gui2/dialogs/restore_library.py index 7ba852fb13..a460460120 100644 --- a/src/calibre/gui2/dialogs/restore_library.py +++ b/src/calibre/gui2/dialogs/restore_library.py @@ -103,7 +103,7 @@ def restore_database(db, parent=None): 'blank list of books.' '

    Do you want to restore the database?')): return False - db.conn.close() + db.close() d = DBRestore(parent, db.library_path) d.exec_() r = d.restorer diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 5e4f75895a..6f57ee060a 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -622,7 +622,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ if olddb is not None: try: if call_close: - olddb.conn.close() + olddb.close() except: import traceback traceback.print_exc() diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index ac1fce5783..f703451d2d 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -130,7 +130,7 @@ class Restore(Thread): restore_all_prefs=True, progress_callback=self.progress_callback) db.commit() - db.conn.close() + db.close() self.progress_callback(None, 1) if 'field_metadata' in prefs: self.progress_callback(_('Finished restoring preferences and column metadata'), 1) @@ -232,7 +232,7 @@ class Restore(Thread): for i,args in enumerate(self.custom_columns.values()): db.create_custom_column(*args) self.progress_callback(_('creating custom column ')+args[0], i+1) - db.conn.close() + db.close() def restore_books(self): self.progress_callback(None, len(self.books)) @@ -252,7 +252,7 @@ class Restore(Thread): db.conn.execute('UPDATE authors SET link=? WHERE name=?', (link, author.replace(',', '|'))) db.conn.commit() - db.conn.close() + db.close() def restore_book(self, book, db): db.create_book_entry(book['mi'], add_duplicates=True, From f0ef5f0387049a96108298c3804f41bcb1a54192 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 21:51:48 +0530 Subject: [PATCH 0256/1154] ... --- src/calibre/db/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 930b99f8bf..a989e45dce 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -139,4 +139,5 @@ Various things that require other things before they can be migrated: 3. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) + 4. Check for mem leaks when switching libraries ''' From f2f411f9399788162e68a56df15d921c3e572c00 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Jul 2013 22:13:32 +0530 Subject: [PATCH 0257/1154] Accidentally ignored commit_dirty_cache() --- src/calibre/db/cache.py | 6 ++++++ src/calibre/db/legacy.py | 6 +----- src/calibre/db/tests/legacy.py | 2 +- src/calibre/gui2/ui.py | 3 ++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 7a17f3468b..7cfe9bc4a6 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -800,6 +800,12 @@ class Cache(object): self.dirtied_sequence = max(new_dirtied.itervalues()) + 1 self.dirtied_cache.update(new_dirtied) + @write_api + def commit_dirty_cache(self): + book_ids = [(x,) for x in self.dirtied_cache] + if book_ids: + self.backend.conn.executemany('INSERT OR IGNORE INTO metadata_dirtied (book) VALUES (?)', book_ids) + @write_api def set_field(self, name, book_id_to_val_map, allow_case_change=True, do_path_update=True): f = self.fields[name] diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 54f1fe5c1d..8cf6007479 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -890,6 +890,7 @@ LibraryDatabase.saved_search_set_all = MT(lambda self, smap:self.new_api.saved_s LibraryDatabase.saved_search_delete = MT(lambda self, x:self.new_api.saved_search_delete(x)) LibraryDatabase.saved_search_add = MT(lambda self, x, y:self.new_api.saved_search_add(x, y)) LibraryDatabase.saved_search_rename = MT(lambda self, x, y:self.new_api.saved_search_rename(x, y)) +LibraryDatabase.commit_dirty_cache = MT(lambda self: self.new_api.commit_dirty_cache()) # Cleaning is not required anymore LibraryDatabase.clean = LibraryDatabase.clean_custom = MT(lambda self:None) LibraryDatabase.clean_standard_field = MT(lambda self, field, commit=False:None) @@ -899,8 +900,3 @@ LibraryDatabase.commit = MT(lambda self:None) del MT - - - - - diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 0c4efb9f6e..f9caa7a304 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -396,7 +396,7 @@ class LegacyTest(BaseTest): # Internal API 'clean_user_categories', 'cleanup_tags', 'books_list_filter', 'conn', 'connect', 'construct_file_name', - 'construct_path_name', 'clear_dirtied', 'commit_dirty_cache', 'initialize_database', 'initialize_dynamic', + 'construct_path_name', 'clear_dirtied', 'initialize_database', 'initialize_dynamic', 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', 'dirty_books_referencing', 'windows_check_if_files_in_use', 'get_metadata_for_dump', 'get_a_dirtied_book', 'dirtied_sequence', diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 6f57ee060a..e2f02eb578 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -876,7 +876,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ try: self.shutdown(write_settings=False) except: - pass + import traceback + traceback.print_exc() e.accept() else: e.ignore() From 26fcfc70f1603dea6e61eac174d88cbc4e027106 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 10:07:49 +0530 Subject: [PATCH 0258/1154] Speed up book list rendering by using the new api to get values --- src/calibre/db/cache.py | 10 +++ src/calibre/db/fields.py | 12 +++- src/calibre/db/tables.py | 2 +- src/calibre/gui2/library/models.py | 111 ++++++++++++++++++++++++++++- 4 files changed, 132 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 7cfe9bc4a6..a478a23664 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -318,6 +318,16 @@ class Cache(object): except (KeyError, IndexError): return default_value + @read_api + def fast_field_for(self, field_obj, book_id, default_value=None): + ' Same as field_for, except that it avoids the extra lookup to get the field object ' + if field_obj.is_composite: + return field_obj.get_value_with_cache(book_id, partial(self._get_metadata, get_user_categories=False)) + try: + return field_obj.for_book(book_id, default_value=default_value) + except (KeyError, IndexError): + return default_value + @read_api def composite_for(self, name, book_id, mi=None, default_value=''): try: diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index e028ff5d99..8c6d18a74d 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -23,6 +23,7 @@ class Field(object): is_many = False is_many_many = False + is_composite = False def __init__(self, name, table): self.name, self.table = name, table @@ -148,6 +149,8 @@ class OneToOneField(Field): class CompositeField(OneToOneField): + is_composite = True + def __init__(self, *args, **kwargs): OneToOneField.__init__(self, *args, **kwargs) @@ -229,6 +232,14 @@ class OnDeviceField(OneToOneField): self.is_multiple = False self.cache = {} self._lock = Lock() + self._metadata = { + 'table':None, 'column':None, 'datatype':'text', 'is_multiple':{}, + 'kind':'field', 'name':_('On Device'), 'search_terms':['ondevice'], + 'is_custom':False, 'is_category':False, 'is_csp': False, 'display':{}} + + @property + def metadata(self): + return self._metadata def clear_caches(self, book_ids=None): with self._lock: @@ -330,7 +341,6 @@ class ManyToManyField(Field): def __init__(self, *args, **kwargs): Field.__init__(self, *args, **kwargs) - self.alphabetical_sort = self.name != 'authors' def for_book(self, book_id, default_value=None): ids = self.table.book_col_map.get(book_id, ()) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 9b9ff4e9e0..3e1db9400f 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -113,7 +113,7 @@ class SizeTable(OneToOneTable): for row in db.conn.execute( 'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data ' 'WHERE data.book=books.id) FROM books'): - self.book_col_map[row[0]] = self.unserialize(row[1]) + self.book_col_map[row[0]] = self.unserialize(row[1] or 0) def update_sizes(self, size_map): self.book_col_map.update(size_map) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 738b42a669..4a1a1a3e88 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -227,7 +227,10 @@ class BooksModel(QAbstractTableModel): # {{{ elif col in self.custom_columns: self.headers[col] = self.custom_columns[col]['name'] - self.build_data_convertors() + if hasattr(self.db, 'new_api'): + self.build_new_data_convertors() + else: + self.build_data_convertors() self.reset() self.database_changed.emit(db) self.stop_metadata_backup() @@ -634,6 +637,112 @@ class BooksModel(QAbstractTableModel): # {{{ img = self.default_image return img + def build_new_data_convertors(self): + + def renderer(field, decorator=False): + idfunc = self.db.id + fffunc = self.db.new_api.fast_field_for + field_obj = self.db.new_api.fields[field] + m = field_obj.metadata.copy() + if 'display' not in m: + m['display'] = {} + dt = m['datatype'] + + if decorator == 'bool': + bt = self.db.new_api.pref('bools_are_tristate') + bn = self.bool_no_icon + by = self.bool_yes_icon + def func(idx): + val = force_to_bool(fffunc(field_obj, idfunc(idx))) + if val is None: + return NONE if bt else bn + return by if val else bn + elif field == 'size': + sz_mult = 1.0/(1024**2) + def func(idx): + val = fffunc(field_obj, idfunc(idx), default_value=0) + ans = u'%.1f' % (val * sz_mult) + if val > 0 and ans == u'0.0': + ans = u'<0.1' + return QVariant(ans) + elif field == 'languages': + def func(idx): + return QVariant(', '.join(calibre_langcode_to_name(x) for x in fffunc(field_obj, idfunc(idx)))) + elif field == 'ondevice' and decorator: + by = self.bool_yes_icon + bb = self.bool_blank_icon + def func(idx): + return by if fffunc(field_obj, idfunc(idx)) else bb + elif dt in {'text', 'comments', 'composite', 'enumeration'}: + if m['is_multiple']: + jv = m['is_multiple']['list_to_ui'] + do_sort = field == 'tags' + if do_sort: + def func(idx): + return QVariant(jv.join(sorted(fffunc(field_obj, idfunc(idx), default_value=()), key=sort_key))) + else: + def func(idx): + return QVariant(jv.join(fffunc(field_obj, idfunc(idx), default_value=()))) + else: + if dt in {'text', 'composite', 'enumeration'} and m['display'].get('use_decorations', False): + def func(idx): + text = fffunc(field_obj, idfunc(idx)) + return QVariant(text) if force_to_bool(text) is None else NONE + else: + def func(idx): + return QVariant(fffunc(field_obj, idfunc(idx), default_value='')) + elif dt == 'datetime': + def func(idx): + return QVariant(fffunc(field_obj, idfunc(idx), default_value=UNDEFINED_QDATETIME)) + elif dt == 'rating': + def func(idx): + return QVariant(int(fffunc(field_obj, idfunc(idx), default_value=0)/2.0)) + elif dt == 'series': + sidx_field = self.db.new_api.fields[field + '_index'] + fffunc = self.db.new_api._fast_field_for + read_lock = self.db.new_api.read_lock + def func(idx): + book_id = idfunc(idx) + with read_lock: + series = fffunc(field_obj, book_id, default_value=False) + if series: + return QVariant('%s [%s]' % (series, fmt_sidx(fffunc(sidx_field, book_id, default_value=1.0)))) + return NONE + elif dt in {'int', 'float'}: + fmt = m['display'].get('number_format', None) + def func(idx): + val = fffunc(field_obj, idfunc(idx)) + if val is None: + return NONE + if fmt: + try: + return QVariant(fmt.format(val)) + except (TypeError, ValueError, AttributeError, IndexError): + pass + return QVariant(val) + else: + def func(idx): + return NONE + + return func + + self.dc = {f:renderer(f) for f in 'title authors size timestamp pubdate last_modified rating publisher tags series ondevice languages'.split()} + self.dc_decorator = {f:renderer(f, True) for f in ('ondevice',)} + + for col in self.custom_columns: + self.dc[col] = renderer(col) + m = self.custom_columns[col] + dt = m['datatype'] + mult = m['is_multiple'] + if dt in {'text', 'composite', 'enumeration'} and not mult and m['display'].get('use_decorations', False): + self.dc_decorator[col] = renderer(col, 'bool') + elif dt == 'bool': + self.dc_decorator[col] = renderer(col, 'bool') + + # build a index column to data converter map, to remove the string lookup in the data loop + self.column_to_dc_map = [self.dc[col] for col in self.column_map] + self.column_to_dc_decorator_map = [self.dc_decorator.get(col, None) for col in self.column_map] + def build_data_convertors(self): def authors(r, idx=-1): au = self.db.data[r][idx] From fb450951533b53a5df956e279f6d32934408503b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 10:26:38 +0530 Subject: [PATCH 0259/1154] Clear the composites cache correctly --- src/calibre/db/cache.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index a478a23664..9a19f4bb73 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -83,7 +83,7 @@ class Cache(object): def __init__(self, backend): self.backend = backend self.fields = {} - self.composites = set() + self.composites = {} self.read_lock, self.write_lock = create_locks() self.format_metadata_cache = defaultdict(dict) self.formatter_template_cache = {} @@ -144,6 +144,11 @@ class Cache(object): def initialize_template_cache(self): self.formatter_template_cache = {} + @write_api + def clear_composite_caches(self, book_ids=None): + for field in self.composites.itervalues(): + field.clear_caches(book_ids=book_ids) + @write_api def clear_caches(self, book_ids=None): self._initialize_template_cache() # Clear the formatter template cache @@ -265,7 +270,7 @@ class Cache(object): for field, table in self.backend.tables.iteritems(): self.fields[field] = create_field(field, table) if table.metadata['datatype'] == 'composite': - self.composites.add(field) + self.composites[field] = self.fields[field] self.fields['ondevice'] = create_field('ondevice', VirtualTable('ondevice')) @@ -788,11 +793,13 @@ class Cache(object): @write_api def update_last_modified(self, book_ids, now=None): - if now is None: - now = nowf() if book_ids: + if now is None: + now = nowf() f = self.fields['last_modified'] f.writer.set_books({book_id:now for book_id in book_ids}, self.backend) + if self.composites: + self._clear_composite_caches(book_ids) @write_api def mark_as_dirty(self, book_ids): @@ -846,10 +853,6 @@ class Cache(object): sf = self.fields[f.name+'_index'] dirtied |= sf.writer.set_books(simap, self.backend, allow_case_change=False) - if dirtied and self.composites: - for name in self.composites: - self.fields[name].clear_caches(book_ids=dirtied) - if dirtied and update_path and do_path_update: self._update_path(dirtied, mark_as_dirtied=False) From 4ffb8e22020bd50bf9250795e68f9deeb679eeba Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 10:28:35 +0530 Subject: [PATCH 0260/1154] Restore getting the size value to old behavior --- src/calibre/db/tables.py | 2 +- src/calibre/gui2/library/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 3e1db9400f..9b9ff4e9e0 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -113,7 +113,7 @@ class SizeTable(OneToOneTable): for row in db.conn.execute( 'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data ' 'WHERE data.book=books.id) FROM books'): - self.book_col_map[row[0]] = self.unserialize(row[1] or 0) + self.book_col_map[row[0]] = self.unserialize(row[1]) def update_sizes(self, size_map): self.book_col_map.update(size_map) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 4a1a1a3e88..5705f57000 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -660,7 +660,7 @@ class BooksModel(QAbstractTableModel): # {{{ elif field == 'size': sz_mult = 1.0/(1024**2) def func(idx): - val = fffunc(field_obj, idfunc(idx), default_value=0) + val = fffunc(field_obj, idfunc(idx), default_value=0) or 0 ans = u'%.1f' % (val * sz_mult) if val > 0 and ans == u'0.0': ans = u'<0.1' From f6d8c8f1cd14f4767ee6b504cdc055cd970340fa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 10:41:36 +0530 Subject: [PATCH 0261/1154] ... --- src/calibre/gui2/library/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 5705f57000..7320e36ee5 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -674,7 +674,7 @@ class BooksModel(QAbstractTableModel): # {{{ def func(idx): return by if fffunc(field_obj, idfunc(idx)) else bb elif dt in {'text', 'comments', 'composite', 'enumeration'}: - if m['is_multiple']: + if m['is_multiple'] and not field_obj.is_composite: jv = m['is_multiple']['list_to_ui'] do_sort = field == 'tags' if do_sort: @@ -717,7 +717,7 @@ class BooksModel(QAbstractTableModel): # {{{ if fmt: try: return QVariant(fmt.format(val)) - except (TypeError, ValueError, AttributeError, IndexError): + except (TypeError, ValueError, AttributeError, IndexError, KeyError): pass return QVariant(val) else: From 9f83d3b59d0fc6b5d0aa2f8a2e1b547714489fd1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 10:55:14 +0530 Subject: [PATCH 0262/1154] Fix refresh_ids() and rendering of size and datetime fields --- src/calibre/db/view.py | 2 +- src/calibre/gui2/library/models.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index ffaa0ce746..a8cd75d2c2 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -321,7 +321,7 @@ class View(object): if self.search_restriction or self.base_restriction: self.search('', return_matches=False) - def refresh_ids(self, db, ids): + def refresh_ids(self, ids): self.cache.clear_caches(book_ids=ids) try: return list(map(self.id_to_index, ids)) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 7320e36ee5..e70122fd36 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -17,7 +17,7 @@ from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_autho from calibre.ebooks.metadata.book.formatter import SafeFormat from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import tweaks, device_prefs, prefs -from calibre.utils.date import dt_factory, qt_to_dt, as_local_time +from calibre.utils.date import dt_factory, qt_to_dt, as_local_time, UNDEFINED_DATE from calibre.utils.icu import sort_key from calibre.utils.search_query_parser import SearchQueryParser from calibre.db.search import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH @@ -661,10 +661,10 @@ class BooksModel(QAbstractTableModel): # {{{ sz_mult = 1.0/(1024**2) def func(idx): val = fffunc(field_obj, idfunc(idx), default_value=0) or 0 + if val is 0: + return NONE ans = u'%.1f' % (val * sz_mult) - if val > 0 and ans == u'0.0': - ans = u'<0.1' - return QVariant(ans) + return QVariant(u'<0.1' if ans == u'0.0' else ans) elif field == 'languages': def func(idx): return QVariant(', '.join(calibre_langcode_to_name(x) for x in fffunc(field_obj, idfunc(idx)))) @@ -693,7 +693,7 @@ class BooksModel(QAbstractTableModel): # {{{ return QVariant(fffunc(field_obj, idfunc(idx), default_value='')) elif dt == 'datetime': def func(idx): - return QVariant(fffunc(field_obj, idfunc(idx), default_value=UNDEFINED_QDATETIME)) + return QVariant(QDateTime(as_local_time(fffunc(field_obj, idfunc(idx), default_value=UNDEFINED_DATE)))) elif dt == 'rating': def func(idx): return QVariant(int(fffunc(field_obj, idfunc(idx), default_value=0)/2.0)) From 067ed5b978089956cc4b5ad608af265c56163a3c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 11:40:56 +0530 Subject: [PATCH 0263/1154] Add tests for composite cache invalidation --- src/calibre/db/tests/writing.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 26f73964df..4f2bacf921 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -543,3 +543,32 @@ class WritingTest(BaseTest): self.assertEqual(c.field_for('#series_index', 1), 3.0) self.assertEqual(c.field_for('#series_index', 2), 4.0) # }}} + + def test_composite(self): # {{{ + ' Test that the composite field cache is properly invalidated on writes ' + cache = self.init_cache() + cache.create_custom_column('tc', 'TC', 'composite', False, display={ + 'composite_template':'{title} {author_sort} {title_sort} {formats} {tags} {series} {series_index}'}) + cache = self.init_cache() + + def test_invalidate(): + c = self.init_cache() + for bid in cache.all_book_ids(): + self.assertEqual(cache.field_for('#tc', bid), c.field_for('#tc', bid)) + + cache.set_field('title', {1:'xx', 3:'yy'}) + test_invalidate() + cache.set_field('series_index', {1:9, 3:11}) + test_invalidate() + cache.rename_items('tags', {cache.get_item_id('tags', 'Tag One'):'xxx', cache.get_item_id('tags', 'News'):'news'}) + test_invalidate() + cache.remove_items('tags', (cache.get_item_id('tags', 'news'),)) + test_invalidate() + cache.set_sort_for_authors({cache.get_item_id('authors', 'Author One'):'meow'}) + test_invalidate() + cache.remove_formats({1:{'FMT1'}}) + test_invalidate() + cache.add_format(1, 'ADD', BytesIO(b'xxxx')) + test_invalidate() + # }}} + From 9134866ad61c505b98d720bd134dcc18bfd5ba7e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 11:44:36 +0530 Subject: [PATCH 0264/1154] ... --- src/calibre/db/__init__.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index a989e45dce..ec98073776 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -132,12 +132,9 @@ def get_db_loader(): ''' Various things that require other things before they can be migrated: - 1. From initialize_dynamic(): Also add custom - columns/categories/searches info into - self.field_metadata. - 2. Port library/restore.py, check_library.py and reinit_db() from debug.py - 3. Check that content server reloading on metadata,db change, metadata + 1. Port library/restore.py, check_library.py and reinit_db() from debug.py + 2. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) - 4. Check for mem leaks when switching libraries + 3. Check for mem leaks when switching libraries ''' From c77613d615b4d20d2278463a73a6277c896c9ff8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 12:40:16 +0530 Subject: [PATCH 0265/1154] Implement reinit_db() with apsw --- src/calibre/db/__init__.py | 2 +- src/calibre/debug.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index ec98073776..0628e7b51b 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -132,7 +132,7 @@ def get_db_loader(): ''' Various things that require other things before they can be migrated: - 1. Port library/restore.py, check_library.py and reinit_db() from debug.py + 1. Port library/restore.py, check_library.py 2. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 7a1fe754fa..090e1d6ced 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -84,9 +84,47 @@ Everything after the -- is passed to the script. return parser +def reinit_db_new(dbpath, callback=None, sql_dump=None): + from calibre.db.backend import Connection + import apsw + import shutil + from io import StringIO + from contextlib import closing + if callback is None: + callback = lambda x, y: None + + with closing(Connection(dbpath)) as conn: + uv = conn.get('PRAGMA user_version;', all=False) + if sql_dump is None: + buf = StringIO() + shell = apsw.Shell(db=conn, stdout=buf) + shell.process_command('.dump') + sql = buf.getvalue().encode('utf-8') + else: + sql = open(sql_dump, 'rb').read() + + dest = dbpath + '.tmp' + callback(1, True) + try: + with closing(Connection(dest)) as conn: + conn.execute(sql) + conn.execute('PRAGMA User_version=%d;'%int(uv)) + os.remove(dbpath) + shutil.copyfile(dest, dbpath) + finally: + callback(1, False) + if os.path.exists(dest): + os.remove(dest) + prints('Database successfully re-initialized') + + def reinit_db(dbpath, callback=None, sql_dump=None): if not os.path.exists(dbpath): raise ValueError(dbpath + ' does not exist') + from calibre.utils.config_base import tweaks + if tweaks.get('use_new_db', False): + return reinit_db_new(dbpath, callback, sql_dump) + from calibre.library.sqlite import connect from contextlib import closing import shutil From 8ac82e6b47912fb549fe52aea72b731ef0cd19c8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 12:54:04 +0530 Subject: [PATCH 0266/1154] pep8 --- src/calibre/gui2/dialogs/check_library.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/dialogs/check_library.py b/src/calibre/gui2/dialogs/check_library.py index 2fc90e5c94..2769a422ac 100644 --- a/src/calibre/gui2/dialogs/check_library.py +++ b/src/calibre/gui2/dialogs/check_library.py @@ -17,7 +17,8 @@ from calibre import prints, as_unicode from calibre.ptempfile import PersistentTemporaryFile from calibre.library.sqlite import DBThread, OperationalError -class DBCheck(QDialog): # {{{ + +class DBCheck(QDialog): # {{{ def __init__(self, parent, db): QDialog.__init__(self, parent) @@ -96,7 +97,6 @@ class DBCheck(QDialog): # {{{ self.error = (as_unicode(e), traceback.format_exc()) self.reject() - def do_one_load(self): if self.rejected: return @@ -338,7 +338,7 @@ class CheckLibraryDialog(QDialog): t = self.log t.clear() - t.setColumnCount(2); + t.setColumnCount(2) t.setHeaderLabels([_('Name'), _('Path from library')]) self.all_items = [] self.top_level_items = {} @@ -396,7 +396,7 @@ class CheckLibraryDialog(QDialog): tl = self.top_level_items['missing_formats'] child_count = tl.childCount() for i in range(0, child_count): - item = tl.child(i); + item = tl.child(i) id = item.data(0, Qt.UserRole).toInt()[0] all = self.db.formats(id, index_is_id=True, verify_formats=False) all = set([f.strip() for f in all.split(',')]) if all else set() @@ -409,7 +409,7 @@ class CheckLibraryDialog(QDialog): tl = self.top_level_items['missing_covers'] child_count = tl.childCount() for i in range(0, child_count): - item = tl.child(i); + item = tl.child(i) id = item.data(0, Qt.UserRole).toInt()[0] self.db.set_has_cover(id, False) @@ -417,7 +417,7 @@ class CheckLibraryDialog(QDialog): tl = self.top_level_items['extra_covers'] child_count = tl.childCount() for i in range(0, child_count): - item = tl.child(i); + item = tl.child(i) id = item.data(0, Qt.UserRole).toInt()[0] self.db.set_has_cover(id, True) @@ -441,3 +441,4 @@ if __name__ == '__main__': from calibre.library import db d = CheckLibraryDialog(None, db()) d.exec_() + From d05935ea2a9eb870c8d12218da3cb8b03ab05582 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 15:14:03 +0530 Subject: [PATCH 0267/1154] Minor fixes --- src/calibre/debug.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 090e1d6ced..906fe2f3c4 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -94,21 +94,21 @@ def reinit_db_new(dbpath, callback=None, sql_dump=None): callback = lambda x, y: None with closing(Connection(dbpath)) as conn: - uv = conn.get('PRAGMA user_version;', all=False) + uv = int(conn.get('PRAGMA user_version;', all=False)) if sql_dump is None: buf = StringIO() shell = apsw.Shell(db=conn, stdout=buf) shell.process_command('.dump') - sql = buf.getvalue().encode('utf-8') + sql = buf.getvalue() else: - sql = open(sql_dump, 'rb').read() + sql = open(sql_dump, 'rb').read().decode('utf-8') dest = dbpath + '.tmp' callback(1, True) try: with closing(Connection(dest)) as conn: conn.execute(sql) - conn.execute('PRAGMA User_version=%d;'%int(uv)) + conn.execute('PRAGMA user_version=%d;'%int(uv)) os.remove(dbpath) shutil.copyfile(dest, dbpath) finally: From 39425a15a33f3bdbdee15494b36bb41d0d9cf296 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 15:46:34 +0530 Subject: [PATCH 0268/1154] New API version of library check --- src/calibre/db/__init__.py | 2 +- src/calibre/db/backend.py | 39 ++++++++++++-- src/calibre/db/cache.py | 4 ++ src/calibre/gui2/actions/choose_library.py | 7 ++- src/calibre/gui2/dialogs/check_library.py | 59 +++++++++++++++++++++- src/calibre/utils/filenames.py | 10 +++- 6 files changed, 112 insertions(+), 9 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 0628e7b51b..1fe7da1e04 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -132,7 +132,7 @@ def get_db_loader(): ''' Various things that require other things before they can be migrated: - 1. Port library/restore.py, check_library.py + 1. Port library/restore.py 2. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 914248fedc..f203f5ed0c 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -16,7 +16,7 @@ import apsw from calibre import isbytestring, force_unicode, prints from calibre.constants import (iswindows, filesystem_encoding, preferred_encoding) -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, TemporaryFile from calibre.db import SPOOL_SIZE from calibre.db.schema_upgrades import SchemaUpgrade from calibre.db.errors import NoSuchFormat @@ -25,8 +25,8 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.utils.icu import sort_key from calibre.utils.config import to_json, from_json, prefs, tweaks from calibre.utils.date import utcfromtimestamp, parse_date -from calibre.utils.filenames import (is_case_sensitive, samefile, hardlink_file, ascii_filename, - WindowsAtomicFolderMove) +from calibre.utils.filenames import ( + is_case_sensitive, samefile, hardlink_file, ascii_filename, WindowsAtomicFolderMove, atomic_rename) from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_tree, delete_file from calibre.utils.formatter_functions import load_user_template_functions @@ -967,10 +967,41 @@ class DB(object): self.conn.execute('UPDATE custom_columns SET mark_for_delete=1 WHERE id=?', (data['num'],)) def close(self): - if self._conn is not None: + if getattr(self, '_conn', None) is not None: self._conn.close() del self._conn + def reopen(self): + self.close() + self._conn = None + self.conn + + def dump_and_restore(self, callback=None, sql=None): + from io import StringIO + from contextlib import closing + if callback is None: + callback = lambda x: x + uv = int(self.user_version) + + if sql is None: + callback(_('Dumping database to SQL') + '...') + buf = StringIO() + shell = apsw.Shell(db=self.conn, stdout=buf) + shell.process_command('.dump') + sql = buf.getvalue() + + with TemporaryFile(suffix='_tmpdb.db', dir=os.path.dirname(self.dbpath)) as tmpdb: + callback(_('Restoring database from SQL') + '...') + with closing(Connection(tmpdb)) as conn: + conn.execute(sql) + conn.execute('PRAGMA user_version=%d;'%uv) + + self.close() + try: + atomic_rename(tmpdb, self.dbpath) + finally: + self.reopen() + @dynamic_property def user_version(self): doc = 'The user version of this database' diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 9a19f4bb73..37d0428046 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1564,6 +1564,10 @@ class Cache(object): def change_search_locations(self, newlocs): self._search_api.change_locations(newlocs) + @write_api + def dump_and_restore(self, callback=None, sql=None): + return self.backend.dump_and_restore(callback=callback, sql=sql) + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index 6857454ae4..f85eb09f37 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -413,14 +413,17 @@ class ChooseLibraryAction(InterfaceAction): self.gui.library_moved(db.library_path, call_close=False) def check_library(self): - from calibre.gui2.dialogs.check_library import CheckLibraryDialog, DBCheck + from calibre.gui2.dialogs.check_library import CheckLibraryDialog, DBCheck, DBCheckNew self.gui.library_view.save_state() m = self.gui.library_view.model() m.stop_metadata_backup() db = m.db db.prefs.disable_setting = True - d = DBCheck(self.gui, db) + if hasattr(db, 'new_api'): + d = DBCheckNew(self.gui, db) + else: + d = DBCheck(self.gui, db) d.start() try: d.conn.close() diff --git a/src/calibre/gui2/dialogs/check_library.py b/src/calibre/gui2/dialogs/check_library.py index 2769a422ac..c1f8ed4f18 100644 --- a/src/calibre/gui2/dialogs/check_library.py +++ b/src/calibre/gui2/dialogs/check_library.py @@ -4,11 +4,12 @@ __docformat__ = 'restructuredtext en' __license__ = 'GPL v3' import os, shutil +from threading import Thread from PyQt4.Qt import (QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel, QPushButton, QDialogButtonBox, QApplication, QTreeWidgetItem, QLineEdit, Qt, QProgressBar, QSize, QTimer, QIcon, QTextEdit, - QSplitter, QWidget) + QSplitter, QWidget, pyqtSignal) from calibre.gui2.dialogs.confirm_delete import confirm from calibre.library.check_library import CheckLibrary, CHECKS @@ -17,6 +18,62 @@ from calibre import prints, as_unicode from calibre.ptempfile import PersistentTemporaryFile from calibre.library.sqlite import DBThread, OperationalError +class DBCheckNew(QDialog): # {{{ + + update_msg = pyqtSignal(object) + + def __init__(self, parent, db): + QDialog.__init__(self, parent) + self.l = QVBoxLayout() + self.setLayout(self.l) + self.l1 = QLabel(_('Checking database integrity') + ' ' + + _('This will take a while, please wait...')) + self.setWindowTitle(_('Checking database integrity')) + self.l1.setWordWrap(True) + self.l.addWidget(self.l1) + self.msg = QLabel('') + self.update_msg.connect(self.msg.setText, type=Qt.QueuedConnection) + self.l.addWidget(self.msg) + self.msg.setWordWrap(True) + self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) + self.l.addWidget(self.bb) + self.bb.rejected.connect(self.reject) + self.resize(self.sizeHint() + QSize(100, 50)) + self.error = None + self.db = db.new_api + self.closed_orig_conn = False + self.rejected = False + + def start(self): + t = self.thread = Thread(target=self.dump_and_restore) + t.daemon = True + t.start() + QTimer.singleShot(100, self.check) + self.exec_() + + def dump_and_restore(self): + try: + self.db.dump_and_restore(self.update_msg.emit) + except Exception as e: + import traceback + self.error = (as_unicode(e), traceback.format_exc()) + + def reject(self): + self.rejected = True + return QDialog.reject(self) + + def check(self): + if self.rejected: + return + if self.thread.is_alive(): + QTimer.singleShot(100, self.check) + else: + self.accept() + + def break_cycles(self): + self.db = self.thread = None + +# }}} class DBCheck(QDialog): # {{{ diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index d756978040..23ac8fd43a 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -383,4 +383,12 @@ def hardlink_file(src, dest): return os.link(src, dest) - +def atomic_rename(oldpath, newpath): + '''Replace the file newpath with the file oldpath. Can fail if the files + are on different volumes. If succeeds, guaranteed to be atomic. newpath may + or may not exist. If it exists, it is replaced. ''' + if iswindows: + import win32file + win32file.MoveFileEx(oldpath, newpath, win32file.MOVEFILE_REPLACE_EXISTING|win32file.MOVEFILE_WRITE_THROUGH) + else: + os.rename(oldpath, newpath) From 1a11c09d3cc72c41a10bdc2877be44a110cdf308 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 09:12:37 +0530 Subject: [PATCH 0269/1154] When reloading on dn modification, re-open the connection This ensures that the content server does not continue to use a deleted metadata.db. --- src/calibre/db/legacy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 8cf6007479..13edf47999 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -117,6 +117,7 @@ class LibraryDatabase(object): def check_if_modified(self): if self.last_modified() > self.last_update_check: + self.backend.reopen() self.new_api.reload_from_db() self.last_update_check = utcnow() From e7777df9aecf94488c47618295a113bf70d92074 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 09:29:52 +0530 Subject: [PATCH 0270/1154] Update taz.de (RSS) --- recipes/taz_rss.recipe | 55 ++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/recipes/taz_rss.recipe b/recipes/taz_rss.recipe index 0535b6ef3a..d4fc0237da 100644 --- a/recipes/taz_rss.recipe +++ b/recipes/taz_rss.recipe @@ -1,3 +1,4 @@ + __license__ = 'GPL v3' __copyright__ = '2013, Alexander Schremmer , Robert Riemann ' @@ -6,10 +7,15 @@ from calibre.web.feeds.news import BasicNewsRecipe class TazRSSRecipe(BasicNewsRecipe): title = u'Taz - die Tageszeitung' - description = u'Taz.de - die tageszeitung' + description = u'Taz.de - die tageszeitung (Anpassung von Robert)' __author__ = 'Alexander Schremmer, Robert Riemann' language = 'de' lang = 'de-DE' + category = 'news, Germany' + timefmt = ' [%a, %d %b %Y]' + publication_type = 'newspaper' + remove_empty_feeds = True + use_embedded_content = False oldest_article = 7 max_articles_per_feed = 100 publisher = 'taz Entwicklungs GmbH & Co. Medien KG' @@ -20,24 +26,37 @@ class TazRSSRecipe(BasicNewsRecipe): 'language': lang, } feeds = [ - (u'Schlagzeilen', u'http://www.taz.de/!p3270;rss/'), - (u'Politik', u'http://www.taz.de/Politik/!p2;rss/'), - (u'Zukunft', u'http://www.taz.de/Zukunft/!p4;rss/'), - (u'Netz', u'http://www.taz.de/Netz/!p5;rss/'), - (u'Debatte', u'http://www.taz.de/Debatte/!p9;rss/'), - (u'Leben', u'http://www.taz.de/Leben/!p10;rss/'), - (u'Sport', u'http://www.taz.de/Sport/!p12;rss/'), - (u'Wahrheit', u'http://www.taz.de/Wahrheit/!p13;rss/'), - (u'Berlin', u'http://www.taz.de/Berlin/!p14;rss/'), - (u'Nord', u'http://www.taz.de/Nord/!p11;rss/') + (u'Schlagzeilen', u'http://www.taz.de/!p3270;rss/'), + (u'Politik', u'http://www.taz.de/Politik/!p2;rss/'), + (u'Zukunft', u'http://www.taz.de/Zukunft/!p4;rss/'), + (u'Netz', u'http://www.taz.de/Netz/!p5;rss/'), + (u'Debatte', u'http://www.taz.de/Debatte/!p9;rss/'), + (u'Leben', u'http://www.taz.de/Leben/!p10;rss/'), + (u'Sport', u'http://www.taz.de/Sport/!p12;rss/'), + (u'Wahrheit', u'http://www.taz.de/Wahrheit/!p13;rss/'), + (u'Berlin', u'http://www.taz.de/Berlin/!p14;rss/'), + (u'Nord', u'http://www.taz.de/Nord/!p11;rss/') ] + # omit articles already linked in Schlagzeilen feed + ignore_duplicate_articles = {'title', 'url'} + + # use the cover presented on the homepage + cover_url = 'http://www.taz.de/digitaz/.s1jpeg320' + keep_only_tags = [dict(name='div', attrs={'class': 'sect sect_article'})] remove_tags = [ - dict(name=['div'], attrs={'class': 'artikelwerbung'}), - dict(name=['ul'], attrs={'class': 'toolbar'}), - # remove: taz paywall - dict(name=['div'], attrs={'id': 'tzi_paywall'}), - # remove: Artikel zum Thema (not working on Kindle) - dict(name=['div'], attrs={'class': re.compile(r".*\bsect_seealso\b.*")}), - dict(name=['div'], attrs={'class': 'sectfoot'}) + dict(name=['div'], attrs={'class': 'artikelwerbung'}), + dict(name=['ul'], attrs={'class': 'toolbar'}), + # remove: taz paywall + dict(name=['div'], attrs={'id': 'tzi_paywall'}), + # remove: Artikel zum Thema (not working on Kindle) + dict(name=['div'], attrs={'class': re.compile(r".*\bsect_seealso\b.*")}), + dict(name=['div'], attrs={'class': 'sectfoot'}) ] + +# with article pictures on Kindle super-slow +# def populate_article_metadata(self, article, soup, first): +# if first and hasattr(self, 'add_toc_thumbnail'): +# picdiv = soup.find('img') +# if picdiv is not None: +# self.add_toc_thumbnail(article,picdiv['src']) From b5bc63617868fd5dd796ed6351d290e818cb13d4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 11:50:20 +0530 Subject: [PATCH 0271/1154] Fix queueing of all dirtied books --- src/calibre/db/legacy.py | 5 ++++- src/calibre/db/tests/legacy.py | 2 +- src/calibre/gui2/actions/choose_library.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 13edf47999..5d41986fce 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -323,7 +323,10 @@ class LibraryDatabase(object): self.notify('delete', [id]) def dirtied(self, book_ids, commit=True): - self.new_api.mark_as_dirty(book_ids) + self.new_api.mark_as_dirty(frozenset(book_ids) if book_ids is not None else book_ids) + + def dirty_queue_length(self): + return self.new_api.dirty_queue_length() def dump_metadata(self, book_ids=None, remove_from_dirtied=True, commit=True, callback=None): self.new_api.dump_metadata(book_ids=book_ids, remove_from_dirtied=remove_from_dirtied, callback=callback) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index f9caa7a304..a5f5f693fe 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -398,7 +398,7 @@ class LegacyTest(BaseTest): 'clean_user_categories', 'cleanup_tags', 'books_list_filter', 'conn', 'connect', 'construct_file_name', 'construct_path_name', 'clear_dirtied', 'initialize_database', 'initialize_dynamic', 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', - 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', 'dirty_books_referencing', + 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_books_referencing', 'windows_check_if_files_in_use', 'get_metadata_for_dump', 'get_a_dirtied_book', 'dirtied_sequence', 'format_filename_cache', 'format_metadata_cache', 'filter', 'create_version1', 'normpath', 'custom_data_adapters', 'custom_table_names', 'custom_columns_in_meta', 'custom_tables', diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index f85eb09f37..4010847109 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -376,7 +376,7 @@ class ChooseLibraryAction(InterfaceAction): dirty_text = 'no' try: dirty_text = \ - unicode(self.gui.library_view.model().db.dirty_queue_length()) + unicode(self.gui.current_db.dirty_queue_length()) except: dirty_text = _('none') info_dialog(self.gui, _('Backup status'), '

    '+ From 4b1a61d7cec2f54ff866f28a13cc868e5464d485 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 11:55:31 +0530 Subject: [PATCH 0272/1154] Fix backups not working in new backend --- src/calibre/db/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/db/backup.py b/src/calibre/db/backup.py index 6410a347c6..18722e1cf8 100644 --- a/src/calibre/db/backup.py +++ b/src/calibre/db/backup.py @@ -26,7 +26,7 @@ class MetadataBackup(Thread): def __init__(self, db, interval=2, scheduling_interval=0.1): Thread.__init__(self) self.daemon = True - self._db = weakref.ref(db) + self._db = weakref.ref(db.new_api) self.stop_running = Event() self.interval = interval self.scheduling_interval = scheduling_interval From 8a5c71f36a71bb2bfd3e437d0aa666fcae319938 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sun, 21 Jul 2013 09:09:02 +0200 Subject: [PATCH 0273/1154] 1) Fix silly regression in template_functions preference that caused all functions to be added to the custom function preference. 2) Change function registration and deregistration to restore function bodies if the name conflict goes away. --- .../gui2/preferences/template_functions.py | 7 +- src/calibre/utils/formatter_functions.py | 67 ++++++++++--------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/calibre/gui2/preferences/template_functions.py b/src/calibre/gui2/preferences/template_functions.py index 6842e0798e..241d5eef8c 100644 --- a/src/calibre/gui2/preferences/template_functions.py +++ b/src/calibre/gui2/preferences/template_functions.py @@ -220,11 +220,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): def commit(self): # formatter_functions().reset_to_builtins() pref_value = [] - for f in self.funcs: - func = self.funcs[f] - pref_value.append((func.name, func.doc, func.arg_count, func.program_text)) + for name, cls in self.funcs.iteritems(): + if name not in self.builtins: + pref_value.append((cls.name, cls.doc, cls.arg_count, cls.program_text)) self.db.prefs.set('user_template_functions', pref_value) - formatter_functions().unregister_functions(self.db.library_id) load_user_template_functions(self.db.library_id, pref_value) return False diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index f8d62c367a..f2b973a1e7 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en' import inspect, re, traceback from math import trunc -from collections import defaultdict from calibre import human_readable from calibre.constants import DEBUG @@ -23,10 +22,16 @@ from calibre.utils.localization import calibre_langcode_to_name, canonicalize_la class FormatterFunctions(object): + error_function_body = ('def evaluate(self, formatter, kwargs, mi, locals):\n' + '\treturn "' + + _('Duplicate user function name {0}. ' + 'Change the name or ensure that the functions are identical') + + '"') + def __init__(self): self._builtins = {} self._functions = {} - self._functions_from_library = defaultdict(list) + self._functions_from_library = {} def register_builtin(self, func_class): if not isinstance(func_class, FormatterFunction): @@ -40,7 +45,7 @@ class FormatterFunctions(object): for a in func_class.aliases: self._functions[a] = func_class - def register_function(self, library_uuid, func_class, replace=False): + def _register_function(self, func_class, replace=False): if not isinstance(func_class, FormatterFunction): raise ValueError('Class %s is not an instance of FormatterFunction'%( func_class.__class__.__name__)) @@ -48,16 +53,36 @@ class FormatterFunctions(object): if not replace and name in self._functions: raise ValueError('Name %s already used'%name) self._functions[name] = func_class - self._functions_from_library[library_uuid].append(name) - def function_exists(self, name): - return self._functions.get(name, None) + def register_functions(self, library_uuid, funcs): + self._functions_from_library[library_uuid] = funcs + self._register_functions() + + def _register_functions(self): + for compiled_funcs in self._functions_from_library.itervalues(): + for cls in compiled_funcs: + f = self._functions.get(cls.name, None) + replace = False + if f is not None: + existing_body = f.program_text + new_body = cls.program_text + if new_body != existing_body: + # Change the body of the template function to one that will + # return an error message. Also change the arg count to + # -1 (variable) to avoid template compilation errors + replace = True + func = [cls.name, '', -1, self.error_function_body.format(cls.name)] + cls = compile_user_function(*func) + else: + continue + formatter_functions()._register_function(cls, replace=replace) def unregister_functions(self, library_uuid): if library_uuid in self._functions_from_library: - for name in self._functions_from_library[library_uuid]: - self._functions.pop(name) + for cls in self._functions_from_library[library_uuid]: + self._functions.pop(cls.name, None) self._functions_from_library.pop(library_uuid) + self._register_functions() def get_builtins(self): return self._builtins @@ -1271,15 +1296,11 @@ class UserFunction(FormatterUserFunction): cls = locals_['UserFunction'](name, doc, arg_count, eval_func) return cls -error_function_body = ('def evaluate(self, formatter, kwargs, mi, locals):\n' - '\treturn "' + - _('Duplicate user function name {0}. ' - 'Change the name or ensure that the functions are identical') - + '"') def load_user_template_functions(library_uuid, funcs): unload_user_template_functions(library_uuid) + compiled_funcs = [] for func in funcs: try: # Force a name conflict to test the logic @@ -1290,26 +1311,10 @@ def load_user_template_functions(library_uuid, funcs): # source. This helps ensure that if the function already is defined # then white space differences don't cause them to compare differently - cls = compile_user_function(*func) - f = formatter_functions().function_exists(cls.name) - replace = False - if f is not None: - existing_body = f.program_text - new_body = cls.program_text - if new_body != existing_body: - # Change the body of the template function to one that will - # return an error message. Also change the arg count to - # -1 (variable) to avoid template compilation errors - replace = True - func[3] = error_function_body.format(func[0]) - func[2] = -1 - cls = compile_user_function(*func) - else: - continue - - formatter_functions().register_function(library_uuid, cls, replace=replace) + compiled_funcs.append(compile_user_function(*func)) except: traceback.print_exc() + formatter_functions().register_functions(library_uuid, compiled_funcs) def unload_user_template_functions(library_uuid): formatter_functions().unregister_functions(library_uuid) \ No newline at end of file From 176da67d82b3425c66996bba36ef1c1bdea34b06 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 12:40:53 +0530 Subject: [PATCH 0274/1154] Database restore now works with the new API --- src/calibre/db/__init__.py | 5 ++--- src/calibre/db/backend.py | 4 ++++ src/calibre/db/cache.py | 21 +++++++++++++++++++++ src/calibre/db/legacy.py | 2 +- src/calibre/gui2/dialogs/restore_library.py | 8 ++++++-- src/calibre/library/cli.py | 5 ++++- 6 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 1fe7da1e04..c1b48cad0c 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -132,9 +132,8 @@ def get_db_loader(): ''' Various things that require other things before they can be migrated: - 1. Port library/restore.py - 2. Check that content server reloading on metadata,db change, metadata + 1. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) - 3. Check for mem leaks when switching libraries + 2. Check for mem leaks when switching libraries ''' diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index f203f5ed0c..9eb49ccd4c 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1525,6 +1525,10 @@ class DB(object): except: pass + def restore_book(self, book_id, path, formats): + self.conn.execute('UPDATE books SET path=? WHERE id=?', (path.replace(os.sep, '/'), book_id)) + vals = [(book_id, fmt, size, name) for fmt, size, name in formats] + self.conn.executemany('INSERT INTO data (book,format,uncompressed_size,name) VALUES (?,?,?,?)', vals) # }}} diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 37d0428046..16e6d59adb 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -423,6 +423,12 @@ class Cache(object): rmap = {icu_lower(v) if isinstance(v, unicode) else v:k for k, v in self.fields[field].table.id_map.iteritems()} return rmap.get(icu_lower(item_name) if isinstance(item_name, unicode) else item_name, None) + @read_api + def get_item_ids(self, field, item_names): + ' Return the item id for item_name (case-insensitive) ' + rmap = {icu_lower(v) if isinstance(v, unicode) else v:k for k, v in self.fields[field].table.id_map.iteritems()} + return {name:rmap.get(icu_lower(name) if isinstance(name, unicode) else name, None) for name in item_names} + @read_api def author_data(self, author_ids=None): ''' @@ -1568,6 +1574,21 @@ class Cache(object): def dump_and_restore(self, callback=None, sql=None): return self.backend.dump_and_restore(callback=callback, sql=sql) + @write_api + def close(self): + self.backend.close() + + @write_api + def restore_book(self, book_id, mi, last_modified, path, formats): + ''' Restore the book entry in the database for a book that already exists on the filesystem ''' + cover = mi.cover + mi.cover = None + self._create_book_entry(mi, add_duplicates=True, + force_id=book_id, apply_import_tags=False, preserve_uuid=True) + self._update_last_modified((book_id,), last_modified) + if cover and os.path.exists(cover): + self._set_field('cover', {book_id:1}) + self.backend.restore_book(book_id, path, formats) # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 5d41986fce..94c35429b2 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -78,7 +78,7 @@ class LibraryDatabase(object): set_saved_searches(self, 'saved_searches') def close(self): - self.backend.close() + self.new_api.close() def break_cycles(self): delattr(self.backend, 'field_metadata') diff --git a/src/calibre/gui2/dialogs/restore_library.py b/src/calibre/gui2/dialogs/restore_library.py index a460460120..58731fa2d0 100644 --- a/src/calibre/gui2/dialogs/restore_library.py +++ b/src/calibre/gui2/dialogs/restore_library.py @@ -8,11 +8,11 @@ __docformat__ = 'restructuredtext en' from PyQt4.Qt import (QDialog, QLabel, QVBoxLayout, QDialogButtonBox, QProgressBar, QSize, QTimer, pyqtSignal, Qt) -from calibre.library.restore import Restore from calibre.gui2 import (error_dialog, question_dialog, warning_dialog, info_dialog) from calibre import force_unicode from calibre.constants import filesystem_encoding +from calibre.utils.config_base import tweaks class DBRestore(QDialog): @@ -42,13 +42,16 @@ class DBRestore(QDialog): self.library_path = library_path self.update_signal.connect(self.do_update, type=Qt.QueuedConnection) + if tweaks.get('use_new_db', False): + from calibre.db.restore import Restore + else: + from calibre.library.restore import Restore self.restorer = Restore(library_path, self) self.restorer.daemon = True # Give the metadata backup thread time to stop QTimer.singleShot(2000, self.start) - def start(self): self.restorer.start() QTimer.singleShot(10, self.update) @@ -133,3 +136,4 @@ def repair_library_at(library_path, parent=None): return True + diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 2c6e7cd777..5a888f672e 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -1214,7 +1214,6 @@ what is found in the OPF files. return parser def command_restore_database(args, dbpath): - from calibre.library.restore import Restore parser = restore_database_option_parser() opts, args = parser.parse_args(args) if len(args) != 0: @@ -1242,6 +1241,10 @@ def command_restore_database(args, dbpath): self.total = float(step) else: prints(msg, '...', '%d%%'%int(100*(step/self.total))) + if tweaks.get('use_new_db', False): + from calibre.db.restore import Restore + else: + from calibre.library.restore import Restore r = Restore(dbpath, progress_callback=Progress()) r.start() r.join() From 7cf5da528f9c37ec51780768d9710cfe1e0fe410 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 12:41:36 +0530 Subject: [PATCH 0275/1154] oops, forgot one file in the last commit --- src/calibre/db/restore.py | 266 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 src/calibre/db/restore.py diff --git a/src/calibre/db/restore.py b/src/calibre/db/restore.py new file mode 100644 index 0000000000..9569dbeab6 --- /dev/null +++ b/src/calibre/db/restore.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import re, os, traceback, shutil +from threading import Thread +from operator import itemgetter + +from calibre.ptempfile import TemporaryDirectory +from calibre.ebooks.metadata.opf2 import OPF +from calibre.db.backend import DB, DBPrefs +from calibre.db.cache import Cache +from calibre.constants import filesystem_encoding +from calibre.utils.date import utcfromtimestamp +from calibre import isbytestring + +NON_EBOOK_EXTENSIONS = frozenset([ + 'jpg', 'jpeg', 'gif', 'png', 'bmp', + 'opf', 'swp', 'swo' + ]) + +class Restorer(Cache): + + def __init__(self, library_path, default_prefs=None, restore_all_prefs=False, progress_callback=lambda x, y:True): + backend = DB(library_path, default_prefs=default_prefs, restore_all_prefs=restore_all_prefs, progress_callback=progress_callback) + Cache.__init__(self, backend) + for x in ('update_path', 'mark_as_dirty'): + setattr(self, x, self.no_op) + setattr(self, '_' + x, self.no_op) + self.init() + + def no_op(self, *args, **kwargs): + pass + +class Restore(Thread): + + def __init__(self, library_path, progress_callback=None): + super(Restore, self).__init__() + if isbytestring(library_path): + library_path = library_path.decode(filesystem_encoding) + self.src_library_path = os.path.abspath(library_path) + self.progress_callback = progress_callback + self.db_id_regexp = re.compile(r'^.* \((\d+)\)$') + self.bad_ext_pat = re.compile(r'[^a-z0-9_]+') + if not callable(self.progress_callback): + self.progress_callback = lambda x, y: x + self.dirs = [] + self.ignored_dirs = [] + self.failed_dirs = [] + self.books = [] + self.conflicting_custom_cols = {} + self.failed_restores = [] + self.mismatched_dirs = [] + self.successes = 0 + self.tb = None + self.authors_links = {} + + @property + def errors_occurred(self): + return (self.failed_dirs or self.mismatched_dirs or + self.conflicting_custom_cols or self.failed_restores) + + @property + def report(self): + ans = '' + failures = list(self.failed_dirs) + [(x['dirpath'], tb) for x, tb in + self.failed_restores] + if failures: + ans += 'Failed to restore the books in the following folders:\n' + for dirpath, tb in failures: + ans += '\t' + dirpath + ' with error:\n' + ans += '\n'.join('\t\t'+x for x in tb.splitlines()) + ans += '\n\n' + + if self.conflicting_custom_cols: + ans += '\n\n' + ans += 'The following custom columns have conflicting definitions ' \ + 'and were not fully restored:\n' + for x in self.conflicting_custom_cols: + ans += '\t#'+x+'\n' + ans += '\tused:\t%s, %s, %s, %s\n'%(self.custom_columns[x][1], + self.custom_columns[x][2], + self.custom_columns[x][3], + self.custom_columns[x][5]) + for coldef in self.conflicting_custom_cols[x]: + ans += '\tother:\t%s, %s, %s, %s\n'%(coldef[1], coldef[2], + coldef[3], coldef[5]) + + if self.mismatched_dirs: + ans += '\n\n' + ans += 'The following folders were ignored:\n' + for x in self.mismatched_dirs: + ans += '\t'+x+'\n' + + return ans + + def run(self): + try: + with TemporaryDirectory('_library_restore') as tdir: + self.library_path = tdir + self.scan_library() + if not self.load_preferences(): + # Something went wrong with preferences restore. Start over + # with a new database and attempt to rebuild the structure + # from the metadata in the opf + dbpath = os.path.join(self.library_path, 'metadata.db') + if os.path.exists(dbpath): + os.remove(dbpath) + self.create_cc_metadata() + self.restore_books() + if self.successes == 0 and len(self.dirs) > 0: + raise Exception(('Something bad happened')) + self.replace_db() + except: + self.tb = traceback.format_exc() + + def load_preferences(self): + self.progress_callback(None, 1) + self.progress_callback(_('Starting restoring preferences and column metadata'), 0) + prefs_path = os.path.join(self.src_library_path, 'metadata_db_prefs_backup.json') + if not os.path.exists(prefs_path): + self.progress_callback(_('Cannot restore preferences. Backup file not found.'), 1) + return False + try: + prefs = DBPrefs.read_serialized(self.src_library_path, recreate_prefs=False) + db = Restorer(self.library_path, default_prefs=prefs, + restore_all_prefs=True, + progress_callback=self.progress_callback) + db.close() + self.progress_callback(None, 1) + if 'field_metadata' in prefs: + self.progress_callback(_('Finished restoring preferences and column metadata'), 1) + return True + self.progress_callback(_('Finished restoring preferences'), 1) + return False + except: + traceback.print_exc() + self.progress_callback(None, 1) + self.progress_callback(_('Restoring preferences and column metadata failed'), 0) + return False + + def scan_library(self): + for dirpath, dirnames, filenames in os.walk(self.src_library_path): + leaf = os.path.basename(dirpath) + m = self.db_id_regexp.search(leaf) + if m is None or 'metadata.opf' not in filenames: + self.ignored_dirs.append(dirpath) + continue + self.dirs.append((dirpath, filenames, m.group(1))) + + self.progress_callback(None, len(self.dirs)) + for i, x in enumerate(self.dirs): + dirpath, filenames, book_id = x + try: + self.process_dir(dirpath, filenames, book_id) + except: + self.failed_dirs.append((dirpath, traceback.format_exc())) + self.progress_callback(_('Processed') + ' ' + dirpath, i+1) + + def is_ebook_file(self, filename): + ext = os.path.splitext(filename)[1] + if not ext: + return False + ext = ext[1:].lower() + if ext in NON_EBOOK_EXTENSIONS or \ + self.bad_ext_pat.search(ext) is not None: + return False + return True + + def process_dir(self, dirpath, filenames, book_id): + book_id = int(book_id) + formats = filter(self.is_ebook_file, filenames) + fmts = [os.path.splitext(x)[1][1:].upper() for x in formats] + sizes = [os.path.getsize(os.path.join(dirpath, x)) for x in formats] + names = [os.path.splitext(x)[0] for x in formats] + opf = os.path.join(dirpath, 'metadata.opf') + mi = OPF(opf, basedir=dirpath).to_book_metadata() + timestamp = os.path.getmtime(opf) + path = os.path.relpath(dirpath, self.src_library_path).replace(os.sep, + '/') + + if int(mi.application_id) == book_id: + self.books.append({ + 'mi': mi, + 'timestamp': timestamp, + 'formats': list(zip(fmts, sizes, names)), + 'id': book_id, + 'dirpath': dirpath, + 'path': path, + }) + else: + self.mismatched_dirs.append(dirpath) + + alm = mi.get('author_link_map', {}) + for author, link in alm.iteritems(): + existing_link, timestamp = self.authors_links.get(author, (None, None)) + if existing_link is None or existing_link != link and timestamp < mi.timestamp: + self.authors_links[author] = (link, mi.timestamp) + + def create_cc_metadata(self): + self.books.sort(key=itemgetter('timestamp')) + self.custom_columns = {} + fields = ('label', 'name', 'datatype', 'is_multiple', 'is_editable', + 'display') + for b in self.books: + for key in b['mi'].custom_field_keys(): + cfm = b['mi'].metadata_for_field(key) + args = [] + for x in fields: + if x in cfm: + if x == 'is_multiple': + args.append(bool(cfm[x])) + else: + args.append(cfm[x]) + if len(args) == len(fields): + # TODO: Do series type columns need special handling? + label = cfm['label'] + if label in self.custom_columns and args != self.custom_columns[label]: + if label not in self.conflicting_custom_cols: + self.conflicting_custom_cols[label] = [] + if self.custom_columns[label] not in self.conflicting_custom_cols[label]: + self.conflicting_custom_cols[label].append(self.custom_columns[label]) + self.custom_columns[label] = args + + db = Restorer(self.library_path) + self.progress_callback(None, len(self.custom_columns)) + if len(self.custom_columns): + for i, args in enumerate(self.custom_columns.values()): + db.create_custom_column(*args) + self.progress_callback(_('Creating custom column ')+args[0], i+1) + db.close() + + def restore_books(self): + self.progress_callback(None, len(self.books)) + self.books.sort(key=itemgetter('id')) + + db = Restorer(self.library_path) + + for i, book in enumerate(self.books): + try: + db.restore_book(book['id'], book['mi'], utcfromtimestamp(book['timestamp']), book['path'], book['formats']) + self.successes += 1 + except: + self.failed_restores.append((book, traceback.format_exc())) + self.progress_callback(book['mi'].title, i+1) + + id_map = db.get_item_ids('authors', [author for author in self.authors_links]) + link_map = {aid:self.authors_links[name][0] for name, aid in id_map.iteritems() if aid is not None} + if link_map: + db.set_link_for_authors(link_map) + db.close() + + def replace_db(self): + dbpath = os.path.join(self.src_library_path, 'metadata.db') + ndbpath = os.path.join(self.library_path, 'metadata.db') + + save_path = self.olddb = os.path.splitext(dbpath)[0]+'_pre_restore.db' + if os.path.exists(save_path): + os.remove(save_path) + os.rename(dbpath, save_path) + shutil.copyfile(ndbpath, dbpath) + + From 30ee950706ee6c3ef7ef4f56240f0635ecc03e3b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 14:20:37 +0530 Subject: [PATCH 0276/1154] pep8 --- src/calibre/gui2/dialogs/metadata_bulk.py | 38 +++++++++++++---------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 0e4b2a1572..007ff3b58f 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -4,6 +4,7 @@ __copyright__ = '2008, Kovid Goyal ' '''Dialog to edit metadata in bulk''' import re, os, inspect +from collections import namedtuple from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \ pyqtSignal, QDialogButtonBox, QInputDialog, QLineEdit, \ @@ -27,7 +28,7 @@ from calibre.utils.date import qt_to_dt from calibre.ptempfile import SpooledTemporaryFile from calibre.db import SPOOL_SIZE -def get_cover_data(stream, ext): # {{{ +def get_cover_data(stream, ext): # {{{ from calibre.ebooks.metadata.meta import get_metadata old = prefs['read_file_metadata'] if not old: @@ -53,7 +54,11 @@ def get_cover_data(stream, ext): # {{{ return cdata, area # }}} -class MyBlockingBusy(QDialog): # {{{ +Settings = namedtuple('Settings', 'remove_all remove add au aus do_aus rating pub do_series do_autonumber do_remove_format ' + 'remove_format do_swap_ta do_remove_conv do_auto_author series do_series_restart series_start_value ' + 'do_title_case cover_action clear_series pubdate adddate do_title_sort languages clear_languages restore_original') + +class MyBlockingBusy(QDialog): # {{{ do_one_signal = pyqtSignal() @@ -71,8 +76,8 @@ class MyBlockingBusy(QDialog): # {{{ self._layout = QVBoxLayout() self.setLayout(self._layout) self.msg_text = msg - self.msg = QLabel(msg+' ') # Ensure dialog is wide enough - #self.msg.setWordWrap(True) + self.msg = QLabel(msg+' ') # Ensure dialog is wide enough + # self.msg.setWordWrap(True) self.font = QFont() self.font.setPointSize(self.font.pointSize() + 8) self.msg.setFont(self.font) @@ -140,7 +145,6 @@ class MyBlockingBusy(QDialog): # {{{ pubdate, adddate, do_title_sort, languages, clear_languages, \ restore_original = self.args - # first loop: All changes that modify the filesystem and commit # immediately. We want to # try hard to keep the DB and the file system in sync, even in the face @@ -199,7 +203,8 @@ class MyBlockingBusy(QDialog): # {{{ for fmt in fmts.split(','): fmtf = self.db.format(id, fmt, index_is_id=True, as_file=True) - if fmtf is None: continue + if fmtf is None: + continue cdata, area = get_cover_data(fmtf, fmt) if cdata: covers.append((cdata, area)) @@ -264,7 +269,7 @@ class MyBlockingBusy(QDialog): # {{{ self.db.set_series(id, series, notify=False, commit=False) if not series: self.db.set_series_index(id, 1.0, notify=False, commit=False) - elif do_autonumber: # is True if do_series_restart is True + elif do_autonumber: # is True if do_series_restart is True self.db.set_series_index(id, next, notify=False, commit=False) elif tweaks['series_index_auto_increment'] != 'no_change': self.db.set_series_index(id, 1.0, notify=False, commit=False) @@ -296,18 +301,18 @@ class MyBlockingBusy(QDialog): # {{{ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): - s_r_functions = { '' : lambda x: x, + s_r_functions = {'' : lambda x: x, _('Lower Case') : lambda x: icu_lower(x), _('Upper Case') : lambda x: icu_upper(x), _('Title Case') : lambda x: titlecase(x), _('Capitalize') : lambda x: capitalize(x), } - s_r_match_modes = [ _('Character match'), + s_r_match_modes = [_('Character match'), _('Regular Expression'), ] - s_r_replace_modes = [ _('Replace field'), + s_r_replace_modes = [_('Replace field'), _('Prepend to field'), _('Append to field'), ] @@ -559,8 +564,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): def s_r_get_field(self, mi, field): if field: if field == '{template}': - v = SafeFormat().safe_format\ - (unicode(self.s_r_template.text()), mi, _('S/R TEMPLATE ERROR'), mi) + v = SafeFormat().safe_format( + unicode(self.s_r_template.text()), mi, _('S/R TEMPLATE ERROR'), mi) return [v] fm = self.db.metadata_for_field(field) if field == 'sort': @@ -605,7 +610,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.template_label.setVisible(False) self.s_r_src_ident_label.setVisible(False) self.s_r_src_ident.setVisible(False) - if idx == 1: # Template + if idx == 1: # Template self.s_r_template.setVisible(True) self.template_label.setVisible(True) elif self.s_r_sf_itemdata(idx) == 'identifiers': @@ -998,7 +1003,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): elif self.cover_from_fmt.isChecked(): cover_action = 'fromfmt' - args = (remove_all, remove, add, au, aus, do_aus, rating, pub, do_series, + args = Settings(remove_all, remove, add, au, aus, do_aus, rating, pub, do_series, do_autonumber, do_remove_format, remove_format, do_swap_ta, do_remove_conv, do_auto_author, series, do_series_restart, series_start_value, do_title_case, cover_action, clear_series, @@ -1146,9 +1151,9 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): set_index(self.search_field, 'search_field') set_text(self.s_r_template, 's_r_template') - self.s_r_template_changed() #simulate gain/loss of focus + self.s_r_template_changed() # simulate gain/loss of focus - set_index(self.s_r_src_ident, 's_r_src_ident'); + set_index(self.s_r_src_ident, 's_r_src_ident') set_text(self.s_r_dst_ident, 's_r_dst_ident') set_text(self.search_for, 'search_for') set_checked(self.case_sensitive, 'case_sensitive') @@ -1179,3 +1184,4 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.starting_from.setValue(1) self.multiple_separator.setText(" ::: ") + From a5e32b4ad44032666e95e67dba5f6458b0c59d14 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 15:37:28 +0530 Subject: [PATCH 0277/1154] Some missing API --- src/calibre/db/legacy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 94c35429b2..6992eab2b3 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -65,6 +65,7 @@ class LibraryDatabase(object): cache.init() self.data = View(cache) self.id = self.data.index_to_id + self.row = self.index = self.data.id_to_index for x in ('get_property', 'count', 'refresh_ids', 'set_marked_ids', 'multisort', 'search', 'search_getting_ids'): setattr(self, x, getattr(self.data, x)) From b2733f6052b79207fb6b6769d3eb2141f23d44f0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 15:48:42 +0530 Subject: [PATCH 0278/1154] Start work on porting the bulk metadata dialog to the new API --- src/calibre/db/cache.py | 6 ++ src/calibre/gui2/dialogs/metadata_bulk.py | 110 +++++++++++++++++++++- 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 16e6d59adb..e6174015c5 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -333,6 +333,12 @@ class Cache(object): except (KeyError, IndexError): return default_value + @read_api + def all_field_for(self, field, book_ids, default_value=None): + ' Same as field_for, except that it operates on multiple books at once ' + field_obj = self.fields[field] + return {book_id:self._fast_field_for(field_obj, book_id, default_value=default_value) for book_id in book_ids} + @read_api def composite_for(self, name, book_id, mi=None, default_value=''): try: diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 007ff3b58f..31ba4d033a 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -5,6 +5,7 @@ __copyright__ = '2008, Kovid Goyal ' import re, os, inspect from collections import namedtuple +from threading import Thread from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \ pyqtSignal, QDialogButtonBox, QInputDialog, QLineEdit, \ @@ -58,6 +59,106 @@ Settings = namedtuple('Settings', 'remove_all remove add au aus do_aus rating pu 'remove_format do_swap_ta do_remove_conv do_auto_author series do_series_restart series_start_value ' 'do_title_case cover_action clear_series pubdate adddate do_title_sort languages clear_languages restore_original') +class MyBlockingBusyNew(QDialog): + + all_done = pyqtSignal() + + def __init__(self, args, ids, db, cc_widgets, s_r_func, + parent=None, window_title=_('Working')): + QDialog.__init__(self, parent) + + self._layout = l = QVBoxLayout() + self.setLayout(l) + + self.msg = QLabel(_('Processing %d books, please wait...') % len(ids)) + self.font = QFont() + self.font.setPointSize(self.font.pointSize() + 8) + self.msg.setFont(self.font) + self.pi = ProgressIndicator(self) + self.pi.setDisplaySize(100) + self._layout.addWidget(self.pi, 0, Qt.AlignHCenter) + self._layout.addSpacing(15) + self._layout.addWidget(self.msg, 0, Qt.AlignHCenter) + self.setWindowTitle(window_title + '...') + self.setMinimumWidth(200) + self.resize(self.sizeHint()) + self.error = None + self.all_done.connect(self.on_all_done, type=Qt.QueuedConnection) + self.args, self.ids, self.s_r_func = args, ids, s_r_func + self.db, self.cc_widgets = db, cc_widgets + + def accept(self): + pass + + def reject(self): + pass + + def on_all_done(self): + QDialog.accept(self) + + def exec_(self): + self.thread = Thread(target=self.do_it) + self.thread.start() + return QDialog.exec_(self) + + def do_it(self): + try: + self.do_all() + except Exception as err: + import traceback + try: + err = unicode(err) + except: + err = repr(err) + self.error = (err, traceback.format_exc()) + + self.all_done.emit() + + def do_all(self): + cache = self.db.new_api + args = self.args + + # Title and authors + if args.do_swap_ta: + title_map = cache.all_field_for('title', self.ids) + authors_map = cache.all_field_for('authors', self.ids) + def new_title(authors): + ans = authors_to_string(authors) + return titlecase(ans) if args.do_title_case else ans + new_title_map = {bid:new_title(authors) for bid, authors in authors_map.iteritems()} + new_authors_map = {bid:string_to_authors(title) for bid, title in title_map.iteritems()} + cache.set_field('authors', new_authors_map) + cache.set_field('title', new_title_map) + + if args.do_title_case and not args.do_swap_ta: + title_map = cache.all_field_for('title', self.ids) + cache.set_field('title', {bid:titlecase(title) for bid, title in title_map.iteritems()}) + + if args.do_title_sort: + lang_map = cache.all_field_for('languages', self.ids) + title_map = cache.all_field_for('title', self.ids) + def get_sort(book_id): + if args.languages: + lang = args.languages[0] + else: + try: + lang = lang_map[book_id][0] + except (KeyError, IndexError, TypeError, AttributeError): + lang = 'eng' + return title_sort(title_map[book_id], lang=lang) + cache.set_field('sort', {bid:get_sort(bid) for bid in self.ids}) + + if args.au: + authors = string_to_authors(args.au) + cache.set_field('authors', {bid:authors for bid in self.ids}) + + if args.do_auto_author: + aus_map = cache.author_sort_strings_for_books(self.ids) + cache.set_field('author_sort', {book_id:' & '.join(aus_map[book_id]) for book_id in aus_map}) + + if args.aus and args.do_aus: + cache.set_field('author_sort', {bid:args.aus for bid in self.ids}) + class MyBlockingBusy(QDialog): # {{{ do_one_signal = pyqtSignal() @@ -1010,7 +1111,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): pubdate, adddate, do_title_sort, languages, clear_languages, restore_original) - bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.') + if hasattr(self.db, 'new_api'): + bb = MyBlockingBusyNew(args, self.ids, self.db, + getattr(self, 'custom_column_widgets', []), + self.do_search_replace, parent=self) + else: + bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.') %len(self.ids), args, self.db, self.ids, getattr(self, 'custom_column_widgets', []), self.do_search_replace, parent=self) @@ -1023,6 +1129,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): finally: self.model.start_metadata_backup() + bb.thread = bb.db = bb.cc_widgets = None + if bb.error is not None: return error_dialog(self, _('Failed'), bb.error[0], det_msg=bb.error[1], From cd509fc9953ee5d153caf905699cc59a3baed661 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 15:52:08 +0530 Subject: [PATCH 0279/1154] ... --- src/calibre/db/legacy.py | 2 +- src/calibre/db/tests/legacy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 6992eab2b3..efb71223a2 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -65,7 +65,7 @@ class LibraryDatabase(object): cache.init() self.data = View(cache) self.id = self.data.index_to_id - self.row = self.index = self.data.id_to_index + self.row = self.data.id_to_index for x in ('get_property', 'count', 'refresh_ids', 'set_marked_ids', 'multisort', 'search', 'search_getting_ids'): setattr(self, x, getattr(self.data, x)) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index a5f5f693fe..2d8bf466f6 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -397,7 +397,7 @@ class LegacyTest(BaseTest): # Internal API 'clean_user_categories', 'cleanup_tags', 'books_list_filter', 'conn', 'connect', 'construct_file_name', 'construct_path_name', 'clear_dirtied', 'initialize_database', 'initialize_dynamic', - 'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat', + 'run_import_plugins', 'vacuum', 'set_path', 'row_factory', 'rows', 'rmtree', 'series_index_pat', 'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_books_referencing', 'windows_check_if_files_in_use', 'get_metadata_for_dump', 'get_a_dirtied_book', 'dirtied_sequence', 'format_filename_cache', 'format_metadata_cache', 'filter', 'create_version1', 'normpath', 'custom_data_adapters', From 9a6c322d8781f25eac6f707931069724cc1374a6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 17:10:00 +0530 Subject: [PATCH 0280/1154] Sort plugins list case insensitively --- src/calibre/gui2/dialogs/plugin_updater.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/dialogs/plugin_updater.py b/src/calibre/gui2/dialogs/plugin_updater.py index c5d79218f9..0b0885dc82 100644 --- a/src/calibre/gui2/dialogs/plugin_updater.py +++ b/src/calibre/gui2/dialogs/plugin_updater.py @@ -276,6 +276,7 @@ class DisplayPluginSortFilterModel(QSortFilterProxyModel): def __init__(self, parent): QSortFilterProxyModel.__init__(self, parent) self.setSortRole(Qt.UserRole) + self.setSortCaseSensitivity(Qt.CaseInsensitive) self.filter_criteria = FILTER_ALL def filterAcceptsRow(self, sourceRow, sourceParent): From 8c24464ba665f2dc2a6cbe5d76ff443981eb076d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 17:10:29 +0530 Subject: [PATCH 0281/1154] Show [newdb] in the status bar when using the new backend --- src/calibre/gui2/init.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index d066f8aa01..feed5bb4ad 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -10,7 +10,7 @@ import functools from PyQt4.Qt import (Qt, QApplication, QStackedWidget, QMenu, QTimer, QSize, QSizePolicy, QStatusBar, QLabel, QFont) -from calibre.utils.config import prefs +from calibre.utils.config import prefs, tweaks from calibre.constants import (isosx, __appname__, preferred_encoding, get_version) from calibre.gui2 import config, is_widescreen, gprefs @@ -161,8 +161,10 @@ class StatusBar(QStatusBar): # {{{ def __init__(self, parent=None): QStatusBar.__init__(self, parent) - self.base_msg = '%s %s' % (__appname__, get_version()) self.version = get_version() + self.base_msg = '%s %s' % (__appname__, self.version) + if tweaks.get('use_new_db', False): + self.base_msg += ' [newdb]' self.device_string = '' self.update_label = UpdateLabel('') self.total = self.current = self.selected = self.library_total = 0 From 17b1cb5115103540136c035c089a967dc6a1095c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 17:31:14 +0530 Subject: [PATCH 0282/1154] Fix remove_formats() not actually removing the files --- src/calibre/db/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index e6174015c5..a669b7017b 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1146,8 +1146,6 @@ class Cache(object): def remove_formats(self, formats_map, db_only=False): table = self.fields['formats'].table formats_map = {book_id:frozenset((f or '').upper() for f in fmts) for book_id, fmts in formats_map.iteritems()} - size_map = table.remove_formats(formats_map, self.backend) - self.fields['size'].table.update_sizes(size_map) for book_id, fmts in formats_map.iteritems(): for fmt in fmts: @@ -1167,6 +1165,8 @@ class Cache(object): if name and path: self.backend.remove_format(book_id, fmt, name, path) + size_map = table.remove_formats(formats_map, self.backend) + self.fields['size'].table.update_sizes(size_map) self._update_last_modified(tuple(formats_map.iterkeys())) @read_api From b38e1c45fc3ac845046b159dc87c3bc35deb014e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 17:40:48 +0530 Subject: [PATCH 0283/1154] Covers and formats --- src/calibre/gui2/dialogs/metadata_bulk.py | 48 ++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 31ba4d033a..0b00574d15 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -159,6 +159,52 @@ class MyBlockingBusyNew(QDialog): if args.aus and args.do_aus: cache.set_field('author_sort', {bid:args.aus for bid in self.ids}) + # Covers + if args.cover_action == 'remove': + cache.set_cover({bid:None for bid in self.ids}) + elif args.cover_action == 'generate': + from calibre.ebooks import calibre_cover + from calibre.ebooks.metadata import fmt_sidx + from calibre.gui2 import config + for book_id in self.ids: + mi = self.db.get_metadata(book_id, index_is_id=True) + series_string = None + if mi.series: + series_string = _('Book %(sidx)s of %(series)s')%dict( + sidx=fmt_sidx(mi.series_index, + use_roman=config['use_roman_numerals_for_series_number']), + series=mi.series) + + cdata = calibre_cover(mi.title, mi.format_field('authors')[-1], + series_string=series_string) + cache.set_cover({book_id:cdata}) + elif args.cover_action == 'fromfmt': + for book_id in self.ids: + fmts = cache.formats(book_id, verify_formats=False) + if fmts: + covers = [] + for fmt in fmts: + fmtf = cache.format(book_id, fmt, as_file=True) + if fmtf is None: + continue + cdata, area = get_cover_data(fmtf, fmt) + if cdata: + covers.append((cdata, area)) + covers.sort(key=lambda x: x[1]) + if covers: + cache.set_cover({book_id:covers[-1][0]}) + + # Formats + if args.do_remove_format: + cache.remove_formats({bid:(args.remove_format,) for bid in self.ids}) + + if args.restore_original: + for book_id in self.ids: + formats = cache.formats(book_id) + originals = tuple(x.upper() for x in formats if x.upper().startswith('ORIGINAL_')) + for ofmt in originals: + cache.restore_original_format(book_id, ofmt) + class MyBlockingBusy(QDialog): # {{{ do_one_signal = pyqtSignal() @@ -438,7 +484,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.initialize_combos() - for f in self.db.all_formats(): + for f in sorted(self.db.all_formats()): self.remove_format.addItem(f) self.remove_format.setCurrentIndex(-1) From 5b4e96e6b16af531394baf9a3e0429df3a26849c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 21:00:00 +0530 Subject: [PATCH 0284/1154] Finish off remaining bulk metadata fields --- src/calibre/gui2/dialogs/metadata_bulk.py | 50 +++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 0b00574d15..f6cce35d67 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -94,11 +94,21 @@ class MyBlockingBusyNew(QDialog): pass def on_all_done(self): + if not self.error: + # The cc widgets can only be accessed in the GUI thread + try: + for w in self.cc_widgets: + w.commit(self.ids) + except Exception as err: + import traceback + self.error = (err, traceback.format_exc()) + self.pi.stopAnimation() QDialog.accept(self) def exec_(self): self.thread = Thread(target=self.do_it) self.thread.start() + self.pi.startAnimation() return QDialog.exec_(self) def do_it(self): @@ -205,6 +215,46 @@ class MyBlockingBusyNew(QDialog): for ofmt in originals: cache.restore_original_format(book_id, ofmt) + # Various fields + if args.rating != -1: + cache.set_field('rating', {bid:args.rating*2 for bid in self.ids}) + + if args.pub: + cache.set_field('publisher', {bid:args.pub for bid in self.ids}) + + if args.clear_series: + cache.set_field('series', {bid:'' for bid in self.ids}) + + if args.pubdate is not None: + cache.set_field('pubdate', {bid:args.pubdate for bid in self.ids}) + + if args.adddate is not None: + cache.set_field('timestamp', {bid:args.adddate for bid in self.ids}) + + if args.do_series: + cache.set_field('series', {bid:args.series for bid in self.ids}) + if not args.series: + cache.set_field('series_index', {bid:1.0 for bid in self.ids}) + else: + sval = args.series_start_value if args.do_series_restart else cache.get_next_series_num_for(args.series) + smap = {bid:((sval + i) if args.do_autonumber else 1.0) for i, bid in enumerate(self.ids)} + if args.do_autonumber or tweaks['series_index_auto_increment'] != 'no_change': + cache.set_field('series_index', smap) + + if args.do_remove_conv: + cache.delete_conversion_options(self.ids) + + if args.clear_languages: + cache.set_field('languages', {bid:() for bid in self.ids}) + elif args.languages: + cache.set_field('languages', {bid:args.languages for bid in self.ids}) + + if args.remove_all: + cache.set_field('tags', {bid:() for bid in self.ids}) + if args.add or args.remove: + self.db.bulk_modify_tags(self.ids, add=args.add, remove=args.remove) + + class MyBlockingBusy(QDialog): # {{{ do_one_signal = pyqtSignal() From 5727ff9fa40318002d1aec509f83805b471cdfcc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Jul 2013 21:32:15 +0530 Subject: [PATCH 0285/1154] Bulk metadata S&R --- src/calibre/gui2/dialogs/metadata_bulk.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index f6cce35d67..51845a3cf2 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -17,7 +17,7 @@ from calibre.ebooks.metadata import string_to_authors, authors_to_string, title_ from calibre.ebooks.metadata.book.formatter import SafeFormat from calibre.gui2.custom_column_widgets import populate_metadata_page from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATETIME, \ - gprefs, question_dialog + gprefs, question_dialog, FunctionDispatcher from calibre.gui2.progress_indicator import ProgressIndicator from calibre.gui2.metadata.basic_widgets import CalendarWidget from calibre.utils.config import dynamic, JSONConfig @@ -63,8 +63,7 @@ class MyBlockingBusyNew(QDialog): all_done = pyqtSignal() - def __init__(self, args, ids, db, cc_widgets, s_r_func, - parent=None, window_title=_('Working')): + def __init__(self, args, ids, db, cc_widgets, s_r_func, do_sr, parent=None, window_title=_('Working')): QDialog.__init__(self, parent) self._layout = l = QVBoxLayout() @@ -84,8 +83,10 @@ class MyBlockingBusyNew(QDialog): self.resize(self.sizeHint()) self.error = None self.all_done.connect(self.on_all_done, type=Qt.QueuedConnection) - self.args, self.ids, self.s_r_func = args, ids, s_r_func + self.args, self.ids = args, ids self.db, self.cc_widgets = db, cc_widgets + self.s_r_func = FunctionDispatcher(s_r_func) + self.do_sr = do_sr def accept(self): pass @@ -254,6 +255,10 @@ class MyBlockingBusyNew(QDialog): if args.add or args.remove: self.db.bulk_modify_tags(self.ids, add=args.add, remove=args.remove) + if self.do_sr: + for book_id in self.ids: + self.s_r_func(book_id) + class MyBlockingBusy(QDialog): # {{{ @@ -1016,7 +1021,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): if not dest: dest = source dfm = self.db.field_metadata[dest] - mi = self.db.get_metadata(id, index_is_id=True,) + mi = self.db.get_metadata(id, index_is_id=True) val = self.s_r_do_regexp(mi) val = self.s_r_do_destination(mi, val) if dfm['is_multiple']: @@ -1208,9 +1213,11 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): restore_original) if hasattr(self.db, 'new_api'): + source = self.s_r_sf_itemdata(None) + do_sr = source and self.s_r_obj bb = MyBlockingBusyNew(args, self.ids, self.db, getattr(self, 'custom_column_widgets', []), - self.do_search_replace, parent=self) + self.do_search_replace, do_sr, parent=self) else: bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.') %len(self.ids), args, self.db, self.ids, From 67b7aa8d117653e4c0940e7b228a28f1fba17a74 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 08:23:47 +0530 Subject: [PATCH 0286/1154] Respect series nunmbering increment tweak when bulk editing series --- src/calibre/db/cache.py | 4 +++- src/calibre/gui2/dialogs/metadata_bulk.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index a669b7017b..d3aea991ac 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1170,7 +1170,7 @@ class Cache(object): self._update_last_modified(tuple(formats_map.iterkeys())) @read_api - def get_next_series_num_for(self, series, field='series'): + def get_next_series_num_for(self, series, field='series', current_indices=False): books = () sf = self.fields[field] if series: @@ -1180,6 +1180,8 @@ class Cache(object): books = book_ids break series_indices = sorted(self._field_for(sf.index_field.name, book_id) for book_id in books) + if current_indices: + return series_indices return _get_next_series_num_for_list(tuple(series_indices), unwrap=False) @read_api diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 51845a3cf2..277e887a09 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -27,7 +27,7 @@ from calibre.utils.config import prefs, tweaks from calibre.utils.magick.draw import identify_data from calibre.utils.date import qt_to_dt from calibre.ptempfile import SpooledTemporaryFile -from calibre.db import SPOOL_SIZE +from calibre.db import SPOOL_SIZE, _get_next_series_num_for_list def get_cover_data(stream, ext): # {{{ from calibre.ebooks.metadata.meta import get_metadata @@ -237,8 +237,15 @@ class MyBlockingBusyNew(QDialog): if not args.series: cache.set_field('series_index', {bid:1.0 for bid in self.ids}) else: - sval = args.series_start_value if args.do_series_restart else cache.get_next_series_num_for(args.series) - smap = {bid:((sval + i) if args.do_autonumber else 1.0) for i, bid in enumerate(self.ids)} + sval = args.series_start_value if args.do_series_restart else list(cache.get_next_series_num_for(args.series, current_indices=True)) + def next_series_num(i): + if args.do_series_restart: + return sval + i + next_num = _get_next_series_num_for_list(sval, unwrap=False) + sval.append(next_num) + return next_num + + smap = {bid:next_series_num(i) for i, bid in enumerate(self.ids)} if args.do_autonumber or tweaks['series_index_auto_increment'] != 'no_change': cache.set_field('series_index', smap) From b4b97a1c17dc7445552feadaf060b880e2d76a58 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 08:35:15 +0530 Subject: [PATCH 0287/1154] More fixes to make bulk series editing behave like before --- src/calibre/gui2/dialogs/metadata_bulk.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 277e887a09..637e8d777e 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -233,11 +233,11 @@ class MyBlockingBusyNew(QDialog): cache.set_field('timestamp', {bid:args.adddate for bid in self.ids}) if args.do_series: + sval = args.series_start_value if args.do_series_restart else list(cache.get_next_series_num_for(args.series, current_indices=True)) cache.set_field('series', {bid:args.series for bid in self.ids}) if not args.series: cache.set_field('series_index', {bid:1.0 for bid in self.ids}) else: - sval = args.series_start_value if args.do_series_restart else list(cache.get_next_series_num_for(args.series, current_indices=True)) def next_series_num(i): if args.do_series_restart: return sval + i @@ -246,8 +246,10 @@ class MyBlockingBusyNew(QDialog): return next_num smap = {bid:next_series_num(i) for i, bid in enumerate(self.ids)} - if args.do_autonumber or tweaks['series_index_auto_increment'] != 'no_change': + if args.do_autonumber: cache.set_field('series_index', smap) + elif tweaks['series_index_auto_increment'] != 'no_change': + cache.set_field('series_index', {bid:1.0 for bid in self.ids}) if args.do_remove_conv: cache.delete_conversion_options(self.ids) From 51cc7275f368d834a5167b2440ebe0b7f0c90edd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 09:06:25 +0530 Subject: [PATCH 0288/1154] ... --- src/calibre/gui2/dialogs/metadata_bulk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 637e8d777e..72ec1440a1 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -59,7 +59,7 @@ Settings = namedtuple('Settings', 'remove_all remove add au aus do_aus rating pu 'remove_format do_swap_ta do_remove_conv do_auto_author series do_series_restart series_start_value ' 'do_title_case cover_action clear_series pubdate adddate do_title_sort languages clear_languages restore_original') -class MyBlockingBusyNew(QDialog): +class MyBlockingBusyNew(QDialog): # {{{ all_done = pyqtSignal() @@ -267,7 +267,7 @@ class MyBlockingBusyNew(QDialog): if self.do_sr: for book_id in self.ids: self.s_r_func(book_id) - +# }}} class MyBlockingBusy(QDialog): # {{{ From b2d5b3abb034b9d53700c1b3d6859fc43d167385 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 09:17:50 +0530 Subject: [PATCH 0289/1154] ... --- src/calibre/gui2/actions/choose_library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index 4010847109..eb34aec6eb 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -498,7 +498,7 @@ class ChooseLibraryAction(InterfaceAction): # import weakref # from PyQt4.Qt import QTimer # self.dbref = weakref.ref(self.gui.library_view.model().db) - # self.before_mem = memory()/1024**2 + # self.before_mem = memory() self.gui.library_moved(loc, allow_rebuild=True) # QTimer.singleShot(5000, self.debug_leak) @@ -514,7 +514,7 @@ class ChooseLibraryAction(InterfaceAction): print r print print 'before:', self.before_mem - print 'after:', memory()/1024**2 + print 'after:', memory() print self.dbref = self.before_mem = None From 4a423ddaae33a7221214d1ec7089fd488c723941 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 09:55:40 +0530 Subject: [PATCH 0290/1154] mem usage behavior when switching libraries appears to be similar for both backends. --- src/calibre/db/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index c1b48cad0c..7a5c8d51d4 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -135,5 +135,4 @@ Various things that require other things before they can be migrated: 1. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) - 2. Check for mem leaks when switching libraries ''' From fa02915eb9a48be03bc9689c8a25c7c344da2fd0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 09:57:29 +0530 Subject: [PATCH 0291/1154] Fix various minor issues with the new API in the content server --- src/calibre/db/cache.py | 2 +- src/calibre/db/view.py | 6 +++++ src/calibre/library/server/browse.py | 39 ++++++++++++++-------------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index d3aea991ac..5a5e43bd70 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1483,7 +1483,7 @@ class Cache(object): if hasattr(f, 'get_books_for_val'): # Composite field return f.get_books_for_val(item_id_or_composite_value, self._get_metadata, self._all_book_ids()) - return self._books_for_field(f.name, item_id_or_composite_value) + return self._books_for_field(f.name, int(item_id_or_composite_value)) @read_api def find_identical_books(self, mi, search_restriction='', book_ids=None): diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index a8cd75d2c2..2a779a5af0 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -127,6 +127,9 @@ class View(object): book_id = id_or_index if index_is_id else self._map_filtered[id_or_index] return self._field_getters[loc](book_id) + def sanitize_sort_field_name(self, field): + return sanitize_sort_field_name(self.field_metadata, field) + @property def field_metadata(self): return self.cache.field_metadata @@ -154,6 +157,9 @@ class View(object): for book_id in sorted(self._map): yield book_id + def tablerow_for_id(self, book_id): + return TableRow(book_id, self) + def get_field_map_field(self, row, col, index_is_id=True): ''' Supports the legacy FIELD_MAP interface for getting metadata. Do not use diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index d25c34d52b..bffeb33829 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -28,7 +28,7 @@ def xml(*args, **kwargs): ans = prepare_string_for_xml(*args, **kwargs) return ans.replace(''', ''') -def render_book_list(ids, prefix, suffix=''): # {{{ +def render_book_list(ids, prefix, suffix=''): # {{{ pages = [] num = len(ids) pos = 0 @@ -113,13 +113,13 @@ def render_book_list(ids, prefix, suffix=''): # {{{ # }}} -def utf8(x): # {{{ +def utf8(x): # {{{ if isinstance(x, unicode): x = x.encode('utf-8') return x # }}} -def render_rating(rating, url_prefix, container='span', prefix=None): # {{{ +def render_rating(rating, url_prefix, container='span', prefix=None): # {{{ if rating < 0.1: return '', '' added = 0 @@ -145,7 +145,7 @@ def render_rating(rating, url_prefix, container='span', prefix=None): # {{{ # }}} -def get_category_items(category, items, datatype, prefix): # {{{ +def get_category_items(category, items, datatype, prefix): # {{{ def item(i): templ = (u'

    ' @@ -179,7 +179,7 @@ def get_category_items(category, items, datatype, prefix): # {{{ # }}} -class Endpoint(object): # {{{ +class Endpoint(object): # {{{ 'Manage encoding, mime-type, last modified, cookies, etc.' def __init__(self, mimetype='text/html; charset=utf-8', sort_type='category'): @@ -194,7 +194,7 @@ class Endpoint(object): # {{{ if 'json' not in eself.mimetype: sort_val = None cookie = cherrypy.request.cookie - if cookie.has_key(eself.sort_cookie_name): + if eself.sort_cookie_name in cookie: sort_val = cookie[eself.sort_cookie_name].value kwargs[eself.sort_kwarg] = sort_val @@ -523,8 +523,6 @@ class BrowseServer(object): items = '\n\n'.join(items) items = u'
    \n{0}
    '.format(items) - - if cats: script = 'toplevel();category(%s);'%script else: @@ -586,12 +584,11 @@ class BrowseServer(object): datatype, self.opts.url_prefix) return json.dumps(entries, ensure_ascii=True) - @Endpoint() def browse_catalog(self, category=None, category_sort=None): 'Entry point for top-level, categories and sub-categories' prefix = '' if self.is_wsgi else self.opts.url_prefix - if category == None: + if category is None: ans = self.browse_toplevel() elif category == 'newest': raise cherrypy.InternalRedirect(prefix + @@ -670,7 +667,10 @@ class BrowseServer(object): ids = self.db.get_books_for_category(q, cid) ids = [x for x in ids if x in all_ids] - items = [self.db.data._data[x] for x in ids] + if hasattr(self.db, 'new_api'): + items = [self.db.data.tablerow_for_id(x) for x in ids] + else: + items = [self.db.data._data[x] for x in ids] if category == 'newest': list_sort = 'timestamp' if dt == 'series': @@ -770,7 +770,7 @@ class BrowseServer(object): if fmts and fmt: other_fmts = [x for x in fmts if x.lower() != fmt.lower()] if other_fmts: - ofmts = [u'
    {3}'\ + ofmts = [u'{3}' .format(f, fname, id_, f.upper(), self.opts.url_prefix) for f in other_fmts] @@ -783,7 +783,7 @@ class BrowseServer(object): if fmt: href = self.opts.url_prefix + '/get/%s/%s_%d.%s'%( fmt, fname, id_, fmt) - rt = xml(_('Read %(title)s in the %(fmt)s format')% \ + rt = xml(_('Read %(title)s in the %(fmt)s format')% {'title':args['title'], 'fmt':fmt.upper()}, True) args['get_button'] = \ @@ -809,7 +809,6 @@ class BrowseServer(object): summs.append(self.browse_summary_template.format(**args)) - raw = json.dumps('\n'.join(summs), ensure_ascii=True) return raw @@ -829,7 +828,7 @@ class BrowseServer(object): args['get_url'] = '' args['formats'] = '' if fmts: - ofmts = [u'{3}'\ + ofmts = [u'{3}' .format(xfmt, fname, id_, xfmt.upper(), self.opts.url_prefix) for xfmt in fmts] ofmts = ', '.join(ofmts) @@ -842,7 +841,7 @@ class BrowseServer(object): field not in displayed_custom_fields: continue if m['datatype'] == 'comments' or field == 'comments' or ( - m['datatype'] == 'composite' and \ + m['datatype'] == 'composite' and m['display'].get('contains_html', False)): val = mi.get(field, '') if val and val.strip(): @@ -924,16 +923,17 @@ class BrowseServer(object): return self.browse_template('').format( title='', script='book();', main=ans) - # }}} - # Search {{{ @Endpoint(sort_type='list') def browse_search(self, query='', list_sort=None): if isbytestring(query): query = query.decode('UTF-8') ids = self.db.search_getting_ids(query.strip(), self.search_restriction) - items = [self.db.data._data[x] for x in ids] + if hasattr(self.db, 'new_api'): + items = [self.db.data.tablerow_for_id(x) for x in ids] + else: + items = [self.db.data._data[x] for x in ids] sort = self.browse_sort_book_list(items, list_sort) ids = [x[0] for x in items] html = render_book_list(ids, self.opts.url_prefix, @@ -945,3 +945,4 @@ class BrowseServer(object): # }}} + From 39d06938166d7398fbafd1f30df45f5519bd122e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 09:58:43 +0530 Subject: [PATCH 0292/1154] Fix backup test broken --- src/calibre/db/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/db/backup.py b/src/calibre/db/backup.py index 18722e1cf8..dc9f338fc9 100644 --- a/src/calibre/db/backup.py +++ b/src/calibre/db/backup.py @@ -26,7 +26,7 @@ class MetadataBackup(Thread): def __init__(self, db, interval=2, scheduling_interval=0.1): Thread.__init__(self) self.daemon = True - self._db = weakref.ref(db.new_api) + self._db = weakref.ref(getattr(db, 'new_api', db)) self.stop_running = Event() self.interval = interval self.scheduling_interval = scheduling_interval From 78bf4d14781de8a50f5021ed105f8cda8d900f4d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 10:02:57 +0530 Subject: [PATCH 0293/1154] Fix the remove_formats test --- src/calibre/db/tests/add_remove.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index 0047a0ec4f..a099ae5e84 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -116,18 +116,20 @@ class AddRemoveTest(BaseTest): # Test full removal of format af(cache.format(1, 'FMT1') is None) at(cache.has_format(1, 'FMT1')) + ap = cache.format_abspath(1, 'FMT1') cache.remove_formats({1:{'FMT1'}}) at(cache.format(1, 'FMT1') is None) af(bool(cache.format_metadata(1, 'FMT1'))) af(bool(cache.format_metadata(1, 'FMT1', allow_cache=False))) af('FMT1' in cache.formats(1)) af(cache.has_format(1, 'FMT1')) + af(os.path.exists(ap)) # Test db only removal at(cache.has_format(1, 'FMT2')) ap = cache.format_abspath(1, 'FMT2') if ap and os.path.exists(ap): - cache.remove_formats({1:{'FMT2'}}) + cache.remove_formats({1:{'FMT2'}}, db_only=True) af(bool(cache.format_metadata(1, 'FMT2'))) af(cache.has_format(1, 'FMT2')) at(os.path.exists(ap)) From f5eda364e6c16bc197e2d8dedef0d286241c1302 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 10:05:09 +0530 Subject: [PATCH 0294/1154] ... --- src/calibre/db/tests/add_remove.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index a099ae5e84..5cbac5deca 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -117,6 +117,7 @@ class AddRemoveTest(BaseTest): af(cache.format(1, 'FMT1') is None) at(cache.has_format(1, 'FMT1')) ap = cache.format_abspath(1, 'FMT1') + at(os.path.exists(ap)) cache.remove_formats({1:{'FMT1'}}) at(cache.format(1, 'FMT1') is None) af(bool(cache.format_metadata(1, 'FMT1'))) From 3b345d3ff50bc5049cc281862c29a04ec2fd5d62 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 10:46:29 +0530 Subject: [PATCH 0295/1154] On windows add a wait when hardlinking to account for broken network filesystems --- src/calibre/utils/filenames.py | 35 +++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index 23ac8fd43a..30645e7380 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -3,7 +3,7 @@ Make strings safe for use as ASCII filenames, while trying to preserve as much meaning as possible. ''' -import os, errno +import os, errno, time from math import ceil from calibre import sanitize_file_name, isbytestring, force_unicode @@ -257,6 +257,19 @@ def samefile(src, dst): os.path.normcase(os.path.abspath(dst))) return samestring +def windows_get_size(path): + ''' On windows file sizes are only accurately stored in the actual file, + not in the directory entry (which could be out of date). So we open the + file, and get the actual size. ''' + import win32file + h = win32file.CreateFile( + path, 0, win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE | win32file.FILE_SHARE_DELETE, + None, win32file.OPEN_EXISTING, 0, None) + try: + return win32file.GetFileSize(h) + finally: + win32file.CloseHandle(h) + def windows_hardlink(src, dest): import win32file, pywintypes try: @@ -264,17 +277,21 @@ def windows_hardlink(src, dest): except pywintypes.error as e: msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) raise Exception(msg % e) + src_size = os.path.getsize(src) # We open and close dest, to ensure its directory entry is updated # see http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx - h = win32file.CreateFile( - dest, 0, win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE | win32file.FILE_SHARE_DELETE, - None, win32file.OPEN_EXISTING, 0, None) - try: - sz = win32file.GetFileSize(h) - finally: - win32file.CloseHandle(h) + for i in range(10): + # If we are on a network filesystem, we have to wait for some indeterminate time, since + # network file systems are the best thing since sliced bread + try: + if windows_get_size(dest) == src_size: + return + except EnvironmentError: + pass + time.sleep(0.3) - if sz != os.path.getsize(src): + sz = windows_get_size(dest) + if sz != src_size: msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) raise Exception(msg % ('hardlink size: %d not the same as source size' % sz)) From 6423745c0ca96fd4a8b8dbbf349b4510c4dad743 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 10:53:13 +0530 Subject: [PATCH 0296/1154] ... --- src/calibre/db/cache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 5a5e43bd70..e49c719a62 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1613,5 +1613,3 @@ class SortKey(object): # {{{ return 0 # }}} - - From 8241d78c16967ae5c2360a185a33fce36148533c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 11:57:48 +0530 Subject: [PATCH 0297/1154] Handle case of bulk series changing index of existing book --- src/calibre/db/cache.py | 6 ++++-- src/calibre/gui2/dialogs/metadata_bulk.py | 10 +++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index e49c719a62..6cdb9419c6 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1179,9 +1179,11 @@ class Cache(object): if q == icu_lower(val): books = book_ids break - series_indices = sorted(self._field_for(sf.index_field.name, book_id) for book_id in books) + idf = sf.index_field + index_map = {book_id:self._fast_field_for(idf, book_id, default_value=1.0) for book_id in books} if current_indices: - return series_indices + return index_map + series_indices = sorted(index_map.itervalues()) return _get_next_series_num_for_list(tuple(series_indices), unwrap=False) @read_api diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 72ec1440a1..c72f1533a5 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -233,19 +233,19 @@ class MyBlockingBusyNew(QDialog): # {{{ cache.set_field('timestamp', {bid:args.adddate for bid in self.ids}) if args.do_series: - sval = args.series_start_value if args.do_series_restart else list(cache.get_next_series_num_for(args.series, current_indices=True)) + sval = args.series_start_value if args.do_series_restart else cache.get_next_series_num_for(args.series, current_indices=True) cache.set_field('series', {bid:args.series for bid in self.ids}) if not args.series: cache.set_field('series_index', {bid:1.0 for bid in self.ids}) else: - def next_series_num(i): + def next_series_num(bid, i): if args.do_series_restart: return sval + i - next_num = _get_next_series_num_for_list(sval, unwrap=False) - sval.append(next_num) + next_num = _get_next_series_num_for_list(sorted(sval.itervalues()), unwrap=False) + sval[bid] = next_num return next_num - smap = {bid:next_series_num(i) for i, bid in enumerate(self.ids)} + smap = {bid:next_series_num(bid, i) for i, bid in enumerate(self.ids)} if args.do_autonumber: cache.set_field('series_index', smap) elif tweaks['series_index_auto_increment'] != 'no_change': From d1609037ab3bf0aecc708b81d11a364e8eec9de4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 12:09:16 +0530 Subject: [PATCH 0298/1154] Handle number like values stored in datetime columns --- src/calibre/db/tables.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 9b9ff4e9e0..f71abe2a6d 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -29,6 +29,12 @@ def _c_convert_timestamp(val): return None try: ret = _c_speedup.parse_date(val.strip()) + except AttributeError: + # If a value like 2001 is stored in the column, apsw will return it as + # an int + if isinstance(val, (int, float)): + return datetime(int(val), 1, 1, tzinfo=tzoffset(None, 0)).astimezone(local_tz) + ret = None except: ret = None if ret is None: From 4f5764b1a81b154cf1dcbcb0e6f34d9a28f8a2d7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 12:20:52 +0530 Subject: [PATCH 0299/1154] Handle undecodable bytes in text columns --- src/calibre/db/tables.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index f71abe2a6d..34de470090 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -52,10 +52,16 @@ class Table(object): self.name, self.metadata = name, metadata self.sort_alpha = metadata.get('is_multiple', False) and metadata.get('display', {}).get('sort_alpha', False) + text = lambda x:x.decode('utf-8', 'replace') if isinstance(x, bytes) else x + # self.unserialize() maps values from the db to python objects self.unserialize = \ { 'datetime': _c_convert_timestamp, + 'text': text, + 'comments': text, + 'series': text, + 'enumeration': text, 'bool': bool }.get( metadata['datatype'], lambda x: x) From dcedc5b87db976d79972cdffece46162f1efe74f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 12:54:56 +0530 Subject: [PATCH 0300/1154] Proper fix for unicode sorting error --- src/calibre/db/fields.py | 2 +- src/calibre/db/tables.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 8c6d18a74d..933a4c861c 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -31,7 +31,7 @@ class Field(object): self.has_text_data = dt in {'text', 'comments', 'series', 'enumeration'} self.table_type = self.table.table_type self._sort_key = (sort_key if dt in ('text', 'series', 'enumeration') else lambda x: x) - self._default_sort_key = '' + self._default_sort_key = b'' if dt in {'int', 'float', 'rating'}: self._default_sort_key = 0 elif dt == 'bool': diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 34de470090..f71abe2a6d 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -52,16 +52,10 @@ class Table(object): self.name, self.metadata = name, metadata self.sort_alpha = metadata.get('is_multiple', False) and metadata.get('display', {}).get('sort_alpha', False) - text = lambda x:x.decode('utf-8', 'replace') if isinstance(x, bytes) else x - # self.unserialize() maps values from the db to python objects self.unserialize = \ { 'datetime': _c_convert_timestamp, - 'text': text, - 'comments': text, - 'series': text, - 'enumeration': text, 'bool': bool }.get( metadata['datatype'], lambda x: x) From d336d75a8161fa0363e8ee69723305a7d1395038 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 12:59:07 +0530 Subject: [PATCH 0301/1154] Add a comment explaining the use of a bytestring for default sort key --- src/calibre/db/fields.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 933a4c861c..403a76707c 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -31,7 +31,13 @@ class Field(object): self.has_text_data = dt in {'text', 'comments', 'series', 'enumeration'} self.table_type = self.table.table_type self._sort_key = (sort_key if dt in ('text', 'series', 'enumeration') else lambda x: x) + + # This will be compared to the output of sort_key() which is a + # bytestring, therefore it is safer to have it be a bytestring. + # Coercing an empty bytestring to unicode will never fail, but the + # output of sort_key cannot be coerced to unicode self._default_sort_key = b'' + if dt in {'int', 'float', 'rating'}: self._default_sort_key = 0 elif dt == 'bool': From e105493d783bd2176c304f1a93a18bcebe7da942 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 13:17:10 +0530 Subject: [PATCH 0302/1154] Micro-optimizations when reading from the db --- src/calibre/db/tables.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index f71abe2a6d..2cd7d1502e 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -53,12 +53,10 @@ class Table(object): self.sort_alpha = metadata.get('is_multiple', False) and metadata.get('display', {}).get('sort_alpha', False) # self.unserialize() maps values from the db to python objects - self.unserialize = \ - { - 'datetime': _c_convert_timestamp, - 'bool': bool - }.get( - metadata['datatype'], lambda x: x) + self.unserialize = { + 'datetime': _c_convert_timestamp, + 'bool': bool + }.get(metadata['datatype'], None) if name == 'authors': # Legacy self.unserialize = lambda x: x.replace('|', ',') if x else None @@ -93,9 +91,13 @@ class OneToOneTable(Table): def read(self, db): self.book_col_map = {} idcol = 'id' if self.metadata['table'] == 'books' else 'book' - for row in db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol, - self.metadata['column'], self.metadata['table'])): - self.book_col_map[row[0]] = self.unserialize(row[1]) + query = db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol, + self.metadata['column'], self.metadata['table'])) + if self.unserialize is None: + self.book_col_map = {row[0]:row[1] for row in query} + else: + us = self.unserialize + self.book_col_map = {row[0]:us(row[1]) for row in query} def remove_books(self, book_ids, db): clean = set() @@ -119,7 +121,7 @@ class SizeTable(OneToOneTable): for row in db.conn.execute( 'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data ' 'WHERE data.book=books.id) FROM books'): - self.book_col_map[row[0]] = self.unserialize(row[1]) + self.book_col_map[row[0]] = row[1] def update_sizes(self, size_map): self.book_col_map.update(size_map) @@ -180,9 +182,13 @@ class ManyToOneTable(Table): self.read_maps(db) def read_id_maps(self, db): - for row in db.conn.execute('SELECT id, {0} FROM {1}'.format( - self.metadata['column'], self.metadata['table'])): - self.id_map[row[0]] = self.unserialize(row[1]) + query = db.conn.execute('SELECT id, {0} FROM {1}'.format( + self.metadata['column'], self.metadata['table'])) + if self.unserialize is None: + self.id_map = {row[0]:row[1] for row in query} + else: + us = self.unserialize + self.id_map = {row[0]:us(row[1]) for row in query} def read_maps(self, db): for row in db.conn.execute( @@ -348,11 +354,14 @@ class AuthorsTable(ManyToManyTable): def read_id_maps(self, db): self.alink_map = {} self.asort_map = {} + self.id_map = {} + us = self.unserialize for row in db.conn.execute( 'SELECT id, name, sort, link FROM authors'): - self.id_map[row[0]] = self.unserialize(row[1]) + val = us(row[1]) + self.id_map[row[0]] = self.unserialize(val) self.asort_map[row[0]] = (row[2] if row[2] else - author_to_author_sort(row[1])) + author_to_author_sort(val)) self.alink_map[row[0]] = row[3] def set_sort_names(self, aus_map, db): From 98924814addee56a5a0dc2633bd8268b7463bc6d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 13:37:52 +0530 Subject: [PATCH 0303/1154] Fix metadata backup not being written if book directory does not exist --- src/calibre/db/backend.py | 9 +++++++-- src/calibre/db/backup.py | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 9eb49ccd4c..db5510d055 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1390,8 +1390,13 @@ class DB(object): def write_backup(self, path, raw): path = os.path.abspath(os.path.join(self.library_path, path, 'metadata.opf')) - with lopen(path, 'wb') as f: - f.write(raw) + try: + with lopen(path, 'wb') as f: + f.write(raw) + except EnvironmentError: + os.makedirs(os.path.dirname(path)) + with lopen(path, 'wb') as f: + f.write(raw) def read_backup(self, path): path = os.path.abspath(os.path.join(self.library_path, path, 'metadata.opf')) diff --git a/src/calibre/db/backup.py b/src/calibre/db/backup.py index dc9f338fc9..1a6ff13412 100644 --- a/src/calibre/db/backup.py +++ b/src/calibre/db/backup.py @@ -100,11 +100,13 @@ class MetadataBackup(Thread): self.db.write_backup(book_id, raw) except: prints('Failed to write backup metadata for id:', book_id, 'once') + traceback.print_exc() self.wait(self.interval) try: self.db.write_backup(book_id, raw) except: prints('Failed to write backup metadata for id:', book_id, 'again, giving up') + traceback.print_exc() return self.db.clear_dirtied(book_id, sequence) From ef50b1a82386ee0a3a72e36bad4e944598c0bb92 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 13:49:21 +0530 Subject: [PATCH 0304/1154] More micro-optimizations --- src/calibre/db/tables.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 2cd7d1502e..412c937544 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -94,10 +94,10 @@ class OneToOneTable(Table): query = db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol, self.metadata['column'], self.metadata['table'])) if self.unserialize is None: - self.book_col_map = {row[0]:row[1] for row in query} + self.book_col_map = dict(query) else: us = self.unserialize - self.book_col_map = {row[0]:us(row[1]) for row in query} + self.book_col_map = {book_id:us(val) for book_id, val in query} def remove_books(self, book_ids, db): clean = set() @@ -117,11 +117,10 @@ class PathTable(OneToOneTable): class SizeTable(OneToOneTable): def read(self, db): - self.book_col_map = {} - for row in db.conn.execute( - 'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data ' - 'WHERE data.book=books.id) FROM books'): - self.book_col_map[row[0]] = row[1] + query = db.conn.execute( + 'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data ' + 'WHERE data.book=books.id) FROM books') + self.book_col_map = dict(query) def update_sizes(self, size_map): self.book_col_map.update(size_map) @@ -185,10 +184,10 @@ class ManyToOneTable(Table): query = db.conn.execute('SELECT id, {0} FROM {1}'.format( self.metadata['column'], self.metadata['table'])) if self.unserialize is None: - self.id_map = {row[0]:row[1] for row in query} + self.id_map = dict(query) else: us = self.unserialize - self.id_map = {row[0]:us(row[1]) for row in query} + self.id_map = {book_id:us(val) for book_id, val in query} def read_maps(self, db): for row in db.conn.execute( From c6bdca08aa6c980b776b7a35e34dc6c9443699e7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 14:10:48 +0530 Subject: [PATCH 0305/1154] Micro-optimization for get_categories() by using __slots__ --- src/calibre/db/categories.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py index f80bb52c08..ebd50750a3 100644 --- a/src/calibre/db/categories.py +++ b/src/calibre/db/categories.py @@ -21,6 +21,11 @@ CATEGORY_SORTS = ('name', 'popularity', 'rating') # This has to be a tuple not class Tag(object): + if tweaks.get('use_new_db', False): + __slots__ = ('name', 'original_name', 'id', 'count', 'state', 'is_hierarchical', + 'is_editable', 'is_searchable', 'id_set', 'avg_rating', 'sort', + 'use_sort_as_name', 'tooltip', 'icon', 'category') + def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None, tooltip=None, icon=None, category=None, id_set=None, is_editable=True, is_searchable=True, use_sort_as_name=False): From 61c170340264612a724371c02eea05774d7ff37b Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Mon, 22 Jul 2013 13:51:53 +0200 Subject: [PATCH 0306/1154] Remove broken code that does nothing useful --- src/calibre/devices/smart_device_app/driver.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 5fe60862e1..f8d679152d 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -140,21 +140,6 @@ class ConnectionListener(Thread): self.driver.listen_socket.settimeout(None) device_socket.settimeout(None) - try: - peer = self.driver.device_socket.getpeername()[0] - attempts = self.drjver.connection_attempts.get(peer, 0) - if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS: - self.driver._debug('too many connection attempts from', peer) - device_socket.close() - device_socket = None -# raise InitialConnectionError(_('Too many connection attempts from %s') % peer) - else: - self.driver.connection_attempts[peer] = attempts + 1 - except InitialConnectionError: - raise - except: - pass - try: self.driver.connection_queue.put_nowait(device_socket) except Queue.Full: From e5007925866fac2a25094f1e18c3af6d8eac940d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Jul 2013 18:47:25 +0530 Subject: [PATCH 0307/1154] Conversion: Fix a regression in the last release that broke conversion of a few files with comments just before a chapter start. See #1188635 --- src/calibre/ebooks/oeb/transforms/structure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/oeb/transforms/structure.py b/src/calibre/ebooks/oeb/transforms/structure.py index 50ee4d011d..e5014194ff 100644 --- a/src/calibre/ebooks/oeb/transforms/structure.py +++ b/src/calibre/ebooks/oeb/transforms/structure.py @@ -35,7 +35,7 @@ def at_start(elem): for x in body.iter(): if x is elem: return True - if getattr(x, 'tag', None) and x.tag.rpartition('}')[-1] in {'img', 'svg'}: + if hasattr(getattr(x, 'tag', None), 'rpartition') and x.tag.rpartition('}')[-1] in {'img', 'svg'}: return False if isspace(getattr(x, 'text', None)) and (x in ancestors or isspace(getattr(x, 'tail', None))): continue From a1581e1433ec7a2f89af51b02a06192a35930007 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 23 Jul 2013 08:16:55 +0530 Subject: [PATCH 0308/1154] Speed up reading the db Principally by storing dates as UTC instead of local time. Includes some micro optimizations in the code paths to build the maps. --- src/calibre/db/__init__.py | 7 +- src/calibre/db/search.py | 4 +- src/calibre/db/tables.py | 144 ++++++++---------- src/calibre/db/tests/profiling.py | 36 +++++ src/calibre/db/tests/reading.py | 18 +++ .../library/catalogs/epub_mobi_builder.py | 4 +- src/calibre/library/cli.py | 2 +- src/calibre/library/server/content.py | 3 +- src/calibre/library/server/mobile.py | 4 +- src/calibre/library/server/opds.py | 3 +- src/calibre/utils/date.py | 91 ++++++++--- src/calibre/utils/speedup.c | 3 +- 12 files changed, 205 insertions(+), 114 deletions(-) create mode 100644 src/calibre/db/tests/profiling.py diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 7a5c8d51d4..bb0f679fd8 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -54,7 +54,7 @@ def _get_series_values(val): pass return (val, None) -def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None): +def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None, convert_to_local_tz=True): ''' Return all metadata stored in the database as a dict. Includes paths to the cover and each format. @@ -66,6 +66,7 @@ def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None): ''' import os from calibre.ebooks.metadata import authors_to_string + from calibre.utils.date import as_local_time backend = getattr(self, 'backend', self) # Works with both old and legacy interfaces if prefix is None: prefix = backend.library_path @@ -88,6 +89,10 @@ def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None): x = {} for field in FIELDS: x[field] = record[self.FIELD_MAP[field]] + if convert_to_local_tz and hasattr(self, 'new_api'): + for tf in ('timestamp', 'pubdate', 'last_modified'): + x[tf] = as_local_time(x[tf]) + data.append(x) x['id'] = db_id x['formats'] = [] diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index f00b7102e7..98886b8fc9 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -13,7 +13,7 @@ from datetime import timedelta from calibre.constants import preferred_encoding from calibre.utils.config_base import prefs -from calibre.utils.date import parse_date, UNDEFINED_DATE, now +from calibre.utils.date import parse_date, UNDEFINED_DATE, now, dt_as_local from calibre.utils.icu import primary_find, sort_key from calibre.utils.localization import lang_map, canonicalize_lang from calibre.utils.search_query_parser import SearchQueryParser, ParseException @@ -211,7 +211,7 @@ class DateSearch(object): # {{{ for v, book_ids in field_iter(): if isinstance(v, (str, unicode)): v = parse_date(v) - if v is not None and relop(v, qd, field_count): + if v is not None and relop(dt_as_local(v), qd, field_count): matches |= book_ids return matches diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 412c937544..c8d974fcf7 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -7,16 +7,37 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from datetime import datetime +from datetime import datetime, timedelta from collections import defaultdict -from dateutil.tz import tzoffset - from calibre.constants import plugins -from calibre.utils.date import parse_date, local_tz, UNDEFINED_DATE +from calibre.utils.date import parse_date, UNDEFINED_DATE, utc_tz from calibre.ebooks.metadata import author_to_author_sort -_c_speedup = plugins['speedup'][0] +_c_speedup = plugins['speedup'][0].parse_date + +def c_parse(val): + try: + year, month, day, hour, minutes, seconds, tzsecs = _c_speedup(val) + except (AttributeError, TypeError): + # If a value like 2001 is stored in the column, apsw will return it as + # an int + if isinstance(val, (int, float)): + return datetime(int(val), 1, 3, tzinfo=utc_tz) + except: + pass + else: + try: + ans = datetime(year, month, day, hour, minutes, seconds, tzinfo=utc_tz) + if tzsecs is not 0: + ans -= timedelta(seconds=tzsecs) + except OverflowError: + ans = UNDEFINED_DATE + return ans + try: + return parse_date(val, as_utc=True, assume_utc=True) + except ValueError: + return UNDEFINED_DATE ONE_ONE, MANY_ONE, MANY_MANY = xrange(3) @@ -24,28 +45,6 @@ class Null: pass null = Null() -def _c_convert_timestamp(val): - if not val: - return None - try: - ret = _c_speedup.parse_date(val.strip()) - except AttributeError: - # If a value like 2001 is stored in the column, apsw will return it as - # an int - if isinstance(val, (int, float)): - return datetime(int(val), 1, 1, tzinfo=tzoffset(None, 0)).astimezone(local_tz) - ret = None - except: - ret = None - if ret is None: - return parse_date(val, as_utc=False) - year, month, day, hour, minutes, seconds, tzsecs = ret - try: - return datetime(year, month, day, hour, minutes, seconds, - tzinfo=tzoffset(None, tzsecs)).astimezone(local_tz) - except OverflowError: - return UNDEFINED_DATE.astimezone(local_tz) - class Table(object): def __init__(self, name, metadata, link_table=None): @@ -54,7 +53,7 @@ class Table(object): # self.unserialize() maps values from the db to python objects self.unserialize = { - 'datetime': _c_convert_timestamp, + 'datetime': c_parse, 'bool': bool }.get(metadata['datatype'], None) if name == 'authors': @@ -89,7 +88,6 @@ class OneToOneTable(Table): table_type = ONE_ONE def read(self, db): - self.book_col_map = {} idcol = 'id' if self.metadata['table'] == 'books' else 'book' query = db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol, self.metadata['column'], self.metadata['table'])) @@ -175,7 +173,7 @@ class ManyToOneTable(Table): def read(self, db): self.id_map = {} - self.col_book_map = {} + self.col_book_map = defaultdict(set) self.book_col_map = {} self.read_id_maps(db) self.read_maps(db) @@ -190,13 +188,13 @@ class ManyToOneTable(Table): self.id_map = {book_id:us(val) for book_id, val in query} def read_maps(self, db): - for row in db.conn.execute( + cbm = self.col_book_map + bcm = self.book_col_map + for book, item_id in db.conn.execute( 'SELECT book, {0} FROM {1}'.format( self.metadata['link_column'], self.link_table)): - if row[1] not in self.col_book_map: - self.col_book_map[row[1]] = set() - self.col_book_map[row[1]].add(row[0]) - self.book_col_map[row[0]] = row[1] + cbm[item_id].add(book) + bcm[book] = item_id def remove_books(self, book_ids, db): clean = set() @@ -272,17 +270,14 @@ class ManyToManyTable(ManyToOneTable): do_clean_on_remove = True def read_maps(self, db): - for row in db.conn.execute( - self.selectq.format(self.metadata['link_column'], self.link_table)): - if row[1] not in self.col_book_map: - self.col_book_map[row[1]] = set() - self.col_book_map[row[1]].add(row[0]) - if row[0] not in self.book_col_map: - self.book_col_map[row[0]] = [] - self.book_col_map[row[0]].append(row[1]) + bcm = defaultdict(list) + cbm = self.col_book_map + for book, item_id in db.conn.execute( + self.selectq.format(self.metadata['link_column'], self.link_table)): + cbm[item_id].add(book) + bcm[book].append(item_id) - for key in tuple(self.book_col_map.iterkeys()): - self.book_col_map[key] = tuple(self.book_col_map[key]) + self.book_col_map = {k:tuple(v) for k, v in bcm.iteritems()} def remove_books(self, book_ids, db): clean = set() @@ -351,17 +346,16 @@ class ManyToManyTable(ManyToOneTable): class AuthorsTable(ManyToManyTable): def read_id_maps(self, db): - self.alink_map = {} - self.asort_map = {} - self.id_map = {} + self.alink_map = lm = {} + self.asort_map = sm = {} + self.id_map = im = {} us = self.unserialize - for row in db.conn.execute( + for aid, name, sort, link in db.conn.execute( 'SELECT id, name, sort, link FROM authors'): - val = us(row[1]) - self.id_map[row[0]] = self.unserialize(val) - self.asort_map[row[0]] = (row[2] if row[2] else - author_to_author_sort(val)) - self.alink_map[row[0]] = row[3] + name = us(name) + im[aid] = name + sm[aid] = (sort or author_to_author_sort(name)) + lm[aid] = link def set_sort_names(self, aus_map, db): aus_map = {aid:(a or '').strip() for aid, a in aus_map.iteritems()} @@ -404,22 +398,20 @@ class FormatsTable(ManyToManyTable): pass def read_maps(self, db): - self.fname_map = defaultdict(dict) - self.size_map = defaultdict(dict) - for row in db.conn.execute('SELECT book, format, name, uncompressed_size FROM data'): - if row[1] is not None: - fmt = row[1].upper() - if fmt not in self.col_book_map: - self.col_book_map[fmt] = set() - self.col_book_map[fmt].add(row[0]) - if row[0] not in self.book_col_map: - self.book_col_map[row[0]] = [] - self.book_col_map[row[0]].append(fmt) - self.fname_map[row[0]][fmt] = row[2] - self.size_map[row[0]][fmt] = row[3] + self.fname_map = fnm = defaultdict(dict) + self.size_map = sm = defaultdict(dict) + self.col_book_map = cbm = defaultdict(set) + bcm = defaultdict(list) - for key in tuple(self.book_col_map.iterkeys()): - self.book_col_map[key] = tuple(sorted(self.book_col_map[key])) + for book, fmt, name, sz in db.conn.execute('SELECT book, format, name, uncompressed_size FROM data'): + if fmt is not None: + fmt = fmt.upper() + cbm[fmt].add(book) + bcm[book].append(fmt) + fnm[book][fmt] = name + sm[book][fmt] = sz + + self.book_col_map = {k:tuple(sorted(v)) for k, v in bcm.iteritems()} def remove_books(self, book_ids, db): clean = ManyToManyTable.remove_books(self, book_ids, db) @@ -485,14 +477,12 @@ class IdentifiersTable(ManyToManyTable): pass def read_maps(self, db): - for row in db.conn.execute('SELECT book, type, val FROM identifiers'): - if row[1] is not None and row[2] is not None: - if row[1] not in self.col_book_map: - self.col_book_map[row[1]] = set() - self.col_book_map[row[1]].add(row[0]) - if row[0] not in self.book_col_map: - self.book_col_map[row[0]] = {} - self.book_col_map[row[0]][row[1]] = row[2] + self.book_col_map = defaultdict(dict) + self.col_book_map = defaultdict(set) + for book, typ, val in db.conn.execute('SELECT book, type, val FROM identifiers'): + if typ is not None and val is not None: + self.col_book_map[typ].add(book) + self.book_col_map[book][typ] = val def remove_books(self, book_ids, db): clean = set() diff --git a/src/calibre/db/tests/profiling.py b/src/calibre/db/tests/profiling.py new file mode 100644 index 0000000000..6cff57f990 --- /dev/null +++ b/src/calibre/db/tests/profiling.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +import os, cProfile +from tempfile import gettempdir + +from calibre.db.legacy import LibraryDatabase + +db = None +def initdb(path): + global db + db = LibraryDatabase(os.path.expanduser(path)) + +def show_stats(path): + from pstats import Stats + s = Stats(path) + s.sort_stats('cumulative') + s.print_stats(30) + +def main(): + stats = os.path.join(gettempdir(), 'read_db.stats') + pr = cProfile.Profile() + pr.enable() + initdb('~/documents/largelib') + pr.disable() + pr.dump_stats(stats) + show_stats(stats) + print ('Stats saved to', stats) + +if __name__ == '__main__': + main() diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index fcf309ea66..eea8cd2eaa 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -385,3 +385,21 @@ class ReadingTest(BaseTest): self.assertFalse(x.has_book(Metadata(title[:1]))) db.close() # }}} + + def test_datetime(self): + ' Test the reading of datetimes stored in the db ' + from calibre.utils.date import parse_date + from calibre.db.tables import c_parse, UNDEFINED_DATE, _c_speedup + + # First test parsing of string to UTC time + for raw in ('2013-07-22 15:18:29+05:30', ' 2013-07-22 15:18:29+00:00', '2013-07-22 15:18:29', '2003-09-21 23:30:00-06:00'): + self.assertTrue(_c_speedup(raw)) + ctime = c_parse(raw) + pytime = parse_date(raw, assume_utc=True) + self.assertEqual(ctime, pytime) + + self.assertEqual(c_parse(2003).year, 2003) + for x in (None, '', 'abc'): + self.assertEqual(UNDEFINED_DATE, c_parse(x)) + + diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index e31b9808b0..50c29cdbab 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -18,7 +18,7 @@ from calibre.ebooks.metadata import author_to_author_sort from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException, \ InvalidGenresSourceFieldException from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.utils.date import format_date, is_date_undefined, now as nowf +from calibre.utils.date import format_date, is_date_undefined, now as nowf, as_local_time from calibre.utils.filenames import ascii_text, shorten_components_to from calibre.utils.icu import capitalize, collation_order, sort_key from calibre.utils.magick.draw import thumbnail @@ -940,7 +940,7 @@ class CatalogBuilder(object): if is_date_undefined(record['pubdate']): this_title['date'] = None else: - this_title['date'] = strftime(u'%B %Y', record['pubdate'].timetuple()) + this_title['date'] = strftime(u'%B %Y', as_local_time(record['pubdate']).timetuple()) this_title['timestamp'] = record['timestamp'] diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 5a888f672e..012d88dace 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -74,7 +74,7 @@ def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, se db.sort(sort_by, ascending) if search_text: db.search(search_text) - data = db.get_data_as_dict(prefix, authors_as_string=True) + data = db.get_data_as_dict(prefix, authors_as_string=True, convert_to_local_tz=False) if limit > -1: data = data[:limit] fields = ['id'] + fields diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index 45a46d6c56..cdc569f8b7 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -10,7 +10,7 @@ import re, os, posixpath import cherrypy from calibre import fit_image, guess_type -from calibre.utils.date import fromtimestamp +from calibre.utils.date import fromtimestamp, as_utc from calibre.library.caches import SortKeyGenerator from calibre.library.save_to_disk import find_plugboard from calibre.ebooks.metadata import authors_to_string @@ -54,6 +54,7 @@ class ContentServer(object): Generates a locale independent, english timestamp from a datetime object ''' + updated = as_utc(updated) lm = updated.strftime('day, %d month %Y %H:%M:%S GMT') day ={0:'Sun', 1:'Mon', 2:'Tue', 3:'Wed', 4:'Thu', 5:'Fri', 6:'Sat'} lm = lm.replace('day', day[int(updated.strftime('%w'))]) diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py index 51bfca204a..767a48a9d9 100644 --- a/src/calibre/library/server/mobile.py +++ b/src/calibre/library/server/mobile.py @@ -19,7 +19,7 @@ from calibre.library.server.utils import strftime, format_tag_string from calibre.ebooks.metadata import fmt_sidx from calibre.constants import __appname__ from calibre import human_readable, isbytestring -from calibre.utils.date import utcfromtimestamp +from calibre.utils.date import utcfromtimestamp, as_local_time from calibre.utils.filenames import ascii_filename from calibre.utils.icu import sort_key @@ -254,7 +254,7 @@ class MobileServer(object): no_tag_count=True) book['title'] = record[FM['title']] for x in ('timestamp', 'pubdate'): - book[x] = strftime('%d %b, %Y', record[FM[x]]) + book[x] = strftime('%d %b, %Y', as_local_time(record[FM[x]])) book['id'] = record[FM['id']] books.append(book) for key in CKEYS: diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index 36a65661d1..b4e5c7b265 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -22,6 +22,7 @@ from calibre.library.server import custom_fields_to_display from calibre.library.server.utils import format_tag_string, Offsets from calibre import guess_type, prepare_string_for_xml as xml from calibre.utils.icu import sort_key +from calibre.utils.date import as_utc BASE_HREFS = { 0 : '/stanza', @@ -58,7 +59,7 @@ ID = E.id ICON = E.icon def UPDATED(dt, *args, **kwargs): - return E.updated(dt.strftime('%Y-%m-%dT%H:%M:%S+00:00'), *args, **kwargs) + return E.updated(as_utc(dt).strftime('%Y-%m-%dT%H:%M:%S+00:00'), *args, **kwargs) LINK = partial(E.link, type='application/atom+xml') NAVLINK = partial(E.link, diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index 1a2289681c..0a8d779af8 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -6,23 +6,47 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re -from datetime import datetime, time, timedelta +import re, time +from datetime import datetime, time as dtime, timedelta from functools import partial -from dateutil.tz import tzlocal, tzutc +from dateutil.tz import tzlocal, tzutc, EPOCHORDINAL from calibre import strftime class SafeLocalTimeZone(tzlocal): - ''' - Assume DST was not in effect for historical dates, if DST - data for the local timezone is not present in the operating system. - ''' def _isdst(self, dt): + # We can't use mktime here. It is unstable when deciding if + # the hour near to a change is DST or not. + # + # timestamp = time.mktime((dt.year, dt.month, dt.day, dt.hour, + # dt.minute, dt.second, dt.weekday(), 0, -1)) + # return time.localtime(timestamp).tm_isdst + # + # The code above yields the following result: + # + #>>> import tz, datetime + #>>> t = tz.tzlocal() + #>>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() + #'BRDT' + #>>> datetime.datetime(2003,2,16,0,tzinfo=t).tzname() + #'BRST' + #>>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() + #'BRST' + #>>> datetime.datetime(2003,2,15,22,tzinfo=t).tzname() + #'BRDT' + #>>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() + #'BRDT' + # + # Here is a more stable implementation: + # try: - return tzlocal._isdst(self, dt) + timestamp = ((dt.toordinal() - EPOCHORDINAL) * 86400 + + dt.hour * 3600 + + dt.minute * 60 + + dt.second) + return time.localtime(timestamp+time.timezone).tm_isdst except ValueError: pass return False @@ -150,6 +174,11 @@ def as_local_time(date_time, assume_utc=True): _local_tz) return date_time.astimezone(_local_tz) +def dt_as_local(dt): + if dt.tzinfo is local_tz: + return dt + return dt.astimezone(local_tz) + def as_utc(date_time, assume_utc=True): if not hasattr(date_time, 'tzinfo'): return date_time @@ -174,24 +203,27 @@ def utcfromtimestamp(stamp): traceback.print_exc() return utcnow() -#### Format date functions +# Format date functions def fd_format_hour(dt, strf, ampm, hr): l = len(hr) h = dt.hour if ampm: h = h%12 - if l == 1: return '%d'%h + if l == 1: + return '%d'%h return '%02d'%h def fd_format_minute(dt, strf, ampm, min): l = len(min) - if l == 1: return '%d'%dt.minute + if l == 1: + return '%d'%dt.minute return '%02d'%dt.minute def fd_format_second(dt, strf, ampm, sec): l = len(sec) - if l == 1: return '%d'%dt.second + if l == 1: + return '%d'%dt.second return '%02d'%dt.second def fd_format_ampm(dt, strf, ampm, ap): @@ -202,20 +234,27 @@ def fd_format_ampm(dt, strf, ampm, ap): def fd_format_day(dt, strf, ampm, dy): l = len(dy) - if l == 1: return '%d'%dt.day - if l == 2: return '%02d'%dt.day - if l == 3: return strf('%a') + if l == 1: + return '%d'%dt.day + if l == 2: + return '%02d'%dt.day + if l == 3: + return strf('%a') return strf('%A') def fd_format_month(dt, strf, ampm, mo): l = len(mo) - if l == 1: return '%d'%dt.month - if l == 2: return '%02d'%dt.month - if l == 3: return strf('%b') + if l == 1: + return '%d'%dt.month + if l == 2: + return '%02d'%dt.month + if l == 3: + return strf('%b') return strf('%B') def fd_format_year(dt, strf, ampm, yr): - if len(yr) == 2: return '%02d'%(dt.year % 100) + if len(yr) == 2: + return '%02d'%(dt.year % 100) return '%04d'%dt.year fd_function_index = { @@ -240,7 +279,7 @@ def format_date(dt, format, assume_utc=False, as_utc=False): format = 'dd MMM yyyy' if not isinstance(dt, datetime): - dt = datetime.combine(dt, time()) + dt = datetime.combine(dt, dtime()) if hasattr(dt, 'tzinfo'): if dt.tzinfo is None: @@ -260,7 +299,7 @@ def format_date(dt, format, assume_utc=False, as_utc=False): '(s{1,2})|(m{1,2})|(h{1,2})|(ap)|(AP)|(d{1,4}|M{1,4}|(?:yyyy|yy))', repl_func, format) -#### Clean date functions +# Clean date functions def cd_has_hour(tt, dt): tt['hour'] = dt.hour @@ -307,7 +346,7 @@ def clean_date_for_sort(dt, format): format = 'yyMd' if not isinstance(dt, datetime): - dt = datetime.combine(dt, time()) + dt = datetime.combine(dt, dtime()) if hasattr(dt, 'tzinfo'): if dt.tzinfo is None: @@ -340,7 +379,7 @@ def replace_months(datestr, clang): u'[sS]eptembre': u'sep', u'[Oo]ctobre': u'oct', u'[nN]ovembre': u'nov', - u'[dD].cembre': u'dec' } + u'[dD].cembre': u'dec'} detoen = { u'[jJ]anuar': u'jan', u'[fF]ebruar': u'feb', @@ -353,7 +392,7 @@ def replace_months(datestr, clang): u'[sS]eptember': u'sep', u'[Oo]ktober': u'oct', u'[nN]ovember': u'nov', - u'[dD]ezember': u'dec' } + u'[dD]ezember': u'dec'} if clang == 'fr': dictoen = frtoen @@ -364,6 +403,8 @@ def replace_months(datestr, clang): for k in dictoen.iterkeys(): tmp = re.sub(k, dictoen[k], datestr) - if tmp != datestr: break + if tmp != datestr: + break return tmp + diff --git a/src/calibre/utils/speedup.c b/src/calibre/utils/speedup.c index 171179a88a..b05d8ee3c1 100644 --- a/src/calibre/utils/speedup.c +++ b/src/calibre/utils/speedup.c @@ -13,12 +13,12 @@ speedup_parse_date(PyObject *self, PyObject *args) { long year, month, day, hour, minute, second, tzh = 0, tzm = 0, sign = 0; size_t len; if(!PyArg_ParseTuple(args, "s", &raw)) return NULL; + while ((*raw == ' ' || *raw == '\t' || *raw == '\n' || *raw == '\r' || *raw == '\f' || *raw == '\v') && *raw != 0) raw++; len = strlen(raw); if (len < 19) Py_RETURN_NONE; orig = raw; - year = strtol(raw, &end, 10); if ((end - raw) != 4) Py_RETURN_NONE; raw += 5; @@ -27,7 +27,6 @@ speedup_parse_date(PyObject *self, PyObject *args) { month = strtol(raw, &end, 10); if ((end - raw) != 2) Py_RETURN_NONE; raw += 3; - day = strtol(raw, &end, 10); if ((end - raw) != 2) Py_RETURN_NONE; From e1ed5f679ffaaeda46342186624903698c2caeba Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 23 Jul 2013 08:47:01 +0530 Subject: [PATCH 0309/1154] Micro-optimization for get_categories() --- src/calibre/db/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 403a76707c..ac96cb837d 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -114,6 +114,7 @@ class Field(object): if not self.is_many: return ans + id_map = self.table.id_map special_sort = hasattr(self, 'category_sort_value') for item_id, item_book_ids in self.table.col_book_map.iteritems(): if book_ids is not None: @@ -122,7 +123,7 @@ class Field(object): ratings = tuple(r for r in (book_rating_map.get(book_id, 0) for book_id in item_book_ids) if r > 0) avg = sum(ratings)/len(ratings) if ratings else 0 - name = self.category_formatter(self.table.id_map[item_id]) + name = self.category_formatter(id_map[item_id]) sval = (self.category_sort_value(item_id, item_book_ids, lang_map) if special_sort else name) c = tag_class(name, id=item_id, sort=sval, avg=avg, From 2da2657345ccfdaf2197d8572ae22a6fd82abe44 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 23 Jul 2013 08:57:10 +0530 Subject: [PATCH 0310/1154] pep8 --- src/calibre/ebooks/metadata/book/base.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 0948ef544d..67c3fb1428 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -80,7 +80,7 @@ class Metadata(object): self.title = title if authors: # List of strings or [] - self.author = list(authors) if authors else []# Needed for backward compatibility + self.author = list(authors) if authors else [] # Needed for backward compatibility self.authors = list(authors) if authors else [] from calibre.ebooks.metadata.book.formatter import SafeFormat self.formatter = SafeFormat() @@ -429,8 +429,7 @@ class Metadata(object): try: src = op[0] dest = op[1] - val = formatter.safe_format\ - (src, other, 'PLUGBOARD TEMPLATE ERROR', other) + val = formatter.safe_format(src, other, 'PLUGBOARD TEMPLATE ERROR', other) if dest == 'tags': self.set(dest, [f.strip() for f in val.split(',') if f.strip()]) elif dest == 'authors': @@ -476,7 +475,7 @@ class Metadata(object): if replace_metadata: # SPECIAL_FIELDS = frozenset(['lpath', 'size', 'comments', 'thumbnail']) for attr in SC_COPYABLE_FIELDS: - setattr(self, attr, getattr(other, attr, 1.0 if \ + setattr(self, attr, getattr(other, attr, 1.0 if attr == 'series_index' else None)) self.tags = other.tags self.cover_data = getattr(other, 'cover_data', @@ -507,8 +506,10 @@ class Metadata(object): if getattr(other, 'cover_data', False): other_cover = other.cover_data[-1] self_cover = self.cover_data[-1] if self.cover_data else '' - if not self_cover: self_cover = '' - if not other_cover: other_cover = '' + if not self_cover: + self_cover = '' + if not other_cover: + other_cover = '' if len(other_cover) > len(self_cover): self.cover_data = other.cover_data @@ -517,7 +518,7 @@ class Metadata(object): meta = other.get_user_metadata(x, make_copy=True) if meta is not None: self_tags = self.get(x, []) - self.set_user_metadata(x, meta) # get... did the deepcopy + self.set_user_metadata(x, meta) # get... did the deepcopy other_tags = other.get(x, []) if meta['datatype'] == 'text' and meta['is_multiple']: # Case-insensitive but case preserving merging @@ -607,7 +608,7 @@ class Metadata(object): # Handle custom series index if key.startswith('#') and key.endswith('_index'): - tkey = key[:-6] # strip the _index + tkey = key[:-6] # strip the _index cmeta = self.get_user_metadata(tkey, make_copy=False) if cmeta and cmeta['datatype'] == 'series': if self.get(tkey): @@ -698,7 +699,7 @@ class Metadata(object): if self.title_sort: fmt('Title sort', self.title_sort) if self.authors: - fmt('Author(s)', authors_to_string(self.authors) + \ + fmt('Author(s)', authors_to_string(self.authors) + ((' [' + self.author_sort + ']') if self.author_sort and self.author_sort != _('Unknown') else '')) if self.publisher: @@ -805,3 +806,4 @@ def field_from_string(field, raw, field_metadata): return val + From 9b57e80e28d87356271db8198f311ac879969edf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 23 Jul 2013 09:09:23 +0530 Subject: [PATCH 0311/1154] Micro-optimize mi.set_identifiers() --- src/calibre/ebooks/metadata/book/base.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 67c3fb1428..4427121f37 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -42,6 +42,8 @@ NULL_VALUES = { field_metadata = FieldMetadata() +ck = lambda typ: icu_lower(typ).strip().replace(':', '').replace(',', '') +cv = lambda val: val.strip().replace(',', '|').replace(':', '|') class Metadata(object): @@ -224,9 +226,9 @@ class Metadata(object): def _clean_identifier(self, typ, val): if typ: - typ = icu_lower(typ).strip().replace(':', '').replace(',', '') + typ = ck(typ) if val: - val = val.strip().replace(',', '|').replace(':', '|') + val = cv(val) return typ, val def set_identifiers(self, identifiers): @@ -234,11 +236,7 @@ class Metadata(object): Set all identifiers. Note that if you previously set ISBN, calling this method will delete it. ''' - cleaned = {} - for key, val in identifiers.iteritems(): - key, val = self._clean_identifier(key, val) - if key and val: - cleaned[key] = val + cleaned = {ck(k):cv(v) for k, v in identifiers.iteritems() if k and v} object.__getattribute__(self, '_data')['identifiers'] = cleaned def set_identifier(self, typ, val): From ef48091c23e0f80707977f778aaf477ea6c40998 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 23 Jul 2013 09:48:26 +0530 Subject: [PATCH 0312/1154] ... --- src/calibre/db/fields.py | 5 +++-- src/calibre/db/tests/profiling.py | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index ac96cb837d..9becf82403 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -163,12 +163,13 @@ class CompositeField(OneToOneField): self._render_cache = {} self._lock = Lock() + self._composite_name = '#' + self.metadata['label'] def render_composite(self, book_id, mi): with self._lock: ans = self._render_cache.get(book_id, None) if ans is None: - ans = mi.get('#'+self.metadata['label']) + ans = mi.get(self._composite_name) with self._lock: self._render_cache[book_id] = ans return ans @@ -186,7 +187,7 @@ class CompositeField(OneToOneField): ans = self._render_cache.get(book_id, None) if ans is None: mi = get_metadata(book_id) - ans = mi.get('#'+self.metadata['label']) + ans = mi.get(self._composite_name) with self._lock: self._render_cache[book_id] = ans return ans diff --git a/src/calibre/db/tests/profiling.py b/src/calibre/db/tests/profiling.py index 6cff57f990..07365b1b75 100644 --- a/src/calibre/db/tests/profiling.py +++ b/src/calibre/db/tests/profiling.py @@ -25,8 +25,12 @@ def show_stats(path): def main(): stats = os.path.join(gettempdir(), 'read_db.stats') pr = cProfile.Profile() + initdb('~/test library') + all_ids = db.new_api.all_book_ids() # noqa pr.enable() - initdb('~/documents/largelib') + for book_id in all_ids: + db.new_api._composite_for('#isbn', book_id) + db.new_api._composite_for('#formats', book_id) pr.disable() pr.dump_stats(stats) show_stats(stats) From bfa4c67dc91b6a24a818c5136b8cc7af5b08ccc1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 23 Jul 2013 13:40:35 +0530 Subject: [PATCH 0313/1154] Search caching for the new backed with a simple LRU cache --- src/calibre/db/cache.py | 23 ++++++- src/calibre/db/search.py | 107 ++++++++++++++++++++++++++------ src/calibre/db/tests/reading.py | 59 +++++++++++++++++- 3 files changed, 166 insertions(+), 23 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 6cdb9419c6..52227a5b84 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -150,12 +150,22 @@ class Cache(object): field.clear_caches(book_ids=book_ids) @write_api - def clear_caches(self, book_ids=None): - self._initialize_template_cache() # Clear the formatter template cache + def clear_search_caches(self): + self._search_api.clear_caches() + + @write_api + def clear_caches(self, book_ids=None, template_cache=True): + if template_cache: + self._initialize_template_cache() # Clear the formatter template cache for field in self.fields.itervalues(): if hasattr(field, 'clear_caches'): field.clear_caches(book_ids=book_ids) # Clear the composite cache and ondevice caches - self.format_metadata_cache.clear() + if book_ids: + for book_id in book_ids: + self.format_metadata_cache.pop(book_id, None) + else: + self.format_metadata_cache.clear() + self._clear_search_caches() @write_api def reload_from_db(self, clear_caches=True): @@ -497,6 +507,8 @@ class Cache(object): @write_api def set_pref(self, name, val): self.backend.prefs.set(name, val) + if name == 'grouped_search_terms': + self._clear_search_caches() @api def get_metadata(self, book_id, @@ -812,6 +824,7 @@ class Cache(object): f.writer.set_books({book_id:now for book_id in book_ids}, self.backend) if self.composites: self._clear_composite_caches(book_ids) + self._clear_search_caches() @write_api def mark_as_dirty(self, book_ids): @@ -1286,6 +1299,7 @@ class Cache(object): continue # Some fields like ondevice do not have tables else: table.remove_books(book_ids, self.backend) + self._clear_caches(book_ids=book_ids, template_cache=False) @read_api def author_sort_strings_for_books(self, book_ids): @@ -1563,10 +1577,12 @@ class Cache(object): @write_api def saved_search_set_all(self, smap): self._search_api.saved_searches.set_all(smap) + self._clear_search_caches() @write_api def saved_search_delete(self, name): self._search_api.saved_searches.delete(name) + self._clear_search_caches() @write_api def saved_search_add(self, name, val): @@ -1575,6 +1591,7 @@ class Cache(object): @write_api def saved_search_rename(self, old_name, new_name): self._search_api.saved_searches.rename(old_name, new_name) + self._clear_search_caches() @write_api def change_search_locations(self, newlocs): diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 98886b8fc9..07b3bde068 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en' import re, weakref from functools import partial from datetime import timedelta +from collections import deque from calibre.constants import preferred_encoding from calibre.utils.config_base import prefs @@ -711,6 +712,47 @@ class Parser(SearchQueryParser): return candidates - matches return matches +class LRUCache(object): + + 'A simple Least-Recently-Used cache' + + def __init__(self, limit=30): + self.item_map = {} + self.age_map = deque() + self.limit = limit + + def _move_up(self, key): + if key != self.age_map[-1]: + self.age_map.remove(key) + self.age_map.append(key) + + def add(self, key, val): + if key in self.item_map: + self._move_up(key) + return + + if len(self.age_map) >= self.limit: + self.item_map.pop(self.age_map.popleft()) + + self.item_map[key] = val + self.age_map.append(key) + + def get(self, key, default=None): + ans = self.item_map.get(key, default) + if ans is not default: + self._move_up(key) + return ans + + def clear(self): + self.item_map.clear() + self.age_map.clear() + + def __contains__(self, key): + return key in self.item_map + + def __len__(self): + return len(self.age_map) + class Search(object): def __init__(self, db, opt_name, all_search_locations=()): @@ -720,46 +762,73 @@ class Search(object): self.bool_search = BooleanSearch() self.keypair_search = KeyPairSearch() self.saved_searches = SavedSearchQueries(db, opt_name) + self.cache = LRUCache() def get_saved_searches(self): return self.saved_searches def change_locations(self, newlocs): + if frozenset(newlocs) != frozenset(self.all_search_locations): + self.clear_caches() self.all_search_locations = newlocs + def clear_caches(self): + self.cache.clear() + def __call__(self, dbcache, query, search_restriction, virtual_fields=None, book_ids=None): ''' Return the set of ids of all records that match the specified query and restriction ''' - q = '' - if not query or not query.strip(): - q = search_restriction - else: - q = query - if search_restriction: - q = u'(%s) and (%s)' % (search_restriction, query) - - all_book_ids = dbcache._all_book_ids(type=set) if book_ids is None else set(book_ids) - if not q: - return all_book_ids - - if not isinstance(q, type(u'')): - q = q.decode('utf-8') - # We construct a new parser instance per search as the parse is not # thread safe. sqp = Parser( - dbcache, all_book_ids, dbcache._pref('grouped_search_terms'), + dbcache, set(), dbcache._pref('grouped_search_terms'), self.date_search, self.num_search, self.bool_search, self.keypair_search, prefs['limit_search_columns'], prefs['limit_search_columns_to'], self.all_search_locations, virtual_fields, self.saved_searches.lookup) - try: - ret = sqp.parse(q) + return self._do_search(sqp, query, search_restriction, dbcache, book_ids=book_ids) finally: sqp.dbcache = sqp.lookup_saved_search = None - return ret + + def _do_search(self, sqp, query, search_restriction, dbcache, book_ids=None): + if isinstance(search_restriction, bytes): + search_restriction = search_restriction.decode('utf-8') + + restricted_ids = all_book_ids = dbcache._all_book_ids(type=set) + if search_restriction and search_restriction.strip(): + cached = self.cache.get(search_restriction.strip()) + if cached is None: + sqp.all_book_ids = all_book_ids if book_ids is None else book_ids + restricted_ids = sqp.parse(search_restriction) + if sqp.all_book_ids is all_book_ids: + self.cache.add(search_restriction.strip(), restricted_ids) + else: + restricted_ids = cached + if book_ids is not None: + restricted_ids = book_ids.intersection(restricted_ids) + elif book_ids is not None: + restricted_ids = book_ids + + if isinstance(query, bytes): + query = query.decode('utf-8') + + if not query or not query.strip(): + return restricted_ids + + if restricted_ids is all_book_ids: + cached = self.cache.get(query.strip()) + if cached is not None: + return cached + + sqp.all_book_ids = restricted_ids + result = sqp.parse(query) + + if sqp.all_book_ids is all_book_ids: + self.cache.add(query.strip(), result) + + return result diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index eea8cd2eaa..28c499e150 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -386,7 +386,7 @@ class ReadingTest(BaseTest): db.close() # }}} - def test_datetime(self): + def test_datetime(self): # {{{ ' Test the reading of datetimes stored in the db ' from calibre.utils.date import parse_date from calibre.db.tables import c_parse, UNDEFINED_DATE, _c_speedup @@ -401,5 +401,62 @@ class ReadingTest(BaseTest): self.assertEqual(c_parse(2003).year, 2003) for x in (None, '', 'abc'): self.assertEqual(UNDEFINED_DATE, c_parse(x)) + # }}} + def test_restrictions(self): # {{{ + ' Test searching with and without restrictions ' + cache = self.init_cache() + self.assertSetEqual(cache.all_book_ids(), cache.search('')) + self.assertSetEqual({1, 2}, cache.search('', 'not authors:=Unknown')) + self.assertSetEqual(set(), cache.search('authors:=Unknown', 'not authors:=Unknown')) + self.assertSetEqual({2}, cache.search('not authors:"=Author Two"', 'not authors:=Unknown')) + self.assertSetEqual({2}, cache.search('not authors:"=Author Two"', book_ids={1, 2})) + self.assertSetEqual({2}, cache.search('not authors:"=Author Two"', 'not authors:=Unknown', book_ids={1,2,3})) + self.assertSetEqual(set(), cache.search('authors:=Unknown', 'not authors:=Unknown', book_ids={1,2,3})) + # }}} + + def test_search_caching(self): # {{{ + ' Test caching of searches ' + from calibre.db.search import LRUCache + class TestCache(LRUCache): + hit_counter = 0 + miss_counter = 0 + def get(self, key, default=None): + ans = LRUCache.get(self, key, default=default) + if ans is not None: + self.hit_counter += 1 + else: + self.miss_counter += 1 + @property + def cc(self): + self.hit_counter = self.miss_counter = 0 + @property + def counts(self): + return self.hit_counter, self.miss_counter + + cache = self.init_cache() + cache._search_api.cache = c = TestCache() + + ae, at = self.assertEqual, self.assertTrue + + def test(hit, result, *args): + c.cc + ae(cache.search(*args), result) + ae(c.counts, (1, 0) if hit else (0, 1)) + c.cc + + test(False, {3}, 'Unknown') + test(True, {3}, 'Unknown') + test(True, {3}, 'Unknown') + cache.set_field('title', {3:'xxx'}) + test(False, {3}, 'Unknown') # cache cleared + test(True, {3}, 'Unknown') + c.limit = 5 + for i in range(6): + test(False, set(), 'nomatch_%s' % i) + test(False, {3}, 'Unknown') # cached search expired + test(False, {3}, '', 'unknown') + test(True, {3}, '', 'unknown') + test(True, {3}, 'Unknown', 'unknown') + # }}} From 5f3c10f91df30f2a8d5c2f5d6ca98c572251a4c0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 23 Jul 2013 16:06:32 +0530 Subject: [PATCH 0314/1154] El Tribuno Salta and Jujuy by Darko Miletic Fixes #1203724 [New recipes for El Tribuno Salta and Jujuy](https://bugs.launchpad.net/calibre/+bug/1203724) --- recipes/eltribuno_jujuy_impreso.recipe | 127 ++++++++++++++++++++++ recipes/eltribuno_salta_impreso.recipe | 127 ++++++++++++++++++++++ recipes/icons/eltribuno_jujuy_impreso.png | Bin 0 -> 592 bytes recipes/icons/eltribuno_salta_impreso.png | Bin 0 -> 592 bytes 4 files changed, 254 insertions(+) create mode 100644 recipes/eltribuno_jujuy_impreso.recipe create mode 100644 recipes/eltribuno_salta_impreso.recipe create mode 100644 recipes/icons/eltribuno_jujuy_impreso.png create mode 100644 recipes/icons/eltribuno_salta_impreso.png diff --git a/recipes/eltribuno_jujuy_impreso.recipe b/recipes/eltribuno_jujuy_impreso.recipe new file mode 100644 index 0000000000..2b725231c9 --- /dev/null +++ b/recipes/eltribuno_jujuy_impreso.recipe @@ -0,0 +1,127 @@ +__license__ = 'GPL v3' +__copyright__ = '2013, Darko Miletic ' +''' +http://www.eltribuno.info/jujuy/edicion_impresa.aspx +''' + +import urllib +from calibre.ptempfile import PersistentTemporaryFile +from calibre.web.feeds.news import BasicNewsRecipe +from collections import OrderedDict + +class ElTribunoJujuyImpreso(BasicNewsRecipe): + title = 'El Tribuno Jujuy (Edición Impresa)' + __author__ = 'Darko Miletic' + description = "Diario principal de Jujuy" + publisher = 'Horizontes S.A.' + category = 'news, politics, Jujuy, Argentina, World' + oldest_article = 2 + language = 'es_AR' + max_articles_per_feed = 250 + no_stylesheets = True + use_embedded_content = False + encoding = 'cp1252' + publication_type = 'newspaper' + delay = 1 + articles_are_obfuscated = True + temp_files = [] + PREFIX = 'http://www.eltribuno.info/jujuy/' + INDEX = PREFIX + 'edicion_impresa.aspx' + PRINTURL = PREFIX + 'nota_print.aspx?%s' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'linearize_tables' : True + } + + keep_only_tags = [dict(name='div' , attrs={'class':['notaHead', 'notaContent']})] + remove_tags = [ + dict(name=['meta','iframe','base','object','embed','link','img']), + dict(name='ul', attrs={'class':'Tabs'}) + ] + + extra_css = """ + body{font-family: Arial,Helvetica,sans-serif} + .notaHead h4{text-transform: uppercase; color: gray} + img{margin-top: 0.8em; display: block} + """ + + def parse_index(self): + feeds = OrderedDict() + soup = None + count = 0 + while (count < 5): + try: + soup = self.index_to_soup(self.INDEX) + count = 5 + except: + print "Retrying download..." + count += 1 + if not soup: + return [] + alink = soup.find('a', href=True, attrs={'class':'ZoomTapa'}) + if alink and 'href' in alink: + self.cover_url = alink['href'] + sections = soup.findAll('div', attrs={'id':lambda x: x and x.startswith('Ediciones')}) + for section in sections: + section_title = 'Sin titulo' + sectiont=section.find('h3', attrs={'class':'NombreSeccion'}) + if sectiont: + section_title = self.tag_to_string(sectiont.span) + + arts = section.findAll('div', attrs={'class':'Noticia NoticiaAB1'}) + for article in arts: + articles = [] + title=self.tag_to_string(article.div.h3.a) + url=article.div.h3.a['href'] + description=self.tag_to_string(article.p) + articles.append({'title':title, 'url':url, 'description':description, 'date':''}) + + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles + + ans = [(key, val) for key, val in feeds.iteritems()] + return ans + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll('a'): + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + str = self.tag_to_string(item) + item.replaceWith(str) + return soup + + def get_masthead_title(self): + return 'El Tribuno' + + def get_obfuscated_article(self, url): + count = 0 + while (count < 10): + try: + response = self.browser.open(url) + html = response.read() + count = 10 + except: + print "Retrying download..." + count += 1 + tfile = PersistentTemporaryFile('_fa.html') + tfile.write(html) + tfile.close() + self.temp_files.append(tfile) + return tfile.name + + def print_version(self, url): + right = url.rpartition('/')[2] + artid = right.partition('-')[0] + params = {'Note':artid} + return (self.PRINTURL % urllib.urlencode(params)) + diff --git a/recipes/eltribuno_salta_impreso.recipe b/recipes/eltribuno_salta_impreso.recipe new file mode 100644 index 0000000000..67cc073a7e --- /dev/null +++ b/recipes/eltribuno_salta_impreso.recipe @@ -0,0 +1,127 @@ +__license__ = 'GPL v3' +__copyright__ = '2013, Darko Miletic ' +''' +http://www.eltribuno.info/salta/edicion_impresa.aspx +''' + +import urllib +from calibre.ptempfile import PersistentTemporaryFile +from calibre.web.feeds.news import BasicNewsRecipe +from collections import OrderedDict + +class ElTribunoSaltaImpreso(BasicNewsRecipe): + title = 'El Tribuno Salta (Edición Impresa)' + __author__ = 'Darko Miletic' + description = "Diario principal de Salta" + publisher = 'Horizontes S.A.' + category = 'news, politics, Salta, Argentina, World' + oldest_article = 2 + language = 'es_AR' + max_articles_per_feed = 250 + no_stylesheets = True + use_embedded_content = False + encoding = 'cp1252' + publication_type = 'newspaper' + delay = 1 + articles_are_obfuscated = True + temp_files = [] + PREFIX = 'http://www.eltribuno.info/salta/' + INDEX = PREFIX + 'edicion_impresa.aspx' + PRINTURL = PREFIX + 'nota_print.aspx?%s' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'linearize_tables' : True + } + + keep_only_tags = [dict(name='div' , attrs={'class':['notaHead', 'notaContent']})] + remove_tags = [ + dict(name=['meta','iframe','base','object','embed','link','img']), + dict(name='ul', attrs={'class':'Tabs'}) + ] + + extra_css = """ + body{font-family: Arial,Helvetica,sans-serif} + .notaHead h4{text-transform: uppercase; color: gray} + img{margin-top: 0.8em; display: block} + """ + + def parse_index(self): + feeds = OrderedDict() + soup = None + count = 0 + while (count < 5): + try: + soup = self.index_to_soup(self.INDEX) + count = 5 + except: + print "Retrying download..." + count += 1 + if not soup: + return [] + alink = soup.find('a', href=True, attrs={'class':'ZoomTapa'}) + if alink and 'href' in alink: + self.cover_url = alink['href'] + sections = soup.findAll('div', attrs={'id':lambda x: x and x.startswith('Ediciones')}) + for section in sections: + section_title = 'Sin titulo' + sectiont=section.find('h3', attrs={'class':'NombreSeccion'}) + if sectiont: + section_title = self.tag_to_string(sectiont.span) + + arts = section.findAll('div', attrs={'class':'Noticia NoticiaAB1'}) + for article in arts: + articles = [] + title=self.tag_to_string(article.div.h3.a) + url=article.div.h3.a['href'] + description=self.tag_to_string(article.p) + articles.append({'title':title, 'url':url, 'description':description, 'date':''}) + + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles + + ans = [(key, val) for key, val in feeds.iteritems()] + return ans + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll('a'): + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + str = self.tag_to_string(item) + item.replaceWith(str) + return soup + + def get_masthead_title(self): + return 'El Tribuno' + + def get_obfuscated_article(self, url): + count = 0 + while (count < 10): + try: + response = self.browser.open(url) + html = response.read() + count = 10 + except: + print "Retrying download..." + count += 1 + tfile = PersistentTemporaryFile('_fa.html') + tfile.write(html) + tfile.close() + self.temp_files.append(tfile) + return tfile.name + + def print_version(self, url): + right = url.rpartition('/')[2] + artid = right.partition('-')[0] + params = {'Note':artid} + return (self.PRINTURL % urllib.urlencode(params)) + diff --git a/recipes/icons/eltribuno_jujuy_impreso.png b/recipes/icons/eltribuno_jujuy_impreso.png new file mode 100644 index 0000000000000000000000000000000000000000..8862b78d0c77b95c435de73b4bf78265bcc95815 GIT binary patch literal 592 zcmV-W0el z0jV;e3a3F)l(;f+?f9$by$7X7%GLGFz2}{CFUhh@5kNu+$pk2+sHAC11Y{2qM5?^N zbzRv_=yNei63n2|>2zMlBu%&5t=H?wQ5u|_eBa!hSzBvdUSfzdrzvGj&QQsXZt!t> zRUNh4=E{nDbp-{z!^0nYd!UKqcrX}nc4SZ-1Ob$2y6)ZI-)wLH1~(z`)04Tp{OR^K z2}AJa^LcK__kCoDSrmPposATQ31`r2uC6vOE>M@27-PDw(_AhWh9Nh^*k{i(78fvGIO?Kk|Kk7O^pc$s|E5 zmMP`XYPDLaRG=^RypPAnp*Rq~dZ9oKgXlW)VzKBr4!02tc%D}-mw)wonevJ>2o!OU z%EAJ=YqgrHszmM+FJgK$8rimu7>d*4M3!X@heLdpN+on-D@59rDsdVlnx;L!kfxK# eL>G|V(Ek_WCy$~!G;g^80000el z0jV;e3a3F)l(;f+?f9$by$7X7%GLGFz2}{CFUhh@5kNu+$pk2+sHAC11Y{2qM5?^N zbzRv_=yNei63n2|>2zMlBu%&5t=H?wQ5u|_eBa!hSzBvdUSfzdrzvGj&QQsXZt!t> zRUNh4=E{nDbp-{z!^0nYd!UKqcrX}nc4SZ-1Ob$2y6)ZI-)wLH1~(z`)04Tp{OR^K z2}AJa^LcK__kCoDSrmPposATQ31`r2uC6vOE>M@27-PDw(_AhWh9Nh^*k{i(78fvGIO?Kk|Kk7O^pc$s|E5 zmMP`XYPDLaRG=^RypPAnp*Rq~dZ9oKgXlW)VzKBr4!02tc%D}-mw)wonevJ>2o!OU z%EAJ=YqgrHszmM+FJgK$8rimu7>d*4M3!X@heLdpN+on-D@59rDsdVlnx;L!kfxK# eL>G|V(Ek_WCy$~!G;g^80000 Date: Wed, 24 Jul 2013 13:03:39 +1000 Subject: [PATCH 0315/1154] Update supported firmware version for Kobo to 2.8.1 No changes necessary except to bump the firmware version. --- src/calibre/devices/kobo/driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index cb325efb07..278798160e 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -35,7 +35,7 @@ class KOBO(USBMS): gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') author = 'Timothy Legge and David Forrester' - version = (2, 0, 12) + version = (2, 0, 13) dbversion = 0 fwversion = 0 @@ -1218,7 +1218,7 @@ class KOBOTOUCH(KOBO): min_dbversion_images_on_sdcard = 77 min_dbversion_activiy = 77 - max_supported_fwversion = (2,6,1) + max_supported_fwversion = (2,8,1) min_fwversion_images_on_sdcard = (2,4,1) has_kepubs = True From f37de3d33c93259fdefea1c720b4af68fbf9202d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 09:29:57 +0530 Subject: [PATCH 0316/1154] Small fixes to get_metadata() --- src/calibre/db/cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 52227a5b84..f927c1714a 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -203,7 +203,7 @@ class Cache(object): mi.author_link_map = aul mi.comments = self._field_for('comments', book_id) mi.publisher = self._field_for('publisher', book_id) - n = nowf() + n = utcnow() mi.timestamp = self._field_for('timestamp', book_id, default_value=n) mi.pubdate = self._field_for('pubdate', book_id, default_value=n) mi.uuid = self._field_for('uuid', book_id, @@ -223,6 +223,7 @@ class Cache(object): mi.format_metadata = FormatMetadata(self, book_id, formats) good_formats = FormatsList(formats, mi.format_metadata) mi.formats = good_formats + mi.db_approx_formats = formats mi.has_cover = _('Yes') if self._field_for('cover', book_id, default_value=False) else '' mi.tags = list(self._field_for('tags', book_id, default_value=())) From 70f1dbb832810bd63f129b4b0deeab2710268969 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 10:57:04 +0530 Subject: [PATCH 0317/1154] Speed up evaluation of composite columns Use a ProxyMetadata object that lazily evaluates its fields on demand, thereby avoiding the overhead of get_metadata() on every composite field evaluation. --- src/calibre/db/cache.py | 17 ++- src/calibre/db/lazy.py | 236 +++++++++++++++++++++++++++++++- src/calibre/db/tests/reading.py | 26 ++++ 3 files changed, 271 insertions(+), 8 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index f927c1714a..f909563fa5 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -23,7 +23,7 @@ from calibre.db.fields import create_field from calibre.db.search import Search from calibre.db.tables import VirtualTable from calibre.db.write import get_series_values -from calibre.db.lazy import FormatMetadata, FormatsList +from calibre.db.lazy import FormatMetadata, FormatsList, ProxyMetadata from calibre.ebooks import check_ebook_format from calibre.ebooks.metadata import string_to_authors, author_to_author_sort, get_title_sort_pat from calibre.ebooks.metadata.book.base import Metadata @@ -338,7 +338,7 @@ class Cache(object): def fast_field_for(self, field_obj, book_id, default_value=None): ' Same as field_for, except that it avoids the extra lookup to get the field object ' if field_obj.is_composite: - return field_obj.get_value_with_cache(book_id, partial(self._get_metadata, get_user_categories=False)) + return field_obj.get_value_with_cache(book_id, self._get_proxy_metadata) try: return field_obj.for_book(book_id, default_value=default_value) except (KeyError, IndexError): @@ -358,8 +358,7 @@ class Cache(object): return default_value if mi is None: - return f.get_value_with_cache(book_id, partial(self._get_metadata, - get_user_categories=False)) + return f.get_value_with_cache(book_id, self._get_proxy_metadata) else: return f.render_composite(book_id, mi) @@ -534,6 +533,10 @@ class Cache(object): return mi + @read_api + def get_proxy_metadata(self, book_id): + return ProxyMetadata(self, book_id) + @api def cover(self, book_id, as_file=False, as_image=False, as_path=False): @@ -781,7 +784,7 @@ class Cache(object): ''' all_book_ids = frozenset(self._all_book_ids() if ids_to_sort is None else ids_to_sort) - get_metadata = partial(self._get_metadata, get_user_categories=False) + get_metadata = self._get_proxy_metadata lang_map = self.fields['languages'].book_value_map fm = {'title':'sort', 'authors':'author_sort'} @@ -1189,7 +1192,7 @@ class Cache(object): sf = self.fields[field] if series: q = icu_lower(series) - for val, book_ids in sf.iter_searchable_values(self._get_metadata, frozenset(self._all_book_ids())): + for val, book_ids in sf.iter_searchable_values(self._get_proxy_metadata, frozenset(self._all_book_ids())): if q == icu_lower(val): books = book_ids break @@ -1499,7 +1502,7 @@ class Cache(object): f = self.fields[category] if hasattr(f, 'get_books_for_val'): # Composite field - return f.get_books_for_val(item_id_or_composite_value, self._get_metadata, self._all_book_ids()) + return f.get_books_for_val(item_id_or_composite_value, self._get_proxy_metadata, self._all_book_ids()) return self._books_for_field(f.name, int(item_id_or_composite_value)) @read_api diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index be9334c056..4b1740154b 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -10,14 +10,19 @@ __docformat__ = 'restructuredtext en' import weakref from functools import wraps from collections import MutableMapping, MutableSequence +from copy import deepcopy +from calibre.ebooks.metadata.book.base import Metadata, SIMPLE_GET, TOP_LEVEL_IDENTIFIERS, NULL_VALUES +from calibre.ebooks.metadata.book.formatter import SafeFormat +from calibre.utils.date import utcnow + +# Lazy format metadata retrieval {{{ ''' Avoid doing stats on all files in a book when getting metadata for that book. Speeds up calibre startup with large libraries/libraries on a network share, with a composite custom column. ''' -# Lazy format metadata retrieval {{{ def resolved(f): @wraps(f) def wrapper(self, *args, **kwargs): @@ -97,3 +102,232 @@ class FormatsList(MutableBase, MutableSequence): # }}} +# Lazy metadata getters {{{ +ga = object.__getattribute__ +sa = object.__setattr__ + +def simple_getter(field, default_value=None): + def func(dbref, book_id, cache): + try: + return cache[field] + except KeyError: + db = dbref() + cache[field] = ret = db.field_for(field, book_id, default_value=default_value) + return ret + return func + +def pp_getter(field, postprocess, default_value=None): + def func(dbref, book_id, cache): + try: + return cache[field] + except KeyError: + db = dbref() + cache[field] = ret = postprocess(db.field_for(field, book_id, default_value=default_value)) + return ret + return func + +def adata_getter(field): + def func(dbref, book_id, cache): + try: + author_ids, adata = cache['adata'] + except KeyError: + db = dbref() + with db.read_lock: + author_ids = db._field_ids_for('authors', book_id) + adata = db._author_data(author_ids) + cache['adata'] = (author_ids, adata) + k = 'sort' if field == 'author_sort_map' else 'link' + return {adata[i]['name']:adata[i][k] for i in author_ids} + return func + +def dt_getter(field): + def func(dbref, book_id, cache): + try: + return cache[field] + except KeyError: + db = dbref() + cache[field] = ret = db.field_for(field, book_id, default_value=utcnow()) + return ret + return func + +def item_getter(field, default_value=None, key=0): + def func(dbref, book_id, cache): + try: + return cache[field] + except KeyError: + db = dbref() + ret = cache[field] = db.field_for(field, book_id, default_value=default_value) + try: + return ret[key] + except (IndexError, KeyError): + return default_value + return func + +def fmt_getter(field): + def func(dbref, book_id, cache): + try: + format_metadata = cache['format_metadata'] + except KeyError: + db = dbref() + format_metadata = {} + for fmt in db.formats(book_id, verify_formats=False): + m = db.format_metadata(book_id, fmt) + if m: + format_metadata[fmt] = m + if field == 'formats': + return list(format_metadata) or None + return format_metadata + return func + +def approx_fmts_getter(dbref, book_id, cache): + try: + return cache['formats'] + except KeyError: + db = dbref() + cache['formats'] = ret = list(db.field_for('formats', book_id)) + return ret + +def series_index_getter(field='series'): + def func(dbref, book_id, cache): + try: + series = getters[field](dbref, book_id, cache) + except KeyError: + series = custom_getter(field, dbref, book_id, cache) + if series: + try: + return cache[field + '_index'] + except KeyError: + db = dbref() + cache[field + '_index'] = ret = db.field_for(field + '_index', book_id, default_value=1.0) + return ret + return func + +def has_cover_getter(dbref, book_id, cache): + try: + return cache['has_cover'] + except KeyError: + db = dbref() + cache['has_cover'] = ret = _('Yes') if db.field_for('cover', book_id, default_value=False) else '' + return ret + +fmt_custom = lambda x:list(x) if isinstance(x, tuple) else x +def custom_getter(field, dbref, book_id, cache): + try: + return cache[field] + except KeyError: + db = dbref() + cache[field] = ret = fmt_custom(db.field_for(field, book_id)) + return ret + +def composite_getter(mi, field, metadata, book_id, cache, formatter, template_cache): + try: + return cache[field] + except KeyError: + ret = cache[field] = formatter.safe_format( + metadata['display']['composite_template'], + mi, + _('TEMPLATE ERROR'), + mi, column_name=field, + template_cache=template_cache).strip() + return ret + +getters = { + 'title':simple_getter('title', _('Unknown')), + 'title_sort':simple_getter('sort', _('Unknown')), + 'authors':pp_getter('authors', list, (_('Unknown'),)), + 'author_sort':simple_getter('author_sort', _('Unknown')), + 'uuid':simple_getter('uuid', 'dummy'), + 'book_size':simple_getter('size', 0), + 'ondevice_col':simple_getter('ondevice', ''), + 'languages':pp_getter('languages', list), + 'language':item_getter('languages', default_value=NULL_VALUES['language']), + 'db_approx_formats': approx_fmts_getter, + 'has_cover': has_cover_getter, + 'tags':pp_getter('tags', list, (_('Unknown'),)), + 'series_index':series_index_getter(), + 'application_id':lambda x, book_id, y: book_id, + 'id':lambda x, book_id, y: book_id, +} + +for field in ('comments', 'publisher', 'identifiers', 'series', 'rating'): + getters[field] = simple_getter(field) + +for field in ('author_sort_map', 'author_link_map'): + getters[field] = adata_getter(field) + +for field in ('timestamp', 'pubdate', 'last_modified'): + getters[field] = dt_getter(field) + +for field in TOP_LEVEL_IDENTIFIERS: + getters[field] = item_getter('identifiers', key=field) + +for field in ('formats', 'format_metadata'): + getters[field] = fmt_getter(field) +# }}} + +class ProxyMetadata(Metadata): + + def __init__(self, db, book_id): + sa(self, 'template_cache', db.formatter_template_cache) + sa(self, 'formatter', SafeFormat()) + sa(self, '_db', weakref.ref(db)) + sa(self, '_book_id', book_id) + sa(self, '_cache', {'user_categories':{}, 'cover_data':(None,None), 'device_collections':[]}) + sa(self, '_user_metadata', db.field_metadata) + + def __getattribute__(self, field): + getter = getters.get(field, None) + if getter is not None: + return getter(ga(self, '_db'), ga(self, '_book_id'), ga(self, '_cache')) + if field in SIMPLE_GET: + return ga(self, '_cache').get(field, None) + try: + return ga(self, field) + except AttributeError: + pass + um = ga(self, '_user_metadata') + d = um.get(field, None) + if d is not None: + dt = d['datatype'] + if dt != 'composite': + if field.endswith('_index') and dt == 'float': + return series_index_getter(field[:-6])(ga(self, '_db'), ga(self, '_book_id'), ga(self, '_cache')) + return custom_getter(field, ga(self, '_db'), ga(self, '_book_id'), ga(self, '_cache')) + return composite_getter(self, field, d, ga(self, '_book_id'), ga(self, '_cache'), ga(self, 'formatter'), ga(self, 'template_cache')) + + try: + return ga(self, '_cache')[field] + except KeyError: + raise AttributeError('Metadata object has no attribute named: %r' % field) + + def __setattr__(self, field, val, extra=None): + cache = ga(self, '_cache') + cache[field] = val + if extra is not None: + cache[field + '_index'] = val + + def get_user_metadata(self, field, make_copy=False): + um = ga(self, '_user_metadata') + try: + ans = um[field] + except KeyError: + pass + else: + if make_copy: + ans = deepcopy(ans) + return ans + + def get_extra(self, field, default=None): + um = ga(self, '_user_metadata') + if field + '_index' in um: + try: + return getattr(self, field + '_index') + except AttributeError: + return default + raise AttributeError( + 'Metadata object has no attribute named: '+ repr(field)) + + def custom_field_keys(self): + um = ga(self, '_user_metadata') + return iter(um.custom_field_keys()) + diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 28c499e150..521139b7b0 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -460,3 +460,29 @@ class ReadingTest(BaseTest): test(True, {3}, 'Unknown', 'unknown') # }}} + def test_proxy_metadata(self): # {{{ + ' Test the ProxyMetadata object used for composite columns ' + from calibre.ebooks.metadata.book.base import STANDARD_METADATA_FIELDS + cache = self.init_cache() + for book_id in cache.all_book_ids(): + mi = cache.get_metadata(book_id, get_user_categories=False) + pmi = cache.get_proxy_metadata(book_id) + self.assertSetEqual(set(mi.custom_field_keys()), set(pmi.custom_field_keys())) + + for field in STANDARD_METADATA_FIELDS | {'#series_index'}: + f = lambda x: x + if field == 'formats': + f = lambda x: x if x is None else set(x) + self.assertEqual(f(getattr(mi, field)), f(getattr(pmi, field)), + 'Standard field: %s not the same for book %s' % (field, book_id)) + self.assertEqual(mi.format_field(field), pmi.format_field(field), + 'Standard field format: %s not the same for book %s' % (field, book_id)) + for field, meta in cache.field_metadata.custom_iteritems(): + if meta['datatype'] != 'composite': + self.assertEqual(f(getattr(mi, field)), f(getattr(pmi, field)), + 'Custom field: %s not the same for book %s' % (field, book_id)) + self.assertEqual(mi.format_field(field), pmi.format_field(field), + 'Custom field format: %s not the same for book %s' % (field, book_id)) + + # }}} + From cbf2bb0c4e58ad787d1a638fc48cc43bb515ac0f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 11:35:48 +0530 Subject: [PATCH 0318/1154] Tentative {virtual_libraries} template support This should allow the use of {virtual_libraries} in custom column templates, with good performance, thanks to search caching. --- src/calibre/db/cache.py | 12 ++++++++++++ src/calibre/db/lazy.py | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index f909563fa5..c8f28a1967 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1620,6 +1620,18 @@ class Cache(object): if cover and os.path.exists(cover): self._set_field('cover', {book_id:1}) self.backend.restore_book(book_id, path, formats) + + @read_api + def virtual_libraries_for_books(self, book_ids): + libraries = tuple(self._pref('virtual_libraries', {}).iterkeys()) + ans = {book_id:[] for book_id in book_ids} + for lib in libraries: + books = self._search(lib) # We deliberately dont use book_ids as we want to use the search cache + for book in book_ids: + if book in books: + ans[book].append(lib) + return {k:tuple(sorted(v, key=sort_key)) for k, v in book_ids} + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index 4b1740154b..74cf5af94b 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -231,6 +231,15 @@ def composite_getter(mi, field, metadata, book_id, cache, formatter, template_ca template_cache=template_cache).strip() return ret +def virtual_libraries_getter(dbref, book_id, cache): + try: + return cache[field] + except KeyError: + db = dbref() + vls = db.virtual_libraries_for_books((book_id,))[book_id] + ret = cache[field] = ', '.join(vls) + return ret + getters = { 'title':simple_getter('title', _('Unknown')), 'title_sort':simple_getter('sort', _('Unknown')), @@ -247,6 +256,7 @@ getters = { 'series_index':series_index_getter(), 'application_id':lambda x, book_id, y: book_id, 'id':lambda x, book_id, y: book_id, + 'virtual_libraries':virtual_libraries_getter, } for field in ('comments', 'publisher', 'identifiers', 'series', 'rating'): From f3af580d5e90280fc8e89ef7db72433a714ce717 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 12:16:08 +0530 Subject: [PATCH 0319/1154] Update taz.de (RSS) --- recipes/taz_rss.recipe | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/recipes/taz_rss.recipe b/recipes/taz_rss.recipe index d4fc0237da..9f308d9739 100644 --- a/recipes/taz_rss.recipe +++ b/recipes/taz_rss.recipe @@ -1,4 +1,3 @@ - __license__ = 'GPL v3' __copyright__ = '2013, Alexander Schremmer , Robert Riemann ' @@ -43,15 +42,14 @@ class TazRSSRecipe(BasicNewsRecipe): # use the cover presented on the homepage cover_url = 'http://www.taz.de/digitaz/.s1jpeg320' - keep_only_tags = [dict(name='div', attrs={'class': 'sect sect_article'})] + no_stylesheets = True # default value is False, but True makes process much faster + keep_only_tags = [ + dict(name=['div'], attrs={'class': re.compile(r".*\bsect_article\b.*")}) + ] remove_tags = [ - dict(name=['div'], attrs={'class': 'artikelwerbung'}), - dict(name=['ul'], attrs={'class': 'toolbar'}), + dict(name=['div'], attrs={'class': 'sectfoot'}), # remove: taz paywall - dict(name=['div'], attrs={'id': 'tzi_paywall'}), - # remove: Artikel zum Thema (not working on Kindle) - dict(name=['div'], attrs={'class': re.compile(r".*\bsect_seealso\b.*")}), - dict(name=['div'], attrs={'class': 'sectfoot'}) + dict(name=['div'], attrs={'id': 'tzi_paywall'}) ] # with article pictures on Kindle super-slow From 7460446395faa44213923320b76ea43fe53fe345 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 12:31:00 +0530 Subject: [PATCH 0320/1154] Handle recursive templates in composite columns --- src/calibre/db/lazy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index 74cf5af94b..1e65a7cc1d 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -223,6 +223,7 @@ def composite_getter(mi, field, metadata, book_id, cache, formatter, template_ca try: return cache[field] except KeyError: + cache[field] = 'RECURSIVE_COMPOSITE FIELD (Metadata) ' + field ret = cache[field] = formatter.safe_format( metadata['display']['composite_template'], mi, From 821179170a390efdbbb1a199d95b1fe165c98b6b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 12:50:26 +0530 Subject: [PATCH 0321/1154] Add get_standard_metadata() to ProxyMetadata --- src/calibre/db/lazy.py | 8 ++++++++ src/calibre/db/tests/reading.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index 1e65a7cc1d..0b73b51ac0 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -342,3 +342,11 @@ class ProxyMetadata(Metadata): um = ga(self, '_user_metadata') return iter(um.custom_field_keys()) + def get_standard_metadata(self, field, make_copy=False): + field_metadata = ga(self, '_user_metadata') + if field in field_metadata and field_metadata[field]['kind'] == 'field': + if make_copy: + return deepcopy(field_metadata[field]) + return field_metadata[field] + return None + diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 521139b7b0..cfb9597550 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -477,6 +477,16 @@ class ReadingTest(BaseTest): 'Standard field: %s not the same for book %s' % (field, book_id)) self.assertEqual(mi.format_field(field), pmi.format_field(field), 'Standard field format: %s not the same for book %s' % (field, book_id)) + def f(x): + try: + return x['label'] + except: + return x + if field not in {'#series_index'}: + v = pmi.get_standard_metadata(field) + self.assertTrue(v is None or isinstance(v, dict)) + self.assertEqual(f(mi.get_standard_metadata(field, False)), f(v), + 'get_standard_metadata() failed for field %s' % field) for field, meta in cache.field_metadata.custom_iteritems(): if meta['datatype'] != 'composite': self.assertEqual(f(getattr(mi, field)), f(getattr(pmi, field)), From dec8572513de07cc389bf5cf3ec9559a8ed92e7b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 12:55:15 +0530 Subject: [PATCH 0322/1154] ... --- src/calibre/db/tests/reading.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index cfb9597550..c242206539 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -479,9 +479,10 @@ class ReadingTest(BaseTest): 'Standard field format: %s not the same for book %s' % (field, book_id)) def f(x): try: - return x['label'] + x.pop('rec_index', None) except: - return x + pass + return x if field not in {'#series_index'}: v = pmi.get_standard_metadata(field) self.assertTrue(v is None or isinstance(v, dict)) From c7f6bc01001f5a37fac763ae1386bdf2a8cefa5a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 13:11:14 +0530 Subject: [PATCH 0323/1154] oops --- src/calibre/db/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index c8f28a1967..39bc905c50 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1630,7 +1630,7 @@ class Cache(object): for book in book_ids: if book in books: ans[book].append(lib) - return {k:tuple(sorted(v, key=sort_key)) for k, v in book_ids} + return {k:tuple(sorted(v, key=sort_key)) for k, v in ans.iteritems()} # }}} From d0b1001ee792f72a85a70aea15c425de551e1bfb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 14:02:06 +0530 Subject: [PATCH 0324/1154] Use a cache for search query parsing --- src/calibre/db/search.py | 12 +++++++++--- src/calibre/utils/search_query_parser.py | 22 ++++++++++++++++------ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 07b3bde068..332842e6ae 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -455,7 +455,7 @@ class Parser(SearchQueryParser): def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, bool_search, keypair_search, limit_search_columns, limit_search_columns_to, - locations, virtual_fields, lookup_saved_search): + locations, virtual_fields, lookup_saved_search, parse_cache): self.dbcache, self.all_book_ids = dbcache, all_book_ids self.all_search_locations = frozenset(locations) self.grouped_search_terms = gst @@ -466,7 +466,7 @@ class Parser(SearchQueryParser): self.virtual_fields = virtual_fields or {} if 'marked' not in self.virtual_fields: self.virtual_fields['marked'] = self - super(Parser, self).__init__(locations, optimize=True, lookup_saved_search=lookup_saved_search) + SearchQueryParser.__init__(self, locations, optimize=True, lookup_saved_search=lookup_saved_search, parse_cache=parse_cache) @property def field_metadata(self): @@ -736,6 +736,7 @@ class LRUCache(object): self.item_map[key] = val self.age_map.append(key) + __setitem__ = add def get(self, key, default=None): ans = self.item_map.get(key, default) @@ -753,6 +754,9 @@ class LRUCache(object): def __len__(self): return len(self.age_map) + def __getitem__(self, key): + return self.get(key) + class Search(object): def __init__(self, db, opt_name, all_search_locations=()): @@ -763,6 +767,7 @@ class Search(object): self.keypair_search = KeyPairSearch() self.saved_searches = SavedSearchQueries(db, opt_name) self.cache = LRUCache() + self.parse_cache = LRUCache(limit=100) def get_saved_searches(self): return self.saved_searches @@ -770,6 +775,7 @@ class Search(object): def change_locations(self, newlocs): if frozenset(newlocs) != frozenset(self.all_search_locations): self.clear_caches() + self.parse_cache.clear() self.all_search_locations = newlocs def clear_caches(self): @@ -788,7 +794,7 @@ class Search(object): self.keypair_search, prefs['limit_search_columns'], prefs['limit_search_columns_to'], self.all_search_locations, - virtual_fields, self.saved_searches.lookup) + virtual_fields, self.saved_searches.lookup, self.parse_cache) try: return self._do_search(sqp, query, search_restriction, dbcache, book_ids=book_ids) finally: diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index 4406785bb1..12ac9b3931 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -300,25 +300,28 @@ class SearchQueryParser(object): failed.append(test[0]) return failed - def __init__(self, locations, test=False, optimize=False, lookup_saved_search=None): + def __init__(self, locations, test=False, optimize=False, lookup_saved_search=None, parse_cache=None): self.sqp_initialize(locations, test=test, optimize=optimize) self.parser = Parser() self.lookup_saved_search = global_lookup_saved_search if lookup_saved_search is None else lookup_saved_search + self.sqp_parse_cache = parse_cache def sqp_change_locations(self, locations): self.sqp_initialize(locations, optimize=self.optimize) + if self.sqp_parse_cache is not None: + self.sqp_parse_cache.clear() def sqp_initialize(self, locations, test=False, optimize=False): self.locations = locations self._tests_failed = False self.optimize = optimize - def parse(self, query): + def parse(self, query, candidates=None): # empty the list of searches used for recursion testing self.recurse_level = 0 self.searches_seen = set([]) candidates = self.universal_set() - return self._parse(query, candidates) + return self._parse(query, candidates=candidates) # this parse is used internally because it doesn't clear the # recursive search test list. However, we permit seeing the @@ -327,9 +330,16 @@ class SearchQueryParser(object): def _parse(self, query, candidates=None): self.recurse_level += 1 try: - res = self.parser.parse(query, self.locations) - except RuntimeError: - raise ParseException(_('Failed to parse query, recursion limit reached: %s')%repr(query)) + res = self.sqp_parse_cache.get(query, None) + except AttributeError: + res = None + if res is None: + try: + res = self.parser.parse(query, self.locations) + except RuntimeError: + raise ParseException(_('Failed to parse query, recursion limit reached: %s')%repr(query)) + if self.sqp_parse_cache is not None: + self.sqp_parse_cache[query] = res if candidates is None: candidates = self.universal_set() t = self.evaluate(res, candidates) From 415dc21588edc32f3ab5e7b781b2df12644dc8e5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 14:09:46 +0530 Subject: [PATCH 0325/1154] Fix handling of recursive templates in ProxyMetadata --- src/calibre/db/lazy.py | 7 ++++++- src/calibre/db/tests/reading.py | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index 0b73b51ac0..566e51b32a 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -12,7 +12,7 @@ from functools import wraps from collections import MutableMapping, MutableSequence from copy import deepcopy -from calibre.ebooks.metadata.book.base import Metadata, SIMPLE_GET, TOP_LEVEL_IDENTIFIERS, NULL_VALUES +from calibre.ebooks.metadata.book.base import Metadata, SIMPLE_GET, TOP_LEVEL_IDENTIFIERS, NULL_VALUES, ALL_METADATA_FIELDS from calibre.ebooks.metadata.book.formatter import SafeFormat from calibre.utils.date import utcnow @@ -350,3 +350,8 @@ class ProxyMetadata(Metadata): return field_metadata[field] return None + def all_field_keys(self): + um = ga(self, '_user_metadata') + return frozenset(ALL_METADATA_FIELDS.union(um.iterkeys())) + + diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index c242206539..dbeda9f6b0 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -495,5 +495,13 @@ class ReadingTest(BaseTest): self.assertEqual(mi.format_field(field), pmi.format_field(field), 'Custom field format: %s not the same for book %s' % (field, book_id)) + # Test handling of recursive templates + cache.create_custom_column('comp2', 'comp2', 'composite', False, display={'composite_template':'{title}'}) + cache.create_custom_column('comp1', 'comp1', 'composite', False, display={'composite_template':'foo{#comp2}'}) + cache.close() + cache = self.init_cache() + mi, pmi = cache.get_metadata(1), cache.get_proxy_metadata(1) + self.assertEqual(mi.get('#comp1'), pmi.get('#comp1')) + # }}} From 1cb2bd10de09b7bc90fffa91466ab6158e4de019 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 18:45:47 +0530 Subject: [PATCH 0326/1154] Update instead of invalidating search caches Updating is cheap when only a few books are affected. --- src/calibre/db/cache.py | 9 ++--- src/calibre/db/search.py | 64 ++++++++++++++++++++++++++++----- src/calibre/db/tests/reading.py | 6 ++++ 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 39bc905c50..23e4498e72 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -150,8 +150,8 @@ class Cache(object): field.clear_caches(book_ids=book_ids) @write_api - def clear_search_caches(self): - self._search_api.clear_caches() + def clear_search_caches(self, book_ids=None): + self._search_api.update_or_clear(self, book_ids) @write_api def clear_caches(self, book_ids=None, template_cache=True): @@ -165,7 +165,7 @@ class Cache(object): self.format_metadata_cache.pop(book_id, None) else: self.format_metadata_cache.clear() - self._clear_search_caches() + self._clear_search_caches(book_ids) @write_api def reload_from_db(self, clear_caches=True): @@ -828,7 +828,7 @@ class Cache(object): f.writer.set_books({book_id:now for book_id in book_ids}, self.backend) if self.composites: self._clear_composite_caches(book_ids) - self._clear_search_caches() + self._clear_search_caches(book_ids) @write_api def mark_as_dirty(self, book_ids): @@ -1407,6 +1407,7 @@ class Cache(object): @write_api def refresh_ondevice(self): self.fields['ondevice'].clear_caches() + self.clear_search_caches() @read_api def tags_older_than(self, tag, delta=None, must_have_tag=None, must_have_authors=None): diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 332842e6ae..3b441e6b2f 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -451,7 +451,7 @@ class SavedSearchQueries(object): # {{{ return sorted(self.queries.iterkeys(), key=sort_key) # }}} -class Parser(SearchQueryParser): +class Parser(SearchQueryParser): # {{{ def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, bool_search, keypair_search, limit_search_columns, limit_search_columns_to, @@ -711,8 +711,9 @@ class Parser(SearchQueryParser): if query == 'false': return candidates - matches return matches +# }}} -class LRUCache(object): +class LRUCache(object): # {{{ 'A simple Least-Recently-Used cache' @@ -748,6 +749,13 @@ class LRUCache(object): self.item_map.clear() self.age_map.clear() + def pop(self, key, default=None): + self.item_map.pop(key, default) + try: + self.age_map.remove(key) + except ValueError: + pass + def __contains__(self, key): return key in self.item_map @@ -757,8 +765,14 @@ class LRUCache(object): def __getitem__(self, key): return self.get(key) + def __iter__(self): + return self.item_map.iteritems() +# }}} + class Search(object): + MAX_CACHE_UPDATE = 50 + def __init__(self, db, opt_name, all_search_locations=()): self.all_search_locations = all_search_locations self.date_search = DateSearch() @@ -778,9 +792,47 @@ class Search(object): self.parse_cache.clear() self.all_search_locations = newlocs + def update_or_clear(self, dbcache, book_ids=None): + if book_ids and (len(book_ids) * len(self.cache)) <= self.MAX_CACHE_UPDATE: + self.update_caches(dbcache, book_ids) + else: + self.clear_caches() + def clear_caches(self): self.cache.clear() + def update_caches(self, dbcache, book_ids): + sqp = self.create_parser(dbcache) + try: + return self._update_caches(sqp, book_ids) + finally: + sqp.dbcache = sqp.lookup_saved_search = None + + def _update_caches(self, sqp, book_ids): + book_ids = sqp.all_book_ids = set(book_ids) + remove = set() + for query, result in self.cache: + try: + matches = sqp.parse(query) + except ParseException: + remove.add(query) + else: + # remove books that no longer match + result.difference_update(book_ids - matches) + # add books that now match but did not before + result.update(matches) + for query in remove: + self.cache.pop(query) + + def create_parser(self, dbcache, virtual_fields=None): + return Parser( + dbcache, set(), dbcache._pref('grouped_search_terms'), + self.date_search, self.num_search, self.bool_search, + self.keypair_search, + prefs['limit_search_columns'], + prefs['limit_search_columns_to'], self.all_search_locations, + virtual_fields, self.saved_searches.lookup, self.parse_cache) + def __call__(self, dbcache, query, search_restriction, virtual_fields=None, book_ids=None): ''' Return the set of ids of all records that match the specified @@ -788,13 +840,7 @@ class Search(object): ''' # We construct a new parser instance per search as the parse is not # thread safe. - sqp = Parser( - dbcache, set(), dbcache._pref('grouped_search_terms'), - self.date_search, self.num_search, self.bool_search, - self.keypair_search, - prefs['limit_search_columns'], - prefs['limit_search_columns_to'], self.all_search_locations, - virtual_fields, self.saved_searches.lookup, self.parse_cache) + sqp = self.create_parser(dbcache, virtual_fields) try: return self._do_search(sqp, query, search_restriction, dbcache, book_ids=book_ids) finally: diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index dbeda9f6b0..2006348862 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -448,6 +448,7 @@ class ReadingTest(BaseTest): test(False, {3}, 'Unknown') test(True, {3}, 'Unknown') test(True, {3}, 'Unknown') + cache._search_api.MAX_CACHE_UPDATE = 0 cache.set_field('title', {3:'xxx'}) test(False, {3}, 'Unknown') # cache cleared test(True, {3}, 'Unknown') @@ -458,6 +459,11 @@ class ReadingTest(BaseTest): test(False, {3}, '', 'unknown') test(True, {3}, '', 'unknown') test(True, {3}, 'Unknown', 'unknown') + cache._search_api.MAX_CACHE_UPDATE = 100 + test(False, {2, 3}, 'title:=xxx or title:"=Title One"') + cache.set_field('publisher', {3:'ppppp', 2:'other'}) + # Test cache update worked + test(True, {2, 3}, 'title:=xxx or title:"=Title One"') # }}} def test_proxy_metadata(self): # {{{ From 474abe15c24466ac14c8de8eae633ef2f50774a0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 18:47:23 +0530 Subject: [PATCH 0327/1154] Invalidate search caches when marked ids change --- src/calibre/db/view.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 2a779a5af0..602a2fbeaf 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -310,6 +310,7 @@ class View(object): a mapping is provided, then the search can be used to search for particular values: ``marked:value`` ''' + old_marked_ids = set(self.marked_ids) if not hasattr(id_dict, 'items'): # Simple list. Make it a dict of string 'true' self.marked_ids = dict.fromkeys(id_dict, u'true') @@ -317,6 +318,9 @@ class View(object): # Ensure that all the items in the dict are text self.marked_ids = dict(izip(id_dict.iterkeys(), imap(unicode, id_dict.itervalues()))) + # This invalidates all searches in the cache even though the cache may + # be shared by multiple views. This is not ideal, but... + self.cache.clear_search_caches(old_marked_ids | set(self.marked_ids)) def refresh(self, field=None, ascending=True): self._map = tuple(self.cache.all_book_ids()) From 114e5cf3a771c64421108ae65e839e6c6b65b5f0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 18:55:39 +0530 Subject: [PATCH 0328/1154] Ensure that removing books does not cause errors when updating search caches --- src/calibre/db/cache.py | 8 +++++--- src/calibre/db/search.py | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 23e4498e72..ef4c8854b4 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -154,7 +154,7 @@ class Cache(object): self._search_api.update_or_clear(self, book_ids) @write_api - def clear_caches(self, book_ids=None, template_cache=True): + def clear_caches(self, book_ids=None, template_cache=True, search_cache=True): if template_cache: self._initialize_template_cache() # Clear the formatter template cache for field in self.fields.itervalues(): @@ -165,7 +165,8 @@ class Cache(object): self.format_metadata_cache.pop(book_id, None) else: self.format_metadata_cache.clear() - self._clear_search_caches(book_ids) + if search_cache: + self._clear_search_caches(book_ids) @write_api def reload_from_db(self, clear_caches=True): @@ -1303,7 +1304,8 @@ class Cache(object): continue # Some fields like ondevice do not have tables else: table.remove_books(book_ids, self.backend) - self._clear_caches(book_ids=book_ids, template_cache=False) + self._search_api.discard_books(book_ids) + self._clear_caches(book_ids=book_ids, template_cache=False, search_cache=False) @read_api def author_sort_strings_for_books(self, book_ids): diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 3b441e6b2f..759b76fdb8 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -808,6 +808,11 @@ class Search(object): finally: sqp.dbcache = sqp.lookup_saved_search = None + def discard_books(self, book_ids): + book_ids = set(book_ids) + for query, result in self.cache: + result.difference_update(book_ids) + def _update_caches(self, sqp, book_ids): book_ids = sqp.all_book_ids = set(book_ids) remove = set() From 9a6073fcfb7110e714c1252034642ae51352e0b9 Mon Sep 17 00:00:00 2001 From: intromatyk Date: Wed, 24 Jul 2013 21:07:29 +0200 Subject: [PATCH 0329/1154] fixed antyweb recipe --- recipes/antyweb.recipe | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/recipes/antyweb.recipe b/recipes/antyweb.recipe index b7d3d2583c..d85ed4adcc 100644 --- a/recipes/antyweb.recipe +++ b/recipes/antyweb.recipe @@ -21,21 +21,9 @@ class AntywebRecipe(BasicNewsRecipe): simultaneous_downloads = 3 keep_only_tags =[] - keep_only_tags.append(dict(name = 'h1', attrs = { 'class' : 'mm-article-title'})) - keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'mm-article-content'})) - - - remove_tags =[] - remove_tags.append(dict(name = 'h2', attrs = {'class' : 'widgettitle'})) - remove_tags.append(dict(name = 'img', attrs = {'class' : 'alignleft'})) - remove_tags.append(dict(name = 'div', attrs = {'class' : 'float: right;margin-left:1em;margin-bottom: 0.5em;padding-bottom: 3px; width: 72px;'})) - remove_tags.append(dict(name = 'img', attrs = {'src' : 'http://antyweb.pl/wp-content/uploads/2011/09/HOSTERSI_testy_pasek600x30.gif'})) - remove_tags.append(dict(name = 'div', attrs = {'class' : 'podwpisowe'})) - - - extra_css = ''' - body {font-family: verdana, arial, helvetica, geneva, sans-serif ;} - ''' + keep_only_tags.append(dict(name = 'h1', attrs = { 'class' : 'entry-title '})) + keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'news-content'})) + extra_css = '''body {font-family: verdana, arial, helvetica, geneva, sans-serif ;}''' feeds = [ (u'Artykuly', u'feed://feeds.feedburner.com/Antyweb?format=xml'), From f555ab52792596815b73a044c19b8b51d42b4c6b Mon Sep 17 00:00:00 2001 From: intromatyk Date: Wed, 24 Jul 2013 22:32:54 +0200 Subject: [PATCH 0330/1154] update Dilbert recipe --- recipes/dilbert.recipe | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/recipes/dilbert.recipe b/recipes/dilbert.recipe index ed2771debf..d64f6e6882 100644 --- a/recipes/dilbert.recipe +++ b/recipes/dilbert.recipe @@ -16,7 +16,7 @@ class DilbertBig(BasicNewsRecipe): oldest_article = 15 max_articles_per_feed = 100 no_stylesheets = True - use_embedded_content = True + use_embedded_content = False encoding = 'utf-8' publisher = 'UNITED FEATURE SYNDICATE, INC.' category = 'comic' @@ -30,25 +30,13 @@ class DilbertBig(BasicNewsRecipe): ,'publisher' : publisher } - feeds = [(u'Dilbert', u'http://feed.dilbert.com/dilbert/daily_strip' )] - - def get_article_url(self, article): - return article.get('feedburner_origlink', None) + feeds = [(u'Dilbert', u'http://feed.dilbert.com/dilbert/daily_strip')] preprocess_regexps = [ (re.compile('strip\..*\.gif', re.DOTALL|re.IGNORECASE), lambda match: 'strip.zoom.gif') ] def preprocess_html(self, soup): - for tag in soup.findAll(name='a'): - if tag['href'].find('http://feedads') >= 0: - tag.extract() - return soup - - extra_css = ''' - h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;} - h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} - img {max-width:100%; min-width:100%;} - p{font-family:Arial,Helvetica,sans-serif;font-size:small;} - body{font-family:Helvetica,Arial,sans-serif;font-size:small;} - ''' + for tag in soup.findAll(name='input'): + image = BeautifulSoup('') + return image From ced9a68a3090f9f8d5c9d281c3cfed2640a6a226 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 07:42:14 +0530 Subject: [PATCH 0331/1154] Update Scientific American --- recipes/scientific_american.recipe | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/recipes/scientific_american.recipe b/recipes/scientific_american.recipe index 2ea865517e..08f0a3b2b7 100644 --- a/recipes/scientific_american.recipe +++ b/recipes/scientific_american.recipe @@ -30,11 +30,13 @@ class ScientificAmerican(BasicNewsRecipe): ,dict(name='p', attrs={'id':'articleDek'}) ,dict(name='p', attrs={'class':'articleInfo'}) ,dict(name='div', attrs={'id':['articleContent']}) - ,dict(name='img', attrs={'src':re.compile(r'/media/inline/blog/Image/', re.DOTALL|re.IGNORECASE)}) + ,dict(name='img', attrs={'src':re.compile(r'/media/inline/blog/Image/', re.DOTALL|re.IGNORECASE)}) ] - remove_tags = [dict(name='a', attrs={'class':'tinyCommentCount'})] - + remove_tags = [dict(name='a', attrs={'class':'tinyCommentCount'}) + ,dict(name='div', attrs={'id':'bigCoverModule'}) + ,dict(name='div', attrs={'class':'addInfo'}) + ] def parse_index(self): soup = self.index_to_soup('http://www.scientificamerican.com/sciammag/') issuetag = soup.find('p',attrs={'id':'articleDek'}) @@ -43,8 +45,9 @@ class ScientificAmerican(BasicNewsRecipe): if img is not None: self.cover_url = img['src'] features, feeds = [], [] - for a in soup.find(attrs={'class':'primaryCol'}).findAll('a',attrs={'title':'Feature'}): - if a is None: continue + for a in soup.find(attrs={'class':'doubleWide'}).find(attrs={'class':'primaryCol'}).findAll('a',attrs={'title':'Feature'}): + if a is None: + continue desc = '' s = a.parent.parent.find(attrs={'class':'dek'}) desc = self.tag_to_string(s) @@ -89,3 +92,4 @@ class ScientificAmerican(BasicNewsRecipe): h2{font-size:large; font-family:Arial,Helvetica,sans-serif;} h3{font-size:x-small;font-family:Arial,Helvetica,sans-serif;} ''' + From be4dbafc9d9fde1e5939bd9c37bb6c3c3d786608 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 08:48:52 +0530 Subject: [PATCH 0332/1154] Viewer: Refactor the bookmarks manager to make it a little prettier --- src/calibre/gui2/viewer/bookmarkmanager.py | 59 +++++------ src/calibre/gui2/viewer/bookmarkmanager.ui | 108 ++++++++++++--------- 2 files changed, 93 insertions(+), 74 deletions(-) diff --git a/src/calibre/gui2/viewer/bookmarkmanager.py b/src/calibre/gui2/viewer/bookmarkmanager.py index c3686bd81e..073087f1e3 100644 --- a/src/calibre/gui2/viewer/bookmarkmanager.py +++ b/src/calibre/gui2/viewer/bookmarkmanager.py @@ -5,8 +5,8 @@ __copyright__ = '2009, John Schember ' import cPickle, os -from PyQt4.Qt import Qt, QDialog, QAbstractTableModel, QVariant, SIGNAL, \ - QModelIndex, QInputDialog, QLineEdit, QFileDialog +from PyQt4.Qt import (Qt, QDialog, QAbstractListModel, QVariant, + QModelIndex, QInputDialog, QLineEdit, QFileDialog, QItemSelectionModel) from calibre.gui2.viewer.bookmarkmanager_ui import Ui_BookmarkManager from calibre.gui2 import NONE @@ -20,28 +20,32 @@ class BookmarkManager(QDialog, Ui_BookmarkManager): self.bookmarks = bookmarks[:] self.set_bookmarks() - self.connect(self.button_revert, SIGNAL('clicked()'), self.set_bookmarks) - self.connect(self.button_delete, SIGNAL('clicked()'), self.delete_bookmark) - self.connect(self.button_edit, SIGNAL('clicked()'), self.edit_bookmark) - self.connect(self.button_export, SIGNAL('clicked()'), self.export_bookmarks) - self.connect(self.button_import, SIGNAL('clicked()'), self.import_bookmarks) + self.button_revert.clicked.connect(lambda :self.set_bookmarks()) + self.button_delete.clicked.connect(self.delete_bookmark) + self.button_edit.clicked.connect(self.edit_bookmark) + self.button_export.clicked.connect(self.export_bookmarks) + self.button_import.clicked.connect(self.import_bookmarks) + self.bookmarks_list.setStyleSheet('QListView::item { padding: 0.5ex }') + self.resize(600, 500) def set_bookmarks(self, bookmarks=None): - if bookmarks == None: + if bookmarks is None: bookmarks = self.bookmarks[:] - self._model = BookmarkTableModel(self, bookmarks) - self.bookmarks_table.setModel(self._model) - self.bookmarks_table.resizeColumnsToContents() + self._model = BookmarkListModel(self, bookmarks) + self.bookmarks_list.setModel(self._model) + if self._model.rowCount(QModelIndex()) > 0: + self.bookmarks_list.selectionModel().select(self._model.index(0), QItemSelectionModel.SelectCurrent) def delete_bookmark(self): - indexes = self.bookmarks_table.selectionModel().selectedIndexes() - if indexes != []: + indexes = list(self.bookmarks_list.selectionModel().selectedIndexes()) + if indexes: self._model.remove_row(indexes[0].row()) def edit_bookmark(self): - indexes = self.bookmarks_table.selectionModel().selectedIndexes() - if indexes != []: - title, ok = QInputDialog.getText(self, _('Edit bookmark'), _('New title for bookmark:'), QLineEdit.Normal, self._model.data(indexes[0], Qt.DisplayRole).toString()) + indexes = list(self.bookmarks_list.selectionModel().selectedIndexes()) + if indexes: + title, ok = QInputDialog.getText(self, _('Edit bookmark'), _( + 'New title for bookmark:'), QLineEdit.Normal, self._model.data(indexes[0], Qt.DisplayRole).toString()) title = QVariant(unicode(title).strip()) if ok and title: self._model.setData(indexes[0], title, Qt.EditRole) @@ -68,7 +72,7 @@ class BookmarkManager(QDialog, Ui_BookmarkManager): with open(filename, 'r') as fileobj: imported = cPickle.load(fileobj) - if imported != None: + if imported is not None: bad = False try: for bm in imported: @@ -86,11 +90,10 @@ class BookmarkManager(QDialog, Ui_BookmarkManager): self.set_bookmarks(bookmarks) -class BookmarkTableModel(QAbstractTableModel): - headers = [_("Name")] +class BookmarkListModel(QAbstractListModel): def __init__(self, parent, bookmarks): - QAbstractTableModel.__init__(self, parent) + QAbstractListModel.__init__(self, parent) self.bookmarks = bookmarks[:] @@ -99,11 +102,6 @@ class BookmarkTableModel(QAbstractTableModel): return 0 return len(self.bookmarks) - def columnCount(self, parent): - if parent and parent.isValid(): - return 0 - return len(self.headers) - def data(self, index, role): if role in (Qt.DisplayRole, Qt.EditRole): ans = self.bookmarks[index.row()]['title'] @@ -114,12 +112,12 @@ class BookmarkTableModel(QAbstractTableModel): if role == Qt.EditRole: bm = self.bookmarks[index.row()] bm['title'] = unicode(value.toString()).strip() - self.emit(SIGNAL("dataChanged(QModelIndex, QModelIndex)"), index, index) + self.dataChanged.emit(index, index) return True return False def flags(self, index): - flags = QAbstractTableModel.flags(self, index) + flags = QAbstractListModel.flags(self, index) flags |= Qt.ItemIsEditable return flags @@ -136,3 +134,10 @@ class BookmarkTableModel(QAbstractTableModel): del self.bookmarks[row] self.endRemoveRows() +if __name__ == '__main__': + from PyQt4.Qt import QApplication + app = QApplication([]) + d = BookmarkManager(None, [{'title':'Bookmark #%d' % i} for i in range(1, 50)]) + d.exec_() + + diff --git a/src/calibre/gui2/viewer/bookmarkmanager.ui b/src/calibre/gui2/viewer/bookmarkmanager.ui index 110b5db841..44c746038c 100644 --- a/src/calibre/gui2/viewer/bookmarkmanager.ui +++ b/src/calibre/gui2/viewer/bookmarkmanager.ui @@ -1,7 +1,8 @@ - + + BookmarkManager - - + + 0 0 @@ -9,80 +10,93 @@ 363 - + Bookmark Manager - - - - + + + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + + + Actions - + - - + + Edit + + + :/images/edit_input.png:/images/edit_input.png + - - + + Delete + + + :/images/trash.png:/images/trash.png + - - + + Reset - - - - - - Export + + + :/images/edit-undo.png:/images/edit-undo.png - - + + + Export + + + + :/images/back.png:/images/back.png + + + + + + Import + + + :/images/forward.png:/images/forward.png + - - - - false - - + + + true - - QAbstractItemView::SingleSelection - - - false - - - - - - - QDialogButtonBox::Cancel|QDialogButtonBox::Ok - - + + + buttonBox @@ -90,11 +104,11 @@ BookmarkManager accept() - + 225 337 - + 225 181 @@ -106,11 +120,11 @@ BookmarkManager reject() - + 225 337 - + 225 181 From ce5821eec85972d55192966b9911f16b660b6341 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 10:29:25 +0530 Subject: [PATCH 0333/1154] Complete rewrite of viewer bookmarks manager Ebook-viewer: Allow re-ordering bookmarks in the bookmarks manager by drag and drop. --- src/calibre/gui2/viewer/bookmarkmanager.py | 127 +++++++++------------ src/calibre/gui2/viewer/bookmarkmanager.ui | 25 ++-- 2 files changed, 71 insertions(+), 81 deletions(-) diff --git a/src/calibre/gui2/viewer/bookmarkmanager.py b/src/calibre/gui2/viewer/bookmarkmanager.py index 073087f1e3..acd00c3a6f 100644 --- a/src/calibre/gui2/viewer/bookmarkmanager.py +++ b/src/calibre/gui2/viewer/bookmarkmanager.py @@ -1,15 +1,17 @@ -from __future__ import with_statement +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) -__license__ = 'GPL v3' -__copyright__ = '2009, John Schember ' +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' import cPickle, os -from PyQt4.Qt import (Qt, QDialog, QAbstractListModel, QVariant, - QModelIndex, QInputDialog, QLineEdit, QFileDialog, QItemSelectionModel) +from PyQt4.Qt import ( + Qt, QDialog, QListWidgetItem, QFileDialog, QItemSelectionModel) from calibre.gui2.viewer.bookmarkmanager_ui import Ui_BookmarkManager -from calibre.gui2 import NONE class BookmarkManager(QDialog, Ui_BookmarkManager): def __init__(self, parent, bookmarks): @@ -17,7 +19,7 @@ class BookmarkManager(QDialog, Ui_BookmarkManager): self.setupUi(self) - self.bookmarks = bookmarks[:] + self.original_bookmarks = bookmarks self.set_bookmarks() self.button_revert.clicked.connect(lambda :self.set_bookmarks()) @@ -26,46 +28,68 @@ class BookmarkManager(QDialog, Ui_BookmarkManager): self.button_export.clicked.connect(self.export_bookmarks) self.button_import.clicked.connect(self.import_bookmarks) self.bookmarks_list.setStyleSheet('QListView::item { padding: 0.5ex }') + self.bookmarks_list.viewport().setAcceptDrops(True) + self.bookmarks_list.setDropIndicatorShown(True) + self.bookmarks_list.itemChanged.connect(self.item_changed) self.resize(600, 500) + self.bookmarks_list.setFocus(Qt.OtherFocusReason) def set_bookmarks(self, bookmarks=None): if bookmarks is None: - bookmarks = self.bookmarks[:] - self._model = BookmarkListModel(self, bookmarks) - self.bookmarks_list.setModel(self._model) - if self._model.rowCount(QModelIndex()) > 0: - self.bookmarks_list.selectionModel().select(self._model.index(0), QItemSelectionModel.SelectCurrent) + bookmarks = self.original_bookmarks + self.bookmarks_list.clear() + for bm in bookmarks: + i = QListWidgetItem(bm['title']) + i.setData(Qt.UserRole, self.bm_to_item(bm)) + i.setFlags(i.flags() | Qt.ItemIsEditable) + self.bookmarks_list.addItem(i) + if len(bookmarks) > 0: + self.bookmarks_list.setCurrentItem(self.bookmarks_list.item(0), QItemSelectionModel.ClearAndSelect) + + def item_changed(self, item): + self.bookmarks_list.blockSignals(True) + title = unicode(item.data(Qt.DisplayRole).toString()) + if not title: + title = _('Unknown') + item.setData(Qt.DisplayRole, title) + bm = self.item_to_bm(item) + bm['title'] = title + item.setData(Qt.UserRole, self.bm_to_item(bm)) + self.bookmarks_list.blockSignals(False) def delete_bookmark(self): - indexes = list(self.bookmarks_list.selectionModel().selectedIndexes()) - if indexes: - self._model.remove_row(indexes[0].row()) + row = self.bookmarks_list.currentRow() + if row > -1: + self.bookmarks_list.takeItem(row) def edit_bookmark(self): - indexes = list(self.bookmarks_list.selectionModel().selectedIndexes()) - if indexes: - title, ok = QInputDialog.getText(self, _('Edit bookmark'), _( - 'New title for bookmark:'), QLineEdit.Normal, self._model.data(indexes[0], Qt.DisplayRole).toString()) - title = QVariant(unicode(title).strip()) - if ok and title: - self._model.setData(indexes[0], title, Qt.EditRole) + item = self.bookmarks_list.currentItem() + if item is not None: + self.bookmarks_list.editItem(item) + + def bm_to_item(self, bm): + return bytearray(cPickle.dumps(bm, -1)) + + def item_to_bm(self, item): + return cPickle.loads(bytes(item.data(Qt.UserRole).toPyObject())) def get_bookmarks(self): - return self._model.bookmarks + l = self.bookmarks_list + return [self.item_to_bm(l.item(i)) for i in xrange(l.count())] def export_bookmarks(self): filename = QFileDialog.getSaveFileName(self, _("Export Bookmarks"), '%s%suntitled.pickle' % (os.getcwdu(), os.sep), _("Saved Bookmarks (*.pickle)")) - if filename == '': + if not filename: return with open(filename, 'w') as fileobj: - cPickle.dump(self._model.bookmarks, fileobj) + cPickle.dump(self.get_bookmarks(), fileobj) def import_bookmarks(self): filename = QFileDialog.getOpenFileName(self, _("Import Bookmarks"), '%s' % os.getcwdu(), _("Pickled Bookmarks (*.pickle)")) - if filename == '': + if not filename: return imported = None @@ -83,61 +107,18 @@ class BookmarkManager(QDialog, Ui_BookmarkManager): pass if not bad: - bookmarks = self._model.bookmarks[:] + bookmarks = self.get_bookmarks() for bm in imported: if bm not in bookmarks and bm['title'] != 'calibre_current_page_bookmark': bookmarks.append(bm) self.set_bookmarks(bookmarks) - -class BookmarkListModel(QAbstractListModel): - - def __init__(self, parent, bookmarks): - QAbstractListModel.__init__(self, parent) - - self.bookmarks = bookmarks[:] - - def rowCount(self, parent): - if parent and parent.isValid(): - return 0 - return len(self.bookmarks) - - def data(self, index, role): - if role in (Qt.DisplayRole, Qt.EditRole): - ans = self.bookmarks[index.row()]['title'] - return NONE if ans is None else QVariant(ans) - return NONE - - def setData(self, index, value, role): - if role == Qt.EditRole: - bm = self.bookmarks[index.row()] - bm['title'] = unicode(value.toString()).strip() - self.dataChanged.emit(index, index) - return True - return False - - def flags(self, index): - flags = QAbstractListModel.flags(self, index) - flags |= Qt.ItemIsEditable - return flags - - def headerData(self, section, orientation, role): - if role != Qt.DisplayRole: - return NONE - if orientation == Qt.Horizontal: - return QVariant(self.headers[section]) - else: - return QVariant(section+1) - - def remove_row(self, row): - self.beginRemoveRows(QModelIndex(), row, row) - del self.bookmarks[row] - self.endRemoveRows() - if __name__ == '__main__': from PyQt4.Qt import QApplication app = QApplication([]) - d = BookmarkManager(None, [{'title':'Bookmark #%d' % i} for i in range(1, 50)]) + d = BookmarkManager(None, [{'title':'Bookmark #%d' % i, 'data':b'xxxxx'} for i in range(1, 5)]) d.exec_() + import pprint + pprint.pprint(d.get_bookmarks()) diff --git a/src/calibre/gui2/viewer/bookmarkmanager.ui b/src/calibre/gui2/viewer/bookmarkmanager.ui index 44c746038c..5d7f6a67d7 100644 --- a/src/calibre/gui2/viewer/bookmarkmanager.ui +++ b/src/calibre/gui2/viewer/bookmarkmanager.ui @@ -14,13 +14,6 @@ Bookmark Manager - - - - QDialogButtonBox::Cancel|QDialogButtonBox::Ok - - - @@ -85,8 +78,24 @@ + + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + - + + + true + + + QAbstractItemView::InternalMove + + + Qt::MoveAction + true From 2b8dc4505d70e5a5cf9d859212790e254adc46ad Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 10:50:39 +0530 Subject: [PATCH 0334/1154] PDF Output: Do not error out when subsetting fails PDF Output: Do not error out when the input document uses a font that cannot be subset, such as the Symbol font. Instead print a warning and embed the full font. Fixes #1203449 [EPUB to PDF conversion fails](https://bugs.launchpad.net/calibre/+bug/1203449) --- src/calibre/ebooks/pdf/render/fonts.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/pdf/render/fonts.py b/src/calibre/ebooks/pdf/render/fonts.py index 9a1167021c..ea8fd03c10 100644 --- a/src/calibre/ebooks/pdf/render/fonts.py +++ b/src/calibre/ebooks/pdf/render/fonts.py @@ -16,7 +16,7 @@ from future_builtins import map from calibre import as_unicode from calibre.ebooks.pdf.render.common import (Array, String, Stream, Dictionary, Name) -from calibre.utils.fonts.sfnt.subset import pdf_subset, UnsupportedFont +from calibre.utils.fonts.sfnt.subset import pdf_subset, UnsupportedFont, NoGlyphs STANDARD_FONTS = { 'Times-Roman', 'Helvetica', 'Courier', 'Symbol', 'Times-Bold', @@ -84,7 +84,6 @@ class CMap(Stream): end ''') - def __init__(self, name, glyph_map, compress=False): Stream.__init__(self, compress) current_map = OrderedDict() @@ -118,7 +117,7 @@ class Font(object): self.font_descriptor = Dictionary({ 'Type': Name('FontDescriptor'), 'FontName': Name('%s+%s'%(self.subset_tag, metrics.postscript_name)), - 'Flags': 0b100, # Symbolic font + 'Flags': 0b100, # Symbolic font 'FontBBox': Array(metrics.pdf_bbox), 'ItalicAngle': metrics.post.italic_angle, 'Ascent': metrics.pdf_ascent, @@ -161,6 +160,11 @@ class Font(object): except UnsupportedFont as e: debug('Subsetting of %s not supported, embedding full font. Error: %s'%( self.metrics.names.get('full_name', 'Unknown'), as_unicode(e))) + except NoGlyphs: + if self.used_glyphs: + debug( + 'Subsetting of %s failed, font appears to have no glyphs for the %d characters it is used with, some text may not be rendered in the PDF' % + (self.metrics.names.get('full_name', 'Unknown'), len(self.used_glyphs))) if self.is_otf: self.font_stream.write(self.metrics.sfnt['CFF '].raw) else: @@ -184,7 +188,7 @@ class Font(object): widths = {g:w for g, w in widths.iteritems() if w != most_common} groups = Array() - for k, g in groupby(enumerate(widths.iterkeys()), lambda (i,x):i-x): + for k, g in groupby(enumerate(widths.iterkeys()), lambda i_x:i_x[0]-i_x[1]): group = list(map(itemgetter(1), g)) gwidths = [widths[g] for g in group] if len(set(gwidths)) == 1 and len(group) > 1: @@ -230,3 +234,4 @@ class FontManager(object): for font in self.fonts: font.embed(self.objects, debug) + From c73123fceea166c7ff1a0d096ad7d576abb4820c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 11:26:38 +0530 Subject: [PATCH 0335/1154] Fix deleting books in newdb --- src/calibre/db/legacy.py | 3 ++- src/calibre/db/view.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index efb71223a2..b2ac69c13a 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -320,8 +320,9 @@ class LibraryDatabase(object): def delete_book(self, book_id, notify=True, commit=True, permanent=False, do_clean=True): self.new_api.remove_books((book_id,), permanent=permanent) + self.data.books_deleted((book_id,)) if notify: - self.notify('delete', [id]) + self.notify('delete', [book_id]) def dirtied(self, book_ids, commit=True): self.new_api.mark_as_dirty(frozenset(book_ids) if book_ids is not None else book_ids) diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 602a2fbeaf..8c99cf41cf 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -339,3 +339,17 @@ class View(object): pass return None + def remove(self, book_id): + try: + self._map = tuple(bid for bid in self._map if bid != book_id) + except ValueError: + pass + try: + self._map_filtered = tuple(bid for bid in self._map_filtered if bid != book_id) + except ValueError: + pass + + def books_deleted(self, ids): + for book_id in ids: + self.remove(book_id) + From 179c28e32850c5be50d5853394f8b18b063d0aae Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 11:45:40 +0530 Subject: [PATCH 0336/1154] Fix adding of books in newdb --- src/calibre/db/legacy.py | 14 +++++++++++--- src/calibre/db/view.py | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index b2ac69c13a..f85340aacd 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -200,7 +200,9 @@ class LibraryDatabase(object): # Adding books {{{ def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None): - return self.new_api.create_book_entry(mi, cover=cover, add_duplicates=add_duplicates, force_id=force_id) + ret = self.new_api.create_book_entry(mi, cover=cover, add_duplicates=add_duplicates, force_id=force_id) + self.data.books_added((ret,)) + return ret def add_books(self, paths, formats, metadata, add_duplicates=True, return_ids=False): books = [(mi, {fmt:path}) for mi, path, fmt in zip(metadata, paths, formats)] @@ -214,6 +216,7 @@ class LibraryDatabase(object): paths.append(path) duplicates = (paths, formats, metadata) ids = book_ids if return_ids else len(book_ids) + self.data.books_added(book_ids) return duplicates or None, ids def import_book(self, mi, formats, notify=True, import_hooks=True, apply_import_tags=True, preserve_uuid=False): @@ -225,6 +228,7 @@ class LibraryDatabase(object): format_map[ext] = path book_ids, duplicates = self.new_api.add_books( [(mi, format_map)], add_duplicates=True, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid, dbapi=self, run_hooks=import_hooks) + self.data.books_added(book_ids) if notify: self.notify('add', book_ids) return book_ids[0] @@ -244,10 +248,14 @@ class LibraryDatabase(object): return recursive_import(self, root, single_book_per_directory=single_book_per_directory, callback=callback, added_ids=added_ids) def add_catalog(self, path, title): - return add_catalog(self.new_api, path, title) + book_id = add_catalog(self.new_api, path, title) + self.data.books_added((book_id,)) + return book_id def add_news(self, path, arg): - return add_news(self.new_api, path, arg) + book_id = add_news(self.new_api, path, arg) + self.data.books_added((book_id,)) + return book_id def add_format(self, index, fmt, stream, index_is_id=False, path=None, notify=True, replace=True, copy_function=None): ''' path and copy_function are ignored by the new API ''' diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 8c99cf41cf..48d53e06a8 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -353,3 +353,8 @@ class View(object): for book_id in ids: self.remove(book_id) + def books_added(self, ids): + ids = tuple(ids) + self._map = ids + self._map + self._map_filtered = ids + self._map_filtered + From 9ae9cf219ecd17d4552e25fcb4c466c5f35778ec Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 12:12:46 +0530 Subject: [PATCH 0337/1154] newdb: Fix reloading of content server on metadata.db change --- src/calibre/db/__init__.py | 2 +- src/calibre/db/legacy.py | 1 + src/calibre/db/view.py | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index bb0f679fd8..ceb6485efc 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -137,7 +137,7 @@ def get_db_loader(): ''' Various things that require other things before they can be migrated: - 1. Check that content server reloading on metadata,db change, metadata + 1. Check that metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) ''' diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index f85340aacd..3da725fe7e 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -120,6 +120,7 @@ class LibraryDatabase(object): if self.last_modified() > self.last_update_check: self.backend.reopen() self.new_api.reload_from_db() + self.data.refresh(clear_caches=False) # caches are already cleared by reload_from_db() self.last_update_check = utcnow() @property diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 48d53e06a8..f679fc2416 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -322,10 +322,11 @@ class View(object): # be shared by multiple views. This is not ideal, but... self.cache.clear_search_caches(old_marked_ids | set(self.marked_ids)) - def refresh(self, field=None, ascending=True): + def refresh(self, field=None, ascending=True, clear_caches=True): self._map = tuple(self.cache.all_book_ids()) self._map_filtered = tuple(self._map) - self.cache.clear_caches() + if clear_caches: + self.cache.clear_caches() if field is not None: self.sort(field, ascending) if self.search_restriction or self.base_restriction: From 6975543207a0b8eae74fa74428815e34061f6fe0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 12:39:03 +0530 Subject: [PATCH 0338/1154] newdb: calibredb add calibre file.ebook tested --- src/calibre/db/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index ceb6485efc..d501883d30 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -137,7 +137,6 @@ def get_db_loader(): ''' Various things that require other things before they can be migrated: - 1. Check that metadata - backup, refresh gui on calibredb add and moving libraries all work (check + 1. Check that metadata backup, and moving libraries all work (check them on windows as well for file locking issues) ''' From 6a488a332033d17bfbf31f51c2237f96a0fcbc40 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Thu, 25 Jul 2013 10:50:58 +0200 Subject: [PATCH 0339/1154] First stage: adding a ProxyMetadata object to get_metadata --- src/calibre/db/cache.py | 3 +++ src/calibre/db/lazy.py | 5 +++-- src/calibre/ebooks/metadata/book/base.py | 6 ++++-- src/calibre/utils/formatter_functions.py | 22 +++++++++++++++++++++- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index ef4c8854b4..1c224bce7e 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -184,6 +184,9 @@ class Cache(object): def _get_metadata(self, book_id, get_user_categories=True): # {{{ mi = Metadata(None, template_cache=self.formatter_template_cache) + + mi._proxy_metadata = ProxyMetadata(self, book_id, formatter=mi.formatter) + author_ids = self._field_ids_for('authors', book_id) adata = self._author_data(author_ids) aut_list = [adata[i] for i in author_ids] diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index 566e51b32a..303d47b697 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -278,9 +278,10 @@ for field in ('formats', 'format_metadata'): class ProxyMetadata(Metadata): - def __init__(self, db, book_id): + def __init__(self, db, book_id, formatter=None): sa(self, 'template_cache', db.formatter_template_cache) - sa(self, 'formatter', SafeFormat()) + if formatter is None: + sa(self, 'formatter', SafeFormat()) sa(self, '_db', weakref.ref(db)) sa(self, '_book_id', book_id) sa(self, '_cache', {'user_categories':{}, 'cover_data':(None,None), 'device_collections':[]}) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 4427121f37..93253b1dcc 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -66,7 +66,8 @@ class Metadata(object): becomes a reserved field name. ''' - def __init__(self, title, authors=(_('Unknown'),), other=None, template_cache=None): + def __init__(self, title, authors=(_('Unknown'),), other=None, template_cache=None, + formatter=None): ''' @param title: title or ``_('Unknown')`` @param authors: List of strings or [] @@ -85,7 +86,8 @@ class Metadata(object): self.author = list(authors) if authors else [] # Needed for backward compatibility self.authors = list(authors) if authors else [] from calibre.ebooks.metadata.book.formatter import SafeFormat - self.formatter = SafeFormat() + if formatter is None: + self.formatter = SafeFormat() self.template_cache = template_cache def is_null(self, field): diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index f2b973a1e7..606fe3b11b 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -1246,6 +1246,26 @@ class BuiltinFinishFormatting(BuiltinFormatterFunction): return val return prefix + formatter._do_format(val, fmt) + suffix +class BuiltinVirtualLibraries(BuiltinFormatterFunction): + name = 'virtual_libraries' + arg_count = 0 + category = 'Get values from metadata' + __doc__ = doc = _('virtual_libraries() -- return a comma-separated list of ' + 'virtual libraries that contain this book. This function ' + 'works only in the GUI. If you want to use these values ' + 'in save-to-disk or send-to-device templates then you ' + 'must make a custom "Column built from other columns", use ' + 'the function in that column\'s template, and use that ' + 'column\'s value in your save/send templates') + + def evaluate(self, formatter, kwargs, mi, locals_): + from calibre.db.lazy import ProxyMetadata + if isinstance(mi, ProxyMetadata): + return mi.virtual_libraries + if hasattr(mi, '_proxy_metadata'): + return mi._proxy_metadata.virtual_libraries + return _('This function can be used only in the GUI') + _formatter_builtins = [ BuiltinAdd(), BuiltinAnd(), BuiltinApproximateFormats(), BuiltinAssign(), BuiltinBooksize(), @@ -1267,7 +1287,7 @@ _formatter_builtins = [ BuiltinStrcmp(), BuiltinStrInList(), BuiltinStrlen(), BuiltinSubitems(), BuiltinSublist(),BuiltinSubstr(), BuiltinSubtract(), BuiltinSwapAroundComma(), BuiltinSwitch(), BuiltinTemplate(), BuiltinTest(), BuiltinTitlecase(), - BuiltinToday(), BuiltinUppercase(), + BuiltinToday(), BuiltinUppercase(), BuiltinVirtualLibraries() ] class FormatterUserFunction(FormatterFunction): From afbfa22fe298390cc2a478e9ac2cee997a933226 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 15:01:42 +0530 Subject: [PATCH 0340/1154] newdb: basic testing complete --- src/calibre/db/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index d501883d30..78f05da8ac 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -135,8 +135,4 @@ def get_db_loader(): errs = (sqlite.Error, DatabaseException) return cls, errs -''' -Various things that require other things before they can be migrated: - 1. Check that metadata backup, and moving libraries all work (check - them on windows as well for file locking issues) -''' + From 8467baea332a099a69ee6e3147da7c68be48833b Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Thu, 25 Jul 2013 11:36:34 +0200 Subject: [PATCH 0341/1154] Use ProxyMetadata in formatter functions for virt libs, ondevice, book_size, and approximate_formats. This saves the assignment in get_metadata() --- src/calibre/db/cache.py | 7 ++-- src/calibre/db/lazy.py | 4 +- src/calibre/gui2/library/models.py | 2 +- src/calibre/utils/formatter_functions.py | 53 ++++++++++++++++-------- 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 1c224bce7e..0f187f0400 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -214,8 +214,8 @@ class Cache(object): default_value='dummy') mi.title_sort = self._field_for('sort', book_id, default_value=_('Unknown')) - mi.book_size = self._field_for('size', book_id, default_value=0) - mi.ondevice_col = self._field_for('ondevice', book_id, default_value='') +# mi.book_size = self._field_for('size', book_id, default_value=0) +# mi.ondevice_col = self._field_for('ondevice', book_id, default_value='') mi.last_modified = self._field_for('last_modified', book_id, default_value=n) formats = self._field_for('formats', book_id) @@ -227,7 +227,7 @@ class Cache(object): mi.format_metadata = FormatMetadata(self, book_id, formats) good_formats = FormatsList(formats, mi.format_metadata) mi.formats = good_formats - mi.db_approx_formats = formats +# mi.db_approx_formats = formats mi.has_cover = _('Yes') if self._field_for('cover', book_id, default_value=False) else '' mi.tags = list(self._field_for('tags', book_id, default_value=())) @@ -1413,6 +1413,7 @@ class Cache(object): def refresh_ondevice(self): self.fields['ondevice'].clear_caches() self.clear_search_caches() + self.clear_composite_caches() @read_api def tags_older_than(self, tag, delta=None, must_have_tag=None, must_have_authors=None): diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index 303d47b697..f522bc1cc1 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -355,4 +355,6 @@ class ProxyMetadata(Metadata): um = ga(self, '_user_metadata') return frozenset(ALL_METADATA_FIELDS.union(um.iterkeys())) - + @property + def _proxy_metadata(self): + return self diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index e70122fd36..5f9e7bfdc6 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -453,7 +453,7 @@ class BooksModel(QAbstractTableModel): # {{{ def get_book_display_info(self, idx): mi = self.db.get_metadata(idx) - mi.size = mi.book_size + mi.size = mi._proxy_metadata.book_size mi.cover_data = ('jpg', self.cover(idx)) mi.id = self.db.id(idx) mi.field_metadata = self.db.field_metadata diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 606fe3b11b..8fcc35e3bb 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -640,15 +640,22 @@ class BuiltinApproximateFormats(BuiltinFormatterFunction): 'although it probably is. ' 'This function can be called in template program mode using ' 'the template "{:\'approximate_formats()\'}". ' - 'Note that format names are always uppercase, as in EPUB.' + 'Note that format names are always uppercase, as in EPUB. ' + 'This function works only in the GUI. If you want to use these values ' + 'in save-to-disk or send-to-device templates then you ' + 'must make a custom "Column built from other columns", use ' + 'the function in that column\'s template, and use that ' + 'column\'s value in your save/send templates' ) def evaluate(self, formatter, kwargs, mi, locals): - fmt_data = mi.get('db_approx_formats', []) - if not fmt_data: - return '' - data = sorted(fmt_data) - return ','.join(v.upper() for v in data) + if hasattr(mi, '_proxy_metadata'): + fmt_data = mi._proxy_metadata.db_approx_formats + if not fmt_data: + return '' + data = sorted(fmt_data) + return ','.join(v.upper() for v in data) + return _('This function can be used only in the GUI') class BuiltinFormatsModtimes(BuiltinFormatterFunction): name = 'formats_modtimes' @@ -902,27 +909,42 @@ class BuiltinBooksize(BuiltinFormatterFunction): name = 'booksize' arg_count = 0 category = 'Get values from metadata' - __doc__ = doc = _('booksize() -- return value of the size field') + __doc__ = doc = _('booksize() -- return value of the size field. ' + 'This function works only in the GUI. If you want to use this value ' + 'in save-to-disk or send-to-device templates then you ' + 'must make a custom "Column built from other columns", use ' + 'the function in that column\'s template, and use that ' + 'column\'s value in your save/send templates') def evaluate(self, formatter, kwargs, mi, locals): - if mi.book_size is not None: + if hasattr(mi, '_proxy_metadata'): try: - return str(mi.book_size) + v = mi._proxy_metadata.book_size + if v is not None: + return str(mi._proxy_metadata.book_size) + return '' except: pass - return '' + return '' + return _('This function can be used only in the GUI') class BuiltinOndevice(BuiltinFormatterFunction): name = 'ondevice' arg_count = 0 category = 'Get values from metadata' __doc__ = doc = _('ondevice() -- return Yes if ondevice is set, otherwise return ' - 'the empty string') + 'the empty string. This function works only in the GUI. If you want to ' + 'use this value in save-to-disk or send-to-device templates then you ' + 'must make a custom "Column built from other columns", use ' + 'the function in that column\'s template, and use that ' + 'column\'s value in your save/send templates') def evaluate(self, formatter, kwargs, mi, locals): - if mi.ondevice_col: - return _('Yes') - return '' + if hasattr(mi, '_proxy_metadata'): + if mi._proxy_metadata.ondevice_col: + return _('Yes') + return '' + return _('This function can be used only in the GUI') class BuiltinSeriesSort(BuiltinFormatterFunction): name = 'series_sort' @@ -1259,9 +1281,6 @@ class BuiltinVirtualLibraries(BuiltinFormatterFunction): 'column\'s value in your save/send templates') def evaluate(self, formatter, kwargs, mi, locals_): - from calibre.db.lazy import ProxyMetadata - if isinstance(mi, ProxyMetadata): - return mi.virtual_libraries if hasattr(mi, '_proxy_metadata'): return mi._proxy_metadata.virtual_libraries return _('This function can be used only in the GUI') From 9c0a1ca2f3a72f6fdb722e3f153a478f91916ba6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 15:24:29 +0530 Subject: [PATCH 0342/1154] Add a button to clear the current virtual library easily --- src/calibre/gui2/layout.py | 8 ++++++++ src/calibre/gui2/search_restriction_mixin.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py index 20c2588bb4..a1c8063dd8 100644 --- a/src/calibre/gui2/layout.py +++ b/src/calibre/gui2/layout.py @@ -184,6 +184,14 @@ class SearchBar(QWidget): # {{{ l.addWidget(x) parent.virtual_library = x + x = QToolButton(self) + x.setIcon(QIcon(I('minus.png'))) + x.setObjectName('clear_vl') + l.addWidget(x) + x.setVisible(False) + x.setToolTip(_('Close the Virtual Library')) + parent.clear_vl = x + x = QLabel(self) x.setObjectName("search_count") l.addWidget(x) diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index ceed4f1928..528289e320 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -316,6 +316,7 @@ class SearchRestrictionMixin(object): self.virtual_library_menu = QMenu() self.virtual_library.clicked.connect(self.virtual_library_clicked) + self.clear_vl.clicked.connect(lambda x: (self.apply_virtual_library(), self.clear_additional_restriction())) self.virtual_library_tooltip = \ _('Use a "virtual library" to show only a subset of the books present in this library') @@ -589,10 +590,12 @@ class SearchRestrictionMixin(object): self.search_count.setStyleSheet( 'QLabel { border-radius: 6px; background-color: %s }' % tweaks['highlight_virtual_library']) + self.clear_vl.setVisible(True) else: # No restriction or not library view t = '' self.search_count.setStyleSheet( 'QLabel { background-color: transparent; }') + self.clear_vl.setVisible(False) self.search_count.setText(t) if __name__ == '__main__': From 6315a05142ac16d017bbf7382674ab5045de0273 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Thu, 25 Jul 2013 12:20:12 +0200 Subject: [PATCH 0343/1154] correctly handle the formatter parameter in the cache and in Metadata --- src/calibre/db/lazy.py | 3 +-- src/calibre/ebooks/metadata/book/base.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index f522bc1cc1..166627c438 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -280,8 +280,7 @@ class ProxyMetadata(Metadata): def __init__(self, db, book_id, formatter=None): sa(self, 'template_cache', db.formatter_template_cache) - if formatter is None: - sa(self, 'formatter', SafeFormat()) + sa(self, 'formatter', SafeFormat() if formatter is None else formatter) sa(self, '_db', weakref.ref(db)) sa(self, '_book_id', book_id) sa(self, '_cache', {'user_categories':{}, 'cover_data':(None,None), 'device_collections':[]}) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 93253b1dcc..9e94678844 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -87,7 +87,7 @@ class Metadata(object): self.authors = list(authors) if authors else [] from calibre.ebooks.metadata.book.formatter import SafeFormat if formatter is None: - self.formatter = SafeFormat() + self.formatter = SafeFormat() if formatter is None else formatter self.template_cache = template_cache def is_null(self, field): From c3301f8cc339c485c7807ac666fd8e73d0172797 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 15:56:47 +0530 Subject: [PATCH 0344/1154] Allow Virtual Library removal confirmation dialog to be skipped --- src/calibre/gui2/search_restriction_mixin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index 528289e320..e8f7cd51ae 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -13,6 +13,7 @@ from PyQt4.Qt import ( QDialogButtonBox, QSize, QVBoxLayout, QListWidget, QStringList, QRadioButton) from calibre.gui2 import error_dialog, question_dialog +from calibre.gui2.dialogs.confirm_delete import confirm from calibre.gui2.widgets import ComboBoxWithHelp from calibre.utils.config_base import tweaks from calibre.utils.icu import sort_key @@ -457,10 +458,9 @@ class SearchRestrictionMixin(object): menu.setEnabled(False) def remove_vl_triggered(self, name=None): - if not question_dialog(self, _('Are you sure?'), - _('Are you sure you want to remove ' - 'the virtual library {0}').format(name), - default_yes=False): + if not confirm( + _('Are you sure you want to remove the virtual library {0}?').format(name), + 'confirm_vl_removal', parent=self): return self._remove_vl(name, reapply=True) From a2eb5554562c4dae34ca8732742d0258ddff9c3b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 16:03:50 +0530 Subject: [PATCH 0345/1154] Scroll the list of tweaks when using the keyboard --- src/calibre/gui2/preferences/tweaks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/preferences/tweaks.py b/src/calibre/gui2/preferences/tweaks.py index 404648b90a..d6be773b9e 100644 --- a/src/calibre/gui2/preferences/tweaks.py +++ b/src/calibre/gui2/preferences/tweaks.py @@ -374,6 +374,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.changed() def current_changed(self, current, previous): + self.tweaks_view.scrollTo(current) tweak = self.tweaks.data(current, Qt.UserRole) self.help.setPlainText(tweak.doc) self.edit_tweak.setPlainText(tweak.edit_text) From 1c5401db7b342648bc340d5c260b265f872b4021 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 16:04:42 +0530 Subject: [PATCH 0346/1154] pep8 --- src/calibre/gui2/preferences/tweaks.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/preferences/tweaks.py b/src/calibre/gui2/preferences/tweaks.py index d6be773b9e..b3122d3940 100644 --- a/src/calibre/gui2/preferences/tweaks.py +++ b/src/calibre/gui2/preferences/tweaks.py @@ -25,7 +25,7 @@ from PyQt4.Qt import (QAbstractListModel, Qt, QStyledItemDelegate, QStyle, ROOT = QModelIndex() -class Delegate(QStyledItemDelegate): # {{{ +class Delegate(QStyledItemDelegate): # {{{ def __init__(self, view): QStyledItemDelegate.__init__(self, view) self.view = view @@ -39,7 +39,7 @@ class Delegate(QStyledItemDelegate): # {{{ # }}} -class Tweak(object): # {{{ +class Tweak(object): # {{{ def __init__(self, name, doc, var_names, defaults, custom): translate = _ @@ -98,7 +98,7 @@ class Tweak(object): # {{{ # }}} -class Tweaks(QAbstractListModel, SearchQueryParser): # {{{ +class Tweaks(QAbstractListModel, SearchQueryParser): # {{{ def __init__(self, parent=None): QAbstractListModel.__init__(self, parent) @@ -190,7 +190,7 @@ class Tweaks(QAbstractListModel, SearchQueryParser): # {{{ if not var_names: raise ValueError('Failed to find any variables for %r'%name) self.tweaks.append(Tweak(name, doc, var_names, defaults, custom)) - #print '\n\n', self.tweaks[-1] + # print '\n\n', self.tweaks[-1] return pos def restore_to_default(self, idx): @@ -289,7 +289,7 @@ class Tweaks(QAbstractListModel, SearchQueryParser): # {{{ # }}} -class PluginTweaks(QDialog): # {{{ +class PluginTweaks(QDialog): # {{{ def __init__(self, raw, parent=None): QDialog.__init__(self, parent) @@ -353,7 +353,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): return True def copy_item_to_clipboard(self, val): - cb = QApplication.clipboard(); + cb = QApplication.clipboard() cb.clear() cb.setText(val) @@ -448,7 +448,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.highlight_index(idx) def highlight_index(self, idx): - if not idx.isValid(): return + if not idx.isValid(): + return self.view.scrollTo(idx) self.view.selectionModel().select(idx, self.view.selectionModel().ClearAndSelect) @@ -473,7 +474,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if __name__ == '__main__': app = QApplication([]) - #Tweaks() - #test_widget + # Tweaks() + # test_widget test_widget('Advanced', 'Tweaks') + From 805421573dd2f6f9be3fdcc8c090075e7d1a7855 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 18:42:24 +0530 Subject: [PATCH 0347/1154] Update mediapart.fr --- recipes/mediapart.recipe | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/recipes/mediapart.recipe b/recipes/mediapart.recipe index a457b713f2..0dbe8bcd72 100644 --- a/recipes/mediapart.recipe +++ b/recipes/mediapart.recipe @@ -1,10 +1,10 @@ __license__ = 'GPL v3' -__copyright__ = '2009, Mathieu Godlewski ; 2010-2012, Louis Gesbert ; 2013, Malah ' +__copyright__ = '2009, Mathieu Godlewski ; 2010-2012, Louis Gesbert ; 2013, Malah ' ''' Mediapart ''' -__author__ = '2009, Mathieu Godlewski ; 2010-2012, Louis Gesbert ; 2013, Malah ' +__author__ = '2009, Mathieu Godlewski ; 2010-2012, Louis Gesbert ; 2013, Malah ' import re from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag @@ -29,19 +29,26 @@ class Mediapart(BasicNewsRecipe): ('Les articles', 'http://www.mediapart.fr/articles/feed'), ] -# -- print-version +# -- full-page-version - conversion_options = { 'smarten_punctuation' : True } + conversion_options = {'smarten_punctuation' : True} - remove_tags = [ dict(name='div', attrs={'class':'print-source_url'}) ] + keep_only_tags = [ + dict(name='div', attrs={'class':'col-left fractal-desktop fractal-10-desktop collapse-7-desktop fractal-tablet fractal-6-tablet collapse-4-tablet'}), + dict(name='div', attrs={'id':'pageFirstContent'}) + ] + remove_tags = [ + dict(name='div', attrs={'id':'lire-aussi'}), + dict(name='div', attrs={'class':'col-right-content'}) + ] def print_version(self, url): raw = self.browser.open(url).read() soup = BeautifulSoup(raw.decode('utf8', 'replace')) - link = soup.find('a', {'href':re.compile('^/print/[0-9]+')}) + link = soup.find('a', {'href':re.compile('^.*?onglet=full$')}) if link is None: return None - return 'http://www.mediapart.fr' + link['href'] + return link['href'] # -- Handle login @@ -62,3 +69,4 @@ class Mediapart(BasicNewsRecipe): legend.insert(0, Tag(soup, 'br', [])) legend.name = 'small' return soup + From 3bd18463700b2124cd5f7be75c4bc856b3695434 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 18:47:14 +0530 Subject: [PATCH 0348/1154] Driver for Surtab Ventos Fixes #1204885 [Trekstor Surftab not recognized](https://bugs.launchpad.net/calibre/+bug/1204885) --- src/calibre/devices/android/driver.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index a0eb021289..055fd32f88 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -55,7 +55,10 @@ class ANDROID(USBMS): 0x040d : {0x8510 : [0x0001], 0x0851 : [0x1]}, # Trekstor - 0x1e68 : {0x006a : [0x0231]}, + 0x1e68 : { + 0x006a : [0x0231], + 0x0062 : [0x222], # Surftab ventos https://bugs.launchpad.net/bugs/1204885 + }, # Motorola 0x22b8 : {0x41d9 : [0x216], 0x2d61 : [0x100], 0x2d67 : [0x100], @@ -220,7 +223,9 @@ class ANDROID(USBMS): 'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD', 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0', 'COBY_MID', 'VS', 'AINOL', 'TOPWISE', 'PAD703', 'NEXT8D12', - 'MEDIATEK', 'KEENHI', 'TECLAST', 'SURFTAB', 'XENTA', 'OBREEY_S'] + 'MEDIATEK', 'KEENHI', 'TECLAST', 'SURFTAB', 'XENTA', 'OBREEY_S', + 'SURFTAB_', + ] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'A953', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', @@ -242,7 +247,7 @@ class ANDROID(USBMS): 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E', 'NOVO7', 'MB526', '_USB#WYK7MSF8KE', 'TABLET_PC', 'F', 'MT65XX_MS', 'ICS', 'E400', '__FILE-STOR_GADG', 'ST80208-1', 'GT-S5660M_CARD', 'XT894', '_USB', - 'PROD_TAB13-201', 'URFPAD2', 'MID1126', + 'PROD_TAB13-201', 'URFPAD2', 'MID1126', 'ST10216-1', ] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', @@ -256,6 +261,7 @@ class ANDROID(USBMS): 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E', 'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD', 'ST80208-1', 'XT894', '_USB', 'PROD_TAB13-201', 'URFPAD2', 'MID1126', 'ANDROID_PLATFORM', + 'ST10216-1', ] OSX_MAIN_MEM = 'Android Device Main Memory' From d7c8bfa91692c6b3e95d0e679c1617dd0003a518 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 25 Jul 2013 20:58:59 +0530 Subject: [PATCH 0349/1154] No names, No jackets by Armin Geller --- recipes/no_names_no_jackets.recipe | 57 ++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 recipes/no_names_no_jackets.recipe diff --git a/recipes/no_names_no_jackets.recipe b/recipes/no_names_no_jackets.recipe new file mode 100644 index 0000000000..672a913270 --- /dev/null +++ b/recipes/no_names_no_jackets.recipe @@ -0,0 +1,57 @@ +# +# Written: July 2013 +# Last Edited: 2013-07-25 +# Version: 1.0 +# Last update: 2013-07-25 +# + +__license__ = 'GPL v3' +__copyright__ = '2013, Armin Geller' + +''' +Fetch blindenbuch.de +''' + +from calibre.web.feeds.recipes import BasicNewsRecipe +class AdvancedUserRecipe1303841067(BasicNewsRecipe): + + title = u'No Names, No Jackets' + __author__ = 'Armin Geller' # AGe 2013-07-25 + description = u'One chapter. Just the writing. Discover something new.' + publisher = 'nonamesnojackets.com/' + publication_type = 'ebook news' + tags = 'Books, Literature, E-Books, US' + timefmt = ' [%a, %d %b %Y]' + publication_type = 'Feed' + language = 'en' + encoding = 'utf-8' + + oldest_article = 14 + max_articles_per_feed = 100 + + no_stylesheets = True + use_embedded_content = False + remove_javascript = True + + conversion_options = {'title' : title, + 'comments' : description, + 'tags' : tags, + 'language' : language, + 'publisher' : publisher, + 'authors' : publisher, + } + +# cover_url = '' +# masthead_url = '' + + extra_css = ''' + h1,h2 {font-weight:bold;font-size:large;} + .entry-meta {font-size: 1em;text-align: left; font-style: italic} + ''' + + keep_only_tags = [ + dict(name='article') + ] + + feeds = [(u'No Names, No Jackets', u'http://www.nonamesnojackets.com/feed/')] + From bc2220a26e1ec3e162bdc3f21ca95734f341a681 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 26 Jul 2013 09:25:58 +0530 Subject: [PATCH 0350/1154] Blindbuch by Armin Geller --- recipes/blind_buch_de.recipe | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 recipes/blind_buch_de.recipe diff --git a/recipes/blind_buch_de.recipe b/recipes/blind_buch_de.recipe new file mode 100644 index 0000000000..9ec24fb5d2 --- /dev/null +++ b/recipes/blind_buch_de.recipe @@ -0,0 +1,63 @@ +# +# Written: July 2013 +# Last Edited: 2013-07-11 +# Version: 1.0 +# Last update: 2013-07-25 +# + +__license__ = 'GPL v3' +__copyright__ = '2013, Armin Geller' + +''' +Fetch blindenbuch.de +''' + +from calibre.web.feeds.recipes import BasicNewsRecipe +class AdvancedUserRecipe1303841067(BasicNewsRecipe): + + title = u'Blindbuch - Bücher neu entdecken' + __author__ = 'Armin Geller' # AGe 2013-07-11 + description = u'Bücher blind präsentiert' + publisher = 'blindbuch.de' + publication_type = 'ebook news' + tags = 'Bücher, Literatur, E-Books, Germany' + timefmt = ' [%a, %d %b %Y]' + publication_type = 'Feed' + language = 'de-DE' + encoding = 'utf-8' + + oldest_article = 14 + max_articles_per_feed = 100 + + no_stylesheets = True + use_embedded_content = False + remove_javascript = True + + conversion_options = {'title' : title, + 'comments' : description, + 'tags' : tags, + 'language' : language, + 'publisher' : publisher, + 'authors' : publisher, + } + + cover_url = 'http://blindbuch.de/img/blindbuch_calibre.png' + masthead_url = 'http://www.blindbuch.de/img/Masterhead.JPG' + + extra_css = ''' + h1{font-weight:bold;font-size:large;} + .post-meta {font-size: 1em;text-align: left; font-style: italic} + ''' + + keep_only_tags = [ + dict(name='article') + ] + + remove_tags = [ + dict(name='div', attrs={'class':['su-spoiler su-spoiler-style-1','post-comments comments',]}), + dict(name='span', attrs={'class':['post-comments comments',]}), + dict(name='div', attrs={'addthis':['title',]}), + ] + + feeds = [(u'Blindbuch', u'http://www.blindbuch.de/feed/')] + From 76b3c38e035f0bac2cad88ba996aba555bb3c808 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 26 Jul 2013 10:47:56 +0530 Subject: [PATCH 0351/1154] Document the danger of using parse_css in the polish books container --- src/calibre/ebooks/oeb/polish/container.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/ebooks/oeb/polish/container.py b/src/calibre/ebooks/oeb/polish/container.py index 63230b899b..6cd9e47b5b 100644 --- a/src/calibre/ebooks/oeb/polish/container.py +++ b/src/calibre/ebooks/oeb/polish/container.py @@ -202,6 +202,9 @@ class Container(object): return data def parse_css(self, data, fname): + ''' WARNING: This modifies the CSS tripping out @page rules, comments + etc. If you wish to write back the css, you should override the + css_preprocessor with a dummy one. ''' from cssutils import CSSParser, log log.setLevel(logging.WARN) log.raiseExceptions = False From ed1d8a30c307bcf0f64a2271dc231f23914aab5b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 26 Jul 2013 11:52:17 +0530 Subject: [PATCH 0352/1154] Polish books: Fix @page rules being removed Book polishing: Fix page margins being removed if an unused font was found during subsetting of embedded fonts. --- src/calibre/ebooks/oeb/polish/container.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/oeb/polish/container.py b/src/calibre/ebooks/oeb/polish/container.py index 6cd9e47b5b..6fb8fcb4dd 100644 --- a/src/calibre/ebooks/oeb/polish/container.py +++ b/src/calibre/ebooks/oeb/polish/container.py @@ -21,7 +21,7 @@ from calibre.customize.ui import (plugin_for_input_format, from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.conversion.plugins.epub_input import ( ADOBE_OBFUSCATION, IDPF_OBFUSCATION, decrypt_font) -from calibre.ebooks.conversion.preprocess import HTMLPreProcessor, CSSPreProcessor +from calibre.ebooks.conversion.preprocess import HTMLPreProcessor, CSSPreProcessor as cssp from calibre.ebooks.mobi import MobiError from calibre.ebooks.mobi.reader.headers import MetadataHeader from calibre.ebooks.mobi.tweak import set_cover @@ -42,6 +42,11 @@ def guess_type(x): OEB_FONTS = {guess_type('a.ttf'), guess_type('b.ttf')} OPF_NAMESPACES = {'opf':OPF2_NS, 'dc':DC11_NS} +class CSSPreProcessor(cssp): + + def __call__(self, data): + return self.MS_PAT.sub(self.ms_sub, data) + class Container(object): ''' @@ -202,14 +207,11 @@ class Container(object): return data def parse_css(self, data, fname): - ''' WARNING: This modifies the CSS tripping out @page rules, comments - etc. If you wish to write back the css, you should override the - css_preprocessor with a dummy one. ''' from cssutils import CSSParser, log log.setLevel(logging.WARN) log.raiseExceptions = False data = self.decode(data) - data = self.css_preprocessor(data, add_namespace=False) + data = self.css_preprocessor(data) parser = CSSParser(loglevel=logging.WARNING, # We dont care about @import rules fetcher=lambda x: (None, None), log=_css_logger) From 517250e05e5ed285199dc26e25aa85e9b7ec77e8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 26 Jul 2013 15:23:43 +0530 Subject: [PATCH 0353/1154] Update Spektrum der Wissenschaft --- recipes/spektrum.recipe | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/recipes/spektrum.recipe b/recipes/spektrum.recipe index fa60b56560..4ce7a7742b 100644 --- a/recipes/spektrum.recipe +++ b/recipes/spektrum.recipe @@ -6,7 +6,7 @@ Fetch RSS-Feeds spektrum.de from calibre.web.feeds.recipes import BasicNewsRecipe class AdvancedUserRecipe1303841067(BasicNewsRecipe): title = u'Spektrum der Wissenschaft' - __author__ = 'Armin Geller, Bratzzo, Rainer Zenz' # Update Bratzzo & AGE 2012-10-12 + __author__ = 'Armin Geller, Bratzzo, Rainer Zenz' # Update AGE 2013-07-26 description = u'German online portal of Spektrum der Wissenschaft' publisher = 'Spektrum der Wissenschaft Verlagsgesellschaft mbH' category = 'science news, Germany' @@ -20,8 +20,8 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): #conversion_options = {'base_font_size': 20} - # cover_url = 'http://upload.wikimedia.org/wikipedia/de/3/3b/Spektrum_der_Wissenschaft_Logo.svg' # old logo - cover_url = 'http://upload.wikimedia.org/wikipedia/de/5/59/Spektrum-cover.jpg' # from Rainer Zenz + # cover_url = 'http://upload.wikimedia.org/wikipedia/de/5/59/Spektrum-cover.jpg' # from Rainer Zenz + cover_url = 'http://www16.zippyshare.com/scaled/52219516/file.html' # AGE 2013-07-26 new cover location masthead_url = 'http://www.spektrum.de/fm/861/spektrum.de.png' @@ -53,5 +53,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): remove_tags = [ dict(attrs={'id':['recommend-article', 'dossierbox', 'cover', 'toc']}), - dict(attrs={'class':['sidebar-box-full clearfix', 'linktotop' ]}), + dict(attrs={'class':['sidebar-box-full clearfix', 'linktotop']}), ] + From c32dc18eb156c63a2b49d65e30375245b997ba32 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 26 Jul 2013 16:52:53 +0530 Subject: [PATCH 0354/1154] McAfee have fixed their SiteAdvisor rating --- manual/faq.rst | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/manual/faq.rst b/manual/faq.rst index 46d675da13..e5a6342cf8 100644 --- a/manual/faq.rst +++ b/manual/faq.rst @@ -840,19 +840,6 @@ If you still cannot get the installer to work and you are on windows, you can us My antivirus program claims |app| is a virus/trojan? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note :: - As of July, 2013 McAfee Site Advisor has started warning that - http://calibre-ebook.com is unsafe, with no stated reason or justification. - McAfee is wrong, the mistake has been reported to them, by several people, - but they have not corrected it. McAfee SiteAdvisor is a notoriously - unreliable service, see for example - `this page `_ or - `this page `_ or - `this Wikipedia entry `_. - We strongly urge you to stop using McAfee products, find a more competent security provider - to give your business to. - Instructions on how to `uninstall McAfee SiteAdvisor `_. - The first thing to check is that you are downloading |app| from the official website: ``_. |app| is a very popular program and unscrupulous people try to setup websites offering it for download to fool From 0ca5ffc49f1432d95ba5519f5b6e457c2dc24f5b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 26 Jul 2013 21:36:18 +0530 Subject: [PATCH 0355/1154] Content server: Fix search query not being fully sanitized in results page Fixes #1205385 [Private bug](https://bugs.launchpad.net/calibre/+bug/1205385) --- src/calibre/library/server/browse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index bffeb33829..ef6b8f3f3c 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -291,7 +291,7 @@ class BrowseServer(object): lp = force_unicode(lp, filesystem_encoding) ans = ans.replace('{library_name}', xml(os.path.basename(lp))) ans = ans.replace('{library_path}', xml(lp, True)) - ans = ans.replace('{initial_search}', initial_search) + ans = ans.replace('{initial_search}', xml(initial_search, attribute=True)) return ans @property From 2a0f6bbeaea9a16b5b1c207cea3ef35afbe00b0b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 27 Jul 2013 07:42:51 +0530 Subject: [PATCH 0356/1154] Update Something Awful --- recipes/something_awful.recipe | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/recipes/something_awful.recipe b/recipes/something_awful.recipe index 006cfdc9b2..636d306a6d 100644 --- a/recipes/something_awful.recipe +++ b/recipes/something_awful.recipe @@ -1,4 +1,3 @@ -import re from calibre.web.feeds.news import BasicNewsRecipe class SomethingAwfulRecipe(BasicNewsRecipe): @@ -6,8 +5,7 @@ class SomethingAwfulRecipe(BasicNewsRecipe): __author__ = 'atordo' description = 'The Internet Makes You Stupid' cover_url = 'http://i.somethingawful.com/core/head-logo-bluegren.png' - masthead_url = 'http://i.somethingawful.com/core/head-logo-bluegren.png' - oldest_article = 7 + oldest_article = 15 max_articles_per_feed = 50 auto_cleanup = False no_stylesheets = True @@ -16,27 +14,21 @@ class SomethingAwfulRecipe(BasicNewsRecipe): use_embedded_content = False remove_empty_feeds = True publication_type = 'magazine' + reverse_article_order = True recursions = 1 - match_regexps = [r'\?page=\d+$'] + match_regexps = [r'^http://www.somethingawful.com/.+/.+/\d{1,2}/$'] - preprocess_regexps = [ - (re.compile(r'.*', re.DOTALL), lambda match: '') - ] - - remove_attributes = [ 'align', 'alt', 'valign' ] + remove_attributes = ['align', 'alt', 'valign'] keep_only_tags = [ - dict(name='div', attrs={'class':'content_area'}) -# ,dict(name='p', attrs={'class':'pagebar'}) - ] - remove_tags = [ - dict(name='div', attrs={'class':['column_box','featurenav','social']}) - ,dict(name='div', attrs={'id':'sidebar'}) - ,dict(name='a', attrs={'class':'curpage'}) + dict(name='div', attrs={'class':'article_head'}) + ,dict(name='div', attrs={'class':'organ article'}) + ,dict(name='ul', attrs={'class':'pager'}) ] extra_css = ''' + .author{font-size:small} .date{font-size:small} .byline{font-size:small} .font_big{font-size:large} .compat5{font-weight:bold} .accentbox{background-color:#E3E3E3; border:solid black} img{margin-bottom:0.4em; display:block; margin-left: auto; margin-right:auto} @@ -53,7 +45,7 @@ class SomethingAwfulRecipe(BasicNewsRecipe): ,('The Great Goon Database', 'http://www.somethingawful.com/rss/great-goon-database.rss.xml') ,('Livejournal Theater', 'http://www.somethingawful.com/rss/livejournal-theater.rss.xml') ,('Joystick Token Healthpack', 'http://www.somethingawful.com/rss/token-healthpack.rss.xml') - #,('Webcam Ward', 'http://www.somethingawful.com/rss/webcam-ward.rss.xml') + ,('Webcam Ward', 'http://www.somethingawful.com/rss/webcam-ward.rss.xml') ,('Features / Articles', 'http://www.somethingawful.com/rss/feature-articles.rss.xml') ,('Guides', 'http://www.somethingawful.com/rss/guides.rss.xml') ,('Legal Threats', 'http://www.somethingawful.com/rss/legal-threats.rss.xml') @@ -77,6 +69,7 @@ class SomethingAwfulRecipe(BasicNewsRecipe): ,('Johnston Checks In', 'http://www.somethingawful.com/rss/levi-johnston.rss.xml') ,('Twitter Tuesday', 'http://www.somethingawful.com/rss/twitter-tuesday.rss.xml') ,('Music Article', 'http://www.somethingawful.com/rss/music-article.rss.xml') + ,('The Everdraed Showcase', 'http://www.somethingawful.com/rss/everdraed-showcase.xml') ,('Reviews [Games]', 'http://www.somethingawful.com/rss/game-reviews.rss.xml') ,('Reviews [Movies]', 'http://www.somethingawful.com/rss/movie-reviews.rss.xml') ,('Rom Pit', 'http://www.somethingawful.com/rss/rom-pit.rss.xml') @@ -92,4 +85,6 @@ class SomethingAwfulRecipe(BasicNewsRecipe): ,('Garbage Day', 'http://www.somethingawful.com/rss/garbage-day.rss.xml') ,('WTF, D&D!?', 'http://www.somethingawful.com/rss/dungeons-and-dragons.rss.xml') ,('Current Releases', 'http://www.somethingawful.com/rss/current-movie-reviews.rss.xml') + ,('Awful Things for Sale', 'http://www.somethingawful.com/rss/awful-things-sale.xml') ] + From 6afb55fdfb0a56d17fbddf2904e5d7094aa20ac3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 27 Jul 2013 08:09:26 +0530 Subject: [PATCH 0357/1154] DOCX Input: Fix conversion breaking for files that use heading style paragraphs to insert line rules --- src/calibre/ebooks/docx/cleanup.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/calibre/ebooks/docx/cleanup.py b/src/calibre/ebooks/docx/cleanup.py index 10bfd9a78f..33f869faca 100644 --- a/src/calibre/ebooks/docx/cleanup.py +++ b/src/calibre/ebooks/docx/cleanup.py @@ -8,7 +8,7 @@ __copyright__ = '2013, Kovid Goyal ' import os -from calibre.ebooks.docx.names import ancestor +from calibre.ebooks.docx.names import XPath def mergeable(previous, current): if previous.tail or current.tail: @@ -100,14 +100,17 @@ def before_count(root, tag, limit=10): def cleanup_markup(log, root, styles, dest_dir, detect_cover): # Move
    s outside paragraphs, if possible. + pancestor = XPath('|'.join('ancestor::%s[1]' % x for x in ('p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'))) for hr in root.xpath('//span/hr'): - p = ancestor(hr, 'p') - descendants = tuple(p.iterdescendants()) - if descendants[-1] is hr: - parent = p.getparent() - idx = parent.index(p) - parent.insert(idx+1, hr) - hr.tail = '\n\t' + p = pancestor(hr) + if p: + p = p[0] + descendants = tuple(p.iterdescendants()) + if descendants[-1] is hr: + parent = p.getparent() + idx = parent.index(p) + parent.insert(idx+1, hr) + hr.tail = '\n\t' # Merge consecutive spans that have the same styling current_run = [] @@ -176,5 +179,3 @@ def cleanup_markup(log, root, styles, dest_dir, detect_cover): img.getparent().remove(img) return path - - From 9386518ff78207b9635161a64f17a251fac4dcf6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 27 Jul 2013 08:45:40 +0530 Subject: [PATCH 0358/1154] version 0.9.41 --- Changelog.yaml | 41 ++++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index 2cbe422226..b379e92416 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,47 @@ # new recipes: # - title: +- version: 0.9.41 + date: 2013-07-27 + + new features: + - title: "Add a button to clear the current virtual library easily" + + - title: "Driver for Surftab Ventos" + tickets: [1204885] + + - title: "Ebook-viewer: Allow re-ordering bookmarks in the bookmarks manager by drag and drop." + + bug fixes: + - title: "DOCX Input: Fix conversion breaking for files that use heading style paragraphs to insert line rules" + + - title: "Content server: Fix last search query not being fully sanitized in results page" + tickets: [1205385] + + - title: "Book polishing: Fix page margins being removed if an unused font was found during subsetting of embedded fonts." + + - title: "PDF Output: Do not error out when the input document uses a font that cannot be subset, such as the Symbol font. Instead print a warning and embed the full font." + tickets: [1203449] + + - title: "Conversion: Fix a regression in the last release that broke conversion of a few files with comments just before a chapter start." + tickets: [1188635] + + improved recipes: + - Something Awful + - Spektrum der Wissenschaft + - mediapart.fr + - Dilbert + - Antyweb + - Scientific American + - taz.de (RSS) + + new recipes: + - title: Blindbuch and No names, No jackets + author: Armin Geller + + - title: El Tribuno Salta and Jujuy + author: Darko Miletic + - version: 0.9.40 date: 2013-07-19 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 1e0b2a1a83..d161a85b3c 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 9, 40) +numeric_version = (0, 9, 41) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " From 748eb921c5294e5f3a2558f968d7684fed0e1d5c Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sat, 27 Jul 2013 10:32:49 +0200 Subject: [PATCH 0359/1154] Prevent exception when deleting a non-existent book. --- src/calibre/library/caches.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index e552ead591..022ba9402e 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -920,7 +920,8 @@ class ResultCache(SearchQueryParser): # {{{ def remove(self, id): try: - self._uuid_map.pop(self._data[id][self._uuid_column_index], None) + if self._data[id] is not None: + self._uuid_map.pop(self._data[id][self._uuid_column_index], None) except IndexError: pass # id is out of bounds -- no uuid in the map to remove try: From d47603bdacfe7fc92b552de59e763b52ef800cb8 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sat, 27 Jul 2013 10:33:14 +0200 Subject: [PATCH 0360/1154] Allow an empty field specifier in calibredb, permitting one to get a list of ids --- src/calibre/library/cli.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 012d88dace..a6ecca3721 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -199,15 +199,18 @@ def command_list(args, dbpath): afields.add('*'+f) if data['datatype'] == 'series': afields.add('*'+f+'_index') - fields = [str(f.strip().lower()) for f in opts.fields.split(',')] - if 'all' in fields: - fields = sorted(list(afields)) - if not set(fields).issubset(afields): - parser.print_help() - print - prints(_('Invalid fields. Available fields:'), - ','.join(sorted(afields)), file=sys.stderr) - return 1 + if opts.fields.strip(): + fields = [str(f.strip().lower()) for f in opts.fields.split(',')] + if 'all' in fields: + fields = sorted(list(afields)) + if not set(fields).issubset(afields): + parser.print_help() + print + prints(_('Invalid fields. Available fields:'), + ','.join(sorted(afields)), file=sys.stderr) + return 1 + else: + fields = [] if not opts.sort_by in afields and opts.sort_by is not None: parser.print_help() From 7f3b6c1f7d30f2183298af23f4fb0ac36b9e3b1d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 28 Jul 2013 09:34:58 +0530 Subject: [PATCH 0361/1154] MOBI metadata: Handle cover/thumbnail exth fields with null pointers MOBI metadata: Do not fail to set metadata in MOBI files if they have EXTH fields with NULL pointers to a cover or thumbnail. Fixes #1205757 [Metadata edits not sticking](https://bugs.launchpad.net/calibre/+bug/1205757) --- src/calibre/ebooks/metadata/mobi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/mobi.py b/src/calibre/ebooks/metadata/mobi.py index 7ad9a01962..ac675b1d84 100644 --- a/src/calibre/ebooks/metadata/mobi.py +++ b/src/calibre/ebooks/metadata/mobi.py @@ -169,10 +169,11 @@ class MetadataUpdater(object): self.timestamp = content elif id == 201: rindex, = self.cover_rindex, = unpack('>I', content) - self.cover_record = self.record(rindex + image_base) + if rindex != 0xffffffff: + self.cover_record = self.record(rindex + image_base) elif id == 202: rindex, = self.thumbnail_rindex, = unpack('>I', content) - if rindex > 0 : + if rindex > 0 and rindex != 0xffffffff: self.thumbnail_record = self.record(rindex + image_base) def patch(self, off, new_record0): From fac9845f90d65dda0c58209a261d6d6199f5171d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 28 Jul 2013 09:35:53 +0530 Subject: [PATCH 0362/1154] pep8 --- src/calibre/ebooks/metadata/mobi.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/metadata/mobi.py b/src/calibre/ebooks/metadata/mobi.py index ac675b1d84..5b45722383 100644 --- a/src/calibre/ebooks/metadata/mobi.py +++ b/src/calibre/ebooks/metadata/mobi.py @@ -265,7 +265,7 @@ class MetadataUpdater(object): # Pad to a 4-byte boundary trail = len(new_record0.getvalue()) % 4 - pad = '\0' * (4 - trail) # Always pad w/ at least 1 byte + pad = '\0' * (4 - trail) # Always pad w/ at least 1 byte new_record0.write(pad) new_record0.write('\0'*(1024*8)) @@ -278,7 +278,8 @@ class MetadataUpdater(object): def hexdump(self, src, length=16): # Diagnostic FILTER=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)]) - N=0; result='' + N=0 + result='' while src: s,src = src[:length],src[length:] hexa = ' '.join(["%02X"%ord(x) for x in s]) @@ -292,7 +293,7 @@ class MetadataUpdater(object): for i in xrange(self.nrecs): offset, a1,a2,a3,a4 = unpack('>LBBBB', self.data[78+i*8:78+i*8+8]) flags, val = a1, a2<<16|a3<<8|a4 - pdbrecords.append( [offset, flags, val] ) + pdbrecords.append([offset, flags, val]) return pdbrecords def update_pdbrecords(self, updated_pdbrecords): @@ -422,7 +423,7 @@ class MetadataUpdater(object): exth.write(data) exth = exth.getvalue() trail = len(exth) % 4 - pad = '\0' * (4 - trail) # Always pad w/ at least 1 byte + pad = '\0' * (4 - trail) # Always pad w/ at least 1 byte exth = ['EXTH', pack('>II', len(exth) + 12, len(recs)), exth, pad] exth = ''.join(exth) @@ -474,7 +475,6 @@ def get_metadata(stream): except ImportError: import Image as PILImage - stream.seek(0) try: raw = stream.read(3) @@ -530,5 +530,3 @@ def get_metadata(stream): im.convert('RGB').save(obuf, format='JPEG') mi.cover_data = ('jpg', obuf.getvalue()) return mi - - From f8ad733e2593d7058a500fe3ca10270c372a9d9d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 28 Jul 2013 09:44:45 +0530 Subject: [PATCH 0363/1154] calibredb: Add a new clone command calibredb: Add a new clone command to create clones of libraries with the same custom columns, virtual libraries, etc. as the current library. --- src/calibre/library/cli.py | 49 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index a6ecca3721..37a690632d 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -11,7 +11,8 @@ import sys, os, cStringIO, re import unicodedata from textwrap import TextWrapper -from calibre import preferred_encoding, prints, isbytestring +from calibre import preferred_encoding, prints, isbytestring, patheq +from calibre.constants import iswindows from calibre.utils.config import OptionParser, prefs, tweaks from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.metadata.book.base import field_from_string @@ -1389,12 +1390,55 @@ def command_list_categories(args, dbpath): else: do_list() + return parser + +def command_clone(args, dbpath): + parser = get_parser(_( + '''\ +%prog clone path/to/new/library + +Create a clone of the current library. This creates a new, empty library that has all the +same custom columns, virtual libraries and other settings as the current library. + +The cloned library will contain no books. If you want to create a full duplicate, including +all books, then simply use your filesystem tools to copy the library folder. + ''')) + opts, args = parser.parse_args(args) + if len(args) < 1: + parser.print_help() + print + prints(_('Error: You must specify the path to the cloned library')) + return 1 + db = get_db(dbpath, opts) + loc = args[0] + if not os.path.exists(loc): + os.makedirs(loc) + loc = os.path.abspath(loc) + + if patheq(loc, db.library_path): + prints(_('The location for the new library is the same as the current library')) + return 1 + empty = not os.listdir(loc) + if not empty: + prints(_('%s is not empty. You must choose an empty directory for the new library.') % loc) + return 1 + from calibre.db import get_db_loader + LibraryDatabase = get_db_loader()[0] + if iswindows and len(loc) > LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT: + prints(_('Path to library too long. Must be less than' + ' %d characters.')%LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT) + return 1 + dbprefs = dict(db.prefs) + db.close() + LibraryDatabase(loc, default_prefs=dbprefs) + COMMANDS = ('list', 'add', 'remove', 'add_format', 'remove_format', 'show_metadata', 'set_metadata', 'export', 'catalog', 'saved_searches', 'add_custom_column', 'custom_columns', 'remove_custom_column', 'set_custom', 'restore_database', - 'check_library', 'list_categories', 'backup_metadata') + 'check_library', 'list_categories', 'backup_metadata', + 'clone') def option_parser(): @@ -1432,3 +1476,4 @@ def main(args=sys.argv): if __name__ == '__main__': sys.exit(main()) + From 49c82ededa935326f84b9164ed7b5a0795eb6d0c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 28 Jul 2013 11:30:41 +0530 Subject: [PATCH 0364/1154] Add has_id() to db.data in the new backend Also speed up has_id() and all_book_ids() --- src/calibre/db/cache.py | 6 +++++- src/calibre/db/legacy.py | 4 ++-- src/calibre/db/view.py | 9 ++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index d5e6bb25aa..0be9f9f9ef 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -401,7 +401,7 @@ class Cache(object): ''' Frozen set of all known book ids. ''' - return type(self.fields['uuid']) + return type(self.fields['uuid'].table.book_col_map) @read_api def all_field_ids(self, name): @@ -1236,6 +1236,10 @@ class Cache(object): return True return False + @read_api + def has_id(self, book_id): + return book_id in self.fields['title'].table.book_col_map + @write_api def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None, apply_import_tags=True, preserve_uuid=False): if mi.tags: diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 3da725fe7e..bd595fb36f 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -143,7 +143,7 @@ class LibraryDatabase(object): self.data.cache.initialize_template_cache() def all_ids(self): - for book_id in self.data.cache.all_book_ids(): + for book_id in self.new_api.all_book_ids(): yield book_id def is_empty(self): @@ -510,7 +510,7 @@ class LibraryDatabase(object): self.new_api._remove_items('tags', tag_ids) def has_id(self, book_id): - return book_id in self.new_api.all_book_ids() + return self.new_api.has_id(book_id) def format(self, index, fmt, index_is_id=False, as_file=False, mode='r+b', as_path=False, preserve_filename=False): book_id = index if index_is_id else self.id(index) diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index f679fc2416..7468ec11b2 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -135,10 +135,13 @@ class View(object): return self.cache.field_metadata def _get_id(self, idx, index_is_id=True): - if index_is_id and idx not in self.cache.all_book_ids(): + if index_is_id and not self.cache.has_id(idx): raise IndexError('No book with id %s present'%idx) return idx if index_is_id else self.index_to_id(idx) + def has_id(self, book_id): + return self.cache.has_id(book_id) + def __getitem__(self, row): return TableRow(self._map_filtered[row], self) @@ -177,7 +180,7 @@ class View(object): def _get(self, field, idx, index_is_id=True, default_value=None, fmt=lambda x:x): id_ = idx if index_is_id else self.index_to_id(idx) - if index_is_id and id_ not in self.cache.all_book_ids(): + if index_is_id and not self.cache.has_id(id_): raise IndexError('No book with id %s present'%idx) return fmt(self.cache.field_for(field, id_, default_value=default_value)) @@ -323,7 +326,7 @@ class View(object): self.cache.clear_search_caches(old_marked_ids | set(self.marked_ids)) def refresh(self, field=None, ascending=True, clear_caches=True): - self._map = tuple(self.cache.all_book_ids()) + self._map = tuple(sorted(self.cache.all_book_ids())) self._map_filtered = tuple(self._map) if clear_caches: self.cache.clear_caches() From 4d90d171923a62dd1b2e4128d0af77c2af5e93c7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 28 Jul 2013 11:33:24 +0530 Subject: [PATCH 0365/1154] Fix test failures caused by removal of virtual fields from get_metadata --- src/calibre/db/tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index dd87ab1583..7a44897600 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -95,7 +95,7 @@ class BaseTest(unittest.TestCase): 'ondevice_col', 'last_modified', 'has_cover', 'cover_data'}.union(allfk1) for attr in all_keys: - if attr == 'user_metadata' or attr in exclude: + if attr in {'user_metadata', 'book_size', 'ondevice_col', 'db_approx_formats'} or attr in exclude: continue attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr) if attr == 'formats': From 0b032222865ef10a6982dfffb723bdd386177837 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 28 Jul 2013 16:41:55 +0530 Subject: [PATCH 0366/1154] ... --- manual/customize.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manual/customize.rst b/manual/customize.rst index 59475e91f2..5dde2b226d 100644 --- a/manual/customize.rst +++ b/manual/customize.rst @@ -34,6 +34,8 @@ Environment variables * ``SYSFS_PATH`` - Use if sysfs is mounted somewhere other than /sys * ``http_proxy`` - Used on linux to specify an HTTP proxy +See `How to set environment variables in windows `_. + Tweaks ------------ From 0356894fca6f2b680e1c39067960fdee8d47476b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 08:35:34 +0530 Subject: [PATCH 0367/1154] Fix apsw not included in linux binary builds --- setup/installer/linux/freeze2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index 6167109d97..184d7229e8 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -16,7 +16,7 @@ SITE_PACKAGES = ['PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize', 'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml', 'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so', 'dbus_bindings.py', '_dbus_glib_bindings.so', 'netifaces.so', '_psutil_posix.so', - '_psutil_linux.so', 'psutil', 'cssselect'] + '_psutil_linux.so', 'psutil', 'cssselect', 'apsw'] QTDIR = '/usr/lib/qt4' QTDLLS = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus') From 6cffe0d105843941c0f06a2b968cc08391bde623 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 08:46:49 +0530 Subject: [PATCH 0368/1154] newdb: Fix undefined custom dates not displaying as blank, depending on time zone --- src/calibre/gui2/library/delegates.py | 24 ++++++++++++------------ src/calibre/utils/date.py | 7 ++++++- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py index ae40352aee..c1b4248c26 100644 --- a/src/calibre/gui2/library/delegates.py +++ b/src/calibre/gui2/library/delegates.py @@ -15,7 +15,7 @@ from calibre.gui2 import UNDEFINED_QDATETIME, error_dialog, rating_font from calibre.constants import iswindows from calibre.gui2.widgets import EnLineEdit from calibre.gui2.complete2 import EditWithComplete -from calibre.utils.date import now, format_date, qt_to_dt +from calibre.utils.date import now, format_date, qt_to_dt, is_date_undefined from calibre.utils.config import tweaks from calibre.utils.formatter import validation_formatter from calibre.utils.icu import sort_key @@ -91,10 +91,10 @@ class DateDelegate(QStyledItemDelegate): # {{{ self.format = default_format def displayText(self, val, locale): - d = val.toDateTime() - if d <= UNDEFINED_QDATETIME: + d = qt_to_dt(val.toDateTime()) + if is_date_undefined(d): return '' - return format_date(qt_to_dt(d, as_utc=False), self.format) + return format_date(d, self.format) def createEditor(self, parent, option, index): return DateTimeEdit(parent, self.format) @@ -110,17 +110,17 @@ class PubDateDelegate(QStyledItemDelegate): # {{{ self.format = 'MMM yyyy' def displayText(self, val, locale): - d = val.toDateTime() - if d <= UNDEFINED_QDATETIME: + d = qt_to_dt(val.toDateTime()) + if is_date_undefined(d): return '' - return format_date(qt_to_dt(d, as_utc=False), self.format) + return format_date(d, self.format) def createEditor(self, parent, option, index): return DateTimeEdit(parent, self.format) def setEditorData(self, editor, index): val = index.data(Qt.EditRole).toDate() - if val == UNDEFINED_QDATETIME.date(): + if is_date_undefined(val): val = QDate(2000, 1, 1) editor.setDate(val) @@ -234,10 +234,10 @@ class CcDateDelegate(QStyledItemDelegate): # {{{ self.format = format def displayText(self, val, locale): - d = val.toDateTime() - if d <= UNDEFINED_QDATETIME: + d = qt_to_dt(val.toDateTime()) + if is_date_undefined(d): return '' - return format_date(qt_to_dt(d, as_utc=False), self.format) + return format_date(d, self.format) def createEditor(self, parent, option, index): return DateTimeEdit(parent, self.format) @@ -253,7 +253,7 @@ class CcDateDelegate(QStyledItemDelegate): # {{{ def setModelData(self, editor, model, index): val = editor.dateTime() - if val <= UNDEFINED_QDATETIME: + if is_date_undefined(val): val = None model.setData(index, QVariant(val), Qt.EditRole) diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index 0a8d779af8..dc8adc6b97 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -75,7 +75,12 @@ def is_date_undefined(qt_or_dt): if d is None: return True if hasattr(d, 'toString'): - d = datetime(d.year(), d.month(), d.day(), tzinfo=utc_tz) + if hasattr(d, 'date'): + d = d.date() + try: + d = datetime(d.year(), d.month(), d.day(), tzinfo=utc_tz) + except ValueError: + return True # Undefined QDate return d.year < UNDEFINED_DATE.year or ( d.year == UNDEFINED_DATE.year and d.month == UNDEFINED_DATE.month and From 2831d39da4fc94cb34decb9f2409689e78e64a19 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 12:51:16 +0530 Subject: [PATCH 0369/1154] newdb: Enable sorting on the marked field for the find duplicated plugin --- src/calibre/db/cache.py | 11 ++++++++--- src/calibre/db/tests/reading.py | 11 +++++++++++ src/calibre/db/view.py | 5 ++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 0be9f9f9ef..6bd7c1a3cf 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -781,7 +781,7 @@ class Cache(object): return ret @read_api - def multisort(self, fields, ids_to_sort=None): + def multisort(self, fields, ids_to_sort=None, virtual_fields=None): ''' Return a list of sorted book ids. If ids_to_sort is None, all book ids are returned. @@ -794,6 +794,7 @@ class Cache(object): else ids_to_sort) get_metadata = self._get_proxy_metadata lang_map = self.fields['languages'].book_value_map + virtual_fields = virtual_fields or {} fm = {'title':'sort', 'authors':'author_sort'} @@ -801,8 +802,12 @@ class Cache(object): 'Handle series type fields' idx = field + '_index' is_series = idx in self.fields - ans = self.fields[fm.get(field, field)].sort_keys_for_books( - get_metadata, lang_map, all_book_ids,) + try: + ans = self.fields[fm.get(field, field)].sort_keys_for_books( + get_metadata, lang_map, all_book_ids) + except KeyError: + ans = virtual_fields[fm.get(field, field)].sort_keys_for_books( + get_metadata, lang_map, all_book_ids) if is_series: idx_ans = self.fields[idx].sort_keys_for_books( get_metadata, lang_map, all_book_ids) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 2006348862..66b1dacec4 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -511,3 +511,14 @@ class ReadingTest(BaseTest): # }}} + def test_marked_field(self): # {{{ + ' Test the marked field ' + db = self.init_legacy() + db.set_marked_ids({3:1, 2:3}) + ids = [1,2,3] + db.multisort([('marked', True)], only_ids=ids) + self.assertListEqual([1, 3, 2], ids) + db.multisort([('marked', False)], only_ids=ids) + self.assertListEqual([2, 3, 1], ids) + # }}} + diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 7468ec11b2..0e8d4081e5 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -30,6 +30,9 @@ class MarkedVirtualField(object): for book_id in candidates: yield self.marked_ids.get(book_id, default_value), {book_id} + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): + return {bid:self.marked_ids.get(bid, None) for bid in all_book_ids} + class TableRow(object): def __init__(self, book_id, view): @@ -219,7 +222,7 @@ class View(object): if not fields: fields = [('timestamp', False)] - sorted_book_ids = self.cache.multisort(fields, ids_to_sort=only_ids) + sorted_book_ids = self.cache.multisort(fields, ids_to_sort=only_ids, virtual_fields={'marked':MarkedVirtualField(self.marked_ids)}) if only_ids is None: self._map = tuple(sorted_book_ids) if len(self._map_filtered) == len(self._map): From c07f612f6b1432ab84dd0eadecbcecb1fd6f8859 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 14:20:38 +0530 Subject: [PATCH 0370/1154] newdb: Do not cache searches that search on virtual fields like marked, since there is no safe way to update such a cache. --- src/calibre/db/search.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 759b76fdb8..d5a620ba58 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -481,12 +481,17 @@ class Parser(SearchQueryParser): # {{{ field = self.dbcache.fields[name] except KeyError: field = self.virtual_fields[name] + self.virtual_field_used = True return field.iter_searchable_values(get_metadata, candidates) def iter_searchable_values(self, *args, **kwargs): - for x in []: + for x in (): yield x, set() + def parse(self, *args, **kwargs): + self.virtual_field_used = False + return SearchQueryParser.parse(self, *args, **kwargs) + def get_matches(self, location, query, candidates=None, allow_recursion=True): # If candidates is not None, it must not be modified. Changing its @@ -852,6 +857,8 @@ class Search(object): sqp.dbcache = sqp.lookup_saved_search = None def _do_search(self, sqp, query, search_restriction, dbcache, book_ids=None): + ''' Do the search, caching the results. Results are cached only if the + search is on the full library and no virtual field is searched on ''' if isinstance(search_restriction, bytes): search_restriction = search_restriction.decode('utf-8') @@ -861,7 +868,7 @@ class Search(object): if cached is None: sqp.all_book_ids = all_book_ids if book_ids is None else book_ids restricted_ids = sqp.parse(search_restriction) - if sqp.all_book_ids is all_book_ids: + if not sqp.virtual_field_used and sqp.all_book_ids is all_book_ids: self.cache.add(search_restriction.strip(), restricted_ids) else: restricted_ids = cached @@ -884,7 +891,7 @@ class Search(object): sqp.all_book_ids = restricted_ids result = sqp.parse(query) - if sqp.all_book_ids is all_book_ids: + if not sqp.virtual_field_used and sqp.all_book_ids is all_book_ids: self.cache.add(query.strip(), result) return result From a68fe4409fcee3e189b8d1db1aa810c1a42da47f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 15:41:01 +0530 Subject: [PATCH 0371/1154] Do not search virtual fields on when searching 'all' Searching virtual fields (such as marked) ona prefix less search disables search caching. --- src/calibre/db/search.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index d5a620ba58..9eeacc24b5 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -619,7 +619,10 @@ class Parser(SearchQueryParser): # {{{ if x.startswith('@'): continue if fm['search_terms'] and x != 'series_sort': - all_locs.add(x) + if x not in self.virtual_fields: + # We dont search virtual fields because if we do, search + # caching will not be used + all_locs.add(x) field_metadata[x] = fm if fm['datatype'] in {'composite', 'text', 'comments', 'series', 'enumeration'}: text_fields.add(x) From 3490af8df7740172e3ca05fa01fa5fc11214c454 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 15:42:10 +0530 Subject: [PATCH 0372/1154] Ensure that undefined datetimes are always written as UNDEFINED_DATE to the db --- src/calibre/db/write.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py index 5b35248353..58c642e59c 100644 --- a/src/calibre/db/write.py +++ b/src/calibre/db/write.py @@ -13,8 +13,8 @@ from datetime import datetime from calibre.constants import preferred_encoding, ispy3 from calibre.ebooks.metadata import author_to_author_sort, title_sort -from calibre.utils.date import (parse_only_date, parse_date, UNDEFINED_DATE, - isoformat) +from calibre.utils.date import ( + parse_only_date, parse_date, UNDEFINED_DATE, isoformat, is_date_undefined) from calibre.utils.localization import canonicalize_lang from calibre.utils.icu import strcmp @@ -67,12 +67,14 @@ def multiple_text(sep, ui_sep, x): def adapt_datetime(x): if isinstance(x, (unicode, bytes)): x = parse_date(x, assume_utc=False, as_utc=False) + if x and is_date_undefined(x): + x = UNDEFINED_DATE return x def adapt_date(x): if isinstance(x, (unicode, bytes)): x = parse_only_date(x) - if x is None: + if x is None or is_date_undefined(x): x = UNDEFINED_DATE return x From fa4506a0197122b7ef908d8e10522091391778c5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 18:36:21 +0530 Subject: [PATCH 0373/1154] oops --- setup/installer/linux/freeze2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index 184d7229e8..deab85b6da 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -16,7 +16,7 @@ SITE_PACKAGES = ['PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize', 'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml', 'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so', 'dbus_bindings.py', '_dbus_glib_bindings.so', 'netifaces.so', '_psutil_posix.so', - '_psutil_linux.so', 'psutil', 'cssselect', 'apsw'] + '_psutil_linux.so', 'psutil', 'cssselect', 'apsw.so'] QTDIR = '/usr/lib/qt4' QTDLLS = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus') From 9c562b35c3956a8ba07d564ca7c59bce6c9bb4c2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 21:20:55 +0530 Subject: [PATCH 0374/1154] ... --- setup/installer/linux/freeze2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index deab85b6da..220f28ba82 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -191,8 +191,10 @@ class LinuxFreeze(Command): if os.path.isdir(x): shutil.copytree(x, self.j(dest, self.b(x)), ignore=ignore_in_lib) - if os.path.isfile(x) and ext in ('.py', '.so'): + elif os.path.isfile(x) and ext in ('.py', '.so'): shutil.copy2(x, dest) + else: + raise ValueError('%s does not exist in site-packages' % x) for x in os.listdir(self.SRC): shutil.copytree(self.j(self.SRC, x), self.j(dest, x), From 0fdbe5a753fad2d241057897809157a06caa2462 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 22:07:12 +0530 Subject: [PATCH 0375/1154] Device drivers: Explicitly fsync() all files when writing to devices --- src/calibre/__init__.py | 4 ++++ src/calibre/devices/android/driver.py | 3 +++ src/calibre/devices/cli.py | 3 ++- src/calibre/devices/cybook/driver.py | 2 ++ src/calibre/devices/hanvon/driver.py | 2 ++ src/calibre/devices/kindle/apnx.py | 3 ++- src/calibre/devices/kindle/driver.py | 3 ++- src/calibre/devices/kobo/driver.py | 12 +++++++----- src/calibre/devices/misc.py | 7 +++++-- src/calibre/devices/nook/driver.py | 2 ++ src/calibre/devices/prs505/driver.py | 2 ++ src/calibre/devices/prs505/sony_cache.py | 5 ++++- src/calibre/devices/prst1/driver.py | 2 ++ src/calibre/devices/udisks.py | 4 ++-- src/calibre/devices/usbms/cli.py | 6 ++++-- src/calibre/devices/usbms/driver.py | 5 ++++- 16 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 5d938ecc55..eaf5102aa1 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -710,3 +710,7 @@ def ipython(user_ns=None): from calibre.utils.ipython import ipython ipython(user_ns=user_ns) +def fsync(fileobj): + fileobj.flush() + os.fsync(fileobj.fileno()) + diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 055fd32f88..28f55751e2 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -8,6 +8,7 @@ import os import cStringIO +from calibre import fsync from calibre.devices.usbms.driver import USBMS HTC_BCDS = [0x100, 0x0222, 0x0224, 0x0226, 0x227, 0x228, 0x229, 0x0231, 0x9999] @@ -400,6 +401,7 @@ class WEBOS(USBMS): with open(os.path.join(path, 'coverCache', filename + '-medium.jpg'), 'wb') as coverfile: coverfile.write(coverdata) + fsync(coverfile) coverdata = getattr(metadata, 'thumbnail', None) if coverdata and coverdata[2]: @@ -423,6 +425,7 @@ class WEBOS(USBMS): with open(os.path.join(path, 'coverCache', filename + '-small.jpg'), 'wb') as coverfile: coverfile.write(coverdata) + fsync(coverfile) diff --git a/src/calibre/devices/cli.py b/src/calibre/devices/cli.py index e1a5375b6b..6defc27541 100755 --- a/src/calibre/devices/cli.py +++ b/src/calibre/devices/cli.py @@ -9,7 +9,7 @@ For usage information run the script. import StringIO, sys, time, os from optparse import OptionParser -from calibre import __version__, __appname__, human_readable +from calibre import __version__, __appname__, human_readable, fsync from calibre.devices.errors import PathError from calibre.devices.errors import ArgumentError, DeviceError, DeviceLocked from calibre.customize.ui import device_plugins @@ -293,6 +293,7 @@ def main(): parser.print_help() return 1 dev.get_file(path, outfile) + fsync(outfile) outfile.close() elif args[1].startswith("dev:"): try: diff --git a/src/calibre/devices/cybook/driver.py b/src/calibre/devices/cybook/driver.py index 0dd88263f0..64bd37b196 100644 --- a/src/calibre/devices/cybook/driver.py +++ b/src/calibre/devices/cybook/driver.py @@ -11,6 +11,7 @@ Device driver for Bookeen's Cybook Gen 3 and Opus and Orizon import os import re +from calibre import fsync from calibre.constants import isunix from calibre.devices.usbms.driver import USBMS import calibre.devices.cybook.t2b as t2b @@ -50,6 +51,7 @@ class CYBOOK(USBMS): coverdata = None with open('%s_6090.t2b' % os.path.join(path, filename), 'wb') as t2bfile: t2b.write_t2b(t2bfile, coverdata) + fsync(t2bfile) @classmethod def can_handle(cls, device_info, debug=False): diff --git a/src/calibre/devices/hanvon/driver.py b/src/calibre/devices/hanvon/driver.py index c967f2c54c..ea13e5a6d0 100644 --- a/src/calibre/devices/hanvon/driver.py +++ b/src/calibre/devices/hanvon/driver.py @@ -9,6 +9,7 @@ Device driver for Hanvon devices ''' import re, os +from calibre import fsync from calibre.devices.usbms.driver import USBMS def is_alex(device_info): @@ -123,6 +124,7 @@ class ALEX(N516): os.makedirs(cdir) with open(cpath, 'wb') as coverfile: coverfile.write(cover) + fsync(coverfile) def delete_books(self, paths, end_session=True): for i, path in enumerate(paths): diff --git a/src/calibre/devices/kindle/apnx.py b/src/calibre/devices/kindle/apnx.py index d6fec9ffd1..faad3c6cd7 100644 --- a/src/calibre/devices/kindle/apnx.py +++ b/src/calibre/devices/kindle/apnx.py @@ -14,7 +14,7 @@ from calibre.ebooks.mobi.reader.mobi6 import MobiReader from calibre.ebooks.pdb.header import PdbHeaderReader from calibre.ebooks.mobi.reader.headers import MetadataHeader from calibre.utils.logging import default_log -from calibre import prints +from calibre import prints, fsync from calibre.constants import DEBUG class APNXBuilder(object): @@ -80,6 +80,7 @@ class APNXBuilder(object): # Write the APNX. with open(apnx_path, 'wb') as apnxf: apnxf.write(apnx) + fsync(apnxf) def generate_apnx(self, pages, apnx_meta): apnx = '' diff --git a/src/calibre/devices/kindle/driver.py b/src/calibre/devices/kindle/driver.py index 461176b95c..4e083fbe04 100644 --- a/src/calibre/devices/kindle/driver.py +++ b/src/calibre/devices/kindle/driver.py @@ -12,7 +12,7 @@ import datetime, os, re, sys, json, hashlib from calibre.devices.kindle.bookmark import Bookmark from calibre.devices.usbms.driver import USBMS -from calibre import strftime +from calibre import strftime, fsync ''' Notes on collections: @@ -410,6 +410,7 @@ class KINDLE2(KINDLE): uuid=mh.exth.uuid, cdetype=mh.exth.cdetype)) with open(thumbfile, 'wb') as f: f.write(coverdata[2]) + fsync(f) def upload_apnx(self, path, filename, metadata, filepath): from calibre.devices.kindle.apnx import APNXBuilder diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 278798160e..feedab0127 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -23,7 +23,7 @@ from calibre.devices.kobo.books import Book from calibre.devices.kobo.books import ImageWrapper from calibre.devices.mime import mime_type_ext from calibre.devices.usbms.driver import USBMS, debug_print -from calibre import prints +from calibre import prints, fsync from calibre.ptempfile import PersistentTemporaryFile from calibre.constants import DEBUG from calibre.utils.config_base import prefs @@ -974,6 +974,7 @@ class KOBO(USBMS): with open(fpath, 'wb') as f: f.write(data) + fsync(f) else: debug_print("ImageID could not be retreived from the database") @@ -1621,7 +1622,7 @@ class KOBOTOUCH(KOBO): debug_print("KoboTouch:books - shelf list:", self.bookshelvelist) opts = self.settings() - + columns = 'Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ImageID, ReadStatus' if self.dbversion >= 16: columns += ', ___ExpirationStatus, FavouritesIndex, Accessibility' @@ -1635,7 +1636,7 @@ class KOBOTOUCH(KOBO): columns += ", Series, SeriesNumber, ___UserID, ExternalId" else: columns += ', null as Series, null as SeriesNumber, ___UserID, null as ExternalId' - + where_clause = '' if self.supports_kobo_archive(): where_clause = (" where BookID is Null " \ @@ -1670,13 +1671,13 @@ class KOBOTOUCH(KOBO): else: where_clause = ' where BookID is Null' - # Note: The card condition should not need the contentId test for the SD card. But the ExternalId does not get set for sideloaded kepubs on the SD card. + # Note: The card condition should not need the contentId test for the SD card. But the ExternalId does not get set for sideloaded kepubs on the SD card. card_condition = '' if self.has_externalid(): card_condition = " AND (externalId IS NOT NULL AND externalId <> '' OR contentId LIKE 'file:///mnt/sd/%')" if oncard == 'carda' else " AND (externalId IS NULL OR externalId = '') AND contentId NOT LIKE 'file:///mnt/sd/%'" else: card_condition = " AND contentId LIKE 'file:///mnt/sd/%'" if oncard == 'carda' else " AND contentId NOT LIKE'file:///mnt/sd/%'" - + query = 'SELECT ' + columns + ' FROM content ' + where_clause + card_condition debug_print("KoboTouch:books - query=", query) @@ -2283,6 +2284,7 @@ class KOBOTOUCH(KOBO): with open(fpath, 'wb') as f: f.write(data) + fsync(f) except Exception as e: err = str(e) debug_print("KoboTouch:_upload_cover - Exception string: %s"%err) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 13ae870fd1..3338a98eac 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -9,8 +9,7 @@ __docformat__ = 'restructuredtext en' import os from calibre.devices.usbms.driver import USBMS -from calibre import prints -prints +from calibre import fsync class PALMPRE(USBMS): @@ -100,6 +99,7 @@ class PDNOVEL(USBMS): if coverdata and coverdata[2]: with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile: coverfile.write(coverdata[2]) + fsync(coverfile) class PDNOVEL_KOBO(PDNOVEL): name = 'Pandigital Kobo device interface' @@ -118,6 +118,7 @@ class PDNOVEL_KOBO(PDNOVEL): os.makedirs(dirpath) with open(os.path.join(dirpath, filename+'.jpg'), 'wb') as coverfile: coverfile.write(coverdata[2]) + fsync(coverfile) class VELOCITYMICRO(USBMS): @@ -190,6 +191,7 @@ class LUMIREAD(USBMS): os.makedirs(pdir) with open(cfilepath+'.jpg', 'wb') as f: f.write(metadata.thumbnail[-1]) + fsync(f) class ALURATEK_COLOR(USBMS): @@ -334,6 +336,7 @@ class NEXTBOOK(USBMS): os.makedirs(thumbnail_dir) with open(os.path.join(thumbnail_dir, filename+'.jpg'), 'wb') as f: f.write(metadata.thumbnail[-1]) + fsync(f) ''' class MOOVYBOOK(USBMS): diff --git a/src/calibre/devices/nook/driver.py b/src/calibre/devices/nook/driver.py index 09924a8204..e24950359c 100644 --- a/src/calibre/devices/nook/driver.py +++ b/src/calibre/devices/nook/driver.py @@ -12,6 +12,7 @@ import os import cStringIO +from calibre import fsync from calibre.constants import isosx from calibre.devices.usbms.driver import USBMS @@ -76,6 +77,7 @@ class NOOK(USBMS): with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile: coverfile.write(coverdata) + fsync(coverfile) def sanitize_path_components(self, components): return [x.replace('#', '_') for x in components] diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index a1c7c78d18..10cb7df22c 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -8,6 +8,7 @@ Device driver for the SONY devices import os, time, re +from calibre import fsync from calibre.devices.usbms.driver import USBMS, debug_print from calibre.devices.prs505 import MEDIA_XML, MEDIA_EXT, CACHE_XML, CACHE_EXT, \ MEDIA_THUMBNAIL, CACHE_THUMBNAIL @@ -142,6 +143,7 @@ class PRS505(USBMS): '''.encode('utf8')) + fsync(f) return True except: import traceback diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index 979940229a..86a1b50582 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -9,7 +9,7 @@ import os, time from base64 import b64decode from datetime import date -from calibre import prints, guess_type, isbytestring +from calibre import prints, guess_type, isbytestring, fsync from calibre.devices.errors import DeviceError from calibre.devices.usbms.driver import debug_print from calibre.constants import DEBUG, preferred_encoding @@ -122,6 +122,7 @@ class XMLCache(object): try: with open(path, 'wb') as f: f.write(EMPTY_EXT_CACHE) + fsync(f) except: pass if os.access(path, os.W_OK): @@ -726,6 +727,7 @@ class XMLCache(object): '') with open(path, 'wb') as f: f.write(raw) + fsync(f) for i, path in self.ext_paths.items(): try: @@ -737,6 +739,7 @@ class XMLCache(object): '') with open(path, 'wb') as f: f.write(raw) + fsync(f) # }}} diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 9c76eb096f..d2482d51a0 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -15,6 +15,7 @@ import os, time, re from contextlib import closing from datetime import date +from calibre import fsync from calibre.devices.errors import DeviceError from calibre.devices.usbms.driver import USBMS, debug_print from calibre.devices.usbms.device import USBDevice @@ -761,6 +762,7 @@ class PRST1(USBMS): with open(thumbnail_file_path, 'wb') as f: f.write(book.thumbnail[-1]) + fsync(f) query = 'UPDATE books SET thumbnail = ? WHERE _id = ?' t = (thumbnail_path, book.bookId,) diff --git a/src/calibre/devices/udisks.py b/src/calibre/devices/udisks.py index b9ab4be498..8156fb227b 100644 --- a/src/calibre/devices/udisks.py +++ b/src/calibre/devices/udisks.py @@ -46,7 +46,7 @@ class UDisks(object): try: return unicode(d.FilesystemMount('', ['auth_no_user_interaction', 'rw', 'noexec', 'nosuid', - 'sync', 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()])) + 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()])) except: # May be already mounted, check mp = node_mountpoint(str(device_node_path)) @@ -123,7 +123,7 @@ class UDisks2(object): def mount(self, device_node_path): d = self.device(device_node_path) mount_options = ['rw', 'noexec', 'nosuid', - 'sync', 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()] + 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()] try: return unicode(d.Mount( { diff --git a/src/calibre/devices/usbms/cli.py b/src/calibre/devices/usbms/cli.py index 4ff9efef8b..6482a5b12b 100644 --- a/src/calibre/devices/usbms/cli.py +++ b/src/calibre/devices/usbms/cli.py @@ -6,6 +6,7 @@ __docformat__ = 'restructuredtext en' import os, shutil, time +from calibre import fsync from calibre.devices.errors import PathError from calibre.utils.filenames import case_preserving_open_file @@ -58,14 +59,15 @@ class CLI(object): dest.seek(0) dest.truncate() shutil.copyfileobj(infile, dest) + fsync(dest) #if not check_transfer(infile, dest): raise Exception('Transfer failed') if close: infile.close() return actual_path def munge_path(self, path): - if path.startswith('/') and not (path.startswith(self._main_prefix) or \ - (self._card_a_prefix and path.startswith(self._card_a_prefix)) or \ + if path.startswith('/') and not (path.startswith(self._main_prefix) or + (self._card_a_prefix and path.startswith(self._card_a_prefix)) or (self._card_b_prefix and path.startswith(self._card_b_prefix))): path = self._main_prefix + path[1:] elif path.startswith('carda:'): diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index f367e549e2..9dbb718fc0 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -14,7 +14,7 @@ import os, time, json, shutil from itertools import cycle from calibre.constants import numeric_version -from calibre import prints, isbytestring +from calibre import prints, isbytestring, fsync from calibre.constants import filesystem_encoding, DEBUG from calibre.devices.usbms.cli import CLI from calibre.devices.usbms.device import Device @@ -85,10 +85,12 @@ class USBMS(CLI, Device): location_code, name) with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: f.write(json.dumps(driveinfo, default=to_json)) + fsync(f) else: driveinfo = self._update_driveinfo_record({}, prefix, location_code, name) with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: f.write(json.dumps(driveinfo, default=to_json)) + fsync(f) return driveinfo def get_device_information(self, end_session=True): @@ -388,6 +390,7 @@ class USBMS(CLI, Device): os.makedirs(self.normalize_path(prefix)) with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f: json_codec.encode_to_file(f, booklists[listid]) + fsync(f) write_prefix(self._main_prefix, 0) write_prefix(self._card_a_prefix, 1) write_prefix(self._card_b_prefix, 2) From 99d9db82f7976ef5a03c1e99894109e256828ee9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 22:17:14 +0530 Subject: [PATCH 0376/1154] ... --- setup/hosting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/hosting.py b/setup/hosting.py index d97373cdbc..273da37706 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -507,7 +507,7 @@ def upload_to_servers(files, version): # {{{ # }}} def upload_to_dbs(files, version): # {{{ - print('Uploading to downloadbestsoftware.com') + print('Uploading to fosshub.com') server = 'www.downloadbestsoft-mirror1.com' rdir = 'release/' check_call(['ssh', 'kovid@%s' % server, 'rm -f release/*']) From e9e3114b230da9e85e7a05b2ecb443c76d305627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Tue, 30 Jul 2013 00:39:25 +0200 Subject: [PATCH 0377/1154] plugin for ebooki.allegro.pl --- src/calibre/customize/builtins.py | 12 +++ .../gui2/store/stores/allegro_plugin.py | 83 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 src/calibre/gui2/store/stores/allegro_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 8fce645c6e..ef18db862c 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1199,6 +1199,17 @@ plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions, #}}} # Store plugins {{{ +class StoreAllegroStore(StoreBase): + name = 'Ebooki Allegro' + author = u'Tomasz Długosz' + description = u'Platforma Grupy Allegro sprzedająca ebooki zabezpieczone znakiem wodnym.' + actual_plugin = 'calibre.gui2.store.stores.allegro_plugin:AllegroStore' + + drm_free_only = True + headquarters = 'PL' + formats = ['EPUB', 'MOBI', 'PDF'] + affiliate = True + class StoreAmazonKindleStore(StoreBase): name = 'Amazon Kindle' description = u'Kindle books from Amazon.' @@ -1675,6 +1686,7 @@ class XinXiiStore(StoreBase): formats = ['EPUB', 'PDF'] plugins += [ + StoreAllegroStore, StoreArchiveOrgStore, StoreAmazonKindleStore, StoreAmazonDEKindleStore, diff --git a/src/calibre/gui2/store/stores/allegro_plugin.py b/src/calibre/gui2/store/stores/allegro_plugin.py new file mode 100644 index 0000000000..f7944e082a --- /dev/null +++ b/src/calibre/gui2/store/stores/allegro_plugin.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +from __future__ import (division, absolute_import, print_function) +store_version = 1 # Needed for dynamic plugin loading + +__license__ = 'GPL 3' +__copyright__ = '2013, Tomasz Długosz ' +__docformat__ = 'restructuredtext en' + +import urllib +from base64 import b64encode +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class AllegroStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/34/58/' + + url = 'http://www.koobe.pl/' + + aff_url = aff_root + str(b64encode(url)) + + detail_url = None + if detail_item: + detail_url = aff_root + str(b64encode(detail_item)) + + if external or self.config.get('open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url if detail_url else aff_url) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + + br = browser() + page=1 + + counter = max_results + while counter: + with closing(br.open('http://ebooki.allegro.pl/szukaj?fraza=' + urllib.quote(query) + '&strona=' + str(page), timeout=timeout)) as f: + doc = html.fromstring(f.read().decode('utf-8')) + for data in doc.xpath('//div[@class="listing-list"]/div[@class="listing-list-item"]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//div[@class="listing-cover-wrapper"]/a/@href')) + if not id: + continue + + cover_url = ''.join(data.xpath('.//div[@class="listing-cover-wrapper"]/a/img/@src')) + title = ''.join(data.xpath('.//div[@class="listing-info"]/div[1]/a/text()')) + author = ', '.join(data.xpath('.//div[@class="listing-info"]/div[2]/a/text()')) + price = ''.join(data.xpath('.//div[@class="book-price"]/text()')) + formats = ', '.join(data.xpath('.//div[@class="listing-buy-formats"]//div[@class="devices-wrapper"]/span[@class="device-label"]/span/text()')) + + counter -= 1 + + s = SearchResult() + s.cover_url = 'http://ebooki.allegro.pl/' + cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.detail_item = 'http://ebooki.allegro.pl/' + id[1:] + s.formats = formats.upper() + s.drm = SearchResult.DRM_UNLOCKED + + yield s + if not doc.xpath('//a[@class="paging-arrow right-paging-arrow"]'): + break + page+=1 From f731f2803b935fec3a0a0163f414afe019023a27 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 30 Jul 2013 09:25:48 +0530 Subject: [PATCH 0378/1154] newdb: Fix sorting of view not stable --- src/calibre/db/cache.py | 5 +++-- src/calibre/db/view.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 6bd7c1a3cf..5c0a7fda59 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -792,6 +792,7 @@ class Cache(object): ''' all_book_ids = frozenset(self._all_book_ids() if ids_to_sort is None else ids_to_sort) + ids_to_sort = all_book_ids if ids_to_sort is None else ids_to_sort get_metadata = self._get_proxy_metadata lang_map = self.fields['languages'].book_value_map virtual_fields = virtual_fields or {} @@ -818,10 +819,10 @@ class Cache(object): if len(sort_keys) == 1: sk = sort_keys[0] - return sorted(all_book_ids, key=lambda i:sk[i], reverse=not + return sorted(ids_to_sort, key=lambda i:sk[i], reverse=not fields[0][1]) else: - return sorted(all_book_ids, key=partial(SortKey, fields, sort_keys)) + return sorted(ids_to_sort, key=partial(SortKey, fields, sort_keys)) @read_api def search(self, query, restriction='', virtual_fields=None, book_ids=None): diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 0e8d4081e5..2a578215ce 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -222,7 +222,9 @@ class View(object): if not fields: fields = [('timestamp', False)] - sorted_book_ids = self.cache.multisort(fields, ids_to_sort=only_ids, virtual_fields={'marked':MarkedVirtualField(self.marked_ids)}) + sorted_book_ids = self.cache.multisort( + fields, ids_to_sort=self._map if only_ids is None else only_ids, + virtual_fields={'marked':MarkedVirtualField(self.marked_ids)}) if only_ids is None: self._map = tuple(sorted_book_ids) if len(self._map_filtered) == len(self._map): From c6daf258be91951ee667ded57829341238d0563f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 30 Jul 2013 10:32:51 +0530 Subject: [PATCH 0379/1154] Factor out the code to update the GUI, so it can be re-used --- src/calibre/gui2/actions/edit_metadata.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 4817db953c..dcf86923dd 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -700,13 +700,7 @@ class EditMetadataAction(InterfaceAction): _('Failed to apply updated metadata for some books' ' in your library. Click "Show Details" to see ' 'details.'), det_msg='\n\n'.join(msg), show=True) - if self.applied_ids: - cr = self.gui.library_view.currentIndex().row() - self.gui.library_view.model().refresh_ids( - list(self.applied_ids), cr) - if self.gui.cover_flow: - self.gui.cover_flow.dataChanged() - self.gui.tags_view.recount() + self.refresh_gui(self.applied_ids) self.apply_id_map = [] self.apply_pd = None @@ -716,5 +710,15 @@ class EditMetadataAction(InterfaceAction): finally: self.apply_callback = None + def refresh_gui(self, book_ids, covers_changed=True, tag_browser_changed=True): + if book_ids: + cr = self.gui.library_view.currentIndex().row() + self.gui.library_view.model().refresh_ids( + list(book_ids), cr) + if covers_changed and self.gui.cover_flow: + self.gui.cover_flow.dataChanged() + if tag_browser_changed: + self.gui.tags_view.recount() + # }}} From 82d7bd3d571ce3464156938657564bb352c2a21c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 30 Jul 2013 10:56:04 +0530 Subject: [PATCH 0380/1154] newdb: Speed up set_metadata() by not committing the db after individual field writes --- src/calibre/db/cache.py | 77 +++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 5c0a7fda59..d311cbadde 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1084,11 +1084,6 @@ class Cache(object): else: raise - for field in ('rating', 'series_index', 'timestamp'): - val = getattr(mi, field) - if val is not None: - protected_set_field(field, val) - # force_changes has no effect on cover manipulation cdata = mi.cover_data[1] if cdata is None and isinstance(mi.cover, basestring) and mi.cover and os.access(mi.cover, os.R_OK): @@ -1099,42 +1094,48 @@ class Cache(object): if cdata is not None: self._set_cover({book_id: cdata}) - for field in ('author_sort', 'publisher', 'series', 'tags', 'comments', - 'languages', 'pubdate'): - val = mi.get(field, None) - if (force_changes and val is not None) or not mi.is_null(field): - protected_set_field(field, val) + with self.backend.conn: # Speed up set_metadata by not operating in autocommit mode + for field in ('rating', 'series_index', 'timestamp'): + val = getattr(mi, field) + if val is not None: + protected_set_field(field, val) - val = mi.get('title_sort', None) - if (force_changes and val is not None) or not mi.is_null('title_sort'): - protected_set_field('sort', val) + for field in ('author_sort', 'publisher', 'series', 'tags', 'comments', + 'languages', 'pubdate'): + val = mi.get(field, None) + if (force_changes and val is not None) or not mi.is_null(field): + protected_set_field(field, val) - # identifiers will always be replaced if force_changes is True - mi_idents = mi.get_identifiers() - if force_changes: - protected_set_field('identifiers', mi_idents) - elif mi_idents: - identifiers = self._field_for('identifiers', book_id, default_value={}) - for key, val in mi_idents.iteritems(): - if val and val.strip(): # Don't delete an existing identifier - identifiers[icu_lower(key)] = val - protected_set_field('identifiers', identifiers) + val = mi.get('title_sort', None) + if (force_changes and val is not None) or not mi.is_null('title_sort'): + protected_set_field('sort', val) - user_mi = mi.get_all_user_metadata(make_copy=False) - fm = self.field_metadata - for key in user_mi.iterkeys(): - if (key in fm and - user_mi[key]['datatype'] == fm[key]['datatype'] and - (user_mi[key]['datatype'] != 'text' or - user_mi[key]['is_multiple'] == fm[key]['is_multiple'])): - val = mi.get(key, None) - if force_changes or val is not None: - protected_set_field(key, val) - idx = key + '_index' - if idx in self.fields: - extra = mi.get_extra(key) - if extra is not None or force_changes: - protected_set_field(idx, extra) + # identifiers will always be replaced if force_changes is True + mi_idents = mi.get_identifiers() + if force_changes: + protected_set_field('identifiers', mi_idents) + elif mi_idents: + identifiers = self._field_for('identifiers', book_id, default_value={}) + for key, val in mi_idents.iteritems(): + if val and val.strip(): # Don't delete an existing identifier + identifiers[icu_lower(key)] = val + protected_set_field('identifiers', identifiers) + + user_mi = mi.get_all_user_metadata(make_copy=False) + fm = self.field_metadata + for key in user_mi.iterkeys(): + if (key in fm and + user_mi[key]['datatype'] == fm[key]['datatype'] and + (user_mi[key]['datatype'] != 'text' or + user_mi[key]['is_multiple'] == fm[key]['is_multiple'])): + val = mi.get(key, None) + if force_changes or val is not None: + protected_set_field(key, val) + idx = key + '_index' + if idx in self.fields: + extra = mi.get_extra(key) + if extra is not None or force_changes: + protected_set_field(idx, extra) @write_api def add_format(self, book_id, fmt, stream_or_path, replace=True, run_hooks=True, dbapi=None): From 2fd8022e9e7fecbcd2c60ca7ec04adf2eae2a68c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 30 Jul 2013 11:46:54 +0530 Subject: [PATCH 0381/1154] Ensure that an error in set_metadata does not cause the memory and disk data to diverge --- src/calibre/db/cache.py | 80 +++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index d311cbadde..4b334fb794 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1094,48 +1094,52 @@ class Cache(object): if cdata is not None: self._set_cover({book_id: cdata}) - with self.backend.conn: # Speed up set_metadata by not operating in autocommit mode - for field in ('rating', 'series_index', 'timestamp'): - val = getattr(mi, field) - if val is not None: - protected_set_field(field, val) + try: + with self.backend.conn: # Speed up set_metadata by not operating in autocommit mode + for field in ('rating', 'series_index', 'timestamp'): + val = getattr(mi, field) + if val is not None: + protected_set_field(field, val) - for field in ('author_sort', 'publisher', 'series', 'tags', 'comments', - 'languages', 'pubdate'): - val = mi.get(field, None) - if (force_changes and val is not None) or not mi.is_null(field): - protected_set_field(field, val) + for field in ('author_sort', 'publisher', 'series', 'tags', 'comments', + 'languages', 'pubdate'): + val = mi.get(field, None) + if (force_changes and val is not None) or not mi.is_null(field): + protected_set_field(field, val) - val = mi.get('title_sort', None) - if (force_changes and val is not None) or not mi.is_null('title_sort'): - protected_set_field('sort', val) + val = mi.get('title_sort', None) + if (force_changes and val is not None) or not mi.is_null('title_sort'): + protected_set_field('sort', val) - # identifiers will always be replaced if force_changes is True - mi_idents = mi.get_identifiers() - if force_changes: - protected_set_field('identifiers', mi_idents) - elif mi_idents: - identifiers = self._field_for('identifiers', book_id, default_value={}) - for key, val in mi_idents.iteritems(): - if val and val.strip(): # Don't delete an existing identifier - identifiers[icu_lower(key)] = val - protected_set_field('identifiers', identifiers) + # identifiers will always be replaced if force_changes is True + mi_idents = mi.get_identifiers() + if force_changes: + protected_set_field('identifiers', mi_idents) + elif mi_idents: + identifiers = self._field_for('identifiers', book_id, default_value={}) + for key, val in mi_idents.iteritems(): + if val and val.strip(): # Don't delete an existing identifier + identifiers[icu_lower(key)] = val + protected_set_field('identifiers', identifiers) - user_mi = mi.get_all_user_metadata(make_copy=False) - fm = self.field_metadata - for key in user_mi.iterkeys(): - if (key in fm and - user_mi[key]['datatype'] == fm[key]['datatype'] and - (user_mi[key]['datatype'] != 'text' or - user_mi[key]['is_multiple'] == fm[key]['is_multiple'])): - val = mi.get(key, None) - if force_changes or val is not None: - protected_set_field(key, val) - idx = key + '_index' - if idx in self.fields: - extra = mi.get_extra(key) - if extra is not None or force_changes: - protected_set_field(idx, extra) + user_mi = mi.get_all_user_metadata(make_copy=False) + fm = self.field_metadata + for key in user_mi.iterkeys(): + if (key in fm and + user_mi[key]['datatype'] == fm[key]['datatype'] and + (user_mi[key]['datatype'] != 'text' or + user_mi[key]['is_multiple'] == fm[key]['is_multiple'])): + val = mi.get(key, None) + if force_changes or val is not None: + protected_set_field(key, val) + idx = key + '_index' + if idx in self.fields: + extra = mi.get_extra(key) + if extra is not None or force_changes: + protected_set_field(idx, extra) + except: + self._reload_from_db() + raise @write_api def add_format(self, book_id, fmt, stream_or_path, replace=True, run_hooks=True, dbapi=None): From 47b927f5bc88cfb248cccb1e10e2d3b263cffb3d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 30 Jul 2013 11:52:51 +0530 Subject: [PATCH 0382/1154] Make ignore_errors also work for setting covers --- src/calibre/db/cache.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 4b334fb794..505342173f 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1085,14 +1085,18 @@ class Cache(object): raise # force_changes has no effect on cover manipulation - cdata = mi.cover_data[1] - if cdata is None and isinstance(mi.cover, basestring) and mi.cover and os.access(mi.cover, os.R_OK): - with lopen(mi.cover, 'rb') as f: - raw = f.read() - if raw: - cdata = raw - if cdata is not None: - self._set_cover({book_id: cdata}) + try: + cdata = mi.cover_data[1] + if cdata is None and isinstance(mi.cover, basestring) and mi.cover and os.access(mi.cover, os.R_OK): + with lopen(mi.cover, 'rb') as f: + cdata = f.read() or None + if cdata is not None: + self._set_cover({book_id: cdata}) + except: + if ignore_errors: + traceback.print_exc() + else: + raise try: with self.backend.conn: # Speed up set_metadata by not operating in autocommit mode From 4dbf49855a39060a4e52a396d2049806f435aa58 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 30 Jul 2013 11:55:58 +0530 Subject: [PATCH 0383/1154] ... --- src/calibre/db/cache.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 505342173f..8d19c7e9f8 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1142,6 +1142,9 @@ class Cache(object): if extra is not None or force_changes: protected_set_field(idx, extra) except: + # sqlite will rollback the entire transaction, thanks to the with + # statement, so we have to re-read everything form the db to ensure + # the db and Cache are in sync self._reload_from_db() raise From 11ab7672662555226d833529742f3b8ec25ceb14 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 30 Jul 2013 13:38:56 +0530 Subject: [PATCH 0384/1154] Fix drag 'n drop of cover onto conversion dialog not working --- src/calibre/gui2/convert/metadata.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/convert/metadata.py b/src/calibre/gui2/convert/metadata.py index 1d354a7881..f0ff1f8af7 100644 --- a/src/calibre/gui2/convert/metadata.py +++ b/src/calibre/gui2/convert/metadata.py @@ -61,6 +61,11 @@ class MetadataWidget(Widget, Ui_Form): self.initialize_options(get_option, get_help, db, book_id) self.connect(self.cover_button, SIGNAL("clicked()"), self.select_cover) self.comment.hide_toolbars() + self.cover.cover_changed.connect(self.change_cover) + + def change_cover(self, data): + self.cover_changed = True + self.cover_data = data def deduce_author_sort(self, *args): au = unicode(self.author.currentText()) @@ -68,7 +73,6 @@ class MetadataWidget(Widget, Ui_Form): authors = string_to_authors(au) self.author_sort.setText(self.db.author_sort_from_authors(authors)) - def initialize_metadata_options(self): self.initialize_combos() self.author.editTextChanged.connect(self.deduce_author_sort) @@ -210,7 +214,6 @@ class MetadataWidget(Widget, Ui_Form): bool(self.opt_prefer_metadata_cover.isChecked()), } - def commit(self, save_defaults=False): ''' Settings are stored in two attributes: `opf_file` and `cover_file`. @@ -242,3 +245,4 @@ class MetadataWidget(Widget, Ui_Form): self.cover_file = cf return recs + From e2db758aceb52dd12c7ec101f3799ea8923f345f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Tue, 30 Jul 2013 10:25:18 +0200 Subject: [PATCH 0385/1154] improve allegro plugin --- src/calibre/gui2/store/stores/allegro_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/stores/allegro_plugin.py b/src/calibre/gui2/store/stores/allegro_plugin.py index f7944e082a..61fc7dc409 100644 --- a/src/calibre/gui2/store/stores/allegro_plugin.py +++ b/src/calibre/gui2/store/stores/allegro_plugin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import (division, absolute_import, print_function) +from __future__ import (division, absolute_import, print_function, unicode_literals) store_version = 1 # Needed for dynamic plugin loading __license__ = 'GPL 3' @@ -27,7 +27,7 @@ class AllegroStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/34/58/' - url = 'http://www.koobe.pl/' + url = 'http://ebooki.allegro.pl/' aff_url = aff_root + str(b64encode(url)) From 60cf6c4c5de5f9d41b4aeee600d0f097ee92d895 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 30 Jul 2013 14:33:20 +0530 Subject: [PATCH 0386/1154] newdb: Fix book_id->row mapping broken when asearch is in effect --- src/calibre/db/tests/legacy.py | 7 +++++++ src/calibre/db/view.py | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 2d8bf466f6..c936367ba4 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -192,6 +192,7 @@ class LegacyTest(BaseTest): 'has_id':[(1,), (2,), (3,), (9999,)], 'id':[(1,), (2,), (0,),], 'index':[(1,), (2,), (3,), ], + 'row':[(1,), (2,), (3,), ], 'is_empty':[()], 'count':[()], 'all_author_names':[()], @@ -254,6 +255,12 @@ class LegacyTest(BaseTest): o = {type('')(k) if isinstance(k, bytes) else k:set(v) if isinstance(v, list) else v for k, v in o.iteritems()} n = {k:set(v) if isinstance(v, list) else v for k, v in n.iteritems()} self.assertEqual(o, n) + + ndb.search('title:Unknown') + db.search('title:Unknown') + self.assertEqual(db.row(3), ndb.row(3)) + self.assertRaises(ValueError, ndb.row, 2) + self.assertRaises(ValueError, db.row, 2) db.close() # }}} diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 2a578215ce..43243318d5 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -178,9 +178,13 @@ class View(object): return self._map_filtered[idx] def id_to_index(self, book_id): - return self._map.index(book_id) + return self._map_filtered.index(book_id) row = index_to_id + def index(self, book_id, cache=False): + x = self._map if cache else self._map_filtered + return x.index(book_id) + def _get(self, field, idx, index_is_id=True, default_value=None, fmt=lambda x:x): id_ = idx if index_is_id else self.index_to_id(idx) if index_is_id and not self.cache.has_id(id_): From 1d9a3dcb5f17e867568d35cc4b301d071e0a6a4c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 30 Jul 2013 17:30:27 +0530 Subject: [PATCH 0387/1154] newdb: Fix displaying the languages column Displaying the languages column for a book with no languages was causing an error. --- src/calibre/db/cache.py | 4 +++- src/calibre/db/fields.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 8d19c7e9f8..283773ac58 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -336,7 +336,7 @@ class Cache(object): except KeyError: return default_value if field.is_multiple: - default_value = {} if name == 'identifiers' else () + default_value = field.default_value try: return field.for_book(book_id, default_value=default_value) except (KeyError, IndexError): @@ -347,6 +347,8 @@ class Cache(object): ' Same as field_for, except that it avoids the extra lookup to get the field object ' if field_obj.is_composite: return field_obj.get_value_with_cache(book_id, self._get_proxy_metadata) + if field_obj.is_multiple: + default_value = field_obj.default_value try: return field_obj.for_book(book_id, default_value=default_value) except (KeyError, IndexError): diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 9becf82403..50fa658b29 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -48,6 +48,7 @@ class Field(object): self._sort_key = lambda x:sort_key(calibre_langcode_to_name(x)) self.is_multiple = (bool(self.metadata['is_multiple']) or self.name == 'formats') + self.default_value = {} if name == 'identifiers' else () if self.is_multiple else None self.category_formatter = type(u'') if dt == 'rating': self.category_formatter = lambda x:'\u2605'*int(x/2) From a6048a6ba74b7601fce9a32272a7315a445922c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Wed, 31 Jul 2013 00:45:25 +0200 Subject: [PATCH 0388/1154] cdp plugin --- src/calibre/customize/builtins.py | 11 +++ src/calibre/gui2/store/stores/cdp_plugin.py | 79 +++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 src/calibre/gui2/store/stores/cdp_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index ef18db862c..3e573be026 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1342,6 +1342,16 @@ class StoreBookotekaStore(StoreBase): headquarters = 'PL' formats = ['EPUB', 'PDF'] +class StoreCdpStore(StoreBase): + name = 'Cdp.pl' + author = u'Tomasz Długosz' + description = u'Ebooki w wielu formatach zabezpieczone znakiem wodnym RuneMark' + actual_plugin = 'calibre.gui2.store.stores.cdp_plugin:CdpStore' + + drm_free_only = True + headquarters = 'PL' + formats = ['EPUB', 'MOBI', 'PDF'] + class StoreChitankaStore(StoreBase): name = u'Моята библиотека' author = 'Alex Stanev' @@ -1700,6 +1710,7 @@ plugins += [ StoreBiblioStore, StoreBookotekaStore, StoreChitankaStore, + StoreCdpStore, StoreDieselEbooksStore, StoreEbookNLStore, StoreEbookpointStore, diff --git a/src/calibre/gui2/store/stores/cdp_plugin.py b/src/calibre/gui2/store/stores/cdp_plugin.py new file mode 100644 index 0000000000..51b8de6448 --- /dev/null +++ b/src/calibre/gui2/store/stores/cdp_plugin.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) +store_version = 1 # Needed for dynamic plugin loading + +__license__ = 'GPL 3' +__copyright__ = '2013, Tomasz Długosz ' +__docformat__ = 'restructuredtext en' + +import re +import urllib +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class CdpStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + + url = 'https://cdp.pl/ksiazki' + + if external or self.config.get('open_external', False): + open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_item if detail_item else url) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + page=1 + + br = browser() + + counter = max_results + + while counter: + with closing(br.open(u'https://cdp.pl/products/search?utf8=✓&keywords=' + urllib.quote_plus(query) + '&page=' + str(page), timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//ul[@id="products"]/li'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//div[@class="product-image"]/a[1]/@href')) + if not id: + continue + if 'ksiazki' not in id: + continue + + cover_url = ''.join(data.xpath('.//div[@class="product-image"]/a[1]/@data-background')) + cover_url = cover_url.split('\'')[1] + title = ''.join(data.xpath('.//div[@class="product-description"]/h2/a/text()')) + author = ''.join(data.xpath('.//div[@class="product-description"]//ul[@class="taxons"]/li[2]/a/text()')) + price = ''.join(data.xpath('.//span[@itemprop="price"]/text()')) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.detail_item = id.strip() + s.drm = SearchResult.DRM_UNLOCKED + #s.formats = formats.upper().strip() + + yield s + if not doc.xpath('//span[@class="next"]/a'): + break + page+=1 From 1c5cb3914106150eff2fe29d2969307c30d62490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Wed, 31 Jul 2013 00:58:30 +0200 Subject: [PATCH 0389/1154] detect formats in cdp --- src/calibre/gui2/store/stores/cdp_plugin.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/stores/cdp_plugin.py b/src/calibre/gui2/store/stores/cdp_plugin.py index 51b8de6448..738ee5e3d5 100644 --- a/src/calibre/gui2/store/stores/cdp_plugin.py +++ b/src/calibre/gui2/store/stores/cdp_plugin.py @@ -62,6 +62,11 @@ class CdpStore(BasicStoreConfig, StorePlugin): author = ''.join(data.xpath('.//div[@class="product-description"]//ul[@class="taxons"]/li[2]/a/text()')) price = ''.join(data.xpath('.//span[@itemprop="price"]/text()')) + with closing(br.open(id.strip(), timeout=timeout/4)) as nf: + idata = html.fromstring(nf.read()) + formats = ', '.join(idata.xpath('//div[@id="product-bonus"]/div/ul/li/text()')) + + counter -= 1 s = SearchResult() @@ -71,7 +76,7 @@ class CdpStore(BasicStoreConfig, StorePlugin): s.price = price s.detail_item = id.strip() s.drm = SearchResult.DRM_UNLOCKED - #s.formats = formats.upper().strip() + s.formats = formats.upper() yield s if not doc.xpath('//span[@class="next"]/a'): From 1e163cbe102411f28a4f42a73c644eefdb881201 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 31 Jul 2013 09:37:26 +0530 Subject: [PATCH 0390/1154] Fix crash when using %e with strftime on windows Fixes #1206705 [strftime crashes under Windows with %e](https://bugs.launchpad.net/calibre/+bug/1206705) --- src/calibre/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index eaf5102aa1..1383fae540 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -509,6 +509,7 @@ def strftime(fmt, t=None): if iswindows: if isinstance(fmt, unicode): fmt = fmt.encode('mbcs') + fmt = fmt.replace(b'%e', b'%#d') ans = plugins['winutil'][0].strftime(fmt, t) else: ans = time.strftime(fmt, t).decode(preferred_encoding, 'replace') From 929fa49d7a6b2195d1452a921f67667d57c480ad Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 31 Jul 2013 13:33:58 +0530 Subject: [PATCH 0391/1154] Move mirror domain name to fosshub.com --- setup/file_hosting_servers.rst | 2 +- setup/hosting.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst index 5b494cf066..ece966490c 100644 --- a/setup/file_hosting_servers.rst +++ b/setup/file_hosting_servers.rst @@ -46,7 +46,7 @@ Services SSH into sourceforge and downloadbestsoftware so that their host keys are stored. - ssh -oStrictHostKeyChecking=no kovid@www.downloadbestsoft-mirror1.com + ssh -oStrictHostKeyChecking=no kovid@mirror1.fosshub.com ssh -oStrictHostKeyChecking=no kovidgoyal,calibre@frs.sourceforge.net ssh -oStrictHostKeyChecking=no files.calibre-ebook.com (and whatever other mirrors are present) diff --git a/setup/hosting.py b/setup/hosting.py index 273da37706..7d16354bd4 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -508,7 +508,7 @@ def upload_to_servers(files, version): # {{{ def upload_to_dbs(files, version): # {{{ print('Uploading to fosshub.com') - server = 'www.downloadbestsoft-mirror1.com' + server = 'mirror1.fosshub.com' rdir = 'release/' check_call(['ssh', 'kovid@%s' % server, 'rm -f release/*']) for x in files: From 34f57e27ebe1f018bb554a60ad0e213a4e7f9ff5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 31 Jul 2013 13:57:47 +0530 Subject: [PATCH 0392/1154] Nicer interface to library metadata backups --- src/calibre/gui2/actions/choose_library.py | 65 +++++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index eb34aec6eb..f50aa95443 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -5,12 +5,12 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, posixpath +import os, posixpath, weakref from functools import partial from PyQt4.Qt import (QMenu, Qt, QInputDialog, QToolButton, QDialog, QDialogButtonBox, QGridLayout, QLabel, QLineEdit, QIcon, QSize, - QCoreApplication, pyqtSignal) + QCoreApplication, pyqtSignal, QVBoxLayout, QTimer) from calibre import isbytestring, sanitize_file_name_unicode from calibre.constants import (filesystem_encoding, iswindows, @@ -152,6 +152,52 @@ class MovedDialog(QDialog): # {{{ QDialog.accept(self) # }}} +class BackupStatus(QDialog): # {{{ + + def __init__(self, gui): + QDialog.__init__(self, gui) + self.l = l = QVBoxLayout(self) + self.msg = QLabel('') + self.msg.setWordWrap(True) + l.addWidget(self.msg) + self.bb = bb = QDialogButtonBox(QDialogButtonBox.Close) + bb.accepted.connect(self.accept) + bb.rejected.connect(self.reject) + b = bb.addButton(_('Queue &all books for backup'), bb.ActionRole) + b.clicked.connect(self.mark_all_dirty) + b.setIcon(QIcon(I('lt.png'))) + l.addWidget(bb) + self.db = weakref.ref(gui.current_db) + self.setResult(9) + self.setWindowTitle(_('Backup status')) + self.update() + self.resize(self.sizeHint()) + + def update(self): + db = self.db() + if db is None: + return + if self.result() != 9: + return + dirty_text = 'no' + try: + dirty_text = '%s' % db.dirty_queue_length() + except: + dirty_text = _('none') + self.msg.setText('

    ' + + _('Book metadata files remaining to be written: %s') % dirty_text) + QTimer.singleShot(1000, self.update) + + def mark_all_dirty(self): + db = self.db() + if hasattr(db, 'new_api'): + db.new_api.mark_as_dirty(db.new_api.all_book_ids()) + else: + db.dirtied(list(db.data.iterallids())) + +# }}} + + class ChooseLibraryAction(InterfaceAction): name = 'Choose Library' @@ -212,10 +258,6 @@ class ChooseLibraryAction(InterfaceAction): 'lt.png', None, None), attr='action_backup_status') ac.triggered.connect(self.backup_status, type=Qt.QueuedConnection) self.maintenance_menu.addAction(ac) - ac = self.create_action(spec=(_('Start backing up metadata of all books'), - 'lt.png', None, None), attr='action_backup_metadata') - ac.triggered.connect(self.mark_dirty, type=Qt.QueuedConnection) - self.maintenance_menu.addAction(ac) ac = self.create_action(spec=(_('Check library'), 'lt.png', None, None), attr='action_check_library') ac.triggered.connect(self.check_library, type=Qt.QueuedConnection) @@ -373,15 +415,8 @@ class ChooseLibraryAction(InterfaceAction): open_local_file(loc) def backup_status(self, location): - dirty_text = 'no' - try: - dirty_text = \ - unicode(self.gui.current_db.dirty_queue_length()) - except: - dirty_text = _('none') - info_dialog(self.gui, _('Backup status'), '

    '+ - _('Book metadata files remaining to be written: %s') % dirty_text, - show=True) + self.__backup_status_dialog = d = BackupStatus(self.gui) + d.show() def mark_dirty(self): db = self.gui.library_view.model().db From ca77e7561724083b3417f343b790fd00b6e0acdd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 31 Jul 2013 17:30:03 +0530 Subject: [PATCH 0393/1154] ... --- manual/faq.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/manual/faq.rst b/manual/faq.rst index e5a6342cf8..9809d4f22c 100644 --- a/manual/faq.rst +++ b/manual/faq.rst @@ -841,9 +841,10 @@ My antivirus program claims |app| is a virus/trojan? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The first thing to check is that you are downloading |app| from the official -website: ``_. |app| is a very popular program -and unscrupulous people try to setup websites offering it for download to fool -the unwary. +website: ``_. Make sure you are clicking the +download links on the left, not the advertisements on the right. |app| is a +very popular program and unscrupulous people try to setup websites offering it +for download to fool the unwary. If you have the official download and your antivirus program is still claiming |app| is a virus, then, your antivirus program is wrong. Antivirus programs use From 66fdd41e703ee33fb4a6727a5f7639fbe69681c5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 31 Jul 2013 17:32:07 +0530 Subject: [PATCH 0394/1154] ... --- src/calibre/library/cli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 37a690632d..325fa6cefe 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -1392,8 +1392,8 @@ def command_list_categories(args, dbpath): return parser -def command_clone(args, dbpath): - parser = get_parser(_( +def clone_option_parser(): + return get_parser(_( '''\ %prog clone path/to/new/library @@ -1403,6 +1403,9 @@ same custom columns, virtual libraries and other settings as the current library The cloned library will contain no books. If you want to create a full duplicate, including all books, then simply use your filesystem tools to copy the library folder. ''')) + +def command_clone(args, dbpath): + parser = clone_option_parser() opts, args = parser.parse_args(args) if len(args) < 1: parser.print_help() From c075b6b401867b769976263919d368724dec1459 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 31 Jul 2013 17:40:52 +0530 Subject: [PATCH 0395/1154] ... --- manual/custom.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manual/custom.py b/manual/custom.py index 30ca28ec96..bffe3d914b 100644 --- a/manual/custom.py +++ b/manual/custom.py @@ -83,7 +83,6 @@ def generate_calibredb_help(preamble, info): global_options = '\n'.join(render_options('calibredb', groups, False, False)) - lines, toc = [], [] for cmd in COMMANDS: args = [] @@ -99,7 +98,7 @@ def generate_calibredb_help(preamble, info): usage = [i for i in usage.replace('%prog', 'calibredb').splitlines()] cmdline = ' '+usage[0] usage = usage[1:] - usage = [i.replace(cmd, ':command:`%s`'%cmd) for i in usage] + usage = [re.sub(r'(%s)([^a-zA-Z0-9])'%cmd, r':command:`\1`\2', i) for i in usage] lines += ['.. code-block:: none', '', cmdline, ''] lines += usage groups = [(None, None, parser.option_list)] @@ -152,7 +151,6 @@ def generate_ebook_convert_help(preamble, info): prog = 'ebook-convert-'+(pl.name.lower().replace(' ', '-')) raw += '\n\n' + '\n'.join(render_options(prog, groups, False, True)) - update_cli_doc(os.path.join('cli', 'ebook-convert.rst'), raw, info) def update_cli_doc(path, raw, info): @@ -200,7 +198,8 @@ def cli_docs(app): for script in entry_points['console_scripts'] + entry_points['gui_scripts']: module = script[script.index('=')+1:script.index(':')].strip() cmd = script[:script.index('=')].strip() - if cmd in ('calibre-complete', 'calibre-parallel'): continue + if cmd in ('calibre-complete', 'calibre-parallel'): + continue module = __import__(module, fromlist=[module.split('.')[-1]]) if hasattr(module, 'option_parser'): documented_cmds.append((cmd, getattr(module, 'option_parser')())) @@ -260,3 +259,4 @@ def setup(app): def finished(app, exception): pass + From 8713ebb0b17c2b8ac6d1cfad9dcc1f4796c1bcbe Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 31 Jul 2013 18:00:10 +0530 Subject: [PATCH 0396/1154] newdb: Fix duplicates during adding creating blank book record Fixes #1206830 [Problem when adding duplicate title](https://bugs.launchpad.net/calibre/+bug/1206830) --- src/calibre/db/legacy.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index bd595fb36f..a97e107c0e 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -202,7 +202,8 @@ class LibraryDatabase(object): # Adding books {{{ def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None): ret = self.new_api.create_book_entry(mi, cover=cover, add_duplicates=add_duplicates, force_id=force_id) - self.data.books_added((ret,)) + if ret is not None: + self.data.books_added((ret,)) return ret def add_books(self, paths, formats, metadata, add_duplicates=True, return_ids=False): @@ -217,7 +218,8 @@ class LibraryDatabase(object): paths.append(path) duplicates = (paths, formats, metadata) ids = book_ids if return_ids else len(book_ids) - self.data.books_added(book_ids) + if book_ids: + self.data.books_added(book_ids) return duplicates or None, ids def import_book(self, mi, formats, notify=True, import_hooks=True, apply_import_tags=True, preserve_uuid=False): @@ -229,7 +231,8 @@ class LibraryDatabase(object): format_map[ext] = path book_ids, duplicates = self.new_api.add_books( [(mi, format_map)], add_duplicates=True, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid, dbapi=self, run_hooks=import_hooks) - self.data.books_added(book_ids) + if book_ids: + self.data.books_added(book_ids) if notify: self.notify('add', book_ids) return book_ids[0] @@ -250,12 +253,14 @@ class LibraryDatabase(object): def add_catalog(self, path, title): book_id = add_catalog(self.new_api, path, title) - self.data.books_added((book_id,)) + if book_id is not None: + self.data.books_added((book_id,)) return book_id def add_news(self, path, arg): book_id = add_news(self.new_api, path, arg) - self.data.books_added((book_id,)) + if book_id is not None: + self.data.books_added((book_id,)) return book_id def add_format(self, index, fmt, stream, index_is_id=False, path=None, notify=True, replace=True, copy_function=None): From bd456d1b860f02c312570586053c9f119997ee91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Wed, 31 Jul 2013 16:53:27 +0200 Subject: [PATCH 0397/1154] add unicode_literals in koobe plugin --- src/calibre/gui2/store/stores/koobe_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/stores/koobe_plugin.py b/src/calibre/gui2/store/stores/koobe_plugin.py index 52bdd41e54..b8b7386593 100644 --- a/src/calibre/gui2/store/stores/koobe_plugin.py +++ b/src/calibre/gui2/store/stores/koobe_plugin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import (division, absolute_import, print_function) +from __future__ import (unicode_literals, division, absolute_import, print_function) store_version = 3 # Needed for dynamic plugin loading __license__ = 'GPL 3' From ccb3eff05b65d4ee5680ee08a4b7b01f4340b50e Mon Sep 17 00:00:00 2001 From: Adam Victor Nazareth Brandizzi Date: Wed, 31 Jul 2013 22:18:40 -0300 Subject: [PATCH 0398/1154] Fix the issue of renaming user categories to the same name it already has resulting in deletion of the category (see https://bugs.launchpad.net/calibre/+bug/1207131) --- src/calibre/gui2/tag_browser/model.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/tag_browser/model.py b/src/calibre/gui2/tag_browser/model.py index 1b291930fd..ea0e5f5f68 100644 --- a/src/calibre/gui2/tag_browser/model.py +++ b/src/calibre/gui2/tag_browser/model.py @@ -971,6 +971,10 @@ class TagsModel(QAbstractItemModel): # {{{ else: nkey = ckey[:dotpos+1] + val nkey_lower = icu_lower(nkey) + + if ckey == nkey: + return True + for c in sorted(user_cats.keys(), key=sort_key): if icu_lower(c).startswith(ckey_lower): if len(c) == len(ckey): From e2c3bb40a51a3f11137a047fbfe0d2d2edbb81e4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 07:41:49 +0530 Subject: [PATCH 0399/1154] newdb: Fix default value for identifiers is mutable --- src/calibre/db/fields.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 50fa658b29..e47e366c9b 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -410,7 +410,10 @@ class IdentifiersField(ManyToManyField): def for_book(self, book_id, default_value=None): ids = self.table.book_col_map.get(book_id, ()) if not ids: - ids = default_value + try: + ids = default_value.copy() # in case default_value is a mutable dict + except AttributeError: + ids = default_value return ids def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): From d62006bc11e16235170e37ff5bfa139e22ec1f92 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 08:23:06 +0530 Subject: [PATCH 0400/1154] QDateTime->datetime faster and more robust Fix editing of book metadata failing when its timestamp is out of range for the system. Fixes #1191599 [Cannot edit simple ebook metadatas](https://bugs.launchpad.net/calibre/+bug/1191599) --- src/calibre/utils/date.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index dc8adc6b97..a0c1c77a7a 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -7,7 +7,7 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import re, time -from datetime import datetime, time as dtime, timedelta +from datetime import datetime, time as dtime, timedelta, MINYEAR, MAXYEAR from functools import partial from dateutil.tz import tzlocal, tzutc, EPOCHORDINAL @@ -141,14 +141,26 @@ def dt_factory(time_t, assume_utc=False, as_utc=True): dt = dt.replace(tzinfo=_utc_tz if assume_utc else _local_tz) return dt.astimezone(_utc_tz if as_utc else _local_tz) +safeyear = lambda x: min(max(x, MINYEAR), MAXYEAR) + def qt_to_dt(qdate_or_qdatetime, as_utc=True): - from PyQt4.Qt import Qt o = qdate_or_qdatetime if hasattr(o, 'toUTC'): # QDateTime - o = unicode(o.toUTC().toString(Qt.ISODate)) - return parse_date(o, assume_utc=True, as_utc=as_utc) - dt = datetime(o.year(), o.month(), o.day()).replace(tzinfo=_local_tz) + o = o.toUTC() + d, t = o.date(), o.time() + try: + ans = datetime(safeyear(d.year()), d.month(), d.day(), t.hour(), t.minute(), t.second(), t.msec()*1000, utc_tz) + except ValueError: + ans = datetime(safeyear(d.year()), d.month(), 1, t.hour(), t.minute(), t.second(), t.msec()*1000, utc_tz) + if not as_utc: + ans = ans.astimezone(local_tz) + return ans + + try: + dt = datetime(safeyear(o.year()), o.month(), o.day()).replace(tzinfo=_local_tz) + except ValueError: + dt = datetime(safeyear(o.year()), o.month(), 1).replace(tzinfo=_local_tz) return dt.astimezone(_utc_tz if as_utc else _local_tz) def fromtimestamp(ctime, as_utc=True): From 49eb89a59aacc719636ddd945e844e63e481c218 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 08:27:30 +0530 Subject: [PATCH 0401/1154] Update PC World --- recipes/pc_world.recipe | 49 ++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/recipes/pc_world.recipe b/recipes/pc_world.recipe index 9b40b2e94b..714c205251 100644 --- a/recipes/pc_world.recipe +++ b/recipes/pc_world.recipe @@ -41,7 +41,7 @@ class pcWorld(BasicNewsRecipe): br = self.get_browser() br.open(url+'&print') - response = br.follow_link(url, nr = 0) + response = br.follow_link(url, nr=0) html = response.read() self.temp_files.append(PersistentTemporaryFile('_fa.html')) @@ -49,10 +49,10 @@ class pcWorld(BasicNewsRecipe): self.temp_files[-1].close() return self.temp_files[-1].name - #keep_only_tags = [ + # keep_only_tags = [ #dict(name='div', attrs={'class':'article'}) #] - #remove_tags = [ + # remove_tags = [ #dict(name='div', attrs={'class':['toolBar','mac_tags','toolBar btmTools','recommend longRecommend','recommend shortRecommend','textAds']}), #dict(name='div', attrs={'id':['sidebar','comments','mac_tags']}), #dict(name='ul', attrs={'class':['tools', 'tools clearfix']}), @@ -60,28 +60,26 @@ class pcWorld(BasicNewsRecipe): #dict(name='p', attrs={'id':'userDesire'}) #] feeds = [ - (u'PCWorld Headlines', u'http://feeds.pcworld.com/pcworld/latestnews'), - (u'How-To', u'http://feeds.pcworld.com/pcworld/update/howto'), - (u'Today@PCWorld', u'http://feeds.pcworld.com/pcworld/blogs/todayatpcw'), - (u'Reviews', u'http://feeds.pcworld.com/pcworld/update/reviews'), - (u'Most Popular Downloads', u'http://feeds.pcworld.com/pcworld/downloads/monthly'), - (u'Answer Lines', u'http://feeds.pcworld.com/pcworld/blogs/answer_line'), - (u'Digital Focus', u'http://feeds.pcworld.com/pcworld/blogs/digital_focus'), - (u'Download this', u'http://feeds.pcworld.com/pcworld/blogs/download_this/'), - (u'Game on', u'http://feeds.pcworld.com/pcworld/blogs/game_on'), - (u'Geek tech', u'http://feeds.pcworld.com/pcworld/blogs/geektech/'), - (u'Hassle free PC', u'http://feeds.pcworld.com/pcworld/blogs/hassle-free_pc'), - (u'Mobile computing', u'http://feeds.pcworld.com/pcworld/blogs/mobile_computing'), - (u'Security alert', u'http://feeds.pcworld.com/pcworld/blogs/security_alert/'), - (u'BizFeed', u'http://feeds.pcworld.com/pcworld/businesscenter/bizfeed/'), - (u'The Cost Cutter', u'http://feeds.pcworld.com/pcworld/businesscenter/cost_cutter/'), - (u'Linux line', u'http://feeds.pcworld.com/pcworld/businesscenter/linuxline/'), - (u'Net Work', u'http://feeds.pcworld.com/pcworld/businesscenter/network/'), - (u'Peer-to-Peer', u'http://feeds.pcworld.com/pcworld/businesscenter/peertopeer/'), - (u'Tech inciter', u'http://feeds.pcworld.com/pcworld/businesscenter/tech_inciter/'), - (u'Gadgets and gear', u'http://feeds.pcworld.com/pcworld/update/gadgets'), - (u'Home Entertainment', u'http://feeds.pcworld.com/pcworld/update/home-entertainment'), - (u'Mobile Devices', u'http://feeds.pcworld.com/pcworld/update/mobile-devices') + (u'All Stories', u'http://www.pcworld.com/index.rss'), + (u'Reviews', u'http://www.pcworld.com/reviews/index.rss'), + (u'How-To', u'http://www.pcworld.com/howto/index.rss'), + (u'Video', u'http://www.pcworld.com/video/index.rss'), + (u'Game On', u'http://www.pcworld.com/column/game-on/index.rss'), + (u'Hassle free PC', u'http://www.pcworld.com/column/hassle-free-pc/index.rss'), + (u'Go Social', u'http://www.pcworld.com/column/go-social/index.rss'), + (u'Linux Line', u'http://www.pcworld.com/column/linux-line/index.rss'), + (u'Net Work', u'http://www.pcworld.com/column/net-work/index.rss'), + (u'Security Alert', u'http://www.pcworld.com/column/security-alert/index.rss'), + (u'Simply Business', u'http://www.pcworld.com/column/simply-business/index.rss'), + (u'Business', u'http://www.pcworld.com/category/business/index.rss'), + (u'Security & Privacy', u'http://www.pcworld.com/category/privacy/index.rss'), + (u'Windows', u'http://www.pcworld.com/category/windows/index.rss'), + (u'Laptops', u'http://www.pcworld.com/category/laptop-computers/index.rss'), + (u'Software', u'http://www.pcworld.com/category/software/index.rss'), + (u'Desktops', u'http://www.pcworld.com/category/desktop-computers/index.rss'), + (u'Printers', u'http://www.pcworld.com/category/printers/index.rss'), + (u'Phones', u'http://www.pcworld.com/category/phones/index.rss'), + (u'Tablets', u'http://www.pcworld.com/category/tablets/index.rss') ] extra_css = ''' @@ -103,3 +101,4 @@ class pcWorld(BasicNewsRecipe): #articleHead p {font-size:15px;font-weight:bold;margin:0px;padding:0px;} #articleHead .date {color:#999;margin:0px 0px 20px;padding:0px;} ''' + From 20f414a7f51e7a7c402e5bf53b2f01a9d5fe8afc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 09:10:40 +0530 Subject: [PATCH 0402/1154] Avoid unnecessary tracebacks when backing up metadata for deleted books --- src/calibre/db/backup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/db/backup.py b/src/calibre/db/backup.py index 1a6ff13412..f0f2b07a54 100644 --- a/src/calibre/db/backup.py +++ b/src/calibre/db/backup.py @@ -81,6 +81,7 @@ class MetadataBackup(Thread): if mi is None: self.db.clear_dirtied(book_id, sequence) + return # Give the GUI thread a chance to do something. Python threads don't # have priorities, so this thread would naturally keep the processor From fda8ce7abb0a007b2b53be2259fc89ea78e36e8d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 09:31:02 +0530 Subject: [PATCH 0403/1154] newdb: get_metadata() should have sorted formats list in mi.formats --- src/calibre/db/cache.py | 2 +- src/calibre/db/lazy.py | 2 +- src/calibre/db/tests/reading.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 283773ac58..28058d7e8a 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -223,7 +223,7 @@ class Cache(object): good_formats = None else: mi.format_metadata = FormatMetadata(self, book_id, formats) - good_formats = FormatsList(formats, mi.format_metadata) + good_formats = FormatsList(sorted(formats), mi.format_metadata) # These three attributes are returned by the db2 get_metadata(), # however, we dont actually use them anywhere other than templates, so # they have been removed, to avoid unnecessary overhead. The templates diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index 166627c438..73260c8ae7 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -175,7 +175,7 @@ def fmt_getter(field): if m: format_metadata[fmt] = m if field == 'formats': - return list(format_metadata) or None + return sorted(format_metadata) or None return format_metadata return func diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 66b1dacec4..56866f5aa7 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -478,7 +478,7 @@ class ReadingTest(BaseTest): for field in STANDARD_METADATA_FIELDS | {'#series_index'}: f = lambda x: x if field == 'formats': - f = lambda x: x if x is None else set(x) + f = lambda x: x if x is None else tuple(x) self.assertEqual(f(getattr(mi, field)), f(getattr(pmi, field)), 'Standard field: %s not the same for book %s' % (field, book_id)) self.assertEqual(mi.format_field(field), pmi.format_field(field), From d38dffa86d5a4d3b2268be61b733db24aa778f80 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 10:35:31 +0530 Subject: [PATCH 0404/1154] Download tags from Amazon When downloading metadata from Amazon, convert the amazon categories into tags. You can turn this off by going to Preferences->Metadata download and configuring the Amazon source. Fixes #1206763 [[Enhancement] Get tags from Amazon](https://bugs.launchpad.net/calibre/+bug/1206763) --- src/calibre/ebooks/metadata/sources/amazon.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index 028bad6922..bc9b67c219 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -162,6 +162,18 @@ class Worker(Thread): # Get details {{{ ''' self.language_names = {'Language', 'Sprache', 'Lingua', 'Idioma', 'Langue', '言語'} + self.tags_xpath = ''' + descendant::h2[ + text() = "Look for Similar Items by Category" or + text() = "Ähnliche Artikel finden" or + text() = "Buscar productos similares por categoría" or + text() = "Ricerca articoli simili per categoria" or + text() = "Rechercher des articles similaires par rubrique" or + text() = "Procure por itens similares por categoria" or + text() = "関連商品を探す" + ]/../descendant::ul/li + ''' + self.ratings_pat = re.compile( r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち|de un máximo de|de) ([\d\.]+)( (stars|Sternen|stelle|estrellas|estrelas)){0,1}') @@ -312,6 +324,11 @@ class Worker(Thread): # Get details {{{ except: self.log.exception('Error parsing series for url: %r'%self.url) + try: + mi.tags = self.parse_tags(root) + except: + self.log.exception('Error parsing tags for url: %r'%self.url) + try: self.cover_url = self.parse_cover(root, raw) except: @@ -490,6 +507,23 @@ class Worker(Thread): # Get details {{{ ans = (s, i) return ans + def parse_tags(self, root): + ans = [] + exclude_tokens = {'kindle', 'a-z'} + exclude = {'special features', 'by authors', 'authors & illustrators', 'books', 'new; used & rental textbooks'} + seen = set() + for li in root.xpath(self.tags_xpath): + for i, a in enumerate(li.iterdescendants('a')): + if i > 0: + # we ignore the first category since it is almost always too broad + raw = (a.text or '').strip().replace(',', ';') + lraw = icu_lower(raw) + tokens = frozenset(lraw.split()) + if raw and lraw not in exclude and not tokens.intersection(exclude_tokens) and lraw not in seen: + ans.append(raw) + seen.add(lraw) + return ans + def parse_cover(self, root, raw=b""): imgs = root.xpath('//img[(@id="prodImage" or @id="original-main-image" or @id="main-image") and @src]') if not imgs: @@ -588,7 +622,7 @@ class Amazon(Source): capabilities = frozenset(['identify', 'cover']) touched_fields = frozenset(['title', 'authors', 'identifier:amazon', 'identifier:isbn', 'rating', 'comments', 'publisher', 'pubdate', - 'languages', 'series']) + 'languages', 'series', 'tags']) has_html_comments = True supports_gzip_transfer_encoding = True From 7cbad00c25bd26ecbc9170222889a21baeba68db Mon Sep 17 00:00:00 2001 From: David Forrester Date: Thu, 1 Aug 2013 12:25:51 +1000 Subject: [PATCH 0405/1154] Append and modify CSS when sending epubs to Kobo devices This allows users to have a "kobo_extra.css" file in the root of their device containing CSS rules. This will be appended to all stylesheets in the epub. As well, if the extra rules contain an @page rule, any existing @page rules will be stripped from the stylesheets. Finally, if any of the extra rules include "widows" and "orphans" settings, these are stripped from the rules in the stylesheets. --- src/calibre/devices/kobo/driver.py | 123 +++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 5 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index feedab0127..89b6b39549 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -28,6 +28,15 @@ from calibre.ptempfile import PersistentTemporaryFile from calibre.constants import DEBUG from calibre.utils.config_base import prefs +EPUB_EXT = '.epub' + + +class DummyCSSPreProcessor(object): + + def __call__(self, data, add_namespace=False): + + return data + class KOBO(USBMS): @@ -35,7 +44,7 @@ class KOBO(USBMS): gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') author = 'Timothy Legge and David Forrester' - version = (2, 0, 13) + version = (2, 1, 0) dbversion = 0 fwversion = 0 @@ -1228,6 +1237,7 @@ class KOBOTOUCH(KOBO): book_class = Book MAX_PATH_LEN = 185 # 250 - (len(" - N3_LIBRARY_SHELF.parsed") + len("F:\.kobo\images\")) + KOBO_EXTRA_CSSFILE = 'kobo_extra.css' EXTRA_CUSTOMIZATION_MESSAGE = [ _('The Kobo Touch from firmware V2.0.0 supports bookshelves.')+\ @@ -1259,6 +1269,11 @@ class KOBOTOUCH(KOBO): 'This is not read by the device from the sideloaded books. ' 'Series information can only be added to the device after the book has been processed by the device. ' 'Enable if you wish to set series information.'), + _('Modify CSS') + + ':::'+_('This allows addition of user CSS rules and removal of some CSS. ' + 'When sending a book, the driver adds the contents of ' + KOBO_EXTRA_CSSFILE + ' to all stylesheets in the ePub. ' + 'This file is searched for in the root directory of the main memory of the device. ' + 'As well as this, if the file contains settings for the "orphans" or "widows", these are removed for all styles in the original stylesheet.'), _('Attempt to support newer firmware') + ':::'+_('Kobo routinely updates the firmware and the ' 'database version. With this option Calibre will attempt ' @@ -1284,6 +1299,7 @@ class KOBOTOUCH(KOBO): False, False, False, + False, u'' ] @@ -1297,8 +1313,9 @@ class KOBOTOUCH(KOBO): OPT_SHOW_PREVIEWS = 7 OPT_SHOW_RECOMMENDATIONS = 8 OPT_UPDATE_SERIES_DETAILS = 9 - OPT_SUPPORT_NEWER_FIRMWARE = 10 - OPT_DEBUGGING_TITLE = 11 + OPT_MODIFY_CSS = 10 + OPT_SUPPORT_NEWER_FIRMWARE = 11 + OPT_DEBUGGING_TITLE = 12 opts = None @@ -1805,11 +1822,41 @@ class KOBOTOUCH(KOBO): debug_print("KoboTouch:imagefilename_from_imageID - no cover image found - ImageID=%s" % (ImageID)) return None + + def get_extra_css(self): + extra_sheet = None + + if self.modifying_css(): + extra_css_path = os.path.join(self._main_prefix, self.KOBO_EXTRA_CSSFILE) + if os.path.exists(extra_css_path): + from cssutils import parseFile as cssparseFile + try: + extra_sheet = cssparseFile(extra_css_path) + debug_print("KoboTouch:get_extra_css: Using extra CSS in {0} ({1} rules)".format(extra_css_path, len(extra_sheet.cssRules))) + except Exception as e: + debug_print("KoboTouch:get_extra_css: Problem parsing extra CSS file {0}".format(extra_css_path)) + debug_print("KoboTouch:get_extra_css: Exception {0}".format(e)) + return extra_sheet + + def upload_books(self, files, names, on_card=None, end_session=True, metadata=None): debug_print('KoboTouch:upload_books - %d books'%(len(files))) debug_print('KoboTouch:upload_books - files=', files) + if self.modifying_epub(): + self.extra_sheet = self.get_extra_css() + i = 0 + for file, n, mi in zip(files, names, metadata): + debug_print("KoboTouch:upload_books: Processing book: {0} by {1}".format(mi.title, " and ".join(mi.authors))) + debug_print("KoboTouch:upload_books: file=%s, name=%s" % (file, n)) + self.report_progress(i / float(len(files)), "Processing book: {0} by {1}".format(mi.title, " and ".join(mi.authors))) + mi.kte_calibre_name = n + self._modify_epub(file, mi) + i += 1 + + self.report_progress(0, 'Working...') + result = super(KOBOTOUCH, self).upload_books(files, names, on_card, end_session, metadata) # debug_print('KoboTouch:upload_books - result=', result) @@ -1848,6 +1895,65 @@ class KOBOTOUCH(KOBO): return result + def _modify_epub(self, file, metadata, container=None): + debug_print("KoboTouch:_modify_epub:Processing {0} - {1}".format(metadata.author_sort, metadata.title)) + + # Currently only modifying CSS, so if no stylesheet, don't do anything + if not self.extra_sheet: + return True + + commit_container = False + if not container: + commit_container = True + try: + from calibre.ebooks.oeb.polish.container import get_container + debug_print("KoboTouch:_modify_epub: creating container") + container = get_container(file) + container.css_preprocessor = DummyCSSPreProcessor() + except Exception as e: + debug_print("KoboTouch:_modify_epub: exception from get_container {0} - {1}".format(metadata.author_sort, metadata.title)) + debug_print("KoboTouch:_modify_epub: exception is: {0}".format(e)) + return False + else: + debug_print("KoboTouch:_modify_epub: received container") + + cssnames = [n for n in container.name_path_map if n.endswith('.css')] + for cssname in cssnames: + newsheet = container.parsed(cssname) + oldrules = len(newsheet.cssRules) + # remove any existing @page rules in epub css + # if css to be appended contains an @page rule + if self.extra_sheet and len([r for r in self.extra_sheet if r.type == r.PAGE_RULE]): + page_rules = [r for r in newsheet if r.type == r.PAGE_RULE] + if len(page_rules) > 0: + debug_print("KoboTouch:_modify_epub:Removing existing @page rules") + for rule in page_rules: + rule.style = '' + # remove any existing widow/orphan settings in epub css + # if css to be appended contains a widow/orphan rule or we there is no extra CSS file + if (len([r for r in self.extra_sheet if r.type == r.STYLE_RULE \ + and (r.style['widows'] or r.style['orphans'])]) > 0): + widow_orphan_rules = [r for r in newsheet if r.type == r.STYLE_RULE \ + and (r.style['widows'] or r.style['orphans'])] + if len(widow_orphan_rules) > 0: + debug_print("KoboTouch:_modify_epub:Removing existing widows/orphans attribs") + for rule in widow_orphan_rules: + rule.style.removeProperty('widows') + rule.style.removeProperty('orphans') + # append all rules from kobo extra css stylesheet + for addrule in [r for r in self.extra_sheet.cssRules]: + newsheet.insertRule(addrule, len(newsheet.cssRules)) + debug_print("KoboTouch:_modify_epub:CSS rules {0} -> {1} ({2})".format(oldrules, len(newsheet.cssRules), cssname)) + container.dirty(cssname) + + if commit_container: + debug_print("KoboTouch:_modify_epub: committing container.") + os.unlink(file) + container.commit(file) + + return True + + def delete_via_sql(self, ContentID, ContentType): imageId = super(KOBOTOUCH, self).delete_via_sql(ContentID, ContentType) @@ -1872,11 +1978,11 @@ class KOBOTOUCH(KOBO): cursor.execute('delete from content where BookID is Null and ContentID =?',t) # Remove the content_settings entry - debug_print('KoboTouch:delete_via_sql: detete from content_settings') + debug_print('KoboTouch:delete_via_sql: delete from content_settings') cursor.execute('delete from content_settings where ContentID =?',t) # Remove the ratings entry - debug_print('KoboTouch:delete_via_sql: detete from ratings') + debug_print('KoboTouch:delete_via_sql: delete from ratings') cursor.execute('delete from ratings where ContentID =?',t) # Remove any entries for the Activity table - removes tile from new home page @@ -2636,6 +2742,13 @@ class KOBOTOUCH(KOBO): opts = self.settings() return opts.extra_customization[self.OPT_KEEP_COVER_ASPECT_RATIO] + def modifying_epub(self): + return self.modifying_css() + + def modifying_css(self): + opts = self.settings() + return opts.extra_customization[self.OPT_MODIFY_CSS] + def supports_bookshelves(self): return self.dbversion >= self.min_supported_dbversion From 08819cb34086a19e304943b35e97a35539682c15 Mon Sep 17 00:00:00 2001 From: David Forrester Date: Thu, 1 Aug 2013 14:53:42 +1000 Subject: [PATCH 0406/1154] Kobo driver CSS modification should use mimetype --- src/calibre/devices/kobo/driver.py | 59 ++++++++++++++++-------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 89b6b39549..ec00ca8d57 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -1917,34 +1917,37 @@ class KOBOTOUCH(KOBO): else: debug_print("KoboTouch:_modify_epub: received container") - cssnames = [n for n in container.name_path_map if n.endswith('.css')] - for cssname in cssnames: - newsheet = container.parsed(cssname) - oldrules = len(newsheet.cssRules) - # remove any existing @page rules in epub css - # if css to be appended contains an @page rule - if self.extra_sheet and len([r for r in self.extra_sheet if r.type == r.PAGE_RULE]): - page_rules = [r for r in newsheet if r.type == r.PAGE_RULE] - if len(page_rules) > 0: - debug_print("KoboTouch:_modify_epub:Removing existing @page rules") - for rule in page_rules: - rule.style = '' - # remove any existing widow/orphan settings in epub css - # if css to be appended contains a widow/orphan rule or we there is no extra CSS file - if (len([r for r in self.extra_sheet if r.type == r.STYLE_RULE \ - and (r.style['widows'] or r.style['orphans'])]) > 0): - widow_orphan_rules = [r for r in newsheet if r.type == r.STYLE_RULE \ - and (r.style['widows'] or r.style['orphans'])] - if len(widow_orphan_rules) > 0: - debug_print("KoboTouch:_modify_epub:Removing existing widows/orphans attribs") - for rule in widow_orphan_rules: - rule.style.removeProperty('widows') - rule.style.removeProperty('orphans') - # append all rules from kobo extra css stylesheet - for addrule in [r for r in self.extra_sheet.cssRules]: - newsheet.insertRule(addrule, len(newsheet.cssRules)) - debug_print("KoboTouch:_modify_epub:CSS rules {0} -> {1} ({2})".format(oldrules, len(newsheet.cssRules), cssname)) - container.dirty(cssname) +# cssnames = [n for n in container.name_path_map if n.endswith('.css')] +# for cssname in cssnames: + from calibre.ebooks.oeb.base import OEB_STYLES + for cssname, mt in container.mime_map.iteritems(): + if mt in OEB_STYLES: + newsheet = container.parsed(cssname) + oldrules = len(newsheet.cssRules) + # remove any existing @page rules in epub css + # if css to be appended contains an @page rule + if self.extra_sheet and len([r for r in self.extra_sheet if r.type == r.PAGE_RULE]): + page_rules = [r for r in newsheet if r.type == r.PAGE_RULE] + if len(page_rules) > 0: + debug_print("KoboTouch:_modify_epub:Removing existing @page rules") + for rule in page_rules: + rule.style = '' + # remove any existing widow/orphan settings in epub css + # if css to be appended contains a widow/orphan rule or we there is no extra CSS file + if (len([r for r in self.extra_sheet if r.type == r.STYLE_RULE \ + and (r.style['widows'] or r.style['orphans'])]) > 0): + widow_orphan_rules = [r for r in newsheet if r.type == r.STYLE_RULE \ + and (r.style['widows'] or r.style['orphans'])] + if len(widow_orphan_rules) > 0: + debug_print("KoboTouch:_modify_epub:Removing existing widows/orphans attribs") + for rule in widow_orphan_rules: + rule.style.removeProperty('widows') + rule.style.removeProperty('orphans') + # append all rules from kobo extra css stylesheet + for addrule in [r for r in self.extra_sheet.cssRules]: + newsheet.insertRule(addrule, len(newsheet.cssRules)) + debug_print("KoboTouch:_modify_epub:CSS rules {0} -> {1} ({2})".format(oldrules, len(newsheet.cssRules), cssname)) + container.dirty(cssname) if commit_container: debug_print("KoboTouch:_modify_epub: committing container.") From 20caf3ac996e6aae26fadcdd29e4ff602521a9d6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 11:21:52 +0530 Subject: [PATCH 0407/1154] pep8 cleanup of kobo/driver.py Kobo driver: Add an option to modify the styling in books being sent to the device, based on a template on the device. Fixes #1207151 [Append and modify CSS when sending epubs to Kobo devices](https://bugs.launchpad.net/calibre/+bug/1207151) --- src/calibre/devices/kobo/driver.py | 244 ++++++++++++++--------------- 1 file changed, 117 insertions(+), 127 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index ec00ca8d57..907a8b309a 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -75,8 +75,8 @@ class KOBO(USBMS): VIRTUAL_BOOK_EXTENSIONS = frozenset(['kobo', '']) EXTRA_CUSTOMIZATION_MESSAGE = [ - _('The Kobo supports several collections including ')+\ - 'Read, Closed, Im_Reading. ' +\ + _('The Kobo supports several collections including ')+ + 'Read, Closed, Im_Reading. ' + _('Create tags for automatic management'), _('Upload covers for books (newer readers)') + ':::'+_('Normally, the KOBO readers get the cover image from the' @@ -220,7 +220,7 @@ class KOBO(USBMS): # Try the Touch version if the image does not exist imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - N3_LIBRARY_FULL.parsed') - #print "Image name Normalized: " + imagename + # print "Image name Normalized: " + imagename if not os.path.exists(imagename): debug_print("Strange - The image name does not exist - title: ", title) if imagename is not None: @@ -248,16 +248,16 @@ class KOBO(USBMS): book = Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=1048576) except: - debug_print("prefix: ", prefix, "lpath: ", lpath, "title: ", title, "authors: ", authors, \ + debug_print("prefix: ", prefix, "lpath: ", lpath, "title: ", title, "authors: ", authors, "mime: ", mime, "date: ", date, "ContentType: ", ContentType, "ImageID: ", ImageID) raise # print 'Update booklist' - book.device_collections = playlist_map.get(lpath,[])# if lpath in playlist_map else [] + book.device_collections = playlist_map.get(lpath,[]) # if lpath in playlist_map else [] if bl.add_book(book, replace_metadata=False): changed = True - except: # Probably a path encoding error + except: # Probably a path encoding error import traceback traceback.print_exc() return changed @@ -280,35 +280,35 @@ class KOBO(USBMS): opts = self.settings() if self.dbversion >= 33: - query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ - 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, IsDownloaded from content where ' \ - 'BookID is Null %(previews)s %(recomendations)s and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \ - if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')', \ - previews=' and Accessibility <> 6' \ - if opts.extra_customization[self.OPT_SHOW_PREVIEWS] == False else '', \ - recomendations=' and IsDownloaded in (\'true\', 1)' \ + query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' + 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, IsDownloaded from content where ' + 'BookID is Null %(previews)s %(recomendations)s and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' + if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')', + previews=' and Accessibility <> 6' + if opts.extra_customization[self.OPT_SHOW_PREVIEWS] == False else '', + recomendations=' and IsDownloaded in (\'true\', 1)' if opts.extra_customization[self.OPT_SHOW_RECOMMENDATIONS] == False else '') elif self.dbversion >= 16 and self.dbversion < 33: - query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ - 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, "1" as IsDownloaded from content where ' \ - 'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \ + query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' + 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, "1" as IsDownloaded from content where ' + 'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')') elif self.dbversion < 16 and self.dbversion >= 14: - query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ - 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded from content where ' \ - 'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \ + query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' + 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded from content where ' + 'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')') elif self.dbversion < 14 and self.dbversion >= 8: - query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ - 'ImageID, ReadStatus, ___ExpirationStatus, "-1" as FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded from content where ' \ - 'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \ + query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' + 'ImageID, ReadStatus, ___ExpirationStatus, "-1" as FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded from content where ' + 'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')') else: query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus, "-1" as ___ExpirationStatus, "-1" as FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded from content where BookID is Null' try: - cursor.execute (query) + cursor.execute(query) except Exception as e: err = str(e) if not ('___ExpirationStatus' in err or 'FavouritesIndex' in err or @@ -349,9 +349,9 @@ class KOBO(USBMS): need_sync = True del bl[idx] - #print "count found in cache: %d, count of files in metadata: %d, need_sync: %s" % \ + # print "count found in cache: %d, count of files in metadata: %d, need_sync: %s" % \ # (len(bl_cache), len(bl), need_sync) - if need_sync: #self.count_found_in_bl != len(bl) or need_sync: + if need_sync: # self.count_found_in_bl != len(bl) or need_sync: if oncard == 'cardb': self.sync_booklists((None, None, bl)) elif oncard == 'carda': @@ -391,7 +391,7 @@ class KOBO(USBMS): # Delete the shortcover_pages first cursor.execute('delete from shortcover_page where shortcoverid in (select ContentID from content where BookID = ?)', t) - #Delete the volume_shortcovers second + # Delete the volume_shortcovers second cursor.execute('delete from volume_shortcovers where volumeid = ?', t) # Delete the rows from content_keys @@ -405,18 +405,18 @@ class KOBO(USBMS): cursor.execute('delete from content where BookID = ?', t) if ContentType == 6: try: - cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0, ___ExpirationStatus=3 ' \ + cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0, ___ExpirationStatus=3 ' 'where BookID is Null and ContentID =?',t) except Exception as e: if 'no such column' not in str(e): raise try: - cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0 ' \ + cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0 ' 'where BookID is Null and ContentID =?',t) except Exception as e: if 'no such column' not in str(e): raise - cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\' ' \ + cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\' ' 'where BookID is Null and ContentID =?',t) else: cursor.execute('delete from content where BookID is Null and ContentID =?',t) @@ -436,7 +436,8 @@ class KOBO(USBMS): path_prefix = '.kobo/images/' path = self._main_prefix + path_prefix + ImageID - file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed', ' - N3_LIBRARY_FULL.parsed', ' - N3_LIBRARY_GRID.parsed', ' - N3_LIBRARY_LIST.parsed', ' - N3_SOCIAL_CURRENTREAD.parsed', ' - N3_FULL.parsed',) + file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed', ' - N3_LIBRARY_FULL.parsed', + ' - N3_LIBRARY_GRID.parsed', ' - N3_LIBRARY_LIST.parsed', ' - N3_SOCIAL_CURRENTREAD.parsed', ' - N3_FULL.parsed',) for ending in file_endings: fpath = path + ending @@ -460,7 +461,7 @@ class KOBO(USBMS): ContentID = self.contentid_from_path(path, ContentType) ImageID = self.delete_via_sql(ContentID, ContentType) - #print " We would now delete the Images for" + ImageID + # print " We would now delete the Images for" + ImageID self.delete_images(ImageID, path) if os.path.exists(path): @@ -493,9 +494,9 @@ class KOBO(USBMS): self.report_progress((i+1) / float(len(paths)), _('Removing books from device metadata listing...')) for bl in booklists: for book in bl: - #print "Book Path: " + book.path + # print "Book Path: " + book.path if path.endswith(book.path): - #print " Remove: " + book.path + # print " Remove: " + book.path bl.remove_book(book) self.report_progress(1.0, _('Removing books from device metadata listing...')) @@ -524,12 +525,12 @@ class KOBO(USBMS): prints('in add_books_to_metadata. Prefix is None!', path, self._main_prefix) continue - #print "Add book to metatdata: " - #print "prefix: " + prefix + # print "Add book to metatdata: " + # print "prefix: " + prefix lpath = path.partition(prefix)[2] if lpath.startswith('/') or lpath.startswith('\\'): lpath = lpath[1:] - #print "path: " + lpath + # print "path: " + lpath book = self.book_class(prefix, lpath, other=info) if book.size is None or book.size == 0: book.size = os.stat(self.normalize_path(path)).st_size @@ -551,12 +552,12 @@ class KOBO(USBMS): if self._card_a_prefix is not None: ContentID = ContentID.replace(self._card_a_prefix, '') - elif ContentType == 999: # HTML Files + elif ContentType == 999: # HTML Files ContentID = path ContentID = ContentID.replace(self._main_prefix, "/mnt/onboard/") if self._card_a_prefix is not None: ContentID = ContentID.replace(self._card_a_prefix, "/mnt/sd/") - else: # ContentType = 16 + else: # ContentType = 16 ContentID = path ContentID = ContentID.replace(self._main_prefix, "file:///mnt/onboard/") if self._card_a_prefix is not None: @@ -574,7 +575,7 @@ class KOBO(USBMS): def get_content_type_from_extension(self, extension): if extension == '.kobo': # Kobo books do not have book files. They do have some images though - #print "kobo book" + # print "kobo book" ContentType = 6 elif extension == '.pdf' or extension == '.epub': # print "ePub or pdf" @@ -585,8 +586,8 @@ class KOBO(USBMS): ContentType = 999 else: ContentType = 901 - else: # if extension == '.html' or extension == '.txt': - ContentType = 901 # Yet another hack: to get around Kobo changing how ContentID is stored + else: # if extension == '.html' or extension == '.txt': + ContentType = 901 # Yet another hack: to get around Kobo changing how ContentID is stored return ContentType def path_from_contentid(self, ContentID, ContentType, MimeType, oncard): @@ -706,7 +707,7 @@ class KOBO(USBMS): query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ContentID not like \'file:///mnt/sd/%\'' try: - cursor.execute (query) + cursor.execute(query) except: debug_print(' Database Exception: Unable to reset ReadStatus list') raise @@ -747,7 +748,7 @@ class KOBO(USBMS): cursor = connection.cursor() try: - cursor.execute (query) + cursor.execute(query) except Exception as e: debug_print(' Database Exception: Unable to reset Shortlist list') if 'no such column' not in str(e): @@ -847,7 +848,7 @@ class KOBO(USBMS): elif category in accessibilitylist.keys(): # Do not manage the Accessibility List pass - else: # No collections + else: # No collections # Since no collections exist the ReadStatus needs to be reset to 0 (Unread) debug_print("No Collections - reseting ReadStatus") self.reset_readstatus(connection, oncard) @@ -857,7 +858,6 @@ class KOBO(USBMS): # debug_print('Finished update_device_database_collections', collections_attributes) - def get_collections_attributes(self): collections = [] opts = self.settings() @@ -1107,15 +1107,15 @@ class KOBO(USBMS): spanTag['style'] = 'font-weight:normal' if bookmark.book_format == 'epub': spanTag.insert(0,NavigableString( - _("


    Book Last Read: %(time)s
    Percentage Read: %(pr)d%%
    ") % \ + _("
    Book Last Read: %(time)s
    Percentage Read: %(pr)d%%
    ") % dict(time=last_read, - #loc=last_read_location, + # loc=last_read_location, pr=percent_read))) else: spanTag.insert(0,NavigableString( - _("
    Book Last Read: %(time)s
    Percentage Read: %(pr)d%%
    ") % \ + _("
    Book Last Read: %(time)s
    Percentage Read: %(pr)d%%
    ") % dict(time=last_read, - #loc=last_read_location, + # loc=last_read_location, pr=percent_read))) divTag.insert(dtc, spanTag) @@ -1131,7 +1131,7 @@ class KOBO(USBMS): for location in sorted(user_notes): if user_notes[location]['type'] == 'Bookmark': annotations.append( - _('Chapter %(chapter)d: %(chapter_title)s
    %(typ)s
    Chapter Progress: %(chapter_progress)s%%
    %(annotation)s

    ') % \ + _('Chapter %(chapter)d: %(chapter_title)s
    %(typ)s
    Chapter Progress: %(chapter_progress)s%%
    %(annotation)s

    ') % dict(chapter=user_notes[location]['chapter'], dl=user_notes[location]['displayed_location'], typ=user_notes[location]['type'], @@ -1140,7 +1140,7 @@ class KOBO(USBMS): annotation=user_notes[location]['annotation'] if user_notes[location]['annotation'] is not None else "")) elif user_notes[location]['type'] == 'Highlight': annotations.append( - _('Chapter %(chapter)d: %(chapter_title)s
    %(typ)s
    Chapter Progress: %(chapter_progress)s%%
    Highlight: %(text)s

    ') % \ + _('Chapter %(chapter)d: %(chapter_title)s
    %(typ)s
    Chapter Progress: %(chapter_progress)s%%
    Highlight: %(text)s

    ') % dict(chapter=user_notes[location]['chapter'], dl=user_notes[location]['displayed_location'], typ=user_notes[location]['type'], @@ -1149,7 +1149,7 @@ class KOBO(USBMS): text=user_notes[location]['text'])) elif user_notes[location]['type'] == 'Annotation': annotations.append( - _('Chapter %(chapter)d: %(chapter_title)s
    %(typ)s
    Chapter Progress: %(chapter_progress)s%%
    Highlight: %(text)s
    Notes: %(annotation)s

    ') % \ + _('Chapter %(chapter)d: %(chapter_title)s
    %(typ)s
    Chapter Progress: %(chapter_progress)s%%
    Highlight: %(text)s
    Notes: %(annotation)s

    ') % dict(chapter=user_notes[location]['chapter'], dl=user_notes[location]['displayed_location'], typ=user_notes[location]['type'], @@ -1159,13 +1159,13 @@ class KOBO(USBMS): annotation=user_notes[location]['annotation'])) else: annotations.append( - _('Chapter %(chapter)d: %(chapter_title)s
    %(typ)s
    Chapter Progress: %(chapter_progress)s%%
    Highlight: %(text)s
    Notes: %(annotation)s

    ') % \ + _('Chapter %(chapter)d: %(chapter_title)s
    %(typ)s
    Chapter Progress: %(chapter_progress)s%%
    Highlight: %(text)s
    Notes: %(annotation)s

    ') % dict(chapter=user_notes[location]['chapter'], dl=user_notes[location]['displayed_location'], typ=user_notes[location]['type'], chapter_title=user_notes[location]['chapter_title'], chapter_progress=user_notes[location]['chapter_progress'], - text=user_notes[location]['text'], \ + text=user_notes[location]['text'], annotation=user_notes[location]['annotation'])) for annotation in annotations: @@ -1240,8 +1240,8 @@ class KOBOTOUCH(KOBO): KOBO_EXTRA_CSSFILE = 'kobo_extra.css' EXTRA_CUSTOMIZATION_MESSAGE = [ - _('The Kobo Touch from firmware V2.0.0 supports bookshelves.')+\ - 'These are created on the Kobo Touch. ' +\ + _('The Kobo Touch from firmware V2.0.0 supports bookshelves.')+ + 'These are created on the Kobo Touch. ' + _('Specify a tags type column for automatic management'), _('Create Bookshelves') + ':::'+_('Create new bookshelves on the Kobo Touch if they do not exist. This is only for firmware V2.0.0 or later.'), @@ -1346,20 +1346,19 @@ class KOBOTOUCH(KOBO): } AURA_HD_COVER_FILE_ENDINGS = { ' - N3_FULL.parsed': [(1080,1440), 0, 99,True,], # Used for screensaver, home screen - ' - N3_LIBRARY_FULL.parsed':[(355, 471), 0, 99,False,], # Used for Details screen - ' - N3_LIBRARY_GRID.parsed':[(149, 198), 0, 99,False,], # Used for library lists + ' - N3_LIBRARY_FULL.parsed':[(355, 471), 0, 99,False,], # Used for Details screen + ' - N3_LIBRARY_GRID.parsed':[(149, 198), 0, 99,False,], # Used for library lists } - #Following are the sizes used with pre2.1.4 firmware + # Following are the sizes used with pre2.1.4 firmware # COVER_FILE_ENDINGS = { -# ' - N3_LIBRARY_FULL.parsed':[(355,530),0, 99,], # Used for Details screen -## ' - N3_LIBRARY_FULL.parsed':[(600,800),0, 99,], -# ' - N3_LIBRARY_GRID.parsed':[(149,233),0, 99,], # Used for library lists +# ' - N3_LIBRARY_FULL.parsed':[(355,530),0, 99,], # Used for Details screen +# ' - N3_LIBRARY_FULL.parsed':[(600,800),0, 99,], +# ' - N3_LIBRARY_GRID.parsed':[(149,233),0, 99,], # Used for library lists # ' - N3_LIBRARY_LIST.parsed':[(60,90),0, 53,], # ' - N3_LIBRARY_SHELF.parsed': [(40,60),0, 52,], -# ' - N3_FULL.parsed':[(600,800),0, 99,], # Used for screensaver if "Full screen" is checked. +# ' - N3_FULL.parsed':[(600,800),0, 99,], # Used for screensaver if "Full screen" is checked. # } - def initialize(self): super(KOBOTOUCH, self).initialize() self.bookshelvelist = [] @@ -1368,7 +1367,6 @@ class KOBOTOUCH(KOBO): self.set_device_name() return super(KOBOTOUCH, self).get_device_information(end_session) - def books(self, oncard=None, end_session=True): debug_print("KoboTouch:books - oncard='%s'"%oncard) from calibre.ebooks.metadata.meta import path_to_ext @@ -1415,7 +1413,7 @@ class KOBOTOUCH(KOBO): debug_print("KoboTouch:books - prefs['manage_device_metadata']=", prefs['manage_device_metadata']) if opts.extra_customization: debugging_title = opts.extra_customization[self.OPT_DEBUGGING_TITLE] - debug_print("KoboTouch:books - set_debugging_title to '%s'" % debugging_title ) + debug_print("KoboTouch:books - set_debugging_title to '%s'" % debugging_title) bl.set_debugging_title(debugging_title) debug_print("KoboTouch:books - length bl=%d"%len(bl)) need_sync = self.parse_metadata_cache(bl, prefix, self.METADATA_CACHE) @@ -1431,7 +1429,8 @@ class KOBOTOUCH(KOBO): # show_debug = authors == 'L. Frank Baum' if show_debug: debug_print("KoboTouch:update_booklist - title='%s'"%title, "ContentType=%s"%ContentType, "isdownloaded=", isdownloaded) - debug_print(" prefix=%s, mime=%s, date=%s, readstatus=%d, MimeType=%s, expired=%d, favouritesindex=%d, accessibility=%d, isdownloaded=%s"% + debug_print( + " prefix=%s, mime=%s, date=%s, readstatus=%d, MimeType=%s, expired=%d, favouritesindex=%d, accessibility=%d, isdownloaded=%s"% (prefix, mime, date, readstatus, MimeType, expired, favouritesindex, accessibility, isdownloaded,)) changed = False try: @@ -1503,7 +1502,7 @@ class KOBOTOUCH(KOBO): # print "Normalized FileName: " + path idx = bl_cache.get(lpath, None) - if idx is not None:# and not (accessibility == 1 and isdownloaded == 'false'): + if idx is not None: # and not (accessibility == 1 and isdownloaded == 'false'): if show_debug: self.debug_index = idx debug_print("KoboTouch:update_booklist - idx=%d"%idx) @@ -1556,7 +1555,7 @@ class KOBOTOUCH(KOBO): if os.path.exists(self.normalize_path(os.path.join(prefix, lpath))): book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID) else: - if isdownloaded == 'true': # A recommendation or preview is OK to not have a file + if isdownloaded == 'true': # A recommendation or preview is OK to not have a file debug_print(" Strange: The file: ", prefix, lpath, " does not exist!") title = "FILE MISSING: " + title book = self.book_class(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=0) @@ -1565,7 +1564,7 @@ class KOBOTOUCH(KOBO): except Exception as e: debug_print("KoboTouch:update_booklist - exception creating book: '%s'"%str(e)) - debug_print(" prefix: ", prefix, "lpath: ", lpath, "title: ", title, "authors: ", authors, \ + debug_print(" prefix: ", prefix, "lpath: ", lpath, "title: ", title, "authors: ", authors, "mime: ", mime, "date: ", date, "ContentType: ", ContentType, "ImageID: ", ImageID) raise @@ -1580,7 +1579,7 @@ class KOBOTOUCH(KOBO): debug_print(" kobo_collections:", kobo_collections) # print 'Update booklist' - book.device_collections = playlist_map.get(lpath,[])# if lpath in playlist_map else [] + book.device_collections = playlist_map.get(lpath,[]) # if lpath in playlist_map else [] book.current_shelves = bookshelves book.kobo_collections = kobo_collections book.contentID = ContentID @@ -1594,7 +1593,7 @@ class KOBOTOUCH(KOBO): if show_debug: debug_print(' book.device_collections', book.device_collections) debug_print(' book.title', book.title) - except: # Probably a path encoding error + except: # Probably a path encoding error import traceback traceback.print_exc() return changed @@ -1656,50 +1655,53 @@ class KOBOTOUCH(KOBO): where_clause = '' if self.supports_kobo_archive(): - where_clause = (" where BookID is Null " \ - " and ((Accessibility = -1 and IsDownloaded in ('true', 1 )) or (Accessibility in (1,2) %(expiry)s) " \ - " %(previews)s %(recomendations)s )" \ + where_clause = (" where BookID is Null " + " and ((Accessibility = -1 and IsDownloaded in ('true', 1 )) or (Accessibility in (1,2) %(expiry)s) " + " %(previews)s %(recomendations)s )" " and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) and ContentType = 6)") % \ - dict(\ - expiry="" if opts.extra_customization[self.OPT_SHOW_ARCHIVED_BOOK_RECORDS] else "and IsDownloaded in ('true', 1)", \ - previews=" or (Accessibility in (6) and ___UserID <> '')" if opts.extra_customization[self.OPT_SHOW_PREVIEWS] else "", \ - recomendations=" or (Accessibility in (-1, 4, 6) and ___UserId = '')" if opts.extra_customization[self.OPT_SHOW_RECOMMENDATIONS] else "" \ + dict( + expiry="" if opts.extra_customization[self.OPT_SHOW_ARCHIVED_BOOK_RECORDS] else "and IsDownloaded in ('true', 1)", + previews=" or (Accessibility in (6) and ___UserID <> '')" if opts.extra_customization[self.OPT_SHOW_PREVIEWS] else "", + recomendations=" or (Accessibility in (-1, 4, 6) and ___UserId = '')" if opts.extra_customization[ + self.OPT_SHOW_RECOMMENDATIONS] else "" ) elif self.supports_series(): - where_clause = (" where BookID is Null " \ - " and ((Accessibility = -1 and IsDownloaded in ('true', 1)) or (Accessibility in (1,2)) %(previews)s %(recomendations)s )" \ + where_clause = (" where BookID is Null " + " and ((Accessibility = -1 and IsDownloaded in ('true', 1)) or (Accessibility in (1,2)) %(previews)s %(recomendations)s )" " and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s)") % \ - dict(\ - expiry=" and ContentType = 6" if opts.extra_customization[self.OPT_SHOW_ARCHIVED_BOOK_RECORDS] else "", \ - previews=" or (Accessibility in (6) and ___UserID <> '')" if opts.extra_customization[self.OPT_SHOW_PREVIEWS] else "", \ - recomendations=" or (Accessibility in (-1, 4, 6) and ___UserId = '')" if opts.extra_customization[self.OPT_SHOW_RECOMMENDATIONS] else "" \ + dict( + expiry=" and ContentType = 6" if opts.extra_customization[self.OPT_SHOW_ARCHIVED_BOOK_RECORDS] else "", + previews=" or (Accessibility in (6) and ___UserID <> '')" if opts.extra_customization[self.OPT_SHOW_PREVIEWS] else "", + recomendations=" or (Accessibility in (-1, 4, 6) and ___UserId = '')" if opts.extra_customization[ + self.OPT_SHOW_RECOMMENDATIONS] else "" ) elif self.dbversion >= 33: where_clause = (' where BookID is Null %(previews)s %(recomendations)s and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s)') % \ - dict(\ - expiry=' and ContentType = 6' if opts.extra_customization[self.OPT_SHOW_ARCHIVED_BOOK_RECORDS] else '', \ - previews=' and Accessibility <> 6' if opts.extra_customization[self.OPT_SHOW_PREVIEWS] == False else '', \ - recomendations=' and IsDownloaded in (\'true\', 1)' if opts.extra_customization[self.OPT_SHOW_RECOMMENDATIONS] == False else ''\ + dict( + expiry=' and ContentType = 6' if opts.extra_customization[self.OPT_SHOW_ARCHIVED_BOOK_RECORDS] else '', + previews=' and Accessibility <> 6' if opts.extra_customization[self.OPT_SHOW_PREVIEWS] == False else '', + recomendations=' and IsDownloaded in (\'true\', 1)' if opts.extra_customization[self.OPT_SHOW_RECOMMENDATIONS] == False else '' ) elif self.dbversion >= 16: - where_clause = (' where BookID is Null ' \ + where_clause = (' where BookID is Null ' 'and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s)') % \ dict(expiry=' and ContentType = 6' if opts.extra_customization[self.OPT_SHOW_ARCHIVED_BOOK_RECORDS] else '') else: where_clause = ' where BookID is Null' - # Note: The card condition should not need the contentId test for the SD card. But the ExternalId does not get set for sideloaded kepubs on the SD card. + # Note: The card condition should not need the contentId test for the SD + # card. But the ExternalId does not get set for sideloaded kepubs on the + # SD card. card_condition = '' if self.has_externalid(): card_condition = " AND (externalId IS NOT NULL AND externalId <> '' OR contentId LIKE 'file:///mnt/sd/%')" if oncard == 'carda' else " AND (externalId IS NULL OR externalId = '') AND contentId NOT LIKE 'file:///mnt/sd/%'" else: card_condition = " AND contentId LIKE 'file:///mnt/sd/%'" if oncard == 'carda' else " AND contentId NOT LIKE'file:///mnt/sd/%'" - query = 'SELECT ' + columns + ' FROM content ' + where_clause + card_condition debug_print("KoboTouch:books - query=", query) try: - cursor.execute (query) + cursor.execute(query) except Exception as e: err = str(e) if not ('___ExpirationStatus' in err @@ -1712,7 +1714,7 @@ class KOBOTOUCH(KOBO): raise query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' 'ImageID, ReadStatus, "-1" as ___ExpirationStatus, "-1" as ' - 'FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded, null as Series, null as SeriesNumber' \ + 'FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded, null as Series, null as SeriesNumber' ' from content where BookID is Null') cursor.execute(query) @@ -1737,7 +1739,8 @@ class KOBOTOUCH(KOBO): bookshelves = get_bookshelvesforbook(connection, row[3]) prefix = self._card_a_prefix if oncard == 'carda' else self._main_prefix - changed = update_booklist(prefix, path, row[0], row[1], mime, row[2], row[3], row[5], row[6], row[7], row[4], row[8], row[9], row[10], row[11], row[12], row[13], row[14], bookshelves) + changed = update_booklist(prefix, path, row[0], row[1], mime, row[2], row[3], row[5], row[ + 6], row[7], row[4], row[8], row[9], row[10], row[11], row[12], row[13], row[14], bookshelves) if changed: need_sync = True @@ -1759,11 +1762,11 @@ class KOBOTOUCH(KOBO): # else: # debug_print("KoboTouch:books - Book in mtadata.calibre, on file system but not database - bl[idx].title:'%s'"%bl[idx].title) - #print "count found in cache: %d, count of files in metadata: %d, need_sync: %s" % \ + # print "count found in cache: %d, count of files in metadata: %d, need_sync: %s" % \ # (len(bl_cache), len(bl), need_sync) # Bypassing the KOBO sync_booklists as that does things we don't need to do # Also forcing sync to see if this solves issues with updating shelves and matching books. - if need_sync or True: #self.count_found_in_bl != len(bl) or need_sync: + if need_sync or True: # self.count_found_in_bl != len(bl) or need_sync: debug_print("KoboTouch:books - about to sync_booklists") if oncard == 'cardb': USBMS.sync_booklists(self, (None, None, bl)) @@ -1786,7 +1789,7 @@ class KOBOTOUCH(KOBO): if oncard == 'cardb': print 'path from_contentid cardb' else: - if (ContentType == "6" or ContentType == "10"): # and MimeType == 'application/x-kobo-epub+zip': + if (ContentType == "6" or ContentType == "10"): # and MimeType == 'application/x-kobo-epub+zip': if path.startswith("file:///mnt/onboard/"): path = self._main_prefix + path.replace("file:///mnt/onboard/", '') elif path.startswith("file:///mnt/sd/"): @@ -1804,7 +1807,6 @@ class KOBOTOUCH(KOBO): return path - def imagefilename_from_imageID(self, prefix, ImageID): show_debug = self.is_debugging_title(ImageID) @@ -1822,10 +1824,9 @@ class KOBOTOUCH(KOBO): debug_print("KoboTouch:imagefilename_from_imageID - no cover image found - ImageID=%s" % (ImageID)) return None - def get_extra_css(self): extra_sheet = None - + if self.modifying_css(): extra_css_path = os.path.join(self._main_prefix, self.KOBO_EXTRA_CSSFILE) if os.path.exists(extra_css_path): @@ -1838,7 +1839,6 @@ class KOBOTOUCH(KOBO): debug_print("KoboTouch:get_extra_css: Exception {0}".format(e)) return extra_sheet - def upload_books(self, files, names, on_card=None, end_session=True, metadata=None): debug_print('KoboTouch:upload_books - %d books'%(len(files))) @@ -1891,13 +1891,11 @@ class KOBOTOUCH(KOBO): except Exception as e: debug_print('KoboTouch:upload_books - Exception: %s'%str(e)) - return result - def _modify_epub(self, file, metadata, container=None): debug_print("KoboTouch:_modify_epub:Processing {0} - {1}".format(metadata.author_sort, metadata.title)) - + # Currently only modifying CSS, so if no stylesheet, don't do anything if not self.extra_sheet: return True @@ -1924,7 +1922,7 @@ class KOBOTOUCH(KOBO): if mt in OEB_STYLES: newsheet = container.parsed(cssname) oldrules = len(newsheet.cssRules) - # remove any existing @page rules in epub css + # remove any existing @page rules in epub css # if css to be appended contains an @page rule if self.extra_sheet and len([r for r in self.extra_sheet if r.type == r.PAGE_RULE]): page_rules = [r for r in newsheet if r.type == r.PAGE_RULE] @@ -1932,11 +1930,11 @@ class KOBOTOUCH(KOBO): debug_print("KoboTouch:_modify_epub:Removing existing @page rules") for rule in page_rules: rule.style = '' - # remove any existing widow/orphan settings in epub css - # if css to be appended contains a widow/orphan rule or we there is no extra CSS file - if (len([r for r in self.extra_sheet if r.type == r.STYLE_RULE \ + # remove any existing widow/orphan settings in epub css + # if css to be appended contains a widow/orphan rule or we there is no extra CSS file + if (len([r for r in self.extra_sheet if r.type == r.STYLE_RULE and (r.style['widows'] or r.style['orphans'])]) > 0): - widow_orphan_rules = [r for r in newsheet if r.type == r.STYLE_RULE \ + widow_orphan_rules = [r for r in newsheet if r.type == r.STYLE_RULE and (r.style['widows'] or r.style['orphans'])] if len(widow_orphan_rules) > 0: debug_print("KoboTouch:_modify_epub:Removing existing widows/orphans attribs") @@ -1956,7 +1954,6 @@ class KOBOTOUCH(KOBO): return True - def delete_via_sql(self, ContentID, ContentType): imageId = super(KOBOTOUCH, self).delete_via_sql(ContentID, ContentType) @@ -1972,7 +1969,7 @@ class KOBOTOUCH(KOBO): cursor = connection.cursor() debug_print('KoboTouch:delete_via_sql: have cursor') t = (ContentID,) - #Delete from the Bookshelf + # Delete from the Bookshelf debug_print('KoboTouch:delete_via_sql: Delete from the Bookshelf') cursor.execute('delete from ShelfContent where ContentID = ?', t) @@ -2044,7 +2041,7 @@ class KOBOTOUCH(KOBO): if self._card_a_prefix is not None: ContentID = ContentID.replace(self._card_a_prefix, "file:///mnt/sd/") - else: # ContentType = 16 + else: # ContentType = 16 debug_print("KoboTouch:contentid_from_path ContentType other than 6 - ContentType='%d'"%ContentType, "path='%s'"%path) ContentID = path ContentID = ContentID.replace(self._main_prefix, "file:///mnt/onboard/") @@ -2108,7 +2105,7 @@ class KOBOTOUCH(KOBO): delete_empty_shelves = opts.extra_customization[self.OPT_DELETE_BOOKSHELVES] and self.supports_bookshelves() update_series_details = opts.extra_customization[self.OPT_UPDATE_SERIES_DETAILS] and self.supports_series() debugging_title = opts.extra_customization[self.OPT_DEBUGGING_TITLE] - debug_print("KoboTouch:update_device_database_collections - set_debugging_title to '%s'" % debugging_title ) + debug_print("KoboTouch:update_device_database_collections - set_debugging_title to '%s'" % debugging_title) booklists.set_debugging_title(debugging_title) else: delete_empty_shelves = False @@ -2213,7 +2210,7 @@ class KOBOTOUCH(KOBO): debug_print(' category not added to book.device_collections', book.device_collections) debug_print("KoboTouch:update_device_database_collections - end for category='%s'"%category) - else: # No collections + else: # No collections # Since no collections exist the ReadStatus needs to be reset to 0 (Unread) debug_print("No Collections - reseting ReadStatus") if self.dbversion < 53: @@ -2247,7 +2244,6 @@ class KOBOTOUCH(KOBO): self.dump_bookshelves(connection) - debug_print('KoboTouch:update_device_database_collections - Finished ') def rebuild_collections(self, booklist, oncard): @@ -2292,7 +2288,6 @@ class KOBOTOUCH(KOBO): except Exception as e: debug_print('KoboTouch: FAILED to upload cover=%s Exception=%s'%(filepath, str(e))) - def imageid_from_contentid(self, ContentID): ImageID = ContentID.replace('/', '_') ImageID = ImageID.replace(' ', '_') @@ -2300,7 +2295,6 @@ class KOBOTOUCH(KOBO): ImageID = ImageID.replace('.', '_') return ImageID - def images_path(self, path): if self._card_a_prefix and os.path.abspath(path).startswith(os.path.abspath(self._card_a_prefix)) and self.supports_covers_on_sdcard(): path_prefix = 'koboExtStorage/images/' @@ -2367,7 +2361,7 @@ class KOBOTOUCH(KOBO): debug_print("KoboTouch:_upload_cover - resize=%s min_dbversion=%d max_dbversion=%d" % (resize, min_dbversion, max_dbversion)) if self.dbversion >= min_dbversion and self.dbversion <= max_dbversion: if show_debug: - debug_print("KoboTouch:_upload_cover - creating cover for ending='%s'"%ending)#, "resize'%s'"%resize) + debug_print("KoboTouch:_upload_cover - creating cover for ending='%s'"%ending) # , "resize'%s'"%resize) fpath = path + ending fpath = self.normalize_path(fpath.replace('/', os.sep)) @@ -2381,9 +2375,9 @@ class KOBOTOUCH(KOBO): width, height, fmt = identify_data(data) cover_aspect = width / height if cover_aspect > 1: - resize = (resize[0], int(resize[0] / cover_aspect )) + resize = (resize[0], int(resize[0] / cover_aspect)) elif cover_aspect < 1: - resize = (int(cover_aspect * resize[1]), resize[1] ) + resize = (int(cover_aspect * resize[1]), resize[1]) # Return the data resized and in Grayscale if # required @@ -2402,7 +2396,7 @@ class KOBOTOUCH(KOBO): debug_print("KoboTouch:_upload_cover - ImageID could not be retrieved from the database") def remove_book_from_device_bookshelves(self, connection, book): - show_debug = self.is_debugging_title(book.title)# or True + show_debug = self.is_debugging_title(book.title) # or True remove_shelf_list = set(book.current_shelves) - set(book.device_collections) @@ -2687,7 +2681,6 @@ class KOBOTOUCH(KOBO): if show_debug: debug_print("KoboTouch:set_series - end") - @classmethod def settings(cls): opts = cls._config().parse() @@ -2710,7 +2703,6 @@ class KOBOTOUCH(KOBO): opts.extra_customization = extra_customization return opts - def isAuraHD(self): return self.detected_device.idProduct in self.AURA_HD_PRODUCT_ID def isGlo(self): @@ -2736,7 +2728,6 @@ class KOBOTOUCH(KOBO): self.__class__.gui_name = device_name return device_name - def copying_covers(self): opts = self.settings() return opts.extra_customization[self.OPT_UPLOAD_COVERS] or opts.extra_customization[self.OPT_KEEP_COVER_ASPECT_RATIO] @@ -2752,7 +2743,6 @@ class KOBOTOUCH(KOBO): opts = self.settings() return opts.extra_customization[self.OPT_MODIFY_CSS] - def supports_bookshelves(self): return self.dbversion >= self.min_supported_dbversion @@ -2806,7 +2796,6 @@ class KOBOTOUCH(KOBO): # Supported database version return True - @classmethod def is_debugging_title(cls, title): if not DEBUG: @@ -2858,3 +2847,4 @@ class KOBOTOUCH(KOBO): cursor.close() debug_print('KoboTouch:dump_bookshelves - end') + From 8352b43e284d041bba8cf9096599b4d1403b96e0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 21:37:43 +0530 Subject: [PATCH 0408/1154] Unused import --- src/calibre/gui2/store/stores/cdp_plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/gui2/store/stores/cdp_plugin.py b/src/calibre/gui2/store/stores/cdp_plugin.py index 738ee5e3d5..19b931b962 100644 --- a/src/calibre/gui2/store/stores/cdp_plugin.py +++ b/src/calibre/gui2/store/stores/cdp_plugin.py @@ -7,7 +7,6 @@ __license__ = 'GPL 3' __copyright__ = '2013, Tomasz Długosz ' __docformat__ = 'restructuredtext en' -import re import urllib from contextlib import closing From e5e2eae97fb06ad4e3777960d14d1a8ff697ce32 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 07:21:27 +0530 Subject: [PATCH 0409/1154] Allow adding multiple books to running calibre Allow adding multiple books to an already running calibre by passing multiple file arguments to calibre.exe. Also change the .desktop file in linux to indicate calibre can accept file arguments. Fixes #1207518 [Calibre is invisible in Ubuntu 13.04 "Open with" dialog](https://bugs.launchpad.net/calibre/+bug/1207518) --- src/calibre/gui2/main.py | 5 +++-- src/calibre/gui2/ui.py | 11 ++++++----- src/calibre/linux.py | 6 +++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index bcfc88d239..fc9c542b92 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -401,8 +401,9 @@ def communicate(opts, args): shutdown_other(t) else: if len(args) > 1: - args[1] = os.path.abspath(args[1]) - t.conn.send('launched:'+repr(args)) + args[1:] = [os.path.abspath(x) if os.path.exists(x) else x for x in args[1:]] + import json + t.conn.send('launched:'+json.dumps(args)) t.conn.close() raise SystemExit(0) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index e2f02eb578..43740876ea 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -520,11 +520,12 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ except Empty: return if msg.startswith('launched:'): - argv = eval(msg[len('launched:'):]) - if len(argv) > 1: - path = os.path.abspath(argv[1]) - if os.access(path, os.R_OK): - self.iactions['Add Books'].add_filesystem_book(path) + import json + argv = json.loads(msg[len('launched:'):]) + if isinstance(argv, (list, tuple)) and len(argv) > 1: + files = [os.path.abspath(p) for p in argv[1:] if not os.path.isdir(p) and os.access(p, os.R_OK)] + if files: + self.iactions['Add Books'].add_filesystem_book(files) self.setWindowState(self.windowState() & ~Qt.WindowMinimized|Qt.WindowActive) self.show_windows() diff --git a/src/calibre/linux.py b/src/calibre/linux.py index 41af312c41..98d68816f3 100644 --- a/src/calibre/linux.py +++ b/src/calibre/linux.py @@ -833,7 +833,7 @@ Name=LRF Viewer GenericName=Viewer for LRF files Comment=Viewer for LRF files (SONY ebook format files) TryExec=lrfviewer -Exec=lrfviewer %F +Exec=lrfviewer %f Icon=calibre-viewer MimeType=application/x-sony-bbeb; Categories=Graphics;Viewer; @@ -847,7 +847,7 @@ Name=E-book Viewer GenericName=Viewer for E-books Comment=Viewer for E-books in all the major formats TryExec=ebook-viewer -Exec=ebook-viewer %F +Exec=ebook-viewer %f Icon=calibre-viewer Categories=Graphics;Viewer; ''' @@ -857,7 +857,7 @@ GUI = '''\ [Desktop Entry] Version=1.0 Type=Application -Name=calibre +Name=calibre %F GenericName=E-book library management Comment=E-book library management: Convert, view, share, catalogue all your e-books TryExec=calibre From c92910ea6e892d88a537b6c088eb16ee6151403b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 07:38:20 +0530 Subject: [PATCH 0410/1154] version 0.9.42 --- Changelog.yaml | 34 ++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index b379e92416..afd0516e1a 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,40 @@ # new recipes: # - title: +- version: 0.9.42 + date: 2013-08-02 + + new features: + - title: "When downloading metadata from Amazon, convert the amazon categories into tags. You can turn this off by going to Preferences->Metadata download and configuring the Amazon source." + tickets: [1206763] + + - title: "Kobo driver: Add an option to modify the styling in books being sent to the device, based on a template on the device." + tickets: [1207151] + + - title: "Get Books: Add support for two more Polish ebook stores: cdp.pl and ebooki.allegro.pl" + + - title: "calibredb: Add a new clone command to create clones of libraries with the same custom columns, virtual libraries, etc. as the current library." + + bug fixes: + - title: "MOBI metadata: Do not fail to set metadata in MOBI files if they have EXTH fields with NULL pointers to a cover or thumbnail." + tickets: [1205757] + + - title: "Fix editing of book metadata failing when its timestamp is out of range for the system." + tickets: [1191599] + + - title: "Fix renaming a user category to the same name it already has erases the user category." + tickets: [1207131] + + - title: "Fix drag 'n drop of cover onto conversion dialog not working" + + - title: "Device drivers: Explicitly fsync() all files when writing to devices, to reduce chances of file corruption if the device is disconnected while jobs are running" + + - title: "Fix calibre not appearing in Ubuntu's 'Open with..' menu" + tickets: [1207518] + + improved recipes: + - PC World + - version: 0.9.41 date: 2013-07-27 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index d161a85b3c..c1ea2b80df 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 9, 41) +numeric_version = (0, 9, 42) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " From 3c777a815421a627453717324d5dbcc7aee79ae1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 16:35:54 +0530 Subject: [PATCH 0411/1154] Update The Scotsman --- recipes/the_scotsman.recipe | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/recipes/the_scotsman.recipe b/recipes/the_scotsman.recipe index 0ea73e70b8..6db64cfcaa 100644 --- a/recipes/the_scotsman.recipe +++ b/recipes/the_scotsman.recipe @@ -17,6 +17,7 @@ class TheScotsman(BasicNewsRecipe): max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False + auto_cleanup = True language = 'en_GB' encoding = 'utf-8' publication_type = 'newspaper' @@ -25,13 +26,13 @@ class TheScotsman(BasicNewsRecipe): extra_css = 'body{font-family: Arial,Helvetica,sans-serif}' - keep_only_tags = [dict(attrs={'class':'editorialSection'})] - remove_tags_after = dict(attrs={'class':'socialBookmarkPanel'}) - remove_tags = [ - dict(name=['meta','iframe','object','embed','link']), - dict(attrs={'class':['secondaryArticlesNav','socialBookmarkPanel']}), - dict(attrs={'id':'relatedArticles'}) - ] + #keep_only_tags = [dict(attrs={'class':'editorialSection'})] + #remove_tags_after = dict(attrs={'class':'socialBookmarkPanel'}) + #remove_tags = [ + #dict(name=['meta','iframe','object','embed','link']), + #dict(attrs={'class':['secondaryArticlesNav','socialBookmarkPanel']}), + #dict(attrs={'id':'relatedArticles'}) + #] remove_attributes = ['lang'] conversion_options = { @@ -55,10 +56,10 @@ class TheScotsman(BasicNewsRecipe): ('Opinion' , 'http://www.scotsman.com/cmlink/1.957054' ) ] - def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - for item in soup.findAll('img'): - if not item.has_key('alt'): - item['alt'] = 'image' - return soup + #def preprocess_html(self, soup): + #for item in soup.findAll(style=True): + #del item['style'] + #for item in soup.findAll('img'): + #if not item.has_key('alt'): + #item['alt'] = 'image' + #return soup \ No newline at end of file From 211aff74d7499f42d8dd5f9be7018bf599a7ed50 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 16:46:15 +0530 Subject: [PATCH 0412/1154] ... --- recipes/the_scotsman.recipe | 50 +++++++++++++------------------------ 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/recipes/the_scotsman.recipe b/recipes/the_scotsman.recipe index 6db64cfcaa..ebffdf2698 100644 --- a/recipes/the_scotsman.recipe +++ b/recipes/the_scotsman.recipe @@ -1,4 +1,3 @@ - __license__ = 'GPL v3' __copyright__ = '2008 - 2011, Darko Miletic ' ''' @@ -12,54 +11,39 @@ class TheScotsman(BasicNewsRecipe): __author__ = 'Darko Miletic' description = 'News from Scotland' publisher = 'Johnston Publishing Ltd.' - category = 'news, politics, Scotland, UK' + category = 'news, politics, Scotland, UK' oldest_article = 2 max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False - auto_cleanup = True language = 'en_GB' encoding = 'utf-8' publication_type = 'newspaper' - remove_empty_feeds = True + remove_empty_feeds = True masthead_url = 'http://www.scotsman.com/webimage/swts_thescotsman_image_e_7_25526!image/3142543874.png_gen/derivatives/default/3142543874.png' extra_css = 'body{font-family: Arial,Helvetica,sans-serif}' - - - #keep_only_tags = [dict(attrs={'class':'editorialSection'})] - #remove_tags_after = dict(attrs={'class':'socialBookmarkPanel'}) - #remove_tags = [ - #dict(name=['meta','iframe','object','embed','link']), - #dict(attrs={'class':['secondaryArticlesNav','socialBookmarkPanel']}), - #dict(attrs={'id':'relatedArticles'}) - #] + + keep_only_tags = [dict(name='div', attrs={'class':'article'})] remove_attributes = ['lang'] - + conversion_options = { 'comment' : description , 'tags' : category , 'publisher' : publisher , 'language' : language } - + feeds = [ - ('Latest News' , 'http://www.scotsman.com/cmlink/1.957140' ), - ('UK' , 'http://www.scotsman.com/cmlink/1.957142' ), - ('Scotland' , 'http://www.scotsman.com/cmlink/1.957141' ), - ('International', 'http://www.scotsman.com/cmlink/1.957143' ), - ('Politics' , 'http://www.scotsman.com/cmlink/1.957044' ), + ('Latest News' , 'http://www.scotsman.com/cmlink/1.957140'), + ('UK' , 'http://www.scotsman.com/cmlink/1.957142'), + ('Scotland' , 'http://www.scotsman.com/cmlink/1.957141'), + ('International', 'http://www.scotsman.com/cmlink/1.957143'), + ('Politics' , 'http://www.scotsman.com/cmlink/1.957044'), ('Arts' , 'http://www.scotsman.com/cmlink/1.1804825'), - ('Entertainment', 'http://www.scotsman.com/cmlink/1.957053' ), - ('Sports' , 'http://www.scotsman.com/cmlink/1.957151' ), - ('Business' , 'http://www.scotsman.com/cmlink/1.957156' ), - ('Features' , 'http://www.scotsman.com/cmlink/1.957149' ), - ('Opinion' , 'http://www.scotsman.com/cmlink/1.957054' ) + ('Entertainment', 'http://www.scotsman.com/cmlink/1.957053'), + ('Sports' , 'http://www.scotsman.com/cmlink/1.957151'), + ('Business' , 'http://www.scotsman.com/cmlink/1.957156'), + ('Features' , 'http://www.scotsman.com/cmlink/1.957149'), + ('Opinion' , 'http://www.scotsman.com/cmlink/1.957054') ] - - #def preprocess_html(self, soup): - #for item in soup.findAll(style=True): - #del item['style'] - #for item in soup.findAll('img'): - #if not item.has_key('alt'): - #item['alt'] = 'image' - #return soup \ No newline at end of file + From ed9e3ab4368965904cf64450c6175d27659bb54b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Fri, 2 Aug 2013 23:34:11 +0200 Subject: [PATCH 0413/1154] move format detection to get_details() --- src/calibre/gui2/store/stores/cdp_plugin.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/store/stores/cdp_plugin.py b/src/calibre/gui2/store/stores/cdp_plugin.py index 738ee5e3d5..1aacd23d00 100644 --- a/src/calibre/gui2/store/stores/cdp_plugin.py +++ b/src/calibre/gui2/store/stores/cdp_plugin.py @@ -62,11 +62,6 @@ class CdpStore(BasicStoreConfig, StorePlugin): author = ''.join(data.xpath('.//div[@class="product-description"]//ul[@class="taxons"]/li[2]/a/text()')) price = ''.join(data.xpath('.//span[@itemprop="price"]/text()')) - with closing(br.open(id.strip(), timeout=timeout/4)) as nf: - idata = html.fromstring(nf.read()) - formats = ', '.join(idata.xpath('//div[@id="product-bonus"]/div/ul/li/text()')) - - counter -= 1 s = SearchResult() @@ -76,9 +71,16 @@ class CdpStore(BasicStoreConfig, StorePlugin): s.price = price s.detail_item = id.strip() s.drm = SearchResult.DRM_UNLOCKED - s.formats = formats.upper() yield s if not doc.xpath('//span[@class="next"]/a'): break page+=1 + + def get_details(self, search_result, timeout): + br = browser() + with closing(br.open(search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + formats = ', '.join(idata.xpath('//div[@id="product-bonus"]/div/ul/li/text()')) + search_result.formats = formats.upper() + return True From 1c7a229ebca9a093e338a84c48fb85b76040c4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Sat, 3 Aug 2013 00:13:02 +0200 Subject: [PATCH 0414/1154] bump up plugin version --- src/calibre/gui2/store/stores/cdp_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/stores/cdp_plugin.py b/src/calibre/gui2/store/stores/cdp_plugin.py index 1aacd23d00..4e696f822f 100644 --- a/src/calibre/gui2/store/stores/cdp_plugin.py +++ b/src/calibre/gui2/store/stores/cdp_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 1 # Needed for dynamic plugin loading +store_version = 2 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2013, Tomasz Długosz ' From 8f1bf67ada75a1c7d28f0ca43fc1ea1793c63aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Sat, 3 Aug 2013 00:20:35 +0200 Subject: [PATCH 0415/1154] move format and DRM status detection to get_details() --- .../gui2/store/stores/legimi_plugin.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/store/stores/legimi_plugin.py b/src/calibre/gui2/store/stores/legimi_plugin.py index 1195866faa..0af9220f75 100644 --- a/src/calibre/gui2/store/stores/legimi_plugin.py +++ b/src/calibre/gui2/store/stores/legimi_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 2 # Needed for dynamic plugin loading +store_version = 3 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2011-2013, Tomasz Długosz ' @@ -45,7 +45,6 @@ class LegimiStore(BasicStoreConfig, StorePlugin): url = 'http://www.legimi.com/pl/ebooki/?szukaj=' + urllib.quote_plus(query) br = browser() - drm_pattern = re.compile("zabezpieczona DRM") counter = max_results with closing(br.open(url, timeout=timeout)) as f: @@ -62,14 +61,6 @@ class LegimiStore(BasicStoreConfig, StorePlugin): title = ''.join(data.xpath('.//span[@class="bookListTitle ellipsis"]/text()')) author = ''.join(data.xpath('.//span[@class="bookListAuthor ellipsis"]/text()')) price = ''.join(data.xpath('.//div[@class="bookListPrice"]/span/text()')) - formats = [] - with closing(br.open(id.strip(), timeout=timeout/4)) as nf: - idata = html.fromstring(nf.read()) - formatlist = idata.xpath('.//div[@id="fullBookFormats"]//span[@class="bookFormat"]/text()') - for x in formatlist: - if x.strip() not in formats: - formats.append(x.strip()) - drm = drm_pattern.search(''.join(idata.xpath('.//div[@id="fullBookFormats"]/p/text()'))) counter -= 1 @@ -79,7 +70,20 @@ class LegimiStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price s.detail_item = 'http://www.legimi.com/' + id.strip() - s.formats = ', '.join(formats) - s.drm = SearchResult.DRM_LOCKED if drm else SearchResult.DRM_UNLOCKED yield s + + def get_details(self, search_result, timeout): + drm_pattern = re.compile("zabezpieczona DRM") + formats = [] + br = browser() + with closing(br.open(search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + formatlist = idata.xpath('.//div[@id="fullBookFormats"]//span[@class="bookFormat"]/text()') + for x in formatlist: + if x.strip() not in formats: + formats.append(x.strip()) + drm = drm_pattern.search(''.join(idata.xpath('.//div[@id="fullBookFormats"]/p/text()'))) + search_result.formats = ', '.join(formats) + search_result.drm = SearchResult.DRM_LOCKED if drm else SearchResult.DRM_UNLOCKED + return True From 57437fb0b0774b54bb9282d2318f93fcaeed1174 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Aug 2013 07:53:25 +0530 Subject: [PATCH 0416/1154] Fix incorrect error being raised when trying load a None object as an image with imagemagick. --- src/calibre/utils/magick/__init__.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/calibre/utils/magick/__init__.py b/src/calibre/utils/magick/__init__.py index 01e0a7bdd8..99ed2d98be 100644 --- a/src/calibre/utils/magick/__init__.py +++ b/src/calibre/utils/magick/__init__.py @@ -60,12 +60,12 @@ class FontMetrics(object): # }}} -class PixelWand(_magick.PixelWand): # {{{ +class PixelWand(_magick.PixelWand): # {{{ pass # }}} -class DrawingWand(_magick.DrawingWand): # {{{ +class DrawingWand(_magick.DrawingWand): # {{{ @dynamic_property def font(self): @@ -117,7 +117,7 @@ class DrawingWand(_magick.DrawingWand): # {{{ # }}} -class Image(_magick.Image): # {{{ +class Image(_magick.Image): # {{{ @property def clone(self): @@ -126,9 +126,9 @@ class Image(_magick.Image): # {{{ return ans def load(self, data): - data = bytes(data) if not data: raise ValueError('Cannot open image from empty data string') + data = bytes(data) return _magick.Image.load(self, data) def open(self, path_or_file): @@ -165,7 +165,6 @@ class Image(_magick.Image): # {{{ self.type_ = val return property(fget=fget, fset=fset, doc=_magick.Image.type_.__doc__) - @dynamic_property def size(self): def fget(self): @@ -181,7 +180,6 @@ class Image(_magick.Image): # {{{ self.size_ = (int(val[0]), int(val[1]), filter, blur) return property(fget=fget, fset=fset, doc=_magick.Image.size_.__doc__) - def save(self, path, format=None): if format is None: ext = os.path.splitext(path)[1] From 27e5f2c94a00bf453fed7700e3302c2ef70d0c65 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Aug 2013 13:37:10 +0530 Subject: [PATCH 0417/1154] Private Eye by Martyn Pritchard --- recipes/private_eye.recipe | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 recipes/private_eye.recipe diff --git a/recipes/private_eye.recipe b/recipes/private_eye.recipe new file mode 100644 index 0000000000..265812125d --- /dev/null +++ b/recipes/private_eye.recipe @@ -0,0 +1,31 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1359406781(BasicNewsRecipe): + title = u'Private Eye' + oldest_article = 15 + max_articles_per_feed = 100 + remove_empty_feeds = True + remove_javascript = True + no_stylesheets = True + ignore_duplicate_articles = {'title'} + language = 'en_GB' + __author__ = 'Martyn Pritchard' + encoding = 'iso-8859-1' + compress_news_images = True + compress_news_images_auto_size = 8 + scale_news_images_to_device = False + scale_news_images = (220, 300) + + def get_cover_url(self): + soup = self.index_to_soup('http://www.private-eye.co.uk') + cov = soup.find(attrs={'width' : '180', 'border' : '0'}) + cover_url = 'http://www.private-eye.co.uk/'+cov['src'] + return cover_url + + keep_only_tags = [dict(name='table', attrs={'width':['100%'], 'border':['0'], 'align': ['center'], 'cellspacing':['0'], 'cellpadding':['0']}), + dict(name='table', attrs={'width':['480'], 'cellspacing':['0'], 'cellpadding':['0']}), + dict(name='table', attrs={'width':['490'], 'border':['0'], 'align': ['left'], 'cellspacing':['0'], 'cellpadding':['1']}), + dict(name='table', attrs={'width':['500'], 'cellspacing':['0'], 'cellpadding':['0']}), + ] + + feeds = [(u'Private Eye', u'http://www.private-eye.co.uk/rss/rss.php')] From 34ed01dbd048b32bcf758a6af9d832f28d493830 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Aug 2013 17:30:45 +0530 Subject: [PATCH 0418/1154] Initial implementation of plugins mirroring --- setup/plugins_mirror.py | 526 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 526 insertions(+) create mode 100644 setup/plugins_mirror.py diff --git a/setup/plugins_mirror.py b/setup/plugins_mirror.py new file mode 100644 index 0000000000..dca9595c75 --- /dev/null +++ b/setup/plugins_mirror.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +import urllib2, re, HTMLParser, zlib, gzip, io, sys, bz2, json, errno, urlparse, os, zipfile, ast, tempfile, glob, fcntl, atexit +from future_builtins import map, zip, filter +from collections import namedtuple +from multiprocessing.pool import ThreadPool +from datetime import datetime +from email.utils import parsedate +from contextlib import closing +from functools import partial +from xml.sax.saxutils import escape, quoteattr + +USER_AGENT = 'calibre' +MR_URL = 'http://www.mobileread.com/forums/' +WORKDIR = '/srv/plugins' if os.path.exists('/srv') else '/t/plugins' +PLUGINS = 'plugins.json.bz2' +INDEX = MR_URL + 'showpost.php?p=1362767&postcount=1' +# INDEX = 'file:///t/raw.html' + +IndexEntry = namedtuple('IndexEntry', 'name url donate history uninstall deprecated thread_id') +u = HTMLParser.HTMLParser().unescape + +def read(url, get_info=False): # {{{ + if url.startswith("file://"): + return urllib2.urlopen(url).read() + opener = urllib2.build_opener() + opener.addheaders = [ + ('User-Agent', USER_AGENT), + ('Accept-Encoding', 'gzip,deflate'), + ] + res = opener.open(url) + info = res.info() + encoding = info.get('Content-Encoding') + raw = res.read() + res.close() + if encoding and encoding.lower() in {'gzip', 'x-gzip', 'deflate'}: + if encoding.lower() == 'deflate': + raw = zlib.decompress(raw) + else: + raw = gzip.GzipFile(fileobj=io.BytesIO(raw)).read() + if get_info: + return raw, info + return raw +# }}} + +def url_to_plugin_id(url, deprecated): + query = urlparse.parse_qs(urlparse.urlparse(url).query) + ans = (query['t'] if 't' in query else query['p'])[0] + if deprecated: + ans += '-deprecated' + return ans + +def parse_index(raw=None): # {{{ + raw = raw or read(INDEX).decode('utf-8', 'replace') + + dep_start = raw.index('>Deprecated/Renamed/Retired Plugins:<') + dpat = re.compile(r'''(?is)Donate\s*:\s*(.+?)<(.+?)''', raw): + deprecated = match.start() > dep_start + donate = uninstall = None + history = False + name, url, rest = u(match.group(2)), u(match.group(1)), match.group(3) + m = dpat.search(rest) + if m is not None: + donate = u(m.group(1)) + for m in key_pat.finditer(rest): + k = m.group(1).lower() + if k == 'history' and m.group(2).strip().lower() in {'yes', 'true'}: + history = True + elif k == 'uninstall': + uninstall = tuple(x.strip() for x in m.group(2).strip().split(',')) + + thread_id = url_to_plugin_id(url, deprecated) + if thread_id in seen: + raise ValueError('thread_id for %s and %s is the same: %s' % (seen[thread_id], name, thread_id)) + seen[thread_id] = name + entry = IndexEntry(name, url, donate, history, uninstall, deprecated, thread_id) + yield entry +# }}} + +def parse_plugin_zip_url(raw): + for m in re.finditer(r'''(?is)]*>([^<>]+?\.zip)\s*<''', raw): + url, name = u(m.group(1)), u(m.group(2).strip()) + if name.lower().endswith('.zip'): + return MR_URL + url, name + return None, None + +def load_plugins_index(): + try: + with open(PLUGINS, 'rb') as f: + raw = f.read() + except IOError as err: + if err.errno == errno.ENOENT: + return {} + raise + return json.loads(bz2.decompress(raw)) + +# Get metadata from plugin zip file {{{ +def convert_node(fields, x, names={}): + name = x.__class__.__name__ + conv = lambda x:convert_node(fields, x, names=names) + if name == 'Str': + return x.s.decode('utf-8') if isinstance(x.s, bytes) else x.s + elif name == 'Num': + return x.n + elif name in {'Set', 'List', 'Tuple'}: + func = {'Set':set, 'List':list, 'Tuple':tuple}[name] + return func(map(conv, x.elts)) + elif name == 'Dict': + keys, values = map(conv, x.keys), map(conv, x.values) + return dict(zip(keys, values)) + elif name == 'Call': + if len(x.args) != 1 and len(x.keywords) != 0: + raise TypeError('Unsupported function call for fields: %s' % (fields,)) + return tuple(map(conv, x.args))[0] + elif name == 'Name': + if x.id not in names: + raise ValueError('Could not find name %s for fields: %s' % (x.id, fields)) + return names[x.id] + raise TypeError('Unknown datatype %s for fields: %s' % (x, fields)) + +Alias = namedtuple('Alias', 'name asname') + +def parse_metadata(raw): + module = ast.parse(raw, filename='__init__.py') + top_level_imports = filter(lambda x:x.__class__.__name__ == 'ImportFrom', ast.iter_child_nodes(module)) + top_level_classes = tuple(filter(lambda x:x.__class__.__name__ == 'ClassDef', ast.iter_child_nodes(module))) + top_level_assigments = filter(lambda x:x.__class__.__name__ == 'Assign', ast.iter_child_nodes(module)) + defaults = {'name':'', 'description':'', 'supported_platforms':['windows', 'osx', 'linux'], + 'version':(1, 0, 0), 'author':'Unknown', 'minimum_calibre_version':(0, 9, 42)} + field_names = set(defaults) + + plugin_import_found = set() + all_imports = [] + for node in top_level_imports: + names = getattr(node, 'names', []) + mod = getattr(node, 'module', None) + if names and mod: + names = [Alias(n.name, getattr(n, 'asname', None)) for n in names] + if mod in { + 'calibre.customize', 'calibre.customize.conversion', + 'calibre.ebooks.metadata.sources.base', 'calibre.ebooks.metadata.covers', + 'calibre.devices.interface', 'calibre.ebooks.metadata.fetch', + } or re.match(r'calibre\.devices\.[a-z0-9]+\.driver', mod) is not None: + inames = {n.asname or n.name for n in names} + inames = {x for x in inames if x.lower() != x} + plugin_import_found |= inames + else: + all_imports.append((mod, [n.name for n in names])) + if not plugin_import_found: + return all_imports + + names = {} + for node in top_level_assigments: + targets = {getattr(t, 'id', None) for t in node.targets} + targets.discard(None) + for x in targets - field_names: + try: + val = convert_node({x}, node.value) + except Exception: + pass + else: + names[x] = val + + def parse_class(node): + class_assigments = filter(lambda x:x.__class__.__name__ == 'Assign', ast.iter_child_nodes(node)) + found = {} + for node in class_assigments: + targets = {getattr(t, 'id', None) for t in node.targets} + targets.discard(None) + fields = field_names.intersection(targets) + if fields: + val = convert_node(fields, node.value, names=names) + for field in fields: + found[field] = val + return found + + if top_level_classes: + for node in top_level_classes: + bases = {getattr(x, 'id', None) for x in node.bases} + if not bases.intersection(plugin_import_found): + continue + found = parse_class(node) + if 'name' in found and 'author' in found: + defaults.update(found) + return defaults + for node in top_level_classes: + found = parse_class(node) + if 'name' in found and 'author' in found and 'version' in found: + defaults.update(found) + return defaults + + raise ValueError('Could not find plugin class') + +def get_plugin_info(raw): + metadata = None + with zipfile.ZipFile(io.BytesIO(raw)) as zf: + names = {x.decode('utf-8') if isinstance(x, bytes) else x : x for x in zf.namelist()} + inits = [x for x in names if x.rpartition('/')[-1] == '__init__.py'] + inits.sort(key=lambda x:x.count('/')) + if inits and inits[0] == '__init__.py': + metadata = names[inits[0]] + else: + # Legacy plugin + for name, val in names.iteritems(): + if name.endswith('plugin.py'): + metadata = val + break + if metadata is None: + raise ValueError('No __init__.py found in plugin') + raw = zf.open(metadata).read() + ans = parse_metadata(raw) + if isinstance(ans, dict): + return ans + # The plugin is importing its base class from somewhere else, le sigh + for mod, _ in ans: + mod = mod.split('.') + if mod[0] == 'calibre_plugins': + mod = mod[2:] + mod = '/'.join(mod) + '.py' + if mod in names: + raw = zf.open(names[mod]).read() + ans = parse_metadata(raw) + if isinstance(ans, dict): + return ans + + raise ValueError('Failed to find plugin class') + + +# }}} + +def update_plugin_from_entry(plugin, entry): + plugin['index_name'] = entry.name + plugin['thread_url'] = entry.url + for x in ('donate', 'history', 'deprecated', 'uninstall', 'thread_id'): + plugin[x] = getattr(entry, x) + +def fetch_plugin(old_index, entry): + lm_map = {plugin['thread_id']:plugin for plugin in old_index.itervalues()} + raw = read(entry.url) + url, name = parse_plugin_zip_url(raw) + plugin = lm_map.get(entry.thread_id, None) + + if plugin is not None: + # Previously downloaded plugin + lm = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]) + request = urllib2.Request(url) + request.get_method = lambda : 'HEAD' + with closing(urllib2.urlopen(request)) as response: + info = response.info() + slm = datetime(*parsedate(info.get('Last-Modified'))[:6]) + if lm >= slm: + # The previously downloaded plugin zip file is up-to-date + update_plugin_from_entry(plugin, entry) + return plugin + + raw, info = read(url, get_info=True) + slm = datetime(*parsedate(info.get('Last-Modified'))[:6]) + plugin = get_plugin_info(raw) + plugin['last_modified'] = slm.isoformat() + plugin['file'] = 'staging_%s.zip' % entry.thread_id + plugin['size'] = len(raw) + plugin['original_url'] = url + update_plugin_from_entry(plugin, entry) + with open(plugin['file'], 'wb') as f: + f.write(raw) + return plugin + +def parallel_fetch(old_index, entry): + try: + return fetch_plugin(old_index, entry) + except Exception: + import traceback + return traceback.format_exc() + +def log(*args, **kwargs): + print (*args, **kwargs) + with open('log', 'a') as f: + kwargs['file'] = f + print (*args, **kwargs) + +def atomic_write(raw, name): + with tempfile.NamedTemporaryFile(dir=os.getcwdu(), delete=False) as f: + f.write(raw) + os.rename(f.name, name) + +def fetch_plugins(old_index): + ans = {} + pool = ThreadPool(processes=10) + entries = tuple(parse_index()) + result = pool.map(partial(parallel_fetch, old_index), entries) + for entry, plugin in zip(entries, result): + if isinstance(plugin, dict): + ans[entry.name] = plugin + else: + if entry.name in old_index: + ans[entry.name] = old_index[entry.name] + log('Failed to get plugin', entry.name, 'at', datetime.utcnow().isoformat(), 'with error:') + log(plugin) + # Move staged files + for plugin in ans.itervalues(): + if plugin['file'].startswith('staging_'): + src = plugin['file'] + plugin['file'] = src.partition('_')[-1] + os.rename(src, plugin['file']) + raw = bz2.compress(json.dumps(ans, sort_keys=True, indent=4, separators=(',', ': '))) + atomic_write(raw, PLUGINS) + # Cleanup any extra .zip files + all_plugin_files = {p['file'] for p in ans.itervalues()} + extra = set(glob.glob('*.zip')) - all_plugin_files + for x in extra: + os.unlink(x) + return ans + +def plugin_to_index(plugin): + title = '

    %s

    ' % ( # noqa + quoteattr(plugin['thread_url']), escape(plugin['name'])) + released = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]).strftime('%e %b, %Y').lstrip() + details = [ + 'Version: %s' % escape('.'.join(map(str, plugin['version']))), + 'Released: %s' % escape(released), + 'Author: %s' % escape(plugin['author']), + 'History: %s' % escape('Yes' if plugin['history'] else 'No'), + 'calibre: %s' % escape('.'.join(map(str, plugin['minimum_calibre_version']))), + 'Platforms: %s' % escape(', '.join(sorted(plugin['supported_platforms']) or ['all'])), + ] + if plugin['uninstall']: + details.append('Uninstall: %s' % escape(', '.join(plugin['uninstall']))) + if plugin['donate']: + details.append('Donate' % quoteattr(plugin['donate'])) + block = [] + for li in details: + if li.startswith('calibre:'): + block.append('
    ') + block.append('
  • %s
  • ' % li) + block = '
      %s
    ' % ('\n'.join(block)) + zipfile = '' % ( + quoteattr(plugin['file']), quoteattr(plugin['name'] + '.zip')) + desc = plugin['description'] or '' + if desc: + desc = '

    %s

    ' % desc + return '%s\n%s\n%s\n%s\n\n' % (title, desc, block, zipfile) + +def create_index(index): + plugins = [] + for name in sorted(index): + plugin = index[name] + if not plugin['deprecated']: + plugins.append( + plugin_to_index(plugin)) + index = '''\ + + +Index of calibre plugins + + + + +

    Index of calibre plugins

    + +%s + +''' % ('\n'.join(plugins)) + raw = index.encode('utf-8') + try: + with open('index.html', 'rb') as f: + oraw = f.read() + except EnvironmentError: + oraw = None + if raw != oraw: + atomic_write(raw, 'index.html') + + +def singleinstance(): + path = os.path.abspath('plugins_mirror_update.lock') + try: + f = open(path, 'w') + fcntl.lockf(f.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB) + f.write(str(os.getpid())) + f.flush() + atexit.register(f.close) + return True + except IOError: + return False + return False + +def main(): + try: + os.chdir(WORKDIR) + except OSError as err: + if err.errno == errno.ENOENT: + try: + os.makedirs(WORKDIR) + except EnvironmentError: + pass + os.chdir(WORKDIR) + else: + raise + if not singleinstance(): + print('Another instance is running or you dont have permission to create lock file, aborting.', file=sys.stderr) + raise SystemExit(1) + open('log', 'w').close() + try: + plugins_index = load_plugins_index() + plugins_index = fetch_plugins(plugins_index) + create_index(plugins_index) + except: + import traceback + log('Failed to run at:', datetime.utcnow().isoformat()) + log(traceback.format_exc()) + raise SystemExit(1) + +def test_parse(): # {{{ + raw = read(INDEX).decode('utf-8', 'replace') + + old_entries = [] + from lxml import html + root = html.fromstring(raw) + list_nodes = root.xpath('//div[@id="post_message_1362767"]/ul/li') + # Add our deprecated plugins which are nested in a grey span + list_nodes.extend(root.xpath('//div[@id="post_message_1362767"]/span/ul/li')) + for list_node in list_nodes: + name = list_node.xpath('a')[0].text_content().strip() + url = list_node.xpath('a/@href')[0].strip() + + description_text = list_node.xpath('i')[0].text_content() + description_parts = description_text.partition('Version:') + + details_text = description_parts[1] + description_parts[2].replace('\r\n','') + details_pairs = details_text.split(';') + details = {} + for details_pair in details_pairs: + pair = details_pair.split(':') + if len(pair) == 2: + key = pair[0].strip().lower() + value = pair[1].strip() + details[key] = value + + donation_node = list_node.xpath('i/span/a/@href') + donate = donation_node[0] if donation_node else None + uninstall = tuple(x.strip() for x in details.get('uninstall', '').strip().split(',') if x.strip()) or None + history = details.get('history', 'No').lower() in ['yes', 'true'] + deprecated = details.get('deprecated', 'No').lower() in ['yes', 'true'] + old_entries.append(IndexEntry(name, url, donate, history, uninstall, deprecated, url_to_plugin_id(url, deprecated))) + + new_entries = tuple(parse_index(raw)) + for i, entry in enumerate(old_entries): + if entry != new_entries[i]: + print ('The new entry: %s != %s' % (new_entries[i], entry)) + raise SystemExit(1) + pool = ThreadPool(processes=20) + urls = [e.url for e in new_entries] + data = pool.map(read, urls) + for url, raw in zip(urls, data): + sys.stdout.flush() + root = html.fromstring(raw) + attachment_nodes = root.xpath('//fieldset/table/tr/td/a') + full_url = None + for attachment_node in attachment_nodes: + filename = attachment_node.text_content().lower() + if filename.find('.zip') != -1: + full_url = MR_URL + attachment_node.attrib['href'] + break + new_url, aname = parse_plugin_zip_url(raw) + if new_url != full_url: + print ('new url (%s): %s != %s for plugin at: %s' % (aname, new_url, full_url, url)) + raise SystemExit(1) + +# }}} + +def test_parse_metadata(): # {{{ + raw = b'''\ +import os +from calibre.customize import FileTypePlugin + +MV = (0, 7, 53) + +class HelloWorld(FileTypePlugin): + + name = _('name') # Name of the plugin + description = {1, 2} + supported_platforms = ['windows', 'osx', 'linux'] # Platforms this plugin will run on + author = u'Acme Inc.' # The author of this plugin + version = {1:'a', 'b':2} + file_types = set(['epub', 'mobi']) # The file types that this plugin will be applied to + on_postprocess = True # Run this plugin after conversion is complete + minimum_calibre_version = MV + ''' + vals = { + 'name':'name', 'description':{1, 2}, + 'supported_platforms':['windows', 'osx', 'linux'], + 'author':'Acme Inc.', 'version':{1:'a', 'b':2}, + 'minimum_calibre_version':(0, 7, 53)} + assert parse_metadata(raw) == vals + buf = io.BytesIO() + with zipfile.ZipFile(buf, 'w') as zf: + zf.writestr('very/lovely.py', raw) + zf.writestr('__init__.py', b'from xxx import yyy\nfrom very.lovely import HelloWorld') + assert get_plugin_info(buf.getvalue()) == vals + +# }}} + +if __name__ == '__main__': + # test_parse_metadata() + main() + From 670184a69aa1b444b1ea736cd7344a1df31bdcd6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Aug 2013 17:39:16 +0530 Subject: [PATCH 0419/1154] Fix file permissions and user agent --- setup/plugins_mirror.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/plugins_mirror.py b/setup/plugins_mirror.py index dca9595c75..24bfae072a 100644 --- a/setup/plugins_mirror.py +++ b/setup/plugins_mirror.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' -import urllib2, re, HTMLParser, zlib, gzip, io, sys, bz2, json, errno, urlparse, os, zipfile, ast, tempfile, glob, fcntl, atexit +import urllib2, re, HTMLParser, zlib, gzip, io, sys, bz2, json, errno, urlparse, os, zipfile, ast, tempfile, glob, fcntl, atexit, stat from future_builtins import map, zip, filter from collections import namedtuple from multiprocessing.pool import ThreadPool @@ -16,7 +16,7 @@ from contextlib import closing from functools import partial from xml.sax.saxutils import escape, quoteattr -USER_AGENT = 'calibre' +USER_AGENT = 'calibre mirror' MR_URL = 'http://www.mobileread.com/forums/' WORKDIR = '/srv/plugins' if os.path.exists('/srv') else '/t/plugins' PLUGINS = 'plugins.json.bz2' @@ -291,6 +291,7 @@ def log(*args, **kwargs): def atomic_write(raw, name): with tempfile.NamedTemporaryFile(dir=os.getcwdu(), delete=False) as f: f.write(raw) + os.fchmod(f.fileno(), stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH) os.rename(f.name, name) def fetch_plugins(old_index): From 8b372eeb44708ec1db8e13c31d130a3c68a1dee7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Aug 2013 17:49:40 +0530 Subject: [PATCH 0420/1154] Update notes on file server deployment --- setup/file_hosting_servers.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst index ece966490c..907e88553f 100644 --- a/setup/file_hosting_servers.rst +++ b/setup/file_hosting_servers.rst @@ -15,7 +15,7 @@ dpkg-reconfigure tzdata set timezone to Asia/Kolkata service cron restart -apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools mosh +apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools mosh git chsh -s /bin/zsh mkdir -p /root/staging /root/work/vim /srv/download /srv/manual @@ -23,6 +23,10 @@ mkdir -p /root/staging /root/work/vim /srv/download /srv/manual scp .zshrc .vimrc server: scp -r ~/work/vim/zsh-syntax-highlighting server:work/vim scp -r ~/work/vim/zsh-history-substring-search server:work/vim +cd /usr/local && git clone https://github.com/kovidgoyal/calibre.git + +Add the following to crontab:: + @hourly /usr/bin/python /usr/local/calibre/setup/plugins_mirror.py If the server has a backup hard-disk, mount it at /mnt/backup and edit /etc/fstab so that it is auto-mounted. Then, add the following to crontab:: From a043e21c8e450b2cfbae037f94ffbe16fe854685 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 3 Aug 2013 18:00:18 -0400 Subject: [PATCH 0421/1154] Store: Change punucation into spaces when using author or title searching. --- src/calibre/gui2/store/search/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/search/models.py b/src/calibre/gui2/store/search/models.py index af2c274bd7..96948c2363 100644 --- a/src/calibre/gui2/store/search/models.py +++ b/src/calibre/gui2/store/search/models.py @@ -451,7 +451,9 @@ class SearchFilter(SearchQueryParser): vals = accessor(sr).split(',') elif locvalue in ('author2', 'title2'): m = self.IN_MATCH - vals = re.sub(r'(^|\s)(and|not|or|a|the|is|of|,)(\s|$)', ' ', accessor(sr)).split(' ') + vals = re.sub(r'(^|\s)(and|not|or|a|the|is|of)(\s|$)', ' ', accessor(sr)) + vals = re.sub('[.,!@#$%^&*\(\)\'"\[\]]', ' ', vals) + vals = vals.split(' ') vals = [x for x in vals if x] final_query = query.lower() else: From b9307dd4dac8531fcaea83727d68f500cee93811 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 3 Aug 2013 18:16:22 -0400 Subject: [PATCH 0422/1154] Further work for dealing with common words and punctuation. --- src/calibre/gui2/store/search/models.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/store/search/models.py b/src/calibre/gui2/store/search/models.py index 96948c2363..bfd38d63d9 100644 --- a/src/calibre/gui2/store/search/models.py +++ b/src/calibre/gui2/store/search/models.py @@ -451,11 +451,15 @@ class SearchFilter(SearchQueryParser): vals = accessor(sr).split(',') elif locvalue in ('author2', 'title2'): m = self.IN_MATCH - vals = re.sub(r'(^|\s)(and|not|or|a|the|is|of)(\s|$)', ' ', accessor(sr)) - vals = re.sub('[.,!@#$%^&*\(\)\'"\[\]]', ' ', vals) + def field_trimmer(field): + field = re.sub(r'(^|\s)(and|not|or|a|the|is|of)(\s|$)', ' ', field) + field = re.sub('[.,!@#$%^&*\(\)\'"\[\]]', ' ', field) + return field + vals = field_trimmer(accessor(sr)) vals = vals.split(' ') vals = [x for x in vals if x] - final_query = query.lower() + final_query = field_trimmer(query.lower()) + final_query = re.sub('[ ]{2,}', ' ', final_query) else: vals = [accessor(sr)] if self._match(final_query, vals, m): From c70b70dd76eb1f7fa8f7a587752aef171fc97e5f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 08:20:37 +0530 Subject: [PATCH 0423/1154] Make the punctuation removal faster and more robust --- src/calibre/gui2/store/search/models.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/store/search/models.py b/src/calibre/gui2/store/search/models.py index af2c274bd7..026cb76d8d 100644 --- a/src/calibre/gui2/store/search/models.py +++ b/src/calibre/gui2/store/search/models.py @@ -6,7 +6,7 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import re +import re, string from operator import attrgetter from PyQt4.Qt import (Qt, QAbstractItemModel, QVariant, QPixmap, QModelIndex, QSize, @@ -325,6 +325,9 @@ class SearchFilter(SearchQueryParser): def __init__(self): SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS) self.srs = set([]) + # remove joiner words surrounded by space or at string boundaries + self.joiner_pat = re.compile(r'(^|\s)(and|not|or|a|the|is|of)(\s|$)', re.IGNORECASE) + self.punctuation_table = {ord(x):' ' for x in string.punctuation} def add_search_result(self, search_result): self.srs.add(search_result) @@ -449,11 +452,10 @@ class SearchFilter(SearchQueryParser): if locvalue == 'format': vals = accessor(sr).split(',') - elif locvalue in ('author2', 'title2'): + elif locvalue in {'author2', 'title2'}: m = self.IN_MATCH - vals = re.sub(r'(^|\s)(and|not|or|a|the|is|of|,)(\s|$)', ' ', accessor(sr)).split(' ') - vals = [x for x in vals if x] - final_query = query.lower() + vals = [x for x in self.field_trimmer(accessor(sr)).split() if x] + final_query = ' '.join(self.field_trimmer(icu_lower(query)).split()) else: vals = [accessor(sr)] if self._match(final_query, vals, m): @@ -464,3 +466,8 @@ class SearchFilter(SearchQueryParser): traceback.print_exc() return matches + def field_trimmer(self, field): + ''' Remove common joiner words and punctuation to improve matching, + punctuation is removed first, so that a.and.b becomes a b ''' + return self.joiner_pat.sub(' ', field.translate(self.punctuation_table)) + From db4ba9133b6b4825e777114f8f1ce47792c983df Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 09:18:18 +0530 Subject: [PATCH 0424/1154] Comments editor: Disable insert link until focused Comments editor: The Insert Link button has no effect until the user clicks inside the comments box, therefore disable it until it is ready, to prevent confusion. Fixes #1208073 [Insert Link not working](https://bugs.launchpad.net/calibre/+bug/1208073) --- src/calibre/gui2/comments_editor.py | 47 ++++++++++++++++++----------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/src/calibre/gui2/comments_editor.py b/src/calibre/gui2/comments_editor.py index c043fccba4..ac8ab13d20 100644 --- a/src/calibre/gui2/comments_editor.py +++ b/src/calibre/gui2/comments_editor.py @@ -18,11 +18,11 @@ from PyQt4.QtWebKit import QWebView, QWebPage from calibre.ebooks.chardet import xml_to_unicode from calibre import xml_replace_entities, prepare_string_for_xml -from calibre.gui2 import open_url +from calibre.gui2 import open_url, error_dialog from calibre.utils.soupparser import fromstring from calibre.utils.config import tweaks -class PageAction(QAction): # {{{ +class PageAction(QAction): # {{{ def __init__(self, wac, icon, text, checkable, view): QAction.__init__(self, QIcon(I(icon+'.png')), text, view) @@ -43,14 +43,15 @@ class PageAction(QAction): # {{{ self.page_action.trigger() def update_state(self, *args): - if sip.isdeleted(self) or sip.isdeleted(self.page_action): return + if sip.isdeleted(self) or sip.isdeleted(self.page_action): + return if self.isCheckable(): self.setChecked(self.page_action.isChecked()) self.setEnabled(self.page_action.isEnabled()) # }}} -class BlockStyleAction(QAction): # {{{ +class BlockStyleAction(QAction): # {{{ def __init__(self, text, name, view): QAction.__init__(self, text, view) @@ -62,7 +63,7 @@ class BlockStyleAction(QAction): # {{{ # }}} -class EditorWidget(QWebView): # {{{ +class EditorWidget(QWebView): # {{{ def __init__(self, parent=None): QWebView.__init__(self, parent) @@ -157,6 +158,8 @@ class EditorWidget(QWebView): # {{{ self.action_insert_link = QAction(QIcon(I('insert-link.png')), _('Insert link'), self) self.action_insert_link.triggered.connect(self.insert_link) + self.pageAction(QWebPage.ToggleBold).changed.connect(self.update_link_action) + self.action_insert_link.setEnabled(False) self.action_clear = QAction(QIcon(I('edit-clear')), _('Clear'), self) self.action_clear.triggered.connect(self.clear_text) @@ -166,6 +169,10 @@ class EditorWidget(QWebView): # {{{ self.setHtml('') self.set_readonly(False) + def update_link_action(self): + wac = self.pageAction(QWebPage.ToggleBold) + self.action_insert_link.setEnabled(wac.isEnabled()) + def set_readonly(self, what): self.readonly = what self.page().setContentEditable(not self.readonly) @@ -202,12 +209,16 @@ class EditorWidget(QWebView): # {{{ url = self.parse_link(unicode(link)) if url.isValid(): url = unicode(url.toString()) + self.setFocus(Qt.OtherFocusReason) if name: self.exec_command('insertHTML', '%s'%(prepare_string_for_xml(url, True), prepare_string_for_xml(name))) else: self.exec_command('createLink', url) + else: + error_dialog(self, _('Invalid URL'), + _('The url %r is invalid') % unicode(link), show=True) def ask_link(self): d = QDialog(self) @@ -362,9 +373,9 @@ class Highlighter(QSyntaxHighlighter): self.colors['doctype'] = QColor(192, 192, 192) self.colors['entity'] = QColor(128, 128, 128) self.colors['tag'] = QColor(136, 18, 128) - self.colors['comment'] = QColor( 35, 110, 37) + self.colors['comment'] = QColor(35, 110, 37) self.colors['attrname'] = QColor(153, 69, 0) - self.colors['attrval'] = QColor( 36, 36, 170) + self.colors['attrval'] = QColor(36, 36, 170) def highlightBlock(self, text): state = self.previousBlockState() @@ -378,8 +389,8 @@ class Highlighter(QSyntaxHighlighter): start = pos while pos < len_: if text.mid(pos, 3) == "-->": - pos += 3; - state = State_Text; + pos += 3 + state = State_Text break else: pos += 1 @@ -422,7 +433,7 @@ class Highlighter(QSyntaxHighlighter): if ch == QChar('>'): state = State_Text break - self.setFormat(start, pos - start, self.colors['tag']); + self.setFormat(start, pos - start, self.colors['tag']) # anywhere after tag name and before tag closing ('>') elif state == State_InsideTag: @@ -489,7 +500,7 @@ class Highlighter(QSyntaxHighlighter): # just stop at non-space or tag delimiter start = pos while pos < len_: - ch = text.at(pos); + ch = text.at(pos) if ch.isSpace(): break if ch in (QChar('>'), QChar('/')): @@ -538,7 +549,7 @@ class Highlighter(QSyntaxHighlighter): state = State_DocType else: state = State_TagStart - break; + break elif ch == QChar('&'): start = pos while pos < len_ and text.at(pos) != QChar(';'): @@ -549,12 +560,11 @@ class Highlighter(QSyntaxHighlighter): else: pos += 1 - self.setCurrentBlockState(state) # }}} -class Editor(QWidget): # {{{ +class Editor(QWidget): # {{{ def __init__(self, parent=None, one_line_toolbar=False): QWidget.__init__(self, parent) @@ -650,13 +660,13 @@ class Editor(QWidget): # {{{ return property(fget=fget, fset=fset) def change_tab(self, index): - #print 'reloading:', (index and self.wyswyg_dirty) or (not index and + # print 'reloading:', (index and self.wyswyg_dirty) or (not index and # self.source_dirty) - if index == 1: # changing to code view + if index == 1: # changing to code view if self.wyswyg_dirty: self.code_edit.setPlainText(self.editor.html) self.wyswyg_dirty = False - elif index == 0: #changing to wyswyg + elif index == 0: # changing to wyswyg if self.source_dirty: self.editor.html = unicode(self.code_edit.toPlainText()) self.source_dirty = False @@ -687,4 +697,5 @@ if __name__ == '__main__': w.show() w.html = 'testing' app.exec_() - #print w.html + # print w.html + From 6db94c833fd48e24ab8a9b584c9f4451925695b4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 10:33:22 +0530 Subject: [PATCH 0425/1154] pep8 --- src/calibre/gui2/preferences/plugins.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index 48aae7c3f5..482f64b706 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -23,7 +23,7 @@ from calibre.utils.search_query_parser import SearchQueryParser from calibre.utils.icu import lower from calibre.constants import iswindows -class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{ +class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{ def __init__(self, show_only_user_plugins=False): QAbstractItemModel.__init__(self) @@ -172,7 +172,6 @@ class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{ return self.index(j, 0, parent) return QModelIndex() - def refresh_plugin(self, plugin, rescan=False): if rescan: self.populate() @@ -191,7 +190,7 @@ class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{ if index.internalId() == 0: if role == Qt.DisplayRole: category = self.categories[index.row()] - return QVariant(_("%(plugin_type)s %(plugins)s")%\ + return QVariant(_("%(plugin_type)s %(plugins)s")% dict(plugin_type=category, plugins=_('plugins'))) else: plugin = self.index_to_plugin(index) @@ -276,7 +275,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): unicode(self.search.currentText()), backwards=True) self.highlight_index(idx) - def toggle_plugin(self, *args): self.modify_plugin(op='toggle') @@ -298,8 +296,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if not path: return path = path[0] - if path and os.access(path, os.R_OK) and path.lower().endswith('.zip'): - if not question_dialog(self, _('Are you sure?'), '

    ' + \ + if path and os.access(path, os.R_OK) and path.lower().endswith('.zip'): + if not question_dialog(self, _('Are you sure?'), '

    ' + _('Installing plugins is a security risk. ' 'Plugins can contain a virus/malware. ' 'Only install it if you got it from a trusted source.' @@ -327,7 +325,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): error_dialog(self, _('No valid plugin path'), _('%s is not a valid plugin path')%path).exec_() - def modify_plugin(self, op=''): index = self.plugin_view.currentIndex() if index.isValid(): @@ -399,7 +396,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): def reload_store_plugins(self): self.gui.load_store_plugins() - if self.gui.iactions.has_key('Store'): + if 'Store' in self.gui.iactions: self.gui.iactions['Store'].load_menu() def check_for_add_to_toolbars(self, plugin): @@ -429,7 +426,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): all_locations.iteritems() if key not in plugin_action.dont_add_to] if not allowed_locations: - return # This plugin doesn't want to live in the GUI + return # This plugin doesn't want to live in the GUI from calibre.gui2.dialogs.choose_plugin_toolbars import ChoosePluginToolbarsDialog d = ChoosePluginToolbarsDialog(self, plugin_action, allowed_locations) @@ -445,3 +442,4 @@ if __name__ == '__main__': app = QApplication([]) test_widget('Advanced', 'Plugins') + From e5a8f45879bb042bc9a52af0405a84de107c875f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 10:34:33 +0530 Subject: [PATCH 0426/1154] Make it clear that disabling plugins is prevented by policy --- src/calibre/gui2/preferences/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index 482f64b706..d7ed371a1e 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -338,7 +338,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if op == 'toggle': if not plugin.can_be_disabled: error_dialog(self,_('Plugin cannot be disabled'), - _('The plugin: %s cannot be disabled')%plugin.name).exec_() + _('Disabling the plugin %s is not allowed')%plugin.name).exec_() return if is_disabled(plugin): enable_plugin(plugin) From ba0ce696f484e6ccbbf73aa9bc090569dd518458 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 12:07:28 +0530 Subject: [PATCH 0427/1154] pep8 --- src/calibre/ebooks/oeb/transforms/cover.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/cover.py b/src/calibre/ebooks/oeb/transforms/cover.py index 8facbab785..9bfb4e30d1 100644 --- a/src/calibre/ebooks/oeb/transforms/cover.py +++ b/src/calibre/ebooks/oeb/transforms/cover.py @@ -59,7 +59,6 @@ class CoverManager(object): ''') - def __init__(self, no_default_cover=False, no_svg_cover=False, preserve_aspect_ratio=False, fixed_size=None): self.no_default_cover = no_default_cover @@ -141,7 +140,7 @@ class CoverManager(object): if width is None or height is None: self.log.warning('Failed to read cover dimensions') width, height = 600, 800 - #if self.preserve_aspect_ratio: + # if self.preserve_aspect_ratio: # width, height = 600, 800 self.svg_template = self.svg_template.replace('__viewbox__', '0 0 %d %d'%(width, height)) @@ -170,6 +169,3 @@ class CoverManager(object): titem = getattr(self.oeb.toc, 'item_that_refers_to_cover', None) if titem is not None: titem.href = item.href - - - From 938fe0fa667254ff64bb6e20b2f6226a1b107570 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 12:35:19 +0530 Subject: [PATCH 0428/1154] EPUB Input: Use raster cover metadata EPUB Input: If the EPUB file identifies an actual cover image in addition to the titlepage html file, use the cover image instead of rendering the titlepage. This is faster and has the advantage that an EPUB to EPUB conversion preserves internal cover structure. --- .../ebooks/conversion/plugins/epub_input.py | 41 ++++++++++++++----- src/calibre/ebooks/metadata/opf2.py | 4 +- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/calibre/ebooks/conversion/plugins/epub_input.py b/src/calibre/ebooks/conversion/plugins/epub_input.py index 5ccf00c643..b40f76a78e 100644 --- a/src/calibre/ebooks/conversion/plugins/epub_input.py +++ b/src/calibre/ebooks/conversion/plugins/epub_input.py @@ -75,6 +75,10 @@ class EPUBInput(InputFormatPlugin): return False def rationalize_cover(self, opf, log): + ''' Ensure that the cover information in the guide is correct. That + means, at most one entry with type="cover" that points to a raster + cover and at most one entry with type="titlepage" that points to an + HTML titlepage. ''' removed = None from lxml import etree guide_cover, guide_elem = None, None @@ -109,24 +113,41 @@ class EPUBInput(InputFormatPlugin): # display in the end spine[0].attrib.pop('linear', None) opf.spine[0].is_linear = True - guide_elem.set('href', 'calibre_raster_cover.jpg') + # Ensure that the guide has a cover entry pointing to a raster cover + # and a titlepage entry pointing to the html titlepage. The titlepage + # entry will be used by the epub output plugin, the raster cover entry + # by other output plugins. + from calibre.ebooks.oeb.base import OPF - t = etree.SubElement(elem[0].getparent(), OPF('item'), - href=guide_elem.get('href'), id='calibre_raster_cover') - t.set('media-type', 'image/jpeg') + + # Search for a raster cover identified in the OPF + raster_cover = opf.raster_cover + + # Set the cover guide entry + if raster_cover is not None: + guide_elem.set('href', raster_cover) + else: + # Render the titlepage to create a raster cover + from calibre.ebooks import render_html_svg_workaround + guide_elem.set('href', 'calibre_raster_cover.jpg') + t = etree.SubElement( + elem[0].getparent(), OPF('item'), href=guide_elem.get('href'), id='calibre_raster_cover') + t.set('media-type', 'image/jpeg') + if os.path.exists(guide_cover): + renderer = render_html_svg_workaround(guide_cover, log) + if renderer is not None: + open('calibre_raster_cover.jpg', 'wb').write( + renderer) + + # Set the titlepage guide entry for elem in list(opf.iterguide()): if elem.get('type', '').lower() == 'titlepage': elem.getparent().remove(elem) + t = etree.SubElement(guide_elem.getparent(), OPF('reference')) t.set('type', 'titlepage') t.set('href', guide_cover) t.set('title', 'Title Page') - from calibre.ebooks import render_html_svg_workaround - if os.path.exists(guide_cover): - renderer = render_html_svg_workaround(guide_cover, log) - if renderer is not None: - open('calibre_raster_cover.jpg', 'wb').write( - renderer) return removed def find_opf(self): diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 77e334dd3e..25d4855980 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -1083,12 +1083,12 @@ class OPF(object): # {{{ for item in self.itermanifest(): if item.get('id', None) == cover_id: mt = item.get('media-type', '') - if 'xml' not in mt: + if mt and 'xml' not in mt and 'html' not in mt: return item.get('href', None) for item in self.itermanifest(): if item.get('href', None) == cover_id: mt = item.get('media-type', '') - if mt.startswith('image/'): + if mt and mt.startswith('image/'): return item.get('href', None) @dynamic_property From 66d897893a73b3c54014fe0025f8dcc7f644d500 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 15:07:03 +0530 Subject: [PATCH 0429/1154] Support adding images into the comments field Allow adding images into the comments field, by clicking on the insert link button in the comments editor in the edit metadata dialog. When generating a metadata jacket during book polishing or conversion, embed any images referenced in the comments. --- src/calibre/ebooks/oeb/polish/jacket.py | 31 +++++++++++-- src/calibre/ebooks/oeb/transforms/jacket.py | 27 +++++++++-- src/calibre/gui2/comments_editor.py | 51 ++++++++++++++++----- 3 files changed, 90 insertions(+), 19 deletions(-) diff --git a/src/calibre/ebooks/oeb/polish/jacket.py b/src/calibre/ebooks/oeb/polish/jacket.py index 8ae65f3f9f..ee795c0011 100644 --- a/src/calibre/ebooks/oeb/polish/jacket.py +++ b/src/calibre/ebooks/oeb/polish/jacket.py @@ -11,14 +11,26 @@ from calibre.customize.ui import output_profiles from calibre.ebooks.conversion.config import load_defaults from calibre.ebooks.oeb.base import XPath, OPF from calibre.ebooks.oeb.polish.cover import find_cover_page -from calibre.ebooks.oeb.transforms.jacket import render_jacket as render +from calibre.ebooks.oeb.transforms.jacket import render_jacket as render, referenced_images -def render_jacket(mi): +def render_jacket(container, jacket): + mi = container.mi ps = load_defaults('page_setup') op = ps.get('output_profile', 'default') opmap = {x.short_name:x for x in output_profiles()} output_profile = opmap.get(op, opmap['default']) - return render(mi, output_profile) + root = render(mi, output_profile) + for img, path in referenced_images(root): + container.log('Embedding referenced image: %s into jacket' % path) + ext = path.rpartition('.')[-1] + jacket_item = container.generate_item('jacket_image.'+ext, id_prefix='jacket_img') + name = container.href_to_name(jacket_item.get('href'), container.opf_name) + with open(path, 'rb') as f: + container.parsed_cache[name] = f.read() + container.commit_item(name) + href = container.name_to_href(name, jacket) + img.set('src', href) + return root def is_legacy_jacket(root): return len(root.xpath( @@ -42,17 +54,25 @@ def find_existing_jacket(container): return name def replace_jacket(container, name): - root = render_jacket(container.mi) + root = render_jacket(container, name) container.parsed_cache[name] = root container.dirty(name) def remove_jacket(container): name = find_existing_jacket(container) if name is not None: + remove_jacket_images(container, name) container.remove_item(name) return True return False +def remove_jacket_images(container, name): + root = container.parsed_cache[name] + for img in root.xpath('//*[local-name() = "img" and @src]'): + iname = container.href_to_name(img.get('src'), name) + if container.has_name(iname): + container.remove_item(iname) + def add_or_replace_jacket(container): name = find_existing_jacket(container) found = True @@ -60,6 +80,9 @@ def add_or_replace_jacket(container): jacket_item = container.generate_item('jacket.xhtml', id_prefix='jacket') name = container.href_to_name(jacket_item.get('href'), container.opf_name) found = False + if found: + remove_jacket_images(container, name) + replace_jacket(container, name) if not found: # Insert new jacket into spine diff --git a/src/calibre/ebooks/oeb/transforms/jacket.py b/src/calibre/ebooks/oeb/transforms/jacket.py index 02abda0927..a9cc10dc3d 100644 --- a/src/calibre/ebooks/oeb/transforms/jacket.py +++ b/src/calibre/ebooks/oeb/transforms/jacket.py @@ -6,12 +6,13 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys +import sys, os from xml.sax.saxutils import escape from lxml import etree from calibre import guess_type, strftime +from calibre.constants import iswindows from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.oeb.base import XPath, XHTML_NS, XHTML, xml2text, urldefrag from calibre.library.comments import comments_to_html @@ -84,9 +85,17 @@ class Jacket(object): alt_comments=comments) id, href = self.oeb.manifest.generate('calibre_jacket', 'jacket.xhtml') - item = self.oeb.manifest.add(id, href, guess_type(href)[0], data=root) - self.oeb.spine.insert(0, item, True) - self.oeb.inserted_metadata_jacket = item + jacket = self.oeb.manifest.add(id, href, guess_type(href)[0], data=root) + self.oeb.spine.insert(0, jacket, True) + self.oeb.inserted_metadata_jacket = jacket + for img, path in referenced_images(root): + self.oeb.log('Embedding referenced image %s into jacket' % path) + ext = path.rpartition('.')[-1].lower() + item_id, href = self.oeb.manifest.generate('jacket_image', 'jacket_img.'+ext) + with open(path, 'rb') as f: + item = self.oeb.manifest.add(item_id, href, guess_type(href)[0], data=f.read()) + item.unload_data_from_memory() + img.set('src', jacket.relhref(item.href)) def remove_existing_jacket(self): for x in self.oeb.spine[:4]: @@ -262,3 +271,13 @@ def linearize_jacket(oeb): e.tag = XHTML('span') break +def referenced_images(root): + for img in XPath('//h:img[@src]')(root): + src = img.get('src') + if src.startswith('file://'): + path = src[7:] + if iswindows and path.startswith('/'): + path = path[1:] + if os.path.exists(path): + yield img, path + diff --git a/src/calibre/gui2/comments_editor.py b/src/calibre/gui2/comments_editor.py index ac8ab13d20..921c8676e9 100644 --- a/src/calibre/gui2/comments_editor.py +++ b/src/calibre/gui2/comments_editor.py @@ -12,15 +12,16 @@ import sip from PyQt4.Qt import (QApplication, QFontInfo, QSize, QWidget, QPlainTextEdit, QToolBar, QVBoxLayout, QAction, QIcon, Qt, QTabWidget, QUrl, QFormLayout, - QSyntaxHighlighter, QColor, QChar, QColorDialog, QMenu, QDialog, - QHBoxLayout, QKeySequence, QLineEdit, QDialogButtonBox) + QSyntaxHighlighter, QColor, QChar, QColorDialog, QMenu, QDialog, QLabel, + QHBoxLayout, QKeySequence, QLineEdit, QDialogButtonBox, QPushButton) from PyQt4.QtWebKit import QWebView, QWebPage from calibre.ebooks.chardet import xml_to_unicode from calibre import xml_replace_entities, prepare_string_for_xml -from calibre.gui2 import open_url, error_dialog +from calibre.gui2 import open_url, error_dialog, choose_files from calibre.utils.soupparser import fromstring from calibre.utils.config import tweaks +from calibre.utils.imghdr import what class PageAction(QAction): # {{{ @@ -156,7 +157,7 @@ class EditorWidget(QWebView): # {{{ self.block_style_actions.append(ac) self.action_insert_link = QAction(QIcon(I('insert-link.png')), - _('Insert link'), self) + _('Insert link or image'), self) self.action_insert_link.triggered.connect(self.insert_link) self.pageAction(QWebPage.ToggleBold).changed.connect(self.update_link_action) self.action_insert_link.setEnabled(False) @@ -203,14 +204,18 @@ class EditorWidget(QWebView): # {{{ self.exec_command('hiliteColor', unicode(col.name())) def insert_link(self, *args): - link, name = self.ask_link() + link, name, is_image = self.ask_link() if not link: return - url = self.parse_link(unicode(link)) + url = self.parse_link(link) if url.isValid(): url = unicode(url.toString()) self.setFocus(Qt.OtherFocusReason) - if name: + if is_image: + self.exec_command('insertHTML', + '%s'%(prepare_string_for_xml(url, True), + prepare_string_for_xml(name or '', True))) + elif name: self.exec_command('insertHTML', '%s'%(prepare_string_for_xml(url, True), prepare_string_for_xml(name))) @@ -218,7 +223,7 @@ class EditorWidget(QWebView): # {{{ self.exec_command('createLink', url) else: error_dialog(self, _('Invalid URL'), - _('The url %r is invalid') % unicode(link), show=True) + _('The url %r is invalid') % link, show=True) def ask_link(self): d = QDialog(self) @@ -227,19 +232,43 @@ class EditorWidget(QWebView): # {{{ d.setLayout(l) d.url = QLineEdit(d) d.name = QLineEdit(d) + d.setMinimumWidth(600) d.bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel) + d.br = b = QPushButton(_('&Browse')) + b.setIcon(QIcon(I('document_open.png'))) + def cf(): + files = choose_files(d, 'select link file', _('Choose file'), select_only_single_file=True) + if files: + d.url.setText(files[0]) + b.clicked.connect(cf) + d.la = la = QLabel(_( + 'Enter a URL. You can also choose to create a link to a file on ' + 'your computer. If the selected file is an image, it will be ' + 'inserted as an image. Note that if you create a link to a file on ' + 'your computer, it will stop working if the file is moved.')) + la.setWordWrap(True) + la.setStyleSheet('QLabel { margin-bottom: 1.5ex }') + l.setWidget(0, l.SpanningRole, la) l.addRow(_('Enter &URL:'), d.url) - l.addRow(_('Enter name (optional):'), d.name) + l.addRow(_('Enter &name (optional):'), d.name) + l.addRow(_('Choose a file on your computer:'), d.br) l.addRow(d.bb) d.bb.accepted.connect(d.accept) d.bb.rejected.connect(d.reject) - link, name = None, None + d.resize(d.sizeHint()) + link, name, is_image = None, None, False if d.exec_() == d.Accepted: link, name = unicode(d.url.text()).strip(), unicode(d.name.text()).strip() - return link, name + if link and os.path.exists(link): + with lopen(link, 'rb') as f: + q = what(f) + is_image = q in {'jpeg', 'png', 'gif'} + return link, name, is_image def parse_link(self, link): link = link.strip() + if link and os.path.exists(link): + return QUrl.fromLocalFile(link) has_schema = re.match(r'^[a-zA-Z]+:', link) if has_schema is not None: url = QUrl(link, QUrl.TolerantMode) From eb9714c7619d83cdee3a7f01071b86d4c5f15a1f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 15:21:54 +0530 Subject: [PATCH 0430/1154] Switch to reading plugin index from plugins mirror --- src/calibre/gui2/dialogs/plugin_updater.py | 144 +++++---------------- 1 file changed, 32 insertions(+), 112 deletions(-) diff --git a/src/calibre/gui2/dialogs/plugin_updater.py b/src/calibre/gui2/dialogs/plugin_updater.py index 0b0885dc82..123a14b829 100644 --- a/src/calibre/gui2/dialogs/plugin_updater.py +++ b/src/calibre/gui2/dialogs/plugin_updater.py @@ -23,10 +23,8 @@ from calibre.gui2 import error_dialog, question_dialog, info_dialog, NONE, open_ from calibre.gui2.preferences.plugins import ConfigWidget from calibre.utils.date import UNDEFINED_DATE, format_date - -MR_URL = 'http://www.mobileread.com/forums/' -MR_INDEX_URL = MR_URL + 'showpost.php?p=1362767&postcount=1' - +SERVER = 'http://plugins.calibre-ebook.com/' +INDEX_URL = '%splugins.json.bz2' % SERVER FILTER_ALL = 0 FILTER_INSTALLED = 1 FILTER_UPDATE_AVAILABLE = 2 @@ -55,33 +53,30 @@ def filter_not_installed_plugins(display_plugin): return not display_plugin.is_installed() def read_available_plugins(raise_error=False): + import json, bz2 display_plugins = [] br = browser() - br.set_handle_gzip(True) try: - raw = br.open_novisit(MR_INDEX_URL).read() + raw = br.open_novisit(INDEX_URL).read() if not raw: return + raw = json.loads(bz2.decompress(raw)) except: if raise_error: raise traceback.print_exc() return - raw = raw.decode('utf-8', errors='replace') - root = html.fromstring(raw) - list_nodes = root.xpath('//div[@id="post_message_1362767"]/ul/li') - # Add our deprecated plugins which are nested in a grey span - list_nodes.extend(root.xpath('//div[@id="post_message_1362767"]/span/ul/li')) - for list_node in list_nodes: + for plugin in raw.itervalues(): try: - display_plugin = DisplayPlugin(list_node) + display_plugin = DisplayPlugin(plugin) get_installed_plugin_status(display_plugin) display_plugins.append(display_plugin) except: if DEBUG: - prints('======= MobileRead Parse Error =======') + prints('======= Plugin Parse Error =======') traceback.print_exc() - prints(html.tostring(list_node)) + import pprint + pprint.pprint(plugin) display_plugins = sorted(display_plugins, key=lambda k: k.name) return display_plugins @@ -189,61 +184,21 @@ class PluginFilterComboBox(QComboBox): class DisplayPlugin(object): - def __init__(self, list_node): - # The html from the index web page looks like this: - ''' -

  • Book Sync
    -Add books to a list to be automatically sent to your device the next time it is connected.
    -Version: 1.1; Released: 02-22-2011; Calibre: 0.7.42; Author: kiwidude;
    -Platforms: Windows, OSX, Linux; History: Yes;
  • - ''' - self.name = list_node.xpath('a')[0].text_content().strip() - self.forum_link = list_node.xpath('a/@href')[0].strip() + def __init__(self, plugin): + self.name = plugin['name'] + self.forum_link = plugin['thread_url'] + self.zip_url = SERVER + plugin['file'] self.installed_version = None - - description_text = list_node.xpath('i')[0].text_content() - description_parts = description_text.partition('Version:') - self.description = description_parts[0].strip() - - details_text = description_parts[1] + description_parts[2].replace('\r\n','') - details_pairs = details_text.split(';') - details = {} - for details_pair in details_pairs: - pair = details_pair.split(':') - if len(pair) == 2: - key = pair[0].strip().lower() - value = pair[1].strip() - details[key] = value - - donation_node = list_node.xpath('i/span/a/@href') - self.donation_link = donation_node[0] if donation_node else None - - self.available_version = self._version_text_to_tuple(details.get('version', None)) - - release_date = details.get('released', '01-01-0101').split('-') - date_parts = [int(re.search(r'(\d+)', x).group(1)) for x in release_date] - self.release_date = datetime.date(date_parts[2], date_parts[0], date_parts[1]) - - self.calibre_required_version = self._version_text_to_tuple(details.get('calibre', None)) - self.author = details.get('author', '') - self.platforms = [p.strip().lower() for p in details.get('platforms', '').split(',')] - # Optional pairing just for plugins which require checking for uninstall first - self.uninstall_plugins = [] - uninstall = details.get('uninstall', None) - if uninstall: - self.uninstall_plugins = [i.strip() for i in uninstall.split(',')] - self.has_changelog = details.get('history', 'No').lower() in ['yes', 'true'] - self.is_deprecated = details.get('deprecated', 'No').lower() in ['yes', 'true'] - - def _version_text_to_tuple(self, version_text): - if version_text: - ver = version_text.split('.') - while len(ver) < 3: - ver.append('0') - ver = [int(re.search(r'(\d+)', x).group(1)) for x in ver] - return tuple(ver) - else: - return None + self.description = plugin['description'] + self.donation_link = plugin['donate'] + self.available_version = tuple(plugin['version']) + self.release_date = datetime.datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]).date() + self.calibre_required_version = plugin['minimum_calibre_version'] + self.author = plugin['author'] + self.platforms = plugin['supported_platforms'] + self.uninstall_plugins = plugin['uninstall'] or [] + self.has_changelog = plugin['history'] + self.is_deprecated = plugin['deprecated'] def is_disabled(self): if self.plugin is None: @@ -456,6 +411,7 @@ class PluginUpdaterDialog(SizePersistedDialog): SizePersistedDialog.__init__(self, gui, 'Plugin Updater plugin:plugin updater dialog') self.gui = gui self.forum_link = None + self.zip_url = None self.model = None self.do_restart = False self._initialize_controls() @@ -475,8 +431,8 @@ class PluginUpdaterDialog(SizePersistedDialog): self._select_and_focus_view() else: error_dialog(self.gui, _('Update Check Failed'), - _('Unable to reach the MobileRead plugins forum index page.'), - det_msg=MR_INDEX_URL, show=True) + _('Unable to reach the plugin index page.'), + det_msg=INDEX_URL, show=True) self.filter_combo.setEnabled(False) # Cause our dialog size to be restored from prefs or created on first usage self.resize_dialog() @@ -599,6 +555,7 @@ class PluginUpdaterDialog(SizePersistedDialog): display_plugin = self.model.display_plugins[actual_idx.row()] self.description.setText(display_plugin.description) self.forum_link = display_plugin.forum_link + self.zip_url = display_plugin.zip_url self.forum_action.setEnabled(bool(self.forum_link)) self.install_button.setEnabled(display_plugin.is_valid_to_install()) self.install_action.setEnabled(self.install_button.isEnabled()) @@ -611,6 +568,7 @@ class PluginUpdaterDialog(SizePersistedDialog): else: self.description.setText('') self.forum_link = None + self.zip_url = None self.forum_action.setEnabled(False) self.install_button.setEnabled(False) self.install_action.setEnabled(False) @@ -703,17 +661,7 @@ class PluginUpdaterDialog(SizePersistedDialog): for name_to_remove in uninstall_names: self._uninstall_plugin(name_to_remove) - if DEBUG: - prints('Locating zip file for %s: %s'% (display_plugin.name, display_plugin.forum_link)) - self.gui.status_bar.showMessage( - _('Locating zip file for %(name)s: %(link)s') % dict( - name=display_plugin.name, link=display_plugin.forum_link)) - plugin_zip_url = self._read_zip_attachment_url(display_plugin.forum_link) - if not plugin_zip_url: - return error_dialog(self.gui, _('Install Plugin Failed'), - _('Unable to locate a plugin zip file for %s') % display_plugin.name, - det_msg=display_plugin.forum_link, show=True) - + plugin_zip_url = display_plugin.zip_url if DEBUG: prints('Downloading plugin zip attachment: ', plugin_zip_url) self.gui.status_bar.showMessage(_('Downloading plugin zip attachment: %s') % plugin_zip_url) @@ -852,39 +800,11 @@ class PluginUpdaterDialog(SizePersistedDialog): prints(html.tostring(spoiler_node)) return None - def _read_zip_attachment_url(self, forum_link): - br = browser() - br.set_handle_gzip(True) - try: - raw = br.open_novisit(forum_link).read() - if not raw: - return None - except: - traceback.print_exc() - return None - raw = raw.decode('utf-8', errors='replace') - root = html.fromstring(raw) - attachment_nodes = root.xpath('//fieldset/table/tr/td/a') - for attachment_node in attachment_nodes: - try: - filename = attachment_node.text_content().lower() - if filename.find('.zip') != -1: - full_url = MR_URL + attachment_node.attrib['href'] - return full_url - except: - if DEBUG: - prints('======= MobileRead Parse Error =======') - traceback.print_exc() - prints(html.tostring(attachment_node)) - return None - def _download_zip(self, plugin_zip_url): from calibre.ptempfile import PersistentTemporaryFile br = browser() - br.set_handle_gzip(True) raw = br.open_novisit(plugin_zip_url).read() - pt = PersistentTemporaryFile('.zip') - pt.write(raw) - pt.close() + with PersistentTemporaryFile('.zip') as pt: + pt.write(raw) return pt.name From ca0ee594f91ca9789bd4c020633226a0c31e9257 Mon Sep 17 00:00:00 2001 From: David Forrester Date: Sun, 4 Aug 2013 21:43:02 +1000 Subject: [PATCH 0431/1154] Delete Activity entry for synced shelves When deleting shelves that have been synced, the Activity entry for the shelf was not being deleted. This left a tile for the shelf on the home screen of the Glo and AuraHD. Fixes #1208159 [Delete Activity entry for synced shelves on Kobo devices](https://bugs.launchpad.net/calibre/+bug/1208159) --- src/calibre/devices/kobo/driver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 907a8b309a..2ef2c5efb7 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -2490,7 +2490,8 @@ class KOBOTOUCH(KOBO): "WHERE Type = 'Shelf' " "AND NOT EXISTS " "(SELECT 1 FROM Shelf " - "WHERE Shelf.Name = Activity.Id)" + "WHERE Shelf.Name = Activity.Id " + "AND Shelf._IsDeleted = 'false')" ) cursor = connection.cursor() From d03bfd7f3d7bdd2045fa24e73f4895230ffe2ac6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 23:48:52 +0530 Subject: [PATCH 0432/1154] Fix holding write lock while calling notify in legacy API --- src/calibre/db/legacy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index a97e107c0e..b794d2c14c 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -436,7 +436,7 @@ class LibraryDatabase(object): if typ: identifiers[typ] = val self.new_api._set_field('identifiers', {book_id:identifiers}) - self.notify('metadata', [book_id]) + self.notify('metadata', [book_id]) def set_isbn(self, book_id, isbn, notify=True, commit=True): self.set_identifier(book_id, 'isbn', isbn, notify=notify, commit=commit) @@ -449,9 +449,9 @@ class LibraryDatabase(object): existing = {icu_lower(x) for x in otags} tags = list(otags) + [x for x in tags if icu_lower(x) not in existing] ret = self.new_api._set_field('tags', {book_id:tags}, allow_case_change=allow_case_change) - if notify: - self.notify('metadata', [book_id]) - return ret + if notify: + self.notify('metadata', [book_id]) + return ret def set_metadata(self, book_id, mi, ignore_errors=False, set_title=True, set_authors=True, commit=True, force_changes=False, notify=True): From 27b36ebefc4e3490fdc658bd9779c3a818734b18 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 00:00:34 +0530 Subject: [PATCH 0433/1154] Fix write_lock being help on db when postimport plugins are run --- src/calibre/db/adding.py | 43 +++++++++++++------------- src/calibre/db/cache.py | 67 ++++++++++++++++++++-------------------- 2 files changed, 56 insertions(+), 54 deletions(-) diff --git a/src/calibre/db/adding.py b/src/calibre/db/adding.py index d777e7065e..0ca8c611d8 100644 --- a/src/calibre/db/adding.py +++ b/src/calibre/db/adding.py @@ -106,26 +106,27 @@ def add_catalog(cache, path, title): from calibre.utils.date import utcnow fmt = os.path.splitext(path)[1][1:].lower() - with lopen(path, 'rb') as stream, cache.write_lock: - matches = cache._search('title:="%s" and tags:="%s"' % (title.replace('"', '\\"'), _('Catalog')), None) - db_id = None - if matches: - db_id = list(matches)[0] - try: - mi = get_metadata(stream, fmt) - mi.authors = ['calibre'] - except: - mi = Metadata(title, ['calibre']) - mi.title, mi.authors = title, ['calibre'] - mi.tags = [_('Catalog')] - mi.pubdate = mi.timestamp = utcnow() - if fmt == 'mobi': - mi.cover, mi.cover_data = None, (None, None) - if db_id is None: - db_id = cache._create_book_entry(mi, apply_import_tags=False) - else: - cache._set_metadata(db_id, mi) - cache._add_format(db_id, fmt, stream) + with lopen(path, 'rb') as stream: + with cache.write_lock: + matches = cache._search('title:="%s" and tags:="%s"' % (title.replace('"', '\\"'), _('Catalog')), None) + db_id = None + if matches: + db_id = list(matches)[0] + try: + mi = get_metadata(stream, fmt) + mi.authors = ['calibre'] + except: + mi = Metadata(title, ['calibre']) + mi.title, mi.authors = title, ['calibre'] + mi.tags = [_('Catalog')] + mi.pubdate = mi.timestamp = utcnow() + if fmt == 'mobi': + mi.cover, mi.cover_data = None, (None, None) + if db_id is None: + db_id = cache._create_book_entry(mi, apply_import_tags=False) + else: + cache._set_metadata(db_id, mi) + cache.add_format(db_id, fmt, stream) # Cant keep write lock since post-import hooks might run return db_id @@ -156,7 +157,7 @@ def add_news(cache, path, arg): mi.timestamp = utcnow() db_id = cache._create_book_entry(mi, apply_import_tags=False) - cache._add_format(db_id, fmt, stream) + cache.add_format(db_id, fmt, stream) # Cant keep write lock since post-import hooks might run if not hasattr(path, 'read'): stream.close() diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 28058d7e8a..7db3a8b279 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -679,11 +679,10 @@ class Cache(object): fmtfile = self.format(book_id, original_fmt, as_file=True) if fmtfile is not None: fmt = original_fmt.partition('_')[2] - with self.write_lock: - with fmtfile: - self._add_format(book_id, fmt, fmtfile, run_hooks=False) - self._remove_formats({book_id:(original_fmt,)}) - return True + with fmtfile: + self.add_format(book_id, fmt, fmtfile, run_hooks=False) + self.remove_formats({book_id:(original_fmt,)}) + return True return False @read_api @@ -1150,38 +1149,40 @@ class Cache(object): self._reload_from_db() raise - @write_api + @api def add_format(self, book_id, fmt, stream_or_path, replace=True, run_hooks=True, dbapi=None): - if run_hooks: - # Run import plugins - npath = run_import_plugins(stream_or_path, fmt) - fmt = os.path.splitext(npath)[-1].lower().replace('.', '').upper() - stream_or_path = lopen(npath, 'rb') - fmt = check_ebook_format(stream_or_path, fmt) + with self.write_lock: + if run_hooks: + # Run import plugins + npath = run_import_plugins(stream_or_path, fmt) + fmt = os.path.splitext(npath)[-1].lower().replace('.', '').upper() + stream_or_path = lopen(npath, 'rb') + fmt = check_ebook_format(stream_or_path, fmt) - fmt = (fmt or '').upper() - self.format_metadata_cache[book_id].pop(fmt, None) - try: - name = self.fields['formats'].format_fname(book_id, fmt) - except: - name = None + fmt = (fmt or '').upper() + self.format_metadata_cache[book_id].pop(fmt, None) + try: + name = self.fields['formats'].format_fname(book_id, fmt) + except: + name = None - if name and not replace: - return False + if name and not replace: + return False - path = self._field_for('path', book_id).replace('/', os.sep) - title = self._field_for('title', book_id, default_value=_('Unknown')) - author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0] - stream = stream_or_path if hasattr(stream_or_path, 'read') else lopen(stream_or_path, 'rb') - size, fname = self.backend.add_format(book_id, fmt, stream, title, author, path) - del stream + path = self._field_for('path', book_id).replace('/', os.sep) + title = self._field_for('title', book_id, default_value=_('Unknown')) + author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0] + stream = stream_or_path if hasattr(stream_or_path, 'read') else lopen(stream_or_path, 'rb') + size, fname = self.backend.add_format(book_id, fmt, stream, title, author, path) + del stream - max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend) - self.fields['size'].table.update_sizes({book_id: max_size}) - self._update_last_modified((book_id,)) + max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend) + self.fields['size'].table.update_sizes({book_id: max_size}) + self._update_last_modified((book_id,)) if run_hooks: - # Run post import plugins + # Run post import plugins, the write lock is released so the plugin + # can call api without a locking violation. run_plugins_on_postimport(dbapi or self, book_id, fmt) stream_or_path.close() @@ -1305,17 +1306,17 @@ class Cache(object): return book_id - @write_api + @api def add_books(self, books, add_duplicates=True, apply_import_tags=True, preserve_uuid=False, run_hooks=True, dbapi=None): duplicates, ids = [], [] for mi, format_map in books: - book_id = self._create_book_entry(mi, add_duplicates=add_duplicates, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid) + book_id = self.create_book_entry(mi, add_duplicates=add_duplicates, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid) if book_id is None: duplicates.append((mi, format_map)) else: ids.append(book_id) for fmt, stream_or_path in format_map.iteritems(): - self._add_format(book_id, fmt, stream_or_path, dbapi=dbapi, run_hooks=run_hooks) + self.add_format(book_id, fmt, stream_or_path, dbapi=dbapi, run_hooks=run_hooks) return ids, duplicates @write_api From a97c86bfe3f0d6096444f435ee0f9340d58e3f85 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 4 Aug 2013 16:33:24 -0400 Subject: [PATCH 0434/1154] TXT Input: Allow shipped markdown extensions to be selected and used. --- .../ebooks/conversion/plugins/txt_input.py | 17 +++++++++-- src/calibre/ebooks/txt/processor.py | 7 ++--- src/calibre/gui2/convert/txt_input.py | 2 +- src/calibre/gui2/convert/txt_input.ui | 28 +++++++++++-------- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/calibre/ebooks/conversion/plugins/txt_input.py b/src/calibre/ebooks/conversion/plugins/txt_input.py index a8e18aad11..876e2c381e 100644 --- a/src/calibre/ebooks/conversion/plugins/txt_input.py +++ b/src/calibre/ebooks/conversion/plugins/txt_input.py @@ -47,8 +47,19 @@ class TXTInput(InputFormatPlugin): OptionRecommendation(name='txt_in_remove_indents', recommended_value=False, help=_('Normally extra space at the beginning of lines is retained. ' 'With this option they will be removed.')), - OptionRecommendation(name="markdown_disable_toc", recommended_value=False, - help=_('Do not insert a Table of Contents into the output text.')), + OptionRecommendation(name="markdown_extensions", recommended_value='footnotes, tables, toc', + help=_('Enable extensions to markdown syntax. Extensions are formatting that is not part ' + 'of the standard format.\n' + 'This should be a comma separated list of exentensions to enable:\n' + '* abbr: Abbreviations.\n' + '* def_list: Definition lists.\n' + '* fenced_code: Alternative code block syntax.\n' + '* footnotes: Footnotes\n.' + '* headerid: Allow ids as part of a header.\n' + '* meta: Metadata in the document.\n' + '* tables: Support tables.\n' + '* toc: Generate a table of contents.\n' + '* wikilinks: Wiki style links.\n')), ]) def convert(self, stream, options, file_ext, log, @@ -178,7 +189,7 @@ class TXTInput(InputFormatPlugin): if options.formatting_type == 'markdown': log.debug('Running text through markdown conversion...') try: - html = convert_markdown(txt, disable_toc=options.markdown_disable_toc) + html = convert_markdown(txt, extensions=[x for x in options.markdown_extensions.split(',') if x.strip()]) except RuntimeError: raise ValueError('This txt file has malformed markup, it cannot be' ' converted by calibre. See http://daringfireball.net/projects/markdown/syntax') diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index 0880eca4ca..cb132d374a 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -17,6 +17,7 @@ from calibre.ebooks.conversion.preprocess import DocAnalysis from calibre.utils.cleantext import clean_ascii_chars HTML_TEMPLATE = u'%s \n%s\n' +VALID_MARKDOWN_EXTENSIONS = ['abbr', 'def_list', 'fenced_code', 'footnotes', 'headerid', 'meta', 'tables', 'toc', 'wikilinks'] def clean_txt(txt): ''' @@ -95,11 +96,9 @@ def convert_basic(txt, title='', epub_split_size_kb=0): return HTML_TEMPLATE % (title, u'\n'.join(lines)) -def convert_markdown(txt, title='', disable_toc=False): +def convert_markdown(txt, title='', extensions=['footnotes', 'tables', 'toc']): from calibre.ebooks.markdown import markdown - extensions=['footnotes', 'tables'] - if not disable_toc: - extensions.append('toc') + extensions = [x for x in extensions if x.lower() in VALID_MARKDOWN_EXTENSIONS] md = markdown.Markdown( extensions, safe_mode=False) diff --git a/src/calibre/gui2/convert/txt_input.py b/src/calibre/gui2/convert/txt_input.py index acdf5f43c0..c4ef46a7e2 100644 --- a/src/calibre/gui2/convert/txt_input.py +++ b/src/calibre/gui2/convert/txt_input.py @@ -16,7 +16,7 @@ class PluginWidget(Widget, Ui_Form): def __init__(self, parent, get_option, get_help, db=None, book_id=None): Widget.__init__(self, parent, - ['paragraph_type', 'formatting_type', 'markdown_disable_toc', + ['paragraph_type', 'formatting_type', 'markdown_extensions', 'preserve_spaces', 'txt_in_remove_indents']) self.db, self.book_id = db, book_id for x in get_option('paragraph_type').option.choices: diff --git a/src/calibre/gui2/convert/txt_input.ui b/src/calibre/gui2/convert/txt_input.ui index 211b03294a..1d98eaccfb 100644 --- a/src/calibre/gui2/convert/txt_input.ui +++ b/src/calibre/gui2/convert/txt_input.ui @@ -6,8 +6,8 @@ 0 0 - 518 - 353 + 588 + 330
    @@ -97,8 +97,21 @@ Markdown - - + + + + + + + + Extensions (comma separated): + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + <p>Markdown is a simple markup language for text files, that allows for advanced formatting. To learn more visit <a href="http://daringfireball.net/projects/markdown">markdown</a>. @@ -111,13 +124,6 @@ - - - - Do not insert Table of Contents into output text when using markdown - - -
    From 2fbcd61fb4950b13d5ca0b264256d1c9bff556fe Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 06:46:11 +0530 Subject: [PATCH 0435/1154] Add basic debug info to unhandled error dialog --- src/calibre/gui2/main_window.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/main_window.py b/src/calibre/gui2/main_window.py index 9646211750..6c8a4876dd 100644 --- a/src/calibre/gui2/main_window.py +++ b/src/calibre/gui2/main_window.py @@ -88,7 +88,6 @@ class MainWindow(QMainWindow): cls.___menu_bar = mb cls.___menu = menu - @classmethod def get_menubar_actions(cls): preferences_action = QAction(QIcon(I('config.png')), _('&Preferences'), None) @@ -108,6 +107,11 @@ class MainWindow(QMainWindow): return try: sio = StringIO.StringIO() + try: + from calibre.debug import print_basic_debug_info + print_basic_debug_info(out=sio) + except: + pass traceback.print_exception(type, value, tb, file=sio) fe = sio.getvalue() prints(fe, file=sys.stderr) From e459dee9c63c89ee3af2787e856bbd5ed7becf01 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 08:05:32 +0530 Subject: [PATCH 0436/1154] Add a tweak to debug newdb locking issues --- src/calibre/db/locking.py | 35 +++++++++++++++++++++++++++++++-- src/calibre/gui2/main_window.py | 2 ++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/locking.py b/src/calibre/db/locking.py index 0791a5ac07..90afbb8ffe 100644 --- a/src/calibre/db/locking.py +++ b/src/calibre/db/locking.py @@ -7,12 +7,17 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import traceback, sys from threading import Lock, Condition, current_thread, RLock from functools import partial from collections import Counter +from calibre.utils.config_base import tweaks class LockingError(RuntimeError): - pass + + def __init__(self, msg, extra=None): + RuntimeError.__init__(self, msg) + self.locking_debug_msg = extra def create_locks(): ''' @@ -37,7 +42,8 @@ def create_locks(): the possibility of deadlocking in this scenario). ''' l = SHLock() - return RWLockWrapper(l), RWLockWrapper(l, is_shared=False) + wrapper = DebugRWLockWrapper if tweaks.get('newdb_debug_locking', False) else RWLockWrapper + return wrapper(l), wrapper(l, is_shared=False) class SHLock(object): # {{{ ''' @@ -217,6 +223,31 @@ class RWLockWrapper(object): def owns_lock(self): return self._shlock.owns_lock() +class DebugRWLockWrapper(RWLockWrapper): + + def __init__(self, *args, **kwargs): + RWLockWrapper.__init__(self, *args, **kwargs) + self._last_acquire = () + + def __enter__(self): + try: + RWLockWrapper.__enter__(self) + except LockingError as e: + if self._last_acquire is None: + raise + et, ei, tb = sys.exc_info() + raise LockingError, LockingError(e.message, extra='Last successful call to acquire():\n' + ''.join(self._last_acquire)), tb + self._last_acquire = traceback.format_stack() + + def release(self): + try: + RWLockWrapper.release(self) + except LockingError as e: + if self._last_acquire is None: + raise + et, ei, tb = sys.exc_info() + raise LockingError, LockingError(e.message, extra='Last successful call to acquire():\n' + ''.join(self._last_acquire)), tb + class RecordLock(object): ''' diff --git a/src/calibre/gui2/main_window.py b/src/calibre/gui2/main_window.py index 6c8a4876dd..150c437148 100644 --- a/src/calibre/gui2/main_window.py +++ b/src/calibre/gui2/main_window.py @@ -113,6 +113,8 @@ class MainWindow(QMainWindow): except: pass traceback.print_exception(type, value, tb, file=sio) + if getattr(value, 'locking_debug_msg', None): + prints(value.locking_debug_msg, file=sio) fe = sio.getvalue() prints(fe, file=sys.stderr) msg = '%s:'%type.__name__ + unicode(str(value), 'utf8', 'replace') From 2e3a0e57bf427025cc1f91a962cfaf4cb3d82677 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 10:37:29 +0530 Subject: [PATCH 0437/1154] Fix book count in tooltip of choose library button not updating Fixes #1208217 [library toolbar button not updated on book deletion](https://bugs.launchpad.net/calibre/+bug/1208217) --- src/calibre/gui2/actions/choose_library.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index f50aa95443..9c6a666a92 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -292,16 +292,22 @@ class ChooseLibraryAction(InterfaceAction): path = path.replace(os.sep, '/') return self.stats.pretty(path) + def update_tooltip(self, count): + tooltip = self.action_spec[2] + '\n\n' + _('{0} [{1} books]').format( + getattr(self, 'last_lname', ''), count) + a = self.qaction + a.setToolTip(tooltip) + a.setStatusTip(tooltip) + a.setWhatsThis(tooltip) + def library_changed(self, db): lname = self.stats.library_used(db) - tooltip = self.action_spec[2] + '\n\n' + _('{0} [{1} books]').format(lname, db.count()) + self.last_lname = lname if len(lname) > 16: lname = lname[:16] + u'…' a = self.qaction a.setText(lname) - a.setToolTip(tooltip) - a.setStatusTip(tooltip) - a.setWhatsThis(tooltip) + self.update_tooltip(db.count()) self.build_menus() state = self.view_state_map.get(self.stats.canonicalize_path( db.library_path), None) @@ -557,7 +563,7 @@ class ChooseLibraryAction(InterfaceAction): self.switch_requested(self.qs_locations[idx]) def count_changed(self, new_count): - pass + self.update_tooltip(new_count) def choose_library(self, *args): if not self.change_library_allowed(): From 8da42d6a700c2edd3b06948954a0e99b9df46a90 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 10:59:36 +0530 Subject: [PATCH 0438/1154] Do not popup an error dialog for invalid message Do not popup an error dialog when an invalid message is received from another instance, unless we are in DEBUG mode. See #1207947 --- src/calibre/gui2/ui.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 43740876ea..6d2c2d2d14 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -19,7 +19,7 @@ from PyQt4.Qt import (Qt, SIGNAL, QTimer, QHelpEvent, QAction, QDialog, QSystemTrayIcon, QApplication) from calibre import prints, force_unicode -from calibre.constants import __appname__, isosx, filesystem_encoding +from calibre.constants import __appname__, isosx, filesystem_encoding, DEBUG from calibre.utils.config import prefs, dynamic from calibre.utils.ipc.server import Server from calibre.db import get_db_loader @@ -521,7 +521,16 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ return if msg.startswith('launched:'): import json - argv = json.loads(msg[len('launched:'):]) + try: + argv = json.loads(msg[len('launched:'):]) + except ValueError: + prints('Failed to decode message from other instance: %r' % msg) + if DEBUG: + error_dialog(self, 'Invalid message', + 'Received an invalid message from other calibre instance.' + ' Do you have multiple versions of calibre installed?', + det_msg='Invalid msg: %r' % msg, show=True) + argv = () if isinstance(argv, (list, tuple)) and len(argv) > 1: files = [os.path.abspath(p) for p in argv[1:] if not os.path.isdir(p) and os.access(p, os.R_OK)] if files: From 34d2b0a64fca3b3b14e2eb36fa35d02f5e35ed83 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 11:06:43 +0530 Subject: [PATCH 0439/1154] Allow refering to bugs without closing them in commit messages --- setup/git_pre_commit_hook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/git_pre_commit_hook.py b/setup/git_pre_commit_hook.py index 8be2640f35..978983ca56 100755 --- a/setup/git_pre_commit_hook.py +++ b/setup/git_pre_commit_hook.py @@ -20,7 +20,7 @@ from lxml import html SENDMAIL = ('/home/kovid/work/env', 'pgp_mail') LAUNCHPAD_BUG = 'https://bugs.launchpad.net/calibre/+bug/%s' GITHUB_BUG = 'https://api.github.com/repos/kovidgoyal/calibre/issues/%s' -BUG_PAT = r'(Fix|Implement|Fixes|Fixed|Implemented)\s+#(\d+)' +BUG_PAT = r'(Fix|Implement|Fixes|Fixed|Implemented|See)\s+#(\d+)' class Bug: @@ -45,7 +45,7 @@ class Bug: summary = json.loads(urllib.urlopen(GITHUB_BUG % bug).read())['title'] if summary: print ('Working on bug:', summary) - if int(bug) > 100000: + if int(bug) > 100000 and action != 'See': self.close_bug(bug, action) return match.group() + ' [%s](%s)' % (summary, LAUNCHPAD_BUG % bug) return match.group() + ' (%s)' % summary From 1d338fa8c43108489f00d0a6ca2a514211c13cab Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 12:09:13 +0530 Subject: [PATCH 0440/1154] ... --- src/calibre/gui2/preferences/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index d7ed371a1e..9329477710 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -337,8 +337,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): plugin = self._plugin_model.index_to_plugin(index) if op == 'toggle': if not plugin.can_be_disabled: - error_dialog(self,_('Plugin cannot be disabled'), - _('Disabling the plugin %s is not allowed')%plugin.name).exec_() + info_dialog(self, _('Plugin cannot be disabled'), + _('Disabling the plugin %s is not allowed')%plugin.name, show=True, show_copy_button=False) return if is_disabled(plugin): enable_plugin(plugin) From 6c630b280ffe86c0ac2928ed8b2c0de683d6d428 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 12:54:35 +0530 Subject: [PATCH 0441/1154] Auto-run .ui file compilation on branch change --- setup/git_post_checkout_hook.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 setup/git_post_checkout_hook.py diff --git a/setup/git_post_checkout_hook.py b/setup/git_post_checkout_hook.py new file mode 100755 index 0000000000..f2e47cf582 --- /dev/null +++ b/setup/git_post_checkout_hook.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +import os, subprocess +base = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +subprocess.check_call(['python', 'setup.py', 'gui'], cwd=base) + From b5f1010d8a21e87c90f25f0bbd122a34e53c699b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 13:23:56 +0530 Subject: [PATCH 0442/1154] Conversion: Fix removal of empty inline tags Conversion: Fix empty inline tags that are the second child of a paragraph causing text change location. Fixes #1207735 [Conversion issue from epub to mobi. Italicized word sometimes moves to end of paragrah.](https://bugs.launchpad.net/calibre/+bug/1207735) --- src/calibre/ebooks/oeb/parse_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/oeb/parse_utils.py b/src/calibre/ebooks/oeb/parse_utils.py index 8bf9c23d98..dd4ee11cc6 100644 --- a/src/calibre/ebooks/oeb/parse_utils.py +++ b/src/calibre/ebooks/oeb/parse_utils.py @@ -399,7 +399,7 @@ def parse_html(data, log=None, decoder=None, preprocessor=None, idx = p.index(a) -1 p.remove(a) if a.tail: - if idx <= 0: + if idx < 0: if p.text is None: p.text = '' p.text += a.tail From 3d7b8b939e9c1a165367b8ad8049c3a8ded2d91d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 30 Jul 2013 14:36:36 +0530 Subject: [PATCH 0443/1154] Start work on cover grid --- resources/images/grid.png | Bin 0 -> 10706 bytes src/calibre/gui2/init.py | 20 ++++++++++---- src/calibre/gui2/library/alternate_views.py | 29 ++++++++++++++++++++ src/calibre/gui2/library/views.py | 2 ++ src/calibre/gui2/widgets.py | 10 +++---- 5 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 resources/images/grid.png create mode 100644 src/calibre/gui2/library/alternate_views.py diff --git a/resources/images/grid.png b/resources/images/grid.png new file mode 100644 index 0000000000000000000000000000000000000000..f1dbf40a65b86fe42f834f3b5dc8959db7f94c9e GIT binary patch literal 10706 zcmeI2dpuO#`}Y;)R(DhiB_=skq;V)p4(X&ploUCPW8CGOac)edQXv`1Oisxl6_Z;+ zm{CGyhK!tY7>CRl)L>?4#vGotyYKHG&wtPN^?H8KUys*oW|pN|?Hv2iwtdTgvv%8?V4u#tK7**Z1|rM_+gD-ajGn%-HujsPE^c*tyYnNqt+*j~b4>OOl1-|L+&74*xojfM=i( zyaIWIQ7bO^jfGz-zel=t35~h@+;ss>R}k*k9x->M$sru=;+<(|zVL8eOU}G50-o2U zTqiT5`iO}zyYM3gjU{Q8efb;^IX7C!OiN3nJC`||#3CLT&KpYJz4I&0j~y@Tvx~1GP^apr8g39e5rQX#qH!g&9oQK@w2-pMEfoCM00+_4n;aMg^@SLE3r#pPa*(68w<-XdWBz=Y<8BJ1P-fN*{ zYk4M-)-gA0H`>r_JWVCRZ$wU?uAW;UTGMZHY_TIdS;VBNoNupL*nG1V5|SpunrwGC z8_T*eg0V?TG5XNZ@Qt3d7t>vR zy&8+;@BFeJL0u7 z)+P6#u~@d6TfP4S$%z^Gc0=2`ksvig$ti52v_g;OjWM3K!M>PqH%`oYQ=_T_>m1@47ZCYw7Qt#7~WL5Ld7alHlbT@@-QJcDVkC@3) zBwjG6XR}zW{ZnJ|ol7FIZLtxN#M(;P40W{4lW6p(|4P4h{%P{rhP`=K zw=Wv$ax1<^j=A#1iPAg%V<;}fc5eavKFMMJG2chyyLI4{ss^!l=fR_f!{brW39oGt1Mg|IUMZO>+*#XXi%nD0}A*MXf#6XOP^hlRzkee%1tFNt}MvqyZe zb;!2ZgvHtN1F!IS`N-6ohzY?toK?dUfg2+}PJjKw?DokkQ)5be|Y;` zyp8sc%RW}mTHLi+CA);%)iV0Y54dl4QC=V#+_i?)Y&b5`vNEY%g)>C*&mR3JVY@~g zF#^t@j*d>~2ca}^DcsU(PN1}yy2;k`ukTvF>V>#dwN;ZYp9bE?Q)p|i+FHhJmCAk2;v2L75A-uDozj7`}nSEyIg z!kSx0R&Zk4gq;O~14>MmK@#ld3%YS$F};J^M*92lU7+>y5X!Ydv^~pq0A;rM)4bs$YU-B1M;LRu zS@*JU3iglg&xW~JePqhg8XGMd&!8i1tMGa?lh8C;XlQ^0UO58Y^Hd$9JS!H?>Y zM%N7`wLR&qSK;QJDL=nXJ~O(GpV3xS;V^S{h<2PiapHu{M!Xy)K)%J#xEub`^n_^R zB+50OloR5w^9nbUc1qMw`I^pq6Cir}S>`^^gZ=kK&taTZ4qi}+?3>BH-y}S@rxjgM z6H#i}YN3durt#~|n60r+lw6Aoi;blp#X5pO)A%{3n^v zFU^_YhzC3;Fq!7}3~=tPLRR(xoK?5aeGBtXGQY16m>l0!h4f!p(;uLth)ax!YP3(j zH39Xm$N`O__wf@waQ~H6_@udh3U}>H+8Oc#xoyUk2YDMfrHLt4vmbBzgGtg;hQP9ttL@J5yvv<6J7%gv`XbiP8azp zI2v7_Pa4<@?i&oa5TRoKl@qd&^N>{!^12`!t#yYduFnN$%jk@UjF`&rjE27b#Bx*0 zCCx{O+4iHWA;Yx(c>&JTR6KA5926WtDM-AV#5Y!B^h5SEF`g+-^k11NXFhXWCp;x2 z!00+l*`==?<$3_>&3Fqg3UoUnMMt1pLSBllV(esvXT4cg%tEG3eL(*6skr=@#o4-uPL8=tY?kfh^y*%C2eD27 zwM#N*1beT=>}S9(H|l9O)}0eQkso0^Q_nwhab?&eu2s^)w{J9ErIeJ#Pv4@DG^6Xr z_&4qMTE@M0Uhp!y9@uE&F{}K``i9ZSb_3eo_Hg^IgCaRC%oF}1;iHZm@_emt0B zwTqZhYWdz>MS+v7e9SAQABS;O2V3@3*SAFP4XR-OaW7C_viFXa`y_L_tt@@+24_51 z?oYS|zG?2G%x&fS${f%b(fLoKTfNuTJkWZ8*o(ro2nQ$)N>98#pj=zAXzG zd~P{U>VUOj-nd%K_e_Z#XowlQwX(MpTTXnPBtunbX@duih zuaslxz3T^~qZy>bhIylEKiKT${2~6kB?oTQjTk4IX_1L!OM@hh_?g^HPyZCX`H ztQ}u)v}J4>Kh|E=D<0@^u_D|2v|qmJN;Bl9tTaP9iC4<`Jf^r~38y1HM;n{_9?c0HYANHNAu* ze=@$Mr9iC4Pn&v0=$aO7;fCn4cO&4LD3dv9Z_7#rHM76nX_8FtA*|7z?oA_}hoXbY zX#TKQ_dB6NO!Nt~W;t7~((OD8#OxPnZxwUi- z0Vi5souQd-K0Vu}g+|bH5iXuWluhpO2@AaKKn)$$y)Gsih5C4oLvEU3vky7#b;P_l zy$hwBQcNYWRk>Sd zUg5}mNov>VSLgW+^d<6+OErWmD(F*s`xML}0&>5aUyOrDDBJ$ODj-L#`U`&_sH~9x zPXUbZKVJa58FaQ_yKK2BtDtV4A!q*&zx@P$4gxu_gWr<_4Y7#v47op{g4`q?n5S|5 zudbq`ARV1G=*cUWCb=kWV@pfR^JTHf(@5QmAucX1HPwqQ#A5SK&c>#*fFcGK7Z)?c z;&YC0&$lQ89cpcfz_Ebfaf7$6ouifaTpX!Y0K?tC8&{s+rJVQ6+B+Y9srH+)qupbV zJL-i-*AJQfJ>yjNwO%*tGy4|_Q>Voz5uM8R(3hma{_)xWn?HpZwzIGyg1*Gg;O<7$ zyL}NZYc-yt1!;v?(~nQ&%}jl?{4y}WZ&7X-oo3fO;S(QlVb9SK>jZnZh8>|bV#U@p zQ}9G`S3G#rd-X}VKBukqnRLX~&k2XY^OyhE3XUVuVirISz5YSc_5sf7CH1%j>`1btINLc+)!^zwd%uP?$H6|UtPGM4f7gYq9P znGyG}CY7yZwi!wmr(Zl*{ZhkfxJPggOytFU6qXa%wr}OoIUSrwdoC<6LI&TAxTKTb zIQ(ielzfa0>2d6Tp>~xiIYY5BLOvJ#j44WD3ngnf>8XL8d54xVl4KRiyM6svmUgpL zopvv8A3xTy{E%RJ&|gf}U}dBv`~gN@eo5#%Pmd??aygX; zySp2HrAesG3dobJP(crmHV&--yrcR;VFfoTX=$NJ;Tqo8(a|yN3GV0IRXTn28sCnl z1Nn(MX$BmTT?jC#UD`rwuTs6!{P{EXF|i~z5vz(>@hu(riSs8fM|vNA0t0R$>QerM z$pZg6D&DdZO%E!#ScO$jqL zR80pl73)vqatXN5dC=SaZolwisv$s>FaXsrzE4?4OwA~Tm5`z>D@E&3A0k|4Ta1!D z_GTYdsEO@$qri~fXnkKlv2|QP>$<1O?&hFo{qGu|xT)*>BAcLP$YH}cVd#8j93Y`4 zK?!is3O-`X_}Z33fIegCBoU}*dgSHReZBfm>u0@9YgTc?Q(X4kH&k2A_L}P@deu15 zet8JMj2wl}erBnO__V$O{LHcR-pBG)Pzc?(lAD5)PuU-~7p>m{1@hb!mGvic`F1^- zmYTH#%i4|oQ5-iZ_IX5%pZI#;@=+?*pjUUMJZWhSV#yoS)C1;Hix`9{#{wLWo&Yg( zLC7)DKh*=RWMJT-E*fT+XInSA%|dl?41 zH>k?DavvLo3wd*(RpnyUkC0$SIByW-12!>^lWS{{AhvU>^fK>UdMaS}2cez9z9?+E zP2Eaw#VS2c^!XPooXVUVo=s=xvjfr$ zl{g9=cEwI6c@7bC$_z)5E3unb#TyEiPJA*qq3Zy$U&uEvU#M~djn7*3&cuso8*t=% zElvmjVZ0z!SF$Ua;rp9?IMXhginTsuhh@b2`V9)5ckkW+QDWL-`Sw@%<{n~cX%5Rk zp#3YkhacLT1qHfsP=M*(pmh<1{c@jWLaBAgPE9DXe#GCo8~l0=2{fXmV*@)<(<7!e z8JPJNlnR6GCfeiM7F1Ew;}yBPwcBTSxIG>KA-z~oydj4=evc2l>c(!Lv1urX6K)4# z8^j>3jb0yRUWufyunZMXruMINLTMOPNCM;I)co4HO4ZDp0HqAQIeH ztpBM%TT>ASQQL<e+E1~XlfLHbE1t3@dyC7PBTW=lKne>U&0rUSE!xHS`k0S+&bJp{ z;lHvu(i}8`xN3aK=Q`_E@5J#eFdLwwt7nhV@5E(*Vb_9e1~j(*n$C`$(VF0B{nURG zXuHBj?7jUcn_5u!OfOhm(VC{3-vypY7hAH;K!V_g-#0z8Bt7a|Szdi~Q@2{aA%mb< z@cn)faBCWfZuzSVR`)^g6O4piOpf#=ke(}Zp8lf+#i~v}IR3Hk*U_o5P3$L{M0EZU zh0<8rAXZS&9nmWK($3u_stvxCzb-0|j4duLSv>gU-%&_ZH4pk>I@x+i0z0|&xL07T z7qOV#fN#>&ESkWQjsqY%1>iO2E2E%Ofj}e)_!cbGv?U-Tml2QyPr5Q8OaGft-_<9F zJWk8L%nF)Wzf87$d?NQ_94r&H<1!TX&$#n^5Zc_dN7*2K9yM?=J3qWh2=`#u&y8$B zYfcXDW_@xK^k&Kl2?WRO4ul441ZL(Q@aQF{vi~gUW`RVlEixc>_-!?Y{Qy)OV=Yrj z84LMY+;{QZ>Zp67HtbL9+>^12*w75rx6NS2dM+)EJ%^U4PO0AIJ65<>3fIl_5iKUc z(b(vp4U5uhY6D?U!n7Km66jf|mmocz=j);+DkOyMa5x*+j74g&m=$<3p-{98gWJW6 zTG;5WnlXme+u#d*ss$3Eo_Zuz$N^!4OCOD-?_MFybk^lX0mce*roGS>Jwu|;f~@o3 z1c%BLtR%j+UrQ#jGZ3vDWr51~qmkgE;eI)d0KvLbrsWM{F2Z z-uqax3Tcv)hpdn0H8wYQ$}6wQ?E6Kc-zBY^9CqQQSXguS1IfeeX8&;SH`72Pd>=v_ zeNOV^&j_W@-DOT?C?b2g;L+>B;uvw%!dtCI1c;7Bj{<~+8iKudVm_;Ih9}z-$dw_3 zy|9Ce9b|Rnc^V86UrwNtrpQU8vA9lKK?R{+3w-ES`o&X;400+(gJ>u0<^<0SDAA0>bNe#$o9jdm+@#gQ2wUtbf5Rb=7Y zJ5Sp~uw|$7|DxdZpSk{1*XkA`58nyeOVud}3H zEmEworO3YIRI5;cp)H}aY%lUjFw~ET@i^>l?K~{k7!hOl)`jM;XEJeJT9&vK2E*{V zTrR*?-sY6z%_KH+NVvtk;V{3W0qE!AEYTZ!jKtl(L;un)qV&-D(pCGB!lzmTrM zn!sVs#=#%Fc6Y)w3cdkZ>=aFQeJS7k6M5lFp=JRtH`l&CK%B7jGr_OUBOWd|@xJ~D z69c~-^F@epq1gl^x*h}KjR`{Vn8CM*E+LhfeL(JI)hlmZ7xB20gT^$ox-~U}`YQNu z^cWI~@lKNhGu|vy>tdIw(a37Vt*eQn1!_i;y$E{dYZ3+aFH2{1mpbp)1ygik3Ez!_8biKvUIJKBq zjQgF@y+BE{PVY64G$}~kyUQ<<72V+1$OCO8j(6&;y@@fyk-P`XUxv^ue6GJtS~frT zEPdZUsMGE{2XY&VppEFd=Nw}>M!zLkf21R?uLOu%dlOx~&0g|i->e=wZC?`$5e7`1 zzUg0ozPGOnWELkX@lZ^lbSsdlIH=;cJ76X)x)N`Q0XS#j@>ac4nchIT&SEVTER)!) zCX1bVqobmt+=DQ@@CEZuRTvBwd8@ChE20}b!#{ACQJ!Un?ZA>+{oXn7%D_eFU4Z%u z_R82ntwv>zLc>Tic4YW@;%Qm-<-AU}FhR4{a4^TJQ!fv=I!;8;&$SKcql=5lWs5u84-YG+%$~CS=pPHZO9iqm^I@oXW zIKZpa@CvFLShP>2k_&k=ESAGplxr%*iY6G~hrR){`#rDZ5W`D1t2(fA;0Ccev^$oARD+PNcO}co_T2^BU_@3L2SeN`SYR$P(s*FBHA5lKtzi$rR5)u zEsU|tyUGzA%`W$h>2o_CD%UZ@-_w#0D1_cn8;y55ZeW3i+BM8 zC>NX0OhB~D1XN@LYAgu5$4znGvSo`J!XHv3HvLDDuzT3;!vdEm1K;FZVQRwjgNSwT zF+{Uqw0E)XkD{Xd#J_bNPokF=zNy7`h#9XMGwlb$>=vLU0ch$0(d%!K#~NrmC0<9+ zu6gW$PcqT%fR_S{J6iy7CCWu&{8yS@gLD_VJ*565$|Yd#oYD#RRxhnRUoJwIS=t6q@0z(W={PD3l7QsD0#N<~P!te-F#>J*f8&(qcLf0+sOfw9-AoMCVE-HJ74rc63E-Ulh{T+4pQ^26HvPf$4{;*<#^1i2aF+UsL zl)nhN^pz?yM6myA3Q*7Fo?JVn`l!Z(wzwKU=7eDGFlh#nMa62Iq?Q1&9S7`#Ug!5H z)tF3;)j83s3P~&Wg;uy&4DtUT<|W(A*DwC3nN3r zc`e{vRZ*G)pYui_ycWxX(pUEK;G&(sX%}+-a0%PfE-ylZe zGN>-*0jc;C3Z}7j9&HN0X%KBsv%JUplO1#>2a{z=FiU41rXWFAK%hI_uKY0muo9?Xs`ds-tJ( zq<&#T_qUoS9mtp;(O{iPrbyoNh}!F(hWR(dAOwYa45C-T_Bk^|@)y&sNS#dnV7WUn z9vlhqt-!7N^5l=!{_Z>7s1f=}!ObKFJP&4vPV129Tq98@MR|nuw*la_pGyd07WYqq zNp`kFsa2I<+oI$Qk)(5{-{$@{&jjJGPvoVB&n6!(9bl&tY7l~*HdG9w+JVN@_Dlz_ zVdZ6U9S|{Qm&L}!9;~J2B`LIOhn|1c0J*=?Z`o>9m+9tXv-$Zyt6sP)VHh7Xik8uI zvJ-91hYP5YT7u&_D{E2YTAbuog7|gyV%g#%rKc4)3B5E{g%uZmQ&xBb^4QFy(~>hZ zxR&PyQA}6NnK=9rclSY;-U?67BYk%h$*yOlfa|_qIxHQ8sy45$_QF}JO8ypO*17at z`N-+Na${G4x8ejBczK}%@2DQ8?76YhzV!R|Jze%X67z64*Lx(NxW^y@HXn@}vCNaa zYy5tLw01P)S)yvcf9i$2_Nec3Nv9hSK|kO?qR6ikE}Tlp-ox5+tEcbvX-*ReBGns? zf8X+tSoG=tsBqywo&T%CmRq$TFaMB`Rn_G9<6Cw8N=THN8)D)g%#B Date: Thu, 1 Aug 2013 12:55:36 +0530 Subject: [PATCH 0444/1154] Shortcut for grid view --- src/calibre/gui2/init.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index 283823158d..2ca9b43729 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' import functools from PyQt4.Qt import (Qt, QApplication, QStackedWidget, QMenu, QTimer, - QSize, QSizePolicy, QStatusBar, QLabel, QFont) + QSize, QSizePolicy, QStatusBar, QLabel, QFont, QAction) from calibre.utils.config import prefs, tweaks from calibre.constants import (isosx, __appname__, preferred_encoding, @@ -243,6 +243,20 @@ class StatusBar(QStatusBar): # {{{ # }}} +class GridViewButton(LayoutButton): # {{{ + + def __init__(self, gui): + sc = _('Shift+Alt+G') + LayoutButton.__init__(self, I('grid.png'), _('Cover Grid'), parent=gui, shortcut=sc) + self.set_state_to_show() + self.action_toggle = QAction(self.icon(), _('Toggle') + ' ' + self.label, self) + gui.addAction(self.action_toggle) + gui.keyboard.register_shortcut('grid view toggle' + self.label, unicode(self.action_toggle.text()), + default_keys=(sc,), action=self.action_toggle) + self.action_toggle.triggered.connect(self.toggle) + +# }}} + class LayoutMixin(object): # {{{ def __init__(self): @@ -278,8 +292,7 @@ class LayoutMixin(object): # {{{ self.status_bar = StatusBar(self) stylename = unicode(self.style().objectName()) - self.grid_view_button = LayoutButton(I('grid.png'), _('Cover Grid'), parent=self, shortcut=_('Shift+Alt+G')) - self.grid_view_button.set_state_to_show() + self.grid_view_button = GridViewButton(self) for x in button_order: button = self.grid_view_button if x == 'gv' else getattr(self, x+'_splitter').button From fe363c95f5b5090afe000b8acf450dfb611d92b0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 13:43:25 +0530 Subject: [PATCH 0445/1154] Wire up the grid view button --- src/calibre/gui2/init.py | 4 ++++ src/calibre/gui2/library/alternate_views.py | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index 2ca9b43729..fe4a4e836e 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -293,6 +293,7 @@ class LayoutMixin(object): # {{{ self.status_bar = StatusBar(self) stylename = unicode(self.style().objectName()) self.grid_view_button = GridViewButton(self) + self.grid_view_button.toggled.connect(self.toggle_grid_view) for x in button_order: button = self.grid_view_button if x == 'gv' else getattr(self, x+'_splitter').button @@ -339,6 +340,9 @@ class LayoutMixin(object): # {{{ self.library_view.currentIndex()) self.library_view.setFocus(Qt.OtherFocusReason) + def toggle_grid_view(self, show): + self.library_view.alternate_views.show_view('grid' if show else None) + def bd_cover_changed(self, id_, cdata): self.library_view.model().db.set_cover(id_, cdata) if self.cover_flow: diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 6970dba94e..0c216d8d2f 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -11,7 +11,8 @@ from PyQt4.Qt import QListView class AlternateViews(object): def __init__(self, main_view): - self.views = {} + self.views = {None:main_view} + self.stack_positions = {None:0} self.current_view = self.main_view = main_view self.stack = None @@ -21,9 +22,17 @@ class AlternateViews(object): def add_view(self, key, view): self.views[key] = view + self.stack_positions[key] = self.stack.count() self.stack.addWidget(view) self.stack.setCurrentIndex(0) + def show_view(self, key=None): + view = self.views[key] + if view is self.current_view: + return + self.stack.setCurrentIndex(self.stack_positions[key]) + self.current_view = view + class GridView(QListView): pass From 0b9fe18d911794512ca1487385d12968e6742a11 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 18:07:14 +0530 Subject: [PATCH 0446/1154] Grid view renders --- src/calibre/db/cache.py | 13 ++ src/calibre/gui2/library/alternate_views.py | 194 +++++++++++++++++++- src/calibre/gui2/library/views.py | 2 + src/calibre/gui2/ui.py | 1 + 4 files changed, 208 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 7db3a8b279..677c3168d5 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -89,6 +89,7 @@ class Cache(object): self.formatter_template_cache = {} self.dirtied_cache = {} self.dirtied_sequence = 0 + self.cover_caches = set() # Implement locking for all simple read/write API methods # An unlocked version of the method is stored with the name starting @@ -1031,9 +1032,21 @@ class Cache(object): path = self._field_for('path', book_id).replace('/', os.sep) self.backend.set_cover(book_id, path, data) + for cc in self.cover_caches: + cc.invalidate(book_id) return self._set_field('cover', { book_id:(0 if data is None else 1) for book_id, data in book_id_data_map.iteritems()}) + @write_api + def add_cover_cache(self, cover_cache): + if not callable(cover_cache.invalidate): + raise ValueError('Cover caches must have an invalidate method') + self.cover_caches.add(cover_cache) + + @write_api + def remove_cover_cache(self, cover_cache): + self.cover_caches.discard(cover_cache) + @write_api def set_metadata(self, book_id, mi, ignore_errors=False, force_changes=False, set_title=True, set_authors=True): diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 0c216d8d2f..19588904a6 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -6,7 +6,15 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' -from PyQt4.Qt import QListView +from time import time +from collections import OrderedDict +from threading import Lock, Event, Thread +from Queue import Queue + +from PyQt4.Qt import ( + QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, QPalette, QColor) + +from calibre import fit_image class AlternateViews(object): @@ -25,6 +33,7 @@ class AlternateViews(object): self.stack_positions[key] = self.stack.count() self.stack.addWidget(view) self.stack.setCurrentIndex(0) + view.setModel(self.main_view._model) def show_view(self, key=None): view = self.views[key] @@ -32,7 +41,188 @@ class AlternateViews(object): return self.stack.setCurrentIndex(self.stack_positions[key]) self.current_view = view + if view is not self.main_view: + view.shown() + + def set_database(self, db, stage=0): + for view in self.views.itervalues(): + if view is not self.main_view: + view.set_database(db, stage=stage) + +class CoverCache(dict): + + def __init__(self, limit=200): + self.items = OrderedDict() + self.lock = Lock() + self.limit = limit + + def invalidate(self, book_id): + with self.lock: + self.items.pop(book_id, None) + + def __call__(self, key): + with self.lock: + ans = self.items.pop(key, False) + if ans is not False: + self.items[key] = ans + if len(self.items) > self.limit: + del self.items[next(self.items.iterkeys())] + + return ans + + def set(self, key, val): + with self.lock: + self.items[key] = val + if len(self.items) > self.limit: + del self.items[next(self.items.iterkeys())] + + def clear(self): + with self.lock: + self.items.clear() + +class CoverDelegate(QStyledItemDelegate): + + def __init__(self, parent, width, height): + super(CoverDelegate, self).__init__(parent) + self.cover_size = QSize(width, height) + self.item_size = self.cover_size + QSize(8, 8) + self.spacing = max(10, min(50, int(0.1 * width))) + self.cover_cache = CoverCache() + self.render_queue = Queue() + + def sizeHint(self, option, index): + return self.item_size + + def paint(self, painter, option, index): + QStyledItemDelegate.paint(self, painter, option, QModelIndex()) # draw the hover and selection highlights + db = index.model().db + try: + book_id = db.id(index.row()) + except (ValueError, IndexError, KeyError): + return + db = db.new_api + cdata = self.cover_cache(book_id) + painter.save() + try: + rect = option.rect + rect.adjust(4, 4, -4, -4) + if cdata is None or cdata is False: + title = db.field_for('title', book_id, default_value='') + painter.drawText(rect, Qt.AlignCenter|Qt.TextWordWrap, title) + if cdata is False: + self.render_queue.put(book_id) + else: + dx = max(0, int((rect.width() - cdata.width())/2.0)) + dy = max(0, rect.height() - cdata.height()) + rect.adjust(dx, dy, -dx, 0) + painter.drawImage(rect, cdata) + finally: + painter.restore() + +def join_with_timeout(q, timeout=2): + q.all_tasks_done.acquire() + try: + endtime = time() + timeout + while q.unfinished_tasks: + remaining = endtime - time() + if remaining <= 0.0: + raise RuntimeError('Waiting for queue to clear timed out') + q.all_tasks_done.wait(remaining) + finally: + q.all_tasks_done.release() class GridView(QListView): - pass + + update_item = pyqtSignal(object) + + def __init__(self, parent): + QListView.__init__(self, parent) + pal = QPalette(self.palette()) + r = g = b = 0x50 + pal.setColor(pal.Base, QColor(r, g, b)) + pal.setColor(pal.Text, QColor(Qt.white if (r + g + b)/3.0 < 128 else Qt.black)) + self.setPalette(pal) + self.setUniformItemSizes(True) + self.setWrapping(True) + self.setFlow(self.LeftToRight) + self.setLayoutMode(self.Batched) + self.setResizeMode(self.Adjust) + self.setSelectionMode(self.ExtendedSelection) + self.delegate = CoverDelegate(self, 135, 180) + self.setItemDelegate(self.delegate) + self.setSpacing(self.delegate.spacing) + self.ignore_render_requests = Event() + self.render_thread = None + self.update_item.connect(self.re_render, type=Qt.QueuedConnection) + + def shown(self): + if self.render_thread is None: + self.render_thread = Thread(target=self.render_covers) + self.render_thread.daemon = True + self.render_thread.start() + + def render_covers(self): + q = self.delegate.render_queue + while True: + book_id = q.get() + try: + if book_id is None: + return + if self.ignore_render_requests.is_set(): + continue + try: + self.render_cover(book_id) + except: + import traceback + traceback.print_exc() + finally: + q.task_done() + + def render_cover(self, book_id): + cdata = self.model().db.new_api.cover(book_id) + if self.ignore_render_requests.is_set(): + return + if cdata is not None: + p = QImage() + p.loadFromData(cdata) + cdata = None + if not p.isNull(): + width, height = p.width(), p.height() + scaled, nwidth, nheight = fit_image(width, height, self.delegate.cover_size.width(), self.delegate.cover_size.height()) + if scaled: + if self.ignore_render_requests.is_set(): + return + p = p.scaled(nwidth, nheight, Qt.IgnoreAspectRatio, Qt.SmoothTransformation) + cdata = p + self.delegate.cover_cache.set(book_id, cdata) + self.update_item.emit(book_id) + + def re_render(self, book_id): + m = self.model() + try: + index = m.db.row(book_id) + except (IndexError, ValueError, KeyError): + return + self.update(m.index(index, 0)) + + def shutdown(self): + self.ignore_render_requests.set() + self.delegate.render_queue.put(None) + + def set_database(self, newdb, stage=0): + if stage == 0: + self.ignore_render_requests.set() + try: + # Use a timeout so that if, for some reason, the render thread + # gets stuck, we dont deadlock, future covers wont get + # rendered, but this is better than a deadlock + join_with_timeout(self.delegate.render_queue) + except RuntimeError: + print ('Cover rendering thread is stuck!') + finally: + self.ignore_render_requests.clear() + else: + self.delegate.cover_cache.clear() + + diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index dc412fc368..257540c53d 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -632,6 +632,7 @@ class BooksView(QTableView): # {{{ # Initialization/Delegate Setup {{{ def set_database(self, db): + self.alternate_views.set_database(db) self.save_state() self._model.set_database(db) self.tags_delegate.set_database(db) @@ -639,6 +640,7 @@ class BooksView(QTableView): # {{{ self.authors_delegate.set_database(db) self.series_delegate.set_auto_complete_function(db.all_series) self.publisher_delegate.set_auto_complete_function(db.all_publishers) + self.alternate_views.set_database(db, stage=1) def database_changed(self, db): for i in range(self.model().columnCount(None)): diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 6d2c2d2d14..8e34c2a84f 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -812,6 +812,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ return True def shutdown(self, write_settings=True): + self.grid_view.shutdown() try: db = self.library_view.model().db cf = db.clean From f3bb9ea4ed7eb181dc69fb246ba29691c63061ce Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 18:13:14 +0530 Subject: [PATCH 0447/1154] Register cover caches with the db --- src/calibre/gui2/library/alternate_views.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 19588904a6..479c071784 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -80,6 +80,9 @@ class CoverCache(dict): with self.lock: self.items.clear() + def __hash__(self): + return id(self) + class CoverDelegate(QStyledItemDelegate): def __init__(self, parent, width, height): @@ -212,6 +215,11 @@ class GridView(QListView): def set_database(self, newdb, stage=0): if stage == 0: self.ignore_render_requests.set() + try: + self.model().db.new_api.remove_cover_cache(self.delegate.cover_cache) + except AttributeError: + pass # db is None + newdb.new_api.add_cover_cache(self.delegate.cover_cache) try: # Use a timeout so that if, for some reason, the render thread # gets stuck, we dont deadlock, future covers wont get From b95b0942da67aec5b3e144e2697de64ef431019c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 18:32:36 +0530 Subject: [PATCH 0448/1154] Remember grid view state --- src/calibre/gui2/init.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index fe4a4e836e..14910d749a 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -248,12 +248,33 @@ class GridViewButton(LayoutButton): # {{{ def __init__(self, gui): sc = _('Shift+Alt+G') LayoutButton.__init__(self, I('grid.png'), _('Cover Grid'), parent=gui, shortcut=sc) - self.set_state_to_show() - self.action_toggle = QAction(self.icon(), _('Toggle') + ' ' + self.label, self) - gui.addAction(self.action_toggle) - gui.keyboard.register_shortcut('grid view toggle' + self.label, unicode(self.action_toggle.text()), - default_keys=(sc,), action=self.action_toggle) - self.action_toggle.triggered.connect(self.toggle) + if tweaks.get('use_new_db', False): + self.set_state_to_show() + self.action_toggle = QAction(self.icon(), _('Toggle') + ' ' + self.label, self) + gui.addAction(self.action_toggle) + gui.keyboard.register_shortcut('grid view toggle' + self.label, unicode(self.action_toggle.text()), + default_keys=(sc,), action=self.action_toggle) + self.action_toggle.triggered.connect(self.toggle) + self.toggled.connect(self.update_state) + self.grid_enabled = True + else: + self.setVisible(False) + self.grid_enabled = False + + def update_state(self, checked): + if checked: + self.set_state_to_hide() + else: + self.set_state_to_show() + + def save_state(self): + if self.grid_enabled: + gprefs['grid view visible'] = bool(self.isChecked()) + + def restore_state(self): + if self.grid_enabled and gprefs.get('grid view visible', False): + self.toggle() + # }}} @@ -366,11 +387,13 @@ class LayoutMixin(object): # {{{ s = getattr(self, x+'_splitter') s.update_desired_state() s.save_state() + self.grid_view_button.save_state() def read_layout_settings(self): # View states are restored automatically when set_database is called for x in ('cb', 'tb', 'bd'): getattr(self, x+'_splitter').restore_state() + self.grid_view_button.restore_state() def update_status_bar(self, *args): v = self.current_view() From 7717fe6d0537854ea1278f2c354ea043bd8b875d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Aug 2013 21:27:53 +0530 Subject: [PATCH 0449/1154] Sync the main and grid views --- src/calibre/gui2/library/alternate_views.py | 64 ++++++++++++++++++++- src/calibre/gui2/library/views.py | 12 ++-- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 479c071784..0aada05b7e 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -6,16 +6,28 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' +import itertools, operator from time import time from collections import OrderedDict from threading import Lock, Event, Thread from Queue import Queue +from functools import wraps from PyQt4.Qt import ( - QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, QPalette, QColor) + QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, + QPalette, QColor, QItemSelection) from calibre import fit_image +def sync(func): + @wraps(func) + def ans(self, *args, **kwargs): + if self.break_link or self.current_view is self.main_view: + return + with self: + return func(self, *args, **kwargs) + return ans + class AlternateViews(object): def __init__(self, main_view): @@ -23,6 +35,8 @@ class AlternateViews(object): self.stack_positions = {None:0} self.current_view = self.main_view = main_view self.stack = None + self.break_link = False + self.main_connected = False def set_stack(self, stack): self.stack = stack @@ -34,6 +48,8 @@ class AlternateViews(object): self.stack.addWidget(view) self.stack.setCurrentIndex(0) view.setModel(self.main_view._model) + view.selectionModel().currentChanged.connect(self.slave_current_changed) + view.selectionModel().selectionChanged.connect(self.slave_selection_changed) def show_view(self, key=None): view = self.views[key] @@ -42,13 +58,45 @@ class AlternateViews(object): self.stack.setCurrentIndex(self.stack_positions[key]) self.current_view = view if view is not self.main_view: + self.main_current_changed(self.main_view.currentIndex()) + self.main_selection_changed() view.shown() + if not self.main_connected: + self.main_connected = True + self.main_view.selectionModel().currentChanged.connect(self.main_current_changed) + self.main_view.selectionModel().selectionChanged.connect(self.main_selection_changed) + view.setFocus(Qt.OtherFocusReason) def set_database(self, db, stage=0): for view in self.views.itervalues(): if view is not self.main_view: view.set_database(db, stage=stage) + def __enter__(self): + self.break_link = True + + def __exit__(self, *args): + self.break_link = False + + @sync + def slave_current_changed(self, current, *args): + self.main_view.set_current_row(current.row(), for_sync=True) + + @sync + def slave_selection_changed(self, *args): + rows = {r.row() for r in self.current_view.selectionModel().selectedIndexes()} + self.main_view.select_rows(rows, using_ids=False, change_current=False, scroll=False) + + @sync + def main_current_changed(self, current, *args): + self.current_view.set_current_row(current.row()) + + @sync + def main_selection_changed(self, *args): + rows = {r.row() for r in self.main_view.selectionModel().selectedIndexes()} + self.current_view.select_rows(rows) + + class CoverCache(dict): def __init__(self, limit=200): @@ -232,5 +280,19 @@ class GridView(QListView): else: self.delegate.cover_cache.clear() + def select_rows(self, rows): + sel = QItemSelection() + sm = self.selectionModel() + m = self.model() + # Create a range based selector for each set of contiguous rows + # as supplying selectors for each individual row causes very poor + # performance if a large number of rows has to be selected. + for k, g in itertools.groupby(enumerate(rows), lambda (i,x):i-x): + group = list(map(operator.itemgetter(1), g)) + sel.merge(QItemSelection(m.index(min(group), 0), m.index(max(group), 0)), sm.Select) + sm.select(sel, sm.ClearAndSelect) + def set_current_row(self, row): + sm = self.selectionModel() + sm.setCurrentIndex(self.model().index(row, 0), sm.NoUpdate) diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 257540c53d..c1292c1b6b 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -863,7 +863,7 @@ class BooksView(QTableView): # {{{ self.scrollTo(self.model().index(row, i), self.PositionAtCenter) break - def set_current_row(self, row=0, select=True): + def set_current_row(self, row=0, select=True, for_sync=False): if row > -1 and row < self.model().rowCount(QModelIndex()): h = self.horizontalHeader() logical_indices = list(range(h.count())) @@ -876,10 +876,14 @@ class BooksView(QTableView): # {{{ pairs.sort(cmp=lambda x,y:cmp(x[1], y[1])) i = pairs[0][0] index = self.model().index(row, i) - self.setCurrentIndex(index) - if select: + if for_sync: sm = self.selectionModel() - sm.select(index, sm.ClearAndSelect|sm.Rows) + sm.setCurrentIndex(index, sm.NoUpdate) + else: + self.setCurrentIndex(index) + if select: + sm = self.selectionModel() + sm.select(index, sm.ClearAndSelect|sm.Rows) def keyPressEvent(self, ev): val = self.horizontalScrollBar().value() From 2e6b37ee7f5c262dc17707c1ccc415265603a71c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 08:39:23 +0530 Subject: [PATCH 0450/1154] Document shortcut for toggling grid view --- manual/gui.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manual/gui.rst b/manual/gui.rst index 127d0062c0..5b3b96152d 100755 --- a/manual/gui.rst +++ b/manual/gui.rst @@ -593,6 +593,8 @@ Calibre has several keyboard shortcuts to save you time and mouse movement. Thes - Toggle Book Details panel * - :kbd:`Alt+Shift+T` - Toggle Tag Browser + * - :kbd:`Alt+Shift+G` + - Toggle Cover Grid * - :kbd:`Alt+A` - Show books by the same author as the current book * - :kbd:`Alt+T` From 3ae7205720d5abddbff0e7bf5f0f960bb5b1294e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 09:49:52 +0530 Subject: [PATCH 0451/1154] Speed up cover rendering by caching QPixmaps --- src/calibre/gui2/library/alternate_views.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 0aada05b7e..02a8ca939c 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -15,7 +15,7 @@ from functools import wraps from PyQt4.Qt import ( QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, - QPalette, QColor, QItemSelection) + QPalette, QColor, QItemSelection, QPixmap) from calibre import fit_image @@ -108,18 +108,22 @@ class CoverCache(dict): with self.lock: self.items.pop(book_id, None) - def __call__(self, key): + def __getitem__(self, key): + ' Must be called in the GUI thread ' with self.lock: - ans = self.items.pop(key, False) + ans = self.items.pop(key, False) # pop() so that item is moved to the top if ans is not False: + if isinstance(ans, QImage): + # Convert to QPixmap, since rendering QPixmap is much + # faster + ans = QPixmap.fromImage(ans) self.items[key] = ans - if len(self.items) > self.limit: - del self.items[next(self.items.iterkeys())] return ans def set(self, key, val): with self.lock: + self.items.pop(key, None) # pop() so that item is moved to the top self.items[key] = val if len(self.items) > self.limit: del self.items[next(self.items.iterkeys())] @@ -152,7 +156,7 @@ class CoverDelegate(QStyledItemDelegate): except (ValueError, IndexError, KeyError): return db = db.new_api - cdata = self.cover_cache(book_id) + cdata = self.cover_cache[book_id] painter.save() try: rect = option.rect @@ -166,7 +170,7 @@ class CoverDelegate(QStyledItemDelegate): dx = max(0, int((rect.width() - cdata.width())/2.0)) dy = max(0, rect.height() - cdata.height()) rect.adjust(dx, dy, -dx, 0) - painter.drawImage(rect, cdata) + painter.drawPixmap(rect, cdata) finally: painter.restore() From d97ca5359329f4b26ec7700909d3907bd199caac Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 09:55:02 +0530 Subject: [PATCH 0452/1154] Scroll per pixel --- src/calibre/gui2/library/alternate_views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 02a8ca939c..5722e9e287 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -203,6 +203,7 @@ class GridView(QListView): self.setLayoutMode(self.Batched) self.setResizeMode(self.Adjust) self.setSelectionMode(self.ExtendedSelection) + self.setVerticalScrollMode(self.ScrollPerPixel) self.delegate = CoverDelegate(self, 135, 180) self.setItemDelegate(self.delegate) self.setSpacing(self.delegate.spacing) From 0b70899dfdd20f3fd9d480e2dc87fc3e16bde8da Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 10:36:32 +0530 Subject: [PATCH 0453/1154] Add context menu to grid view --- src/calibre/gui2/library/alternate_views.py | 13 +++++++++++++ src/calibre/gui2/library/views.py | 1 + 2 files changed, 14 insertions(+) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 5722e9e287..237404e32c 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -96,6 +96,11 @@ class AlternateViews(object): rows = {r.row() for r in self.main_view.selectionModel().selectedIndexes()} self.current_view.select_rows(rows) + def set_context_menu(self, menu): + for view in self.views.itervalues(): + if view is not self.main_view: + view.set_context_menu(menu) + class CoverCache(dict): @@ -210,6 +215,7 @@ class GridView(QListView): self.ignore_render_requests = Event() self.render_thread = None self.update_item.connect(self.re_render, type=Qt.QueuedConnection) + self.context_menu = None def shown(self): if self.render_thread is None: @@ -301,3 +307,10 @@ class GridView(QListView): sm = self.selectionModel() sm.setCurrentIndex(self.model().index(row, 0), sm.NoUpdate) + def set_context_menu(self, menu): + self.context_menu = menu + + def contextMenuEvent(self, event): + if self.context_menu is not None: + self.context_menu.popup(event.globalPos()) + event.accept() diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index c1292c1b6b..2a653d7b1d 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -696,6 +696,7 @@ class BooksView(QTableView): # {{{ def set_context_menu(self, menu, edit_collections_action): self.setContextMenuPolicy(Qt.DefaultContextMenu) self.context_menu = menu + self.alternate_views.set_context_menu(menu) self.edit_collections_action = edit_collections_action def contextMenuEvent(self, event): From f17708760d26bfdf997dce5462335e1c773f768f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 10:41:09 +0530 Subject: [PATCH 0454/1154] Double click to read in grid view --- src/calibre/gui2/library/alternate_views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 237404e32c..eee5bc3e6e 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -215,6 +215,7 @@ class GridView(QListView): self.ignore_render_requests = Event() self.render_thread = None self.update_item.connect(self.re_render, type=Qt.QueuedConnection) + self.doubleClicked.connect(parent.iactions['View'].view_triggered) self.context_menu = None def shown(self): From 0c427b046b3ffc41f8ab51cfb0cb4d51a8e6a086 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 11:18:38 +0530 Subject: [PATCH 0455/1154] Add sorting to grid view context menu --- src/calibre/gui2/library/alternate_views.py | 29 ++++++++++++++++++--- src/calibre/gui2/library/views.py | 7 +++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index eee5bc3e6e..66e59fe7fd 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -11,11 +11,11 @@ from time import time from collections import OrderedDict from threading import Lock, Event, Thread from Queue import Queue -from functools import wraps +from functools import wraps, partial from PyQt4.Qt import ( QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, - QPalette, QColor, QItemSelection, QPixmap) + QPalette, QColor, QItemSelection, QPixmap, QMenu) from calibre import fit_image @@ -50,6 +50,7 @@ class AlternateViews(object): view.setModel(self.main_view._model) view.selectionModel().currentChanged.connect(self.slave_current_changed) view.selectionModel().selectionChanged.connect(self.slave_selection_changed) + view.sort_requested.connect(self.main_view.sort_by_named_field) def show_view(self, key=None): view = self.views[key] @@ -194,6 +195,7 @@ def join_with_timeout(q, timeout=2): class GridView(QListView): update_item = pyqtSignal(object) + sort_requested = pyqtSignal(object, object) def __init__(self, parent): QListView.__init__(self, parent) @@ -216,6 +218,7 @@ class GridView(QListView): self.render_thread = None self.update_item.connect(self.re_render, type=Qt.QueuedConnection) self.doubleClicked.connect(parent.iactions['View'].view_triggered) + self.gui = parent self.context_menu = None def shown(self): @@ -313,5 +316,25 @@ class GridView(QListView): def contextMenuEvent(self, event): if self.context_menu is not None: - self.context_menu.popup(event.globalPos()) + lv = self.gui.library_view + menu = self._temp_menu = QMenu(self) + sm = QMenu(_('Sort by'), menu) + db = self.model().db + for col in lv.visible_columns: + m = db.metadata_for_field(col) + last = self.model().sorted_on + ascending = True + extra = '' + if last[0] == col: + ascending = not last[1] + extra = ' [%s]' % _('reverse') + sm.addAction('%s%s' % (m.get('name', col), extra), partial(self.do_sort, col, ascending)) + + for ac in self.context_menu.actions(): + menu.addAction(ac) + menu.addMenu(sm) + menu.popup(event.globalPos()) event.accept() + + def do_sort(self, column, ascending): + self.sort_requested.emit(column, ascending) diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 2a653d7b1d..41ef245edb 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -845,6 +845,13 @@ class BooksView(QTableView): # {{{ def column_map(self): return self._model.column_map + @property + def visible_columns(self): + h = self.horizontalHeader() + logical_indices = (x for x in xrange(h.count()) if not h.isSectionHidden(x)) + rmap = {i:x for i, x in enumerate(self.column_map)} + return (rmap[h.visualIndex(x)] for x in logical_indices if h.visualIndex(x) > -1) + def refresh_book_details(self): idx = self.currentIndex() if idx.isValid(): From 5522a2bf94f86947b2f14e916e75498701b46108 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 13:21:00 +0530 Subject: [PATCH 0456/1154] Drag 'n drop for the grid view Refactor the D'nD code from the main view so that it can be re-used for the grid view directly. --- src/calibre/gui2/library/alternate_views.py | 175 +++++++++++++++++++- src/calibre/gui2/library/views.py | 156 +---------------- 2 files changed, 181 insertions(+), 150 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 66e59fe7fd..921299e074 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -6,7 +6,8 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' -import itertools, operator +import itertools, operator, os +from types import MethodType from time import time from collections import OrderedDict from threading import Lock, Event, Thread @@ -15,10 +16,171 @@ from functools import wraps, partial from PyQt4.Qt import ( QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, - QPalette, QColor, QItemSelection, QPixmap, QMenu) + QPalette, QColor, QItemSelection, QPixmap, QMenu, QApplication, QMimeData, + QUrl, QDrag, QPoint, QPainter, QRect) from calibre import fit_image +from calibre.utils.config import prefs +# Drag 'n Drop {{{ +def dragMoveEvent(self, event): + event.acceptProposedAction() + +def event_has_mods(self, event=None): + mods = event.modifiers() if event is not None else \ + QApplication.keyboardModifiers() + return mods & Qt.ControlModifier or mods & Qt.ShiftModifier + +def mousePressEvent(base_class, self, event): + ep = event.pos() + if self.indexAt(ep) in self.selectionModel().selectedIndexes() and \ + event.button() == Qt.LeftButton and not self.event_has_mods(): + self.drag_start_pos = ep + return base_class.mousePressEvent(self, event) + +def drag_icon(self, cover, multiple): + cover = cover.scaledToHeight(120, Qt.SmoothTransformation) + if multiple: + base_width = cover.width() + base_height = cover.height() + base = QImage(base_width+21, base_height+21, + QImage.Format_ARGB32_Premultiplied) + base.fill(QColor(255, 255, 255, 0).rgba()) + p = QPainter(base) + rect = QRect(20, 0, base_width, base_height) + p.fillRect(rect, QColor('white')) + p.drawRect(rect) + rect.moveLeft(10) + rect.moveTop(10) + p.fillRect(rect, QColor('white')) + p.drawRect(rect) + rect.moveLeft(0) + rect.moveTop(20) + p.fillRect(rect, QColor('white')) + p.save() + p.setCompositionMode(p.CompositionMode_SourceAtop) + p.drawImage(rect.topLeft(), cover) + p.restore() + p.drawRect(rect) + p.end() + cover = base + return QPixmap.fromImage(cover) + +def drag_data(self): + m = self.model() + db = m.db + rows = self.selectionModel().selectedIndexes() + selected = list(map(m.id, rows)) + ids = ' '.join(map(str, selected)) + md = QMimeData() + md.setData('application/calibre+from_library', ids) + fmt = prefs['output_format'] + + def url_for_id(i): + try: + ans = db.format_path(i, fmt, index_is_id=True) + except: + ans = None + if ans is None: + fmts = db.formats(i, index_is_id=True) + if fmts: + fmts = fmts.split(',') + else: + fmts = [] + for f in fmts: + try: + ans = db.format_path(i, f, index_is_id=True) + except: + ans = None + if ans is None: + ans = db.abspath(i, index_is_id=True) + return QUrl.fromLocalFile(ans) + + md.setUrls([url_for_id(i) for i in selected]) + drag = QDrag(self) + col = self.selectionModel().currentIndex().column() + try: + md.column_name = self.column_map[col] + except AttributeError: + md.column_name = 'title' + drag.setMimeData(md) + cover = self.drag_icon(m.cover(self.currentIndex().row()), + len(selected) > 1) + drag.setHotSpot(QPoint(-15, -15)) + drag.setPixmap(cover) + return drag + +def mouseMoveEvent(base_class, self, event): + if not self.drag_allowed: + return + if self.drag_start_pos is None: + return base_class.mouseMoveEvent(self, event) + + if self.event_has_mods(): + self.drag_start_pos = None + return + + if not (event.buttons() & Qt.LeftButton) or \ + (event.pos() - self.drag_start_pos).manhattanLength() \ + < QApplication.startDragDistance(): + return + + index = self.indexAt(event.pos()) + if not index.isValid(): + return + drag = self.drag_data() + drag.exec_(Qt.CopyAction) + self.drag_start_pos = None + +def dragEnterEvent(self, event): + if int(event.possibleActions() & Qt.CopyAction) + \ + int(event.possibleActions() & Qt.MoveAction) == 0: + return + paths = self.paths_from_event(event) + + if paths: + event.acceptProposedAction() + +def dropEvent(self, event): + paths = self.paths_from_event(event) + event.setDropAction(Qt.CopyAction) + event.accept() + self.files_dropped.emit(paths) + +def paths_from_event(self, event): + ''' + Accept a drop event and return a list of paths that can be read from + and represent files with extensions. + ''' + md = event.mimeData() + if md.hasFormat('text/uri-list') and not \ + md.hasFormat('application/calibre+from_library'): + urls = [unicode(u.toLocalFile()) for u in md.urls()] + return [u for u in urls if os.path.splitext(u)[1] and + os.path.exists(u)] + +def setup_dnd_interface(cls_or_self): + if isinstance(cls_or_self, type): + cls = cls_or_self + base_class = cls.__bases__[0] + fmap = globals() + for x in ( + 'dragMoveEvent', 'event_has_mods', 'mousePressEvent', 'mouseMoveEvent', + 'drag_data', 'drag_icon', 'dragEnterEvent', 'dropEvent', 'paths_from_event'): + func = fmap[x] + if x in {'mouseMoveEvent', 'mousePressEvent'}: + func = partial(func, base_class) + setattr(cls, x, MethodType(func, None, cls)) + else: + self = cls_or_self + self.drag_allowed = True + self.drag_start_pos = None + self.setDragEnabled(True) + self.setDragDropOverwriteMode(False) + self.setDragDropMode(self.DragDrop) +# }}} + +# Manage slave views {{{ def sync(func): @wraps(func) def ans(self, *args, **kwargs): @@ -51,6 +213,7 @@ class AlternateViews(object): view.selectionModel().currentChanged.connect(self.slave_current_changed) view.selectionModel().selectionChanged.connect(self.slave_selection_changed) view.sort_requested.connect(self.main_view.sort_by_named_field) + view.files_dropped.connect(self.main_view.files_dropped) def show_view(self, key=None): view = self.views[key] @@ -101,8 +264,9 @@ class AlternateViews(object): for view in self.views.itervalues(): if view is not self.main_view: view.set_context_menu(menu) +# }}} - +# Caching and rendering of covers {{{ class CoverCache(dict): def __init__(self, limit=200): @@ -191,14 +355,17 @@ def join_with_timeout(q, timeout=2): q.all_tasks_done.wait(remaining) finally: q.all_tasks_done.release() +# }}} class GridView(QListView): update_item = pyqtSignal(object) sort_requested = pyqtSignal(object, object) + files_dropped = pyqtSignal(object) def __init__(self, parent): QListView.__init__(self, parent) + setup_dnd_interface(self) pal = QPalette(self.palette()) r = g = b = 0x50 pal.setColor(pal.Base, QColor(r, g, b)) @@ -338,3 +505,5 @@ class GridView(QListView): def do_sort(self, column, ascending): self.sort_requested.emit(column, ascending) + +setup_dnd_interface(GridView) diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 41ef245edb..254709c209 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -5,21 +5,22 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, itertools, operator +import itertools, operator from functools import partial from future_builtins import map from collections import OrderedDict -from PyQt4.Qt import (QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, QFont, - QModelIndex, QIcon, QItemSelection, QMimeData, QDrag, QApplication, QStyle, - QPoint, QPixmap, QUrl, QImage, QPainter, QColor, QRect, QHeaderView, QStyleOptionHeader) +from PyQt4.Qt import ( + QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, QFont, QModelIndex, + QIcon, QItemSelection, QMimeData, QDrag, QStyle, QPoint, QUrl, QHeaderView, + QStyleOptionHeader) from calibre.gui2.library.delegates import (RatingDelegate, PubDateDelegate, TextDelegate, DateDelegate, CompleteDelegate, CcTextDelegate, CcBoolDelegate, CcCommentsDelegate, CcDateDelegate, CcTemplateDelegate, CcEnumDelegate, CcNumberDelegate, LanguagesDelegate) from calibre.gui2.library.models import BooksModel, DeviceBooksModel -from calibre.gui2.library.alternate_views import AlternateViews +from calibre.gui2.library.alternate_views import AlternateViews, setup_dnd_interface from calibre.utils.config import tweaks, prefs from calibre.gui2 import error_dialog, gprefs from calibre.gui2.library import DEFAULT_SORT @@ -163,11 +164,7 @@ class BooksView(QTableView): # {{{ else: self.setEditTriggers(self.DoubleClicked|self.editTriggers()) - self.drag_allowed = True - self.setDragEnabled(True) - self.setDragDropOverwriteMode(False) - self.setDragDropMode(self.DragDrop) - self.drag_start_pos = None + setup_dnd_interface(self) self.setAlternatingRowColors(True) self.setSelectionBehavior(self.SelectRows) self.setShowGrid(False) @@ -704,143 +701,6 @@ class BooksView(QTableView): # {{{ event.accept() # }}} - # Drag 'n Drop {{{ - @classmethod - def paths_from_event(cls, event): - ''' - Accept a drop event and return a list of paths that can be read from - and represent files with extensions. - ''' - md = event.mimeData() - if md.hasFormat('text/uri-list') and not \ - md.hasFormat('application/calibre+from_library'): - urls = [unicode(u.toLocalFile()) for u in md.urls()] - return [u for u in urls if os.path.splitext(u)[1] and - os.path.exists(u)] - - def drag_icon(self, cover, multiple): - cover = cover.scaledToHeight(120, Qt.SmoothTransformation) - if multiple: - base_width = cover.width() - base_height = cover.height() - base = QImage(base_width+21, base_height+21, - QImage.Format_ARGB32_Premultiplied) - base.fill(QColor(255, 255, 255, 0).rgba()) - p = QPainter(base) - rect = QRect(20, 0, base_width, base_height) - p.fillRect(rect, QColor('white')) - p.drawRect(rect) - rect.moveLeft(10) - rect.moveTop(10) - p.fillRect(rect, QColor('white')) - p.drawRect(rect) - rect.moveLeft(0) - rect.moveTop(20) - p.fillRect(rect, QColor('white')) - p.save() - p.setCompositionMode(p.CompositionMode_SourceAtop) - p.drawImage(rect.topLeft(), cover) - p.restore() - p.drawRect(rect) - p.end() - cover = base - return QPixmap.fromImage(cover) - - def drag_data(self): - m = self.model() - db = m.db - rows = self.selectionModel().selectedRows() - selected = list(map(m.id, rows)) - ids = ' '.join(map(str, selected)) - md = QMimeData() - md.setData('application/calibre+from_library', ids) - fmt = prefs['output_format'] - - def url_for_id(i): - try: - ans = db.format_path(i, fmt, index_is_id=True) - except: - ans = None - if ans is None: - fmts = db.formats(i, index_is_id=True) - if fmts: - fmts = fmts.split(',') - else: - fmts = [] - for f in fmts: - try: - ans = db.format_path(i, f, index_is_id=True) - except: - ans = None - if ans is None: - ans = db.abspath(i, index_is_id=True) - return QUrl.fromLocalFile(ans) - - md.setUrls([url_for_id(i) for i in selected]) - drag = QDrag(self) - col = self.selectionModel().currentIndex().column() - md.column_name = self.column_map[col] - drag.setMimeData(md) - cover = self.drag_icon(m.cover(self.currentIndex().row()), - len(selected) > 1) - drag.setHotSpot(QPoint(-15, -15)) - drag.setPixmap(cover) - return drag - - def event_has_mods(self, event=None): - mods = event.modifiers() if event is not None else \ - QApplication.keyboardModifiers() - return mods & Qt.ControlModifier or mods & Qt.ShiftModifier - - def mousePressEvent(self, event): - ep = event.pos() - if self.indexAt(ep) in self.selectionModel().selectedIndexes() and \ - event.button() == Qt.LeftButton and not self.event_has_mods(): - self.drag_start_pos = ep - return QTableView.mousePressEvent(self, event) - - def mouseMoveEvent(self, event): - if not self.drag_allowed: - return - if self.drag_start_pos is None: - return QTableView.mouseMoveEvent(self, event) - - if self.event_has_mods(): - self.drag_start_pos = None - return - - if not (event.buttons() & Qt.LeftButton) or \ - (event.pos() - self.drag_start_pos).manhattanLength() \ - < QApplication.startDragDistance(): - return - - index = self.indexAt(event.pos()) - if not index.isValid(): - return - drag = self.drag_data() - drag.exec_(Qt.CopyAction) - self.drag_start_pos = None - - def dragEnterEvent(self, event): - if int(event.possibleActions() & Qt.CopyAction) + \ - int(event.possibleActions() & Qt.MoveAction) == 0: - return - paths = self.paths_from_event(event) - - if paths: - event.acceptProposedAction() - - def dragMoveEvent(self, event): - event.acceptProposedAction() - - def dropEvent(self, event): - paths = self.paths_from_event(event) - event.setDropAction(Qt.CopyAction) - event.accept() - self.files_dropped.emit(paths) - - # }}} - @property def column_map(self): return self._model.column_map @@ -1048,6 +908,8 @@ class BooksView(QTableView): # {{{ # }}} +setup_dnd_interface(BooksView) + class DeviceBooksView(BooksView): # {{{ def __init__(self, parent): From ae3aa445962724fd6eb18a802b036e0210abe610 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 14:30:25 +0530 Subject: [PATCH 0457/1154] Port preserve_view_state to work with the grid view --- src/calibre/gui2/library/alternate_views.py | 10 ++++++- src/calibre/gui2/library/views.py | 31 +++++++++++++++------ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 921299e074..6e6912c9eb 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -374,7 +374,9 @@ class GridView(QListView): self.setUniformItemSizes(True) self.setWrapping(True) self.setFlow(self.LeftToRight) - self.setLayoutMode(self.Batched) + # We cannot set layout mode to batched, because that breaks + # restore_vpos() + # self.setLayoutMode(self.Batched) self.setResizeMode(self.Adjust) self.setSelectionMode(self.ExtendedSelection) self.setVerticalScrollMode(self.ScrollPerPixel) @@ -506,4 +508,10 @@ class GridView(QListView): def do_sort(self, column, ascending): self.sort_requested.emit(column, ascending) + def restore_vpos(self, vpos): + self.verticalScrollBar().setValue(vpos) + + def restore_hpos(self, hpos): + pass + setup_dnd_interface(GridView) diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 254709c209..06e6e43fcd 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -87,18 +87,24 @@ class PreserveViewState(object): # {{{ require_selected_ids=True): self.view = view self.require_selected_ids = require_selected_ids - self.selected_ids = set() - self.current_id = None self.preserve_hpos = preserve_hpos self.preserve_vpos = preserve_vpos + self.init_vals() + + def init_vals(self): + self.selected_ids = set() + self.current_id = None self.vscroll = self.hscroll = 0 + self.original_view = None def __enter__(self): + self.init_vals() try: + view = self.original_view = self.view.alternate_views.current_view self.selected_ids = self.view.get_selected_ids() self.current_id = self.view.current_id - self.vscroll = self.view.verticalScrollBar().value() - self.hscroll = self.view.horizontalScrollBar().value() + self.vscroll = view.verticalScrollBar().value() + self.hscroll = view.horizontalScrollBar().value() except: import traceback traceback.print_exc() @@ -110,10 +116,19 @@ class PreserveViewState(object): # {{{ if self.selected_ids: self.view.select_rows(self.selected_ids, using_ids=True, scroll=False, change_current=self.current_id is None) - if self.preserve_vpos: - self.view.verticalScrollBar().setValue(self.vscroll) - if self.preserve_hpos: - self.view.horizontalScrollBar().setValue(self.hscroll) + view = self.original_view + if self.view.alternate_views.current_view is view: + if self.preserve_vpos: + if hasattr(view, 'restore_vpos'): + view.restore_vpos(self.vscroll) + else: + view.verticalScrollBar().setValue(self.vscroll) + if self.preserve_hpos: + if hasattr(view, 'restore_hpos'): + view.restore_hpos(self.hscroll) + else: + view.horizontalScrollBar().setValue(self.hscroll) + self.init_vals() @dynamic_property def state(self): From 5737ead466ed7f50180d2f983cc27b65654b0f30 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 15:10:35 +0530 Subject: [PATCH 0458/1154] pep8 --- src/calibre/gui2/__init__.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 7b88a943bb..157ade7020 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -114,20 +114,20 @@ defs['refresh_book_list_on_bulk_edit'] = True del defs # }}} -NONE = QVariant() #: Null value to return from the data function of item models +NONE = QVariant() # : Null value to return from the data function of item models UNDEFINED_QDATETIME = QDateTime(UNDEFINED_DATE) ALL_COLUMNS = ['title', 'ondevice', 'authors', 'size', 'timestamp', 'rating', 'publisher', 'tags', 'series', 'pubdate'] -def _config(): # {{{ +def _config(): # {{{ c = Config('gui', 'preferences for the calibre GUI') c.add_opt('send_to_storage_card_by_default', default=False, help=_('Send file to storage card instead of main memory by default')) c.add_opt('confirm_delete', default=False, help=_('Confirm before deleting')) c.add_opt('main_window_geometry', default=None, - help=_('Main window geometry')) # value QVariant.toByteArray + help=_('Main window geometry')) # value QVariant.toByteArray c.add_opt('new_version_notification', default=True, help=_('Notify when a new version is available')) c.add_opt('use_roman_numerals_for_series_number', default=True, @@ -579,9 +579,9 @@ class FileDialog(QObject): filters=[], add_all_files_filter=True, parent=None, - modal = True, - name = '', - mode = QFileDialog.ExistingFiles, + modal=True, + name='', + mode=QFileDialog.ExistingFiles, default_dir='~', no_save_dir=False ): @@ -610,7 +610,7 @@ class FileDialog(QObject): if not isinstance(initial_dir, basestring): initial_dir = os.path.expanduser(default_dir) self.selected_files = [] - use_native_dialog = not os.environ.has_key('CALIBRE_NO_NATIVE_FILEDIALOGS') + use_native_dialog = 'CALIBRE_NO_NATIVE_FILEDIALOGS' not in os.environ with SanitizeLibraryPath(): opts = QFileDialog.Option() if not use_native_dialog: @@ -630,7 +630,8 @@ class FileDialog(QObject): ftext, "", opts) for f in fs: f = unicode(f) - if not f: continue + if not f: + continue if not os.path.exists(f): # QFileDialog for some reason quotes spaces # on linux if there is more than one space in a row @@ -919,7 +920,6 @@ class Application(QApplication): self.file_event_hook(self._file_open_paths) self._file_open_paths = [] - def load_translations(self): if self._translator is not None: self.removeTranslator(self._translator) @@ -1060,3 +1060,4 @@ _df = os.environ.get('CALIBRE_DEVELOP_FROM', None) if _df and os.path.exists(_df): build_forms(_df) + From 0541a118e6eac6f680d89d7ad76251366d05ce36 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Aug 2013 16:23:24 +0530 Subject: [PATCH 0459/1154] Make the cover grid customizable --- src/calibre/gui2/__init__.py | 4 + src/calibre/gui2/library/alternate_views.py | 56 +++++- src/calibre/gui2/preferences/look_feel.py | 31 +++- src/calibre/gui2/preferences/look_feel.ui | 178 ++++++++++++++++++++ 4 files changed, 255 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 157ade7020..9c28b0ebe9 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -111,6 +111,10 @@ defs['tags_browser_category_icons'] = {} defs['cover_browser_reflections'] = True defs['extra_row_spacing'] = 0 defs['refresh_book_list_on_bulk_edit'] = True +defs['cover_grid_width'] = 0 +defs['cover_grid_height'] = 0 +defs['cover_grid_color'] = (80, 80, 80) +defs['cover_grid_cache_size'] = 200 del defs # }}} diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 6e6912c9eb..f43abb58cb 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -20,8 +20,11 @@ from PyQt4.Qt import ( QUrl, QDrag, QPoint, QPainter, QRect) from calibre import fit_image +from calibre.gui2 import gprefs from calibre.utils.config import prefs +CM_TO_INCH = 0.393701 + # Drag 'n Drop {{{ def dragMoveEvent(self, event): event.acceptProposedAction() @@ -305,15 +308,39 @@ class CoverCache(dict): def __hash__(self): return id(self) + def set_limit(self, limit): + with self.lock: + self.limit = limit + if len(self.items) > self.limit: + extra = len(self.items) - self.limit + remove = tuple(self.iterkeys())[:extra] + for k in remove: + del self.items[k] + class CoverDelegate(QStyledItemDelegate): - def __init__(self, parent, width, height): + def __init__(self, parent): super(CoverDelegate, self).__init__(parent) + self.set_dimensions() + self.cover_cache = CoverCache(limit=gprefs['cover_grid_cache_size']) + self.render_queue = Queue() + + def set_dimensions(self): + width = self.original_width = gprefs['cover_grid_width'] + height = self.original_height = gprefs['cover_grid_height'] + + if height < 0.1: + height = max(185, QApplication.instance().desktop().availableGeometry(self.parent()).height() / 5.0) + else: + height *= self.parent().logicalDpiY() * CM_TO_INCH + + if width < 0.1: + width = 0.75 * height + else: + width *= self.parent().logicalDpiX() * CM_TO_INCH self.cover_size = QSize(width, height) self.item_size = self.cover_size + QSize(8, 8) self.spacing = max(10, min(50, int(0.1 * width))) - self.cover_cache = CoverCache() - self.render_queue = Queue() def sizeHint(self, option, index): return self.item_size @@ -366,11 +393,7 @@ class GridView(QListView): def __init__(self, parent): QListView.__init__(self, parent) setup_dnd_interface(self) - pal = QPalette(self.palette()) - r = g = b = 0x50 - pal.setColor(pal.Base, QColor(r, g, b)) - pal.setColor(pal.Text, QColor(Qt.white if (r + g + b)/3.0 < 128 else Qt.black)) - self.setPalette(pal) + self.set_color() self.setUniformItemSizes(True) self.setWrapping(True) self.setFlow(self.LeftToRight) @@ -380,7 +403,7 @@ class GridView(QListView): self.setResizeMode(self.Adjust) self.setSelectionMode(self.ExtendedSelection) self.setVerticalScrollMode(self.ScrollPerPixel) - self.delegate = CoverDelegate(self, 135, 180) + self.delegate = CoverDelegate(self) self.setItemDelegate(self.delegate) self.setSpacing(self.delegate.spacing) self.ignore_render_requests = Event() @@ -390,6 +413,21 @@ class GridView(QListView): self.gui = parent self.context_menu = None + def set_color(self): + r, g, b = gprefs['cover_grid_color'] + pal = QPalette() + pal.setColor(pal.Base, QColor(r, g, b)) + pal.setColor(pal.Text, QColor(Qt.white if (r + g + b)/3.0 < 128 else Qt.black)) + self.setPalette(pal) + + def refresh_settings(self): + if gprefs['cover_grid_width'] != self.delegate.original_width or gprefs['cover_grid_height'] != self.delegate.original_height: + self.delegate.set_dimensions() + self.setSpacing(self.delegate.spacing) + self.delegate.cover_cache.clear() + self.set_color() + self.delegate.cover_cache.set_limit(gprefs['cover_grid_cache_size']) + def shown(self): if self.render_thread is None: self.render_thread = Thread(target=self.render_covers) diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index b3021cf446..114f44d181 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -5,8 +5,8 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import (QApplication, QFont, QFontInfo, QFontDialog, - QAbstractListModel, Qt, QIcon, QKeySequence) +from PyQt4.Qt import (QApplication, QFont, QFontInfo, QFontDialog, QColorDialog, + QAbstractListModel, Qt, QIcon, QKeySequence, QPalette, QColor) from calibre.gui2.preferences import ConfigWidgetBase, test_widget, CommaSeparatedList from calibre.gui2.preferences.look_feel_ui import Ui_Form @@ -18,7 +18,7 @@ from calibre.utils.icu import sort_key from calibre.gui2.book_details import get_field_list from calibre.gui2.preferences.coloring import EditRules -class DisplayedFields(QAbstractListModel): # {{{ +class DisplayedFields(QAbstractListModel): # {{{ def __init__(self, db, parent=None): QAbstractListModel.__init__(self, parent) @@ -110,6 +110,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('tag_browser_old_look', gprefs, restart_required=True) r('bd_show_cover', gprefs) r('bd_overlay_cover_size', gprefs) + r('cover_grid_width', gprefs) + r('cover_grid_height', gprefs) + r('cover_grid_cache_size', gprefs) r('cover_flow_queue_length', config, restart_required=True) r('cover_browser_reflections', gprefs) @@ -123,7 +126,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): lang = get_lang() if lang is None or lang not in available_translations(): lang = 'en' - items = [(l, get_esc_lang(l)) for l in available_translations() \ + items = [(l, get_esc_lang(l)) for l in available_translations() if l != lang] if lang != 'en': items.append(('en', get_esc_lang('en'))) @@ -170,7 +173,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('categories_using_hierarchy', db.prefs, setting=CommaSeparatedList, choices=sorted(list(choices), key=sort_key)) - self.current_font = self.initial_font = None self.change_font_button.clicked.connect(self.change_font) @@ -197,6 +199,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): keys = [unicode(x.toString(QKeySequence.NativeText)) for x in keys] self.fs_help_msg.setText(unicode(self.fs_help_msg.text())%( _(' or ').join(keys))) + self.cover_grid_color_button.clicked.connect(self.change_cover_grid_color) def initialize(self): ConfigWidgetBase.initialize(self) @@ -215,6 +218,12 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): mi=None self.edit_rules.initialize(db.field_metadata, db.prefs, mi, 'column_color_rules') self.icon_rules.initialize(db.field_metadata, db.prefs, mi, 'column_icon_rules') + self.set_cg_color(gprefs['cover_grid_color']) + + def set_cg_color(self, val): + pal = QPalette() + pal.setColor(QPalette.Window, QColor(*val)) + self.cover_grid_color_label.setPalette(pal) def restore_defaults(self): ConfigWidgetBase.restore_defaults(self) @@ -227,6 +236,15 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.edit_rules.clear() self.icon_rules.clear() self.changed_signal.emit() + self.set_cg_color(gprefs.defaults['cover_grid_color']) + + def change_cover_grid_color(self): + col = QColorDialog.getColor(self.cover_grid_color_label.palette().color(QPalette.Window), + self.gui, _('Choose background color for cover grid')) + if col.isValid(): + col = tuple(col.getRgb())[:3] + self.set_cg_color(col) + self.changed_signal.emit() def build_font_obj(self): font_info = self.current_font @@ -286,6 +304,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.display_model.commit() self.edit_rules.commit(self.gui.current_db.prefs) self.icon_rules.commit(self.gui.current_db.prefs) + gprefs['cover_grid_color'] = tuple(self.cover_grid_color_label.palette().color(QPalette.Window).getRgb())[:3] return rr def refresh_gui(self, gui): @@ -296,9 +315,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if hasattr(gui.cover_flow, 'setShowReflections'): gui.cover_flow.setShowReflections(gprefs['cover_browser_reflections']) gui.library_view.refresh_row_sizing() + gui.grid_view.refresh_settings() if __name__ == '__main__': from calibre.gui2 import Application app = Application([]) test_widget('Interface', 'Look & Feel') + diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index 086c012e17..d962dad0eb 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -223,6 +223,184 @@ + + + + :/images/grid.png:/images/grid.png + + + Cover Grid + + + + + + + + Cover &Width: + + + opt_cover_grid_width + + + + + + + The width of displayed covers + + + cm + + + 1 + + + + + + + Cover &Height: + + + opt_cover_grid_height + + + + + + + The height of displayed covers + + + cm + + + 1 + + + + + + + A value of zero means set automatically based on screen size + + + true + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + Background color for the cover grid: + + + + + + + + 50 + 50 + + + + true + + + + + + + + + + Change &color + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + Number of covers to cache in &memory: + + + opt_cover_grid_cache_size + + + + + + + 3000 + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + Qt::Vertical + + + + 20 + 355 + + + + + + From a914e95d9f218f79f727a38c8c37e0acdc986627 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 16:07:24 +0530 Subject: [PATCH 0460/1154] Ensure that QPixmap objects are never deleted on non-GUI threads --- src/calibre/gui2/library/alternate_views.py | 26 +++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index f43abb58cb..2494890202 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -10,7 +10,7 @@ import itertools, operator, os from types import MethodType from time import time from collections import OrderedDict -from threading import Lock, Event, Thread +from threading import Lock, Event, Thread, current_thread from Queue import Queue from functools import wraps, partial @@ -276,17 +276,29 @@ class CoverCache(dict): self.items = OrderedDict() self.lock = Lock() self.limit = limit + self.pixmap_staging = [] + self.gui_thread = current_thread() + + def clear_staging(self): + ' Must be called in the GUI thread ' + self.pixmap_staging = [] def invalidate(self, book_id): with self.lock: - self.items.pop(book_id, None) + self._pop(book_id) + + def _pop(self, book_id): + val = self.items.pop(book_id, None) + if type(val) is QPixmap and current_thread() is not self.gui_thread: + self.pixmap_staging.append(val) def __getitem__(self, key): ' Must be called in the GUI thread ' with self.lock: + self.clear_staging() ans = self.items.pop(key, False) # pop() so that item is moved to the top if ans is not False: - if isinstance(ans, QImage): + if type(ans) is QImage: # Convert to QPixmap, since rendering QPixmap is much # faster ans = QPixmap.fromImage(ans) @@ -296,13 +308,16 @@ class CoverCache(dict): def set(self, key, val): with self.lock: - self.items.pop(key, None) # pop() so that item is moved to the top + self._pop(key) # pop() so that item is moved to the top self.items[key] = val if len(self.items) > self.limit: del self.items[next(self.items.iterkeys())] def clear(self): with self.lock: + if current_thread() is not self.gui_thread: + pixmaps = (x for x in self.items.itervalues() if type(x) is QPixmap) + self.pixmap_staging.extend(pixmaps) self.items.clear() def __hash__(self): @@ -315,7 +330,7 @@ class CoverCache(dict): extra = len(self.items) - self.limit remove = tuple(self.iterkeys())[:extra] for k in remove: - del self.items[k] + self._pop(k) class CoverDelegate(QStyledItemDelegate): @@ -471,6 +486,7 @@ class GridView(QListView): self.update_item.emit(book_id) def re_render(self, book_id): + self.delegate.cover_cache.clear_staging() m = self.model() try: index = m.db.row(book_id) From 85ef13a6ff252795db01aa7b20521e96db0db8f7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 19:06:19 +0530 Subject: [PATCH 0461/1154] Animate double clicks in the grid view --- src/calibre/gui2/library/alternate_views.py | 44 +++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 2494890202..92b023303b 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -17,10 +17,10 @@ from functools import wraps, partial from PyQt4.Qt import ( QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, QPalette, QColor, QItemSelection, QPixmap, QMenu, QApplication, QMimeData, - QUrl, QDrag, QPoint, QPainter, QRect) + QUrl, QDrag, QPoint, QPainter, QRect, pyqtProperty, QPropertyAnimation, QEasingCurve) from calibre import fit_image -from calibre.gui2 import gprefs +from calibre.gui2 import gprefs, config from calibre.utils.config import prefs CM_TO_INCH = 0.393701 @@ -334,11 +334,24 @@ class CoverCache(dict): class CoverDelegate(QStyledItemDelegate): + @pyqtProperty(float) + def animated_size(self): + return self._animated_size + + @animated_size.setter + def animated_size(self, val): + self._animated_size = val + def __init__(self, parent): super(CoverDelegate, self).__init__(parent) + self._animated_size = 1.0 + self.animation = QPropertyAnimation(self, 'animated_size', self) + self.animation.setEasingCurve(QEasingCurve.OutInCirc) + self.animation.setDuration(500) self.set_dimensions() self.cover_cache = CoverCache(limit=gprefs['cover_grid_cache_size']) self.render_queue = Queue() + self.animating = None def set_dimensions(self): width = self.original_width = gprefs['cover_grid_width'] @@ -356,6 +369,9 @@ class CoverDelegate(QStyledItemDelegate): self.cover_size = QSize(width, height) self.item_size = self.cover_size + QSize(8, 8) self.spacing = max(10, min(50, int(0.1 * width))) + self.animation.setStartValue(1.0) + self.animation.setKeyValueAt(0.5, 0.5) + self.animation.setEndValue(1.0) def sizeHint(self, option, index): return self.item_size @@ -379,6 +395,8 @@ class CoverDelegate(QStyledItemDelegate): if cdata is False: self.render_queue.put(book_id) else: + if self.animating is not None and self.animating.row() == index.row(): + cdata = cdata.scaled(cdata.size() * self._animated_size) dx = max(0, int((rect.width() - cdata.width())/2.0)) dy = max(0, rect.height() - cdata.height()) rect.adjust(dx, dy, -dx, 0) @@ -419,15 +437,35 @@ class GridView(QListView): self.setSelectionMode(self.ExtendedSelection) self.setVerticalScrollMode(self.ScrollPerPixel) self.delegate = CoverDelegate(self) + self.delegate.animation.valueChanged.connect(self.animation_value_changed) + self.delegate.animation.finished.connect(self.animation_done) self.setItemDelegate(self.delegate) self.setSpacing(self.delegate.spacing) self.ignore_render_requests = Event() self.render_thread = None self.update_item.connect(self.re_render, type=Qt.QueuedConnection) - self.doubleClicked.connect(parent.iactions['View'].view_triggered) + self.doubleClicked.connect(self.double_clicked) + self.setCursor(Qt.PointingHandCursor) self.gui = parent self.context_menu = None + def double_clicked(self, index): + d = self.delegate + if d.animating is None and not config['disable_animations']: + d.animating = index + d.animation.start() + self.gui.iactions['View'].view_triggered(index) + + def animation_value_changed(self, value): + if self.delegate.animating is not None: + self.update(self.delegate.animating) + + def animation_done(self): + if self.delegate.animating is not None: + idx = self.delegate.animating + self.delegate.animating = None + self.update(idx) + def set_color(self): r, g, b = gprefs['cover_grid_color'] pal = QPalette() From fb0410b04ca586cf889dfb4529641c8e13fee0e0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 22:26:44 +0530 Subject: [PATCH 0462/1154] D'nD fixes Fix dragging a single book in the book list using the multi-book icon. Also use a class decorator for the API, looks nicer :) --- src/calibre/gui2/library/alternate_views.py | 5 +++-- src/calibre/gui2/library/views.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 92b023303b..3a8cd2130b 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -73,7 +73,7 @@ def drag_data(self): m = self.model() db = m.db rows = self.selectionModel().selectedIndexes() - selected = list(map(m.id, rows)) + selected = list(set(map(m.id, rows))) ids = ' '.join(map(str, selected)) md = QMimeData() md.setData('application/calibre+from_library', ids) @@ -174,6 +174,7 @@ def setup_dnd_interface(cls_or_self): if x in {'mouseMoveEvent', 'mousePressEvent'}: func = partial(func, base_class) setattr(cls, x, MethodType(func, None, cls)) + return cls else: self = cls_or_self self.drag_allowed = True @@ -417,6 +418,7 @@ def join_with_timeout(q, timeout=2): q.all_tasks_done.release() # }}} +@setup_dnd_interface class GridView(QListView): update_item = pyqtSignal(object) @@ -606,4 +608,3 @@ class GridView(QListView): def restore_hpos(self, hpos): pass -setup_dnd_interface(GridView) diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 06e6e43fcd..0b22999b27 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -144,6 +144,7 @@ class PreserveViewState(object): # {{{ # }}} +@setup_dnd_interface class BooksView(QTableView): # {{{ files_dropped = pyqtSignal(object) @@ -923,8 +924,6 @@ class BooksView(QTableView): # {{{ # }}} -setup_dnd_interface(BooksView) - class DeviceBooksView(BooksView): # {{{ def __init__(self, parent): From 579fd8a48568faa413354b94a0168f87b06c7b01 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Aug 2013 23:41:56 +0530 Subject: [PATCH 0463/1154] ... --- src/calibre/gui2/library/alternate_views.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 3a8cd2130b..4c4997b0d7 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -72,8 +72,7 @@ def drag_icon(self, cover, multiple): def drag_data(self): m = self.model() db = m.db - rows = self.selectionModel().selectedIndexes() - selected = list(set(map(m.id, rows))) + selected = self.get_selected_ids() ids = ' '.join(map(str, selected)) md = QMimeData() md.setData('application/calibre+from_library', ids) @@ -602,6 +601,10 @@ class GridView(QListView): def do_sort(self, column, ascending): self.sort_requested.emit(column, ascending) + def get_selected_ids(self): + m = self.model() + return [m.id(i) for i in self.selectionModel().selectedIndexes()] + def restore_vpos(self, vpos): self.verticalScrollBar().setValue(vpos) From 1121be459f054a76bc1cbf459b0a78a52ba30014 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 07:00:45 +0530 Subject: [PATCH 0464/1154] Allow running with olddb --- src/calibre/gui2/library/alternate_views.py | 2 ++ src/calibre/gui2/preferences/look_feel.py | 6 +++++- src/calibre/gui2/preferences/look_feel.ui | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 4c4997b0d7..0b3fc86b0a 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -538,6 +538,8 @@ class GridView(QListView): self.delegate.render_queue.put(None) def set_database(self, newdb, stage=0): + if not hasattr(newdb, 'new_api'): + return if stage == 0: self.ignore_render_requests.set() try: diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index 114f44d181..e0bd8ead36 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -13,7 +13,7 @@ from calibre.gui2.preferences.look_feel_ui import Ui_Form from calibre.gui2 import config, gprefs, qt_app, NONE from calibre.utils.localization import (available_translations, get_language, get_lang) -from calibre.utils.config import prefs +from calibre.utils.config import prefs, tweaks from calibre.utils.icu import sort_key from calibre.gui2.book_details import get_field_list from calibre.gui2.preferences.coloring import EditRules @@ -200,6 +200,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.fs_help_msg.setText(unicode(self.fs_help_msg.text())%( _(' or ').join(keys))) self.cover_grid_color_button.clicked.connect(self.change_cover_grid_color) + if not tweaks.get('use_new_db', False): + for i in range(self.tabWidget.count()): + if self.tabWidget.widget(i) is self.cover_grid_tab: + self.tabWidget.removeTab(i) def initialize(self): ConfigWidgetBase.initialize(self) diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index d962dad0eb..9e4851d73e 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -223,7 +223,7 @@ - + :/images/grid.png:/images/grid.png From 0e64b0c886f84a1b069164816b8fde226470f3d9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 14:08:54 +0530 Subject: [PATCH 0465/1154] Make spacing between covers adjustable --- src/calibre/gui2/__init__.py | 1 + src/calibre/gui2/library/alternate_views.py | 12 ++- src/calibre/gui2/preferences/look_feel.py | 1 + src/calibre/gui2/preferences/look_feel.ui | 94 +++++++++++---------- 4 files changed, 61 insertions(+), 47 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 9c28b0ebe9..bec9a3177c 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -115,6 +115,7 @@ defs['cover_grid_width'] = 0 defs['cover_grid_height'] = 0 defs['cover_grid_color'] = (80, 80, 80) defs['cover_grid_cache_size'] = 200 +defs['cover_grid_spacing'] = 0 del defs # }}} diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 0b3fc86b0a..260b2460ea 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -368,11 +368,18 @@ class CoverDelegate(QStyledItemDelegate): width *= self.parent().logicalDpiX() * CM_TO_INCH self.cover_size = QSize(width, height) self.item_size = self.cover_size + QSize(8, 8) - self.spacing = max(10, min(50, int(0.1 * width))) + self.calculate_spacing() self.animation.setStartValue(1.0) self.animation.setKeyValueAt(0.5, 0.5) self.animation.setEndValue(1.0) + def calculate_spacing(self): + spc = self.original_spacing = gprefs['cover_grid_spacing'] + if spc < 0.1: + self.spacing = max(10, min(50, int(0.1 * self.original_width))) + else: + self.spacing = self.parent().logicalDpiX() * CM_TO_INCH * spc + def sizeHint(self, option, index): return self.item_size @@ -479,6 +486,9 @@ class GridView(QListView): self.delegate.set_dimensions() self.setSpacing(self.delegate.spacing) self.delegate.cover_cache.clear() + if gprefs['cover_grid_spacing'] != self.delegate.original_spacing: + self.delegate.calculate_spacing() + self.setSpacing(self.delegate.spacing) self.set_color() self.delegate.cover_cache.set_limit(gprefs['cover_grid_cache_size']) diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index e0bd8ead36..80521abf9f 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -113,6 +113,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('cover_grid_width', gprefs) r('cover_grid_height', gprefs) r('cover_grid_cache_size', gprefs) + r('cover_grid_spacing', gprefs) r('cover_flow_queue_length', config, restart_required=True) r('cover_browser_reflections', gprefs) diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index 9e4851d73e..d704e1a214 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -231,8 +231,8 @@ Cover Grid - - + + @@ -272,6 +272,9 @@ The height of displayed covers + + Automatic + cm @@ -305,13 +308,16 @@ - + Background color for the cover grid: + + cover_grid_color_button + @@ -352,52 +358,48 @@ - - - - - - Number of covers to cache in &memory: - - - opt_cover_grid_cache_size - - - - - - - 3000 - - - - - - - Qt::Horizontal - - - - 40 - 20 - - - - - + + + + Number of covers to cache in &memory: + + + opt_cover_grid_cache_size + + - - - - Qt::Vertical + + + + 3000 - - - 20 - 355 - + + + + + + &Spacing between covers: - + + opt_cover_grid_spacing + + + + + + + The spacing between covers. A value of zero means calculate automatically based on cover size. + + + Automatic + + + cm + + + 1 + + From 10b4b9d39c89be4913733e0ab71fe8e0a932f5f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 14:13:05 +0530 Subject: [PATCH 0466/1154] Show authors in addition to title when cover not available --- src/calibre/gui2/library/alternate_views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 260b2460ea..0e88953426 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -398,7 +398,8 @@ class CoverDelegate(QStyledItemDelegate): rect.adjust(4, 4, -4, -4) if cdata is None or cdata is False: title = db.field_for('title', book_id, default_value='') - painter.drawText(rect, Qt.AlignCenter|Qt.TextWordWrap, title) + authors = ' & '.join(db.field_for('authors', book_id, default_value=())) + painter.drawText(rect, Qt.AlignCenter|Qt.TextWordWrap, '%s\n\n%s' % (title, authors)) if cdata is False: self.render_queue.put(book_id) else: From 3cf7988c256c4547111e76cd2df9b02ce19f4010 Mon Sep 17 00:00:00 2001 From: sylvaindurand Date: Mon, 5 Aug 2013 16:04:04 +0200 Subject: [PATCH 0467/1154] Typography, error message and description - french typography (apostrophes, guillemets); - error message if using a free account (which can't download the newspaper); - description. --- recipes/le_monde_sub.recipe | 55 +++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/recipes/le_monde_sub.recipe b/recipes/le_monde_sub.recipe index dc9fa9d36f..e55d71e0a7 100644 --- a/recipes/le_monde_sub.recipe +++ b/recipes/le_monde_sub.recipe @@ -13,7 +13,7 @@ class LeMonde(BasicNewsRecipe): title = u'Le Monde: Édition abonnés' __author__ = 'Sylvain Durand' - description = u'Disponible du lundi au samedi à partir de 14 heures environ, avec tous ses cahiers.' + description = u'La version papier du quotidien Le Monde, disponible du lundi au samedi à partir de 14 heures environ, avec tous ses cahiers.' language = 'fr' encoding = 'utf8' @@ -65,28 +65,41 @@ class LeMonde(BasicNewsRecipe): url = time.strftime(self.journal_url,self.date) soup = self.index_to_soup(url).sommaire sections = [] - for sec in soup.findAll("section"): - articles = [] - if sec['cahier'] != "Le Monde": - for col in sec.findAll("fnts"): - col.extract() - if sec['cahier']=="Le Monde Magazine": - continue - for art in sec.findAll("art"): - if art.txt.string and art.ttr.string: - if art.find(['url']): - art.insert(6,'
    ') - if art.find(['lgd']) and art.find(['lgd']).string: - art.insert(7,'
    '+art.find(['lgd']).string+'
    ') - article = ""+unicode(art)+"" - article = article.replace('','').replace(' oC ','°C ') - article = article.replace('srttr>','h3>').replace('ssttr>','h2>').replace('ttr>','h1>') - f = PersistentTemporaryFile() - f.write(article) - articles.append({'title':art.ttr.string,'url':"file:///"+f.name}) - sections.append((sec['nom'], articles)) + try: + for sec in soup.findAll("section"): + articles = [] + if sec['cahier'] != "Le Monde": + for col in sec.findAll("fnts"): + col.extract() + if sec['cahier']=="Le Monde Magazine": + continue + for art in sec.findAll("art"): + if art.txt.string and art.ttr.string: + if art.find(['url']): + art.insert(6,'
    ') + if art.find(['lgd']) and art.find(['lgd']).string: + art.insert(7,'
    '+art.find(['lgd']).string+'
    ') + + def guillemets(match): + if match.group(1) == u"=": + return match.group(0) + return u'%s« %s »' % (match.group(1), match.group(2)) + + article = ""+unicode(art)+"" + article = article.replace('','').replace(' oC ','°C ') + article = article.replace('srttr>','h3>').replace('ssttr>','h2>').replace('ttr>','h1>') + article = article.replace("'" , u'\u2019') + article = re.sub('(.|^)"([^"]+)"', guillemets, article) + + f = PersistentTemporaryFile() + f.write(article) + articles.append({'title':art.ttr.string,'url':"file:///"+f.name}) + sections.append((sec['nom'], articles)) + except AttributeError: + self.log("Vos identifiants sont incorrects, ou votre abonnement LeMonde.fr ne vous permet pas de télécharger le journal.") return sections + def preprocess_html(self, soup): for lgd in soup.findAll(id="lgd"): lgd.contents[-1].extract() From 71c7489cc1b618edce7058820464fd9cab0ec9d9 Mon Sep 17 00:00:00 2001 From: Sylvain Durand Date: Mon, 5 Aug 2013 17:09:22 +0200 Subject: [PATCH 0468/1154] =?UTF-8?q?New=20icon=20for=20Le=20Monde:=20Edit?= =?UTF-8?q?ion=20abonn=C3=A9s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- recipes/icons/le_monde_sub.png | Bin 0 -> 510 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 recipes/icons/le_monde_sub.png diff --git a/recipes/icons/le_monde_sub.png b/recipes/icons/le_monde_sub.png new file mode 100644 index 0000000000000000000000000000000000000000..a9b1717f77efa5e7346e8a9211e077f1896d6fae GIT binary patch literal 510 zcmVn>j(e<0gFjQ zK~y-)#go6zVNn#tf9IZyjzr@L6cRMzU*Qq(8WNFEQi(*N(@=Q>jn*58R-xb>{0WVF zuQkQQoe^JVCi#k4&Dl9Sd!4=4UdI698*un0LV`#n5}vVG3}Xz|TCBAIxUP%qx?dAm zYw3lxxh4p%^`~5CS zIv$Ve_xm!&=%t?kWwTiU>h*e(q*kk?*=#0B>UO&-l}g%fx0+6;uR;L|HQ4X>03;F# zf*>H7Ome+m34(xlJWf8Jr&ugb%707*qoM6N<$f&}W< AJOBUy literal 0 HcmV?d00001 From c5baa30f7ba469f50616e493e32154fd968cd006 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 21:07:41 +0530 Subject: [PATCH 0469/1154] ... --- src/calibre/gui2/preferences/look_feel.ui | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index d704e1a214..2438c2f9c1 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -249,6 +249,9 @@ The width of displayed covers + + Automatic + cm @@ -283,16 +286,6 @@ - - - - A value of zero means set automatically based on screen size - - - true - - - From 602c3d22a55f32aafaa3c377a86de2e67246ee17 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 21:43:29 +0530 Subject: [PATCH 0470/1154] newdb: Handle duplicate values when setting many-many fields Fixes an error when merging book records that have the same tags. --- src/calibre/db/tests/writing.py | 3 +++ src/calibre/db/write.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 4f2bacf921..96e2bb0c34 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -292,6 +292,9 @@ class WritingTest(BaseTest): ae(c.field_for('sort', 1), 'Moose, The') ae(c.field_for('sort', 2), 'Cat') + # Test setting with the same value repeated + ae(sf('tags', {3: ('a', 'b', 'a')}), {3}) + # }}} def test_dirtied(self): # {{{ diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py index 58c642e59c..5f486445db 100644 --- a/src/calibre/db/write.py +++ b/src/calibre/db/write.py @@ -337,6 +337,14 @@ def many_one(book_id_val_map, db, field, allow_case_change, *args): # }}} # Many-Many fields {{{ + +def uniq(vals): + ' Remove all duplicates from vals, while preserving order. Items in vals must be hashable ' + vals = vals or () + seen = set() + seen_add = seen.add + return tuple(x for x in vals if x not in seen and not seen_add(x)) + def many_many(book_id_val_map, db, field, allow_case_change, *args): dirtied = set() m = field.metadata @@ -349,6 +357,7 @@ def many_many(book_id_val_map, db, field, allow_case_change, *args): rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()} val_map = {} case_changes = {} + book_id_val_map = {k:uniq(vals) for k, vals in book_id_val_map.iteritems()} for vals in book_id_val_map.itervalues(): for val in vals: get_db_id(val, db, m, table, kmap, rid_map, allow_case_change, From 8d3fd8b05c3ff2bc054e81af757326997fa21eeb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 22:07:24 +0530 Subject: [PATCH 0471/1154] newdb: Fix true/false searches on numeric columns --- src/calibre/db/search.py | 3 +++ src/calibre/db/tests/reading.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 9eeacc24b5..6a6fbb0a56 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -290,8 +290,11 @@ class NumericSearch(object): # {{{ raise ParseException( _('Non-numeric value in query: {0}').format(query)) + qfalse = query == 'false' for val, book_ids in field_iter(): if val is None: + if qfalse: + matches |= book_ids continue try: v = cast(val) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 56866f5aa7..c9d29475f6 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -224,8 +224,8 @@ class ReadingTest(BaseTest): 'rating:3', 'rating:>2', 'rating:=2', 'rating:true', 'rating:false', 'rating:>4', 'tags:#<2', 'tags:#>7', 'cover:false', 'cover:true', '#float:>11', '#float:<1k', - '#float:10.01', 'series_index:1', 'series_index:<3', 'id:1', - 'id:>2', + '#float:10.01', '#float:false', 'series_index:1', + 'series_index:<3', 'id:1', 'id:>2', # Bool tests '#yesno:true', '#yesno:false', '#yesno:yes', '#yesno:no', From 768cd4d887f5509063686aaa404aaffc5fc28c9a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 22:33:05 +0530 Subject: [PATCH 0472/1154] Plugin updater dialog: Use index names Use index names as the key for plugins rather than names from the plugin zip file, as deprecated plugins sometimes have the same name in the zip file, but different names in the index. --- src/calibre/gui2/dialogs/plugin_updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/plugin_updater.py b/src/calibre/gui2/dialogs/plugin_updater.py index 123a14b829..0e95116866 100644 --- a/src/calibre/gui2/dialogs/plugin_updater.py +++ b/src/calibre/gui2/dialogs/plugin_updater.py @@ -185,7 +185,7 @@ class PluginFilterComboBox(QComboBox): class DisplayPlugin(object): def __init__(self, plugin): - self.name = plugin['name'] + self.name = plugin['index_name'] self.forum_link = plugin['thread_url'] self.zip_url = SERVER + plugin['file'] self.installed_version = None From beca8ecf790d9a6958d5544534cbc84fdd71f168 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 07:37:16 +0530 Subject: [PATCH 0473/1154] A lot more verbose lock debugging --- src/calibre/db/locking.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/calibre/db/locking.py b/src/calibre/db/locking.py index 90afbb8ffe..822168eb08 100644 --- a/src/calibre/db/locking.py +++ b/src/calibre/db/locking.py @@ -227,28 +227,24 @@ class DebugRWLockWrapper(RWLockWrapper): def __init__(self, *args, **kwargs): RWLockWrapper.__init__(self, *args, **kwargs) - self._last_acquire = () def __enter__(self): - try: - RWLockWrapper.__enter__(self) - except LockingError as e: - if self._last_acquire is None: - raise - et, ei, tb = sys.exc_info() - raise LockingError, LockingError(e.message, extra='Last successful call to acquire():\n' + ''.join(self._last_acquire)), tb - self._last_acquire = traceback.format_stack() + print ('#' * 120, file=sys.stderr) + print ('acquire called: thread id:', current_thread(), 'shared:', self._is_shared, file=sys.stderr) + traceback.print_stack() + RWLockWrapper.__enter__(self) + print ('acquire done: thread id:', current_thread(), file=sys.stderr) + print ('_' * 120, file=sys.stderr) def release(self): - try: - RWLockWrapper.release(self) - except LockingError as e: - if self._last_acquire is None: - raise - et, ei, tb = sys.exc_info() - raise LockingError, LockingError(e.message, extra='Last successful call to acquire():\n' + ''.join(self._last_acquire)), tb + print ('*' * 120, file=sys.stderr) + print ('release called: thread id:', current_thread(), 'shared:', self._is_shared, file=sys.stderr) + traceback.print_stack() + RWLockWrapper.release(self) + print ('release done: thread id:', current_thread(), 'is_shared:', self._shlock.is_shared, 'is_exclusive:', self._shlock.is_exclusive, file=sys.stderr) + print ('_' * 120, file=sys.stderr) -class RecordLock(object): +class RecordLock(object): # {{{ ''' Lock records identified by hashable ids. To use @@ -326,6 +322,7 @@ class RecordLock(object): def _return_lock(self, lock): self._free_locks.append(lock) +# }}} # Tests {{{ if __name__ == '__main__': From cf293a6232ec54415ac231100e63b468d15b8a4a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 07:39:04 +0530 Subject: [PATCH 0474/1154] Remove the unused RecordLock class --- src/calibre/db/locking.py | 115 +------------------------------------- 1 file changed, 1 insertion(+), 114 deletions(-) diff --git a/src/calibre/db/locking.py b/src/calibre/db/locking.py index 822168eb08..ac871a5869 100644 --- a/src/calibre/db/locking.py +++ b/src/calibre/db/locking.py @@ -8,9 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' import traceback, sys -from threading import Lock, Condition, current_thread, RLock -from functools import partial -from collections import Counter +from threading import Lock, Condition, current_thread from calibre.utils.config_base import tweaks class LockingError(RuntimeError): @@ -244,86 +242,6 @@ class DebugRWLockWrapper(RWLockWrapper): print ('release done: thread id:', current_thread(), 'is_shared:', self._shlock.is_shared, 'is_exclusive:', self._shlock.is_exclusive, file=sys.stderr) print ('_' * 120, file=sys.stderr) -class RecordLock(object): # {{{ - - ''' - Lock records identified by hashable ids. To use - - rl = RecordLock() - - with rl.lock(some_id): - # do something - - This will lock the record identified by some_id exclusively. The lock is - recursive, which means that you can lock the same record multiple times in - the same thread. - - This class co-operates with the SHLock class. If you try to lock a record - in a thread that already holds the SHLock, a LockingError is raised. This - is to prevent the possibility of a cross-lock deadlock. - - A cross-lock deadlock is still possible if you first lock a record and then - acquire the SHLock, but the usage pattern for this lock makes this highly - unlikely (this lock should be acquired immediately before any file I/O on - files in the library and released immediately after). - ''' - - class Wrap(object): - - def __init__(self, release): - self.release = release - - def __enter__(self): - return self - - def __exit__(self, *args, **kwargs): - self.release() - self.release = None - - def __init__(self, sh_lock): - self._lock = Lock() - # This is for recycling lock objects. - self._free_locks = [RLock()] - self._records = {} - self._counter = Counter() - self.sh_lock = sh_lock - - def lock(self, record_id): - if self.sh_lock.owns_lock(): - raise LockingError('Current thread already holds a shared lock,' - ' you cannot also ask for record lock as this could cause a' - ' deadlock.') - with self._lock: - l = self._records.get(record_id, None) - if l is None: - l = self._take_lock() - self._records[record_id] = l - self._counter[record_id] += 1 - l.acquire() - return RecordLock.Wrap(partial(self.release, record_id)) - - def release(self, record_id): - with self._lock: - l = self._records.pop(record_id, None) - if l is None: - raise LockingError('No lock acquired for record %r'%record_id) - l.release() - self._counter[record_id] -= 1 - if self._counter[record_id] > 0: - self._records[record_id] = l - else: - self._return_lock(l) - - def _take_lock(self): - try: - return self._free_locks.pop() - except IndexError: - return RLock() - - def _return_lock(self, lock): - self._free_locks.append(lock) -# }}} - # Tests {{{ if __name__ == '__main__': import time, random, unittest @@ -490,37 +408,6 @@ if __name__ == '__main__': self.assertFalse(lock.is_shared) self.assertFalse(lock.is_exclusive) - def test_record_lock(self): - shlock = SHLock() - lock = RecordLock(shlock) - - shlock.acquire() - self.assertRaises(LockingError, lock.lock, 1) - shlock.release() - with lock.lock(1): - with lock.lock(1): - pass - - def dolock(): - with lock.lock(1): - time.sleep(0.1) - - t = Thread(target=dolock) - t.daemon = True - with lock.lock(1): - t.start() - t.join(0.2) - self.assertTrue(t.is_alive()) - t.join(0.11) - self.assertFalse(t.is_alive()) - - t = Thread(target=dolock) - t.daemon = True - with lock.lock(2): - t.start() - t.join(0.11) - self.assertFalse(t.is_alive()) - suite = unittest.TestLoader().loadTestsFromTestCase(TestLock) unittest.TextTestRunner(verbosity=2).run(suite) From 666248454b6c22b89fc2022accc8d001d00da656 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 07:44:19 +0530 Subject: [PATCH 0475/1154] Move lock testing into the main test suite --- src/calibre/db/locking.py | 170 ------------------------------- src/calibre/db/tests/locking.py | 174 ++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 170 deletions(-) create mode 100644 src/calibre/db/tests/locking.py diff --git a/src/calibre/db/locking.py b/src/calibre/db/locking.py index ac871a5869..9245ecae84 100644 --- a/src/calibre/db/locking.py +++ b/src/calibre/db/locking.py @@ -242,174 +242,4 @@ class DebugRWLockWrapper(RWLockWrapper): print ('release done: thread id:', current_thread(), 'is_shared:', self._shlock.is_shared, 'is_exclusive:', self._shlock.is_exclusive, file=sys.stderr) print ('_' * 120, file=sys.stderr) -# Tests {{{ -if __name__ == '__main__': - import time, random, unittest - from threading import Thread - - class TestLock(unittest.TestCase): - """Testcases for Lock classes.""" - - def test_owns_locks(self): - lock = SHLock() - self.assertFalse(lock.owns_lock()) - lock.acquire(shared=True) - self.assertTrue(lock.owns_lock()) - lock.release() - self.assertFalse(lock.owns_lock()) - lock.acquire(shared=False) - self.assertTrue(lock.owns_lock()) - lock.release() - self.assertFalse(lock.owns_lock()) - - done = [] - def test(): - if not lock.owns_lock(): - done.append(True) - lock.acquire() - t = Thread(target=test) - t.daemon = True - t.start() - t.join(1) - self.assertEqual(len(done), 1) - lock.release() - - def test_multithread_deadlock(self): - lock = SHLock() - def two_shared(): - r = RWLockWrapper(lock) - with r: - time.sleep(0.2) - with r: - pass - def one_exclusive(): - time.sleep(0.1) - w = RWLockWrapper(lock, is_shared=False) - with w: - pass - threads = [Thread(target=two_shared), Thread(target=one_exclusive)] - for t in threads: - t.daemon = True - t.start() - for t in threads: - t.join(5) - live = [t for t in threads if t.is_alive()] - self.assertListEqual(live, [], 'ShLock hung') - - def test_upgrade(self): - lock = SHLock() - lock.acquire(shared=True) - self.assertRaises(LockingError, lock.acquire, shared=False) - lock.release() - - def test_downgrade(self): - lock = SHLock() - lock.acquire(shared=False) - self.assertRaises(LockingError, lock.acquire, shared=True) - lock.release() - - def test_recursive(self): - lock = SHLock() - lock.acquire(shared=True) - lock.acquire(shared=True) - self.assertEqual(lock.is_shared, 2) - lock.release() - lock.release() - self.assertFalse(lock.is_shared) - lock.acquire(shared=False) - lock.acquire(shared=False) - self.assertEqual(lock.is_exclusive, 2) - lock.release() - lock.release() - self.assertFalse(lock.is_exclusive) - - def test_release(self): - lock = SHLock() - self.assertRaises(LockingError, lock.release) - - def get_lock(shared): - lock.acquire(shared=shared) - time.sleep(1) - lock.release() - - threads = [Thread(target=get_lock, args=(x,)) for x in (True, - False)] - for t in threads: - t.daemon = True - t.start() - self.assertRaises(LockingError, lock.release) - t.join(2) - self.assertFalse(t.is_alive()) - self.assertFalse(lock.is_shared) - self.assertFalse(lock.is_exclusive) - - def test_acquire(self): - lock = SHLock() - - def get_lock(shared): - lock.acquire(shared=shared) - time.sleep(1) - lock.release() - - shared = Thread(target=get_lock, args=(True,)) - shared.daemon = True - shared.start() - time.sleep(0.1) - self.assertTrue(lock.acquire(shared=True, blocking=False)) - lock.release() - self.assertFalse(lock.acquire(shared=False, blocking=False)) - lock.acquire(shared=False) - self.assertFalse(shared.is_alive()) - lock.release() - self.assertTrue(lock.acquire(shared=False, blocking=False)) - lock.release() - - exclusive = Thread(target=get_lock, args=(False,)) - exclusive.daemon = True - exclusive.start() - time.sleep(0.1) - self.assertFalse(lock.acquire(shared=False, blocking=False)) - self.assertFalse(lock.acquire(shared=True, blocking=False)) - lock.acquire(shared=True) - self.assertFalse(exclusive.is_alive()) - lock.release() - lock.acquire(shared=False) - lock.release() - lock.acquire(shared=True) - lock.release() - self.assertFalse(lock.is_shared) - self.assertFalse(lock.is_exclusive) - - def test_contention(self): - lock = SHLock() - done = [] - def lots_of_acquires(): - for _ in xrange(1000): - shared = random.choice([True,False]) - lock.acquire(shared=shared) - lock.acquire(shared=shared) - time.sleep(random.random() * 0.0001) - lock.release() - time.sleep(random.random() * 0.0001) - lock.acquire(shared=shared) - time.sleep(random.random() * 0.0001) - lock.release() - lock.release() - done.append(True) - threads = [Thread(target=lots_of_acquires) for _ in xrange(10)] - for t in threads: - t.daemon = True - t.start() - for t in threads: - t.join(20) - live = [t for t in threads if t.is_alive()] - self.assertListEqual(live, [], 'ShLock hung') - self.assertEqual(len(done), len(threads), 'SHLock locking failed') - self.assertFalse(lock.is_shared) - self.assertFalse(lock.is_exclusive) - - suite = unittest.TestLoader().loadTestsFromTestCase(TestLock) - unittest.TextTestRunner(verbosity=2).run(suite) - -# }}} diff --git a/src/calibre/db/tests/locking.py b/src/calibre/db/tests/locking.py new file mode 100644 index 0000000000..a9dcb43ae4 --- /dev/null +++ b/src/calibre/db/tests/locking.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +import time, random +from threading import Thread +from calibre.db.tests.base import BaseTest +from calibre.db.locking import SHLock, RWLockWrapper, LockingError + +class TestLock(BaseTest): + """Tests for db locking """ + + def test_owns_locks(self): + lock = SHLock() + self.assertFalse(lock.owns_lock()) + lock.acquire(shared=True) + self.assertTrue(lock.owns_lock()) + lock.release() + self.assertFalse(lock.owns_lock()) + lock.acquire(shared=False) + self.assertTrue(lock.owns_lock()) + lock.release() + self.assertFalse(lock.owns_lock()) + + done = [] + def test(): + if not lock.owns_lock(): + done.append(True) + lock.acquire() + t = Thread(target=test) + t.daemon = True + t.start() + t.join(1) + self.assertEqual(len(done), 1) + lock.release() + + def test_multithread_deadlock(self): + lock = SHLock() + def two_shared(): + r = RWLockWrapper(lock) + with r: + time.sleep(0.2) + with r: + pass + def one_exclusive(): + time.sleep(0.1) + w = RWLockWrapper(lock, is_shared=False) + with w: + pass + threads = [Thread(target=two_shared), Thread(target=one_exclusive)] + for t in threads: + t.daemon = True + t.start() + for t in threads: + t.join(5) + live = [t for t in threads if t.is_alive()] + self.assertListEqual(live, [], 'ShLock hung') + + def test_upgrade(self): + lock = SHLock() + lock.acquire(shared=True) + self.assertRaises(LockingError, lock.acquire, shared=False) + lock.release() + + def test_downgrade(self): + lock = SHLock() + lock.acquire(shared=False) + self.assertRaises(LockingError, lock.acquire, shared=True) + lock.release() + + def test_recursive(self): + lock = SHLock() + lock.acquire(shared=True) + lock.acquire(shared=True) + self.assertEqual(lock.is_shared, 2) + lock.release() + lock.release() + self.assertFalse(lock.is_shared) + lock.acquire(shared=False) + lock.acquire(shared=False) + self.assertEqual(lock.is_exclusive, 2) + lock.release() + lock.release() + self.assertFalse(lock.is_exclusive) + + def test_release(self): + lock = SHLock() + self.assertRaises(LockingError, lock.release) + + def get_lock(shared): + lock.acquire(shared=shared) + time.sleep(1) + lock.release() + + threads = [Thread(target=get_lock, args=(x,)) for x in (True, + False)] + for t in threads: + t.daemon = True + t.start() + self.assertRaises(LockingError, lock.release) + t.join(2) + self.assertFalse(t.is_alive()) + self.assertFalse(lock.is_shared) + self.assertFalse(lock.is_exclusive) + + def test_acquire(self): + lock = SHLock() + + def get_lock(shared): + lock.acquire(shared=shared) + time.sleep(1) + lock.release() + + shared = Thread(target=get_lock, args=(True,)) + shared.daemon = True + shared.start() + time.sleep(0.1) + self.assertTrue(lock.acquire(shared=True, blocking=False)) + lock.release() + self.assertFalse(lock.acquire(shared=False, blocking=False)) + lock.acquire(shared=False) + self.assertFalse(shared.is_alive()) + lock.release() + self.assertTrue(lock.acquire(shared=False, blocking=False)) + lock.release() + + exclusive = Thread(target=get_lock, args=(False,)) + exclusive.daemon = True + exclusive.start() + time.sleep(0.1) + self.assertFalse(lock.acquire(shared=False, blocking=False)) + self.assertFalse(lock.acquire(shared=True, blocking=False)) + lock.acquire(shared=True) + self.assertFalse(exclusive.is_alive()) + lock.release() + lock.acquire(shared=False) + lock.release() + lock.acquire(shared=True) + lock.release() + self.assertFalse(lock.is_shared) + self.assertFalse(lock.is_exclusive) + + def test_contention(self): + lock = SHLock() + done = [] + def lots_of_acquires(): + for _ in xrange(1000): + shared = random.choice([True,False]) + lock.acquire(shared=shared) + lock.acquire(shared=shared) + time.sleep(random.random() * 0.0001) + lock.release() + time.sleep(random.random() * 0.0001) + lock.acquire(shared=shared) + time.sleep(random.random() * 0.0001) + lock.release() + lock.release() + done.append(True) + threads = [Thread(target=lots_of_acquires) for _ in xrange(10)] + for t in threads: + t.daemon = True + t.start() + for t in threads: + t.join(20) + live = [t for t in threads if t.is_alive()] + self.assertListEqual(live, [], 'ShLock hung') + self.assertEqual(len(done), len(threads), 'SHLock locking failed') + self.assertFalse(lock.is_shared) + self.assertFalse(lock.is_exclusive) + From 562f8f16b469c2f62f30115d1863e00a18a34bfd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 11:39:09 +0530 Subject: [PATCH 0476/1154] newdb: Fix move_library() broken on case-insenstive file systems get_top_level_move_items() was not checking that the items it returned actually existed on case-insensitive file systems. This bug is actually present in olddb as well. --- src/calibre/db/backend.py | 2 +- src/calibre/db/tests/legacy.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index db5510d055..05a8887576 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1485,7 +1485,7 @@ class DB(object): if not self.is_case_sensitive: for x in items: path_map[x.lower()] = x - items = set(path_map) + items = {x.lower() for x in items} paths = {x.lower() for x in paths} items = items.intersection(paths) return items, path_map diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index c936367ba4..55610f6401 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -167,7 +167,6 @@ class LegacyTest(BaseTest): for meth, args in { 'find_identical_books': [(Metadata('title one', ['author one']),), (Metadata('unknown'),), (Metadata('xxxx'),)], - 'get_top_level_move_items': [()], 'get_books_for_category': [('tags', newstag), ('#formats', 'FMT1')], 'get_next_series_num_for': [('A Series One',)], 'get_id_from_uuid':[('ddddd',), (db.uuid(1, True),)], @@ -241,6 +240,10 @@ class LegacyTest(BaseTest): for a in args: self.assertEqual(fmt(getattr(db, meth)(*a)), fmt(getattr(ndb, meth)(*a)), 'The method: %s() returned different results for argument %s' % (meth, a)) + def f(x, y): # get_top_level_move_items is broken in the old db on case-insensitive file systems + x.discard('metadata_db_prefs_backup.json') + return x, y + self.assertEqual(f(*db.get_top_level_move_items()), f(*ndb.get_top_level_move_items())) d1, d2 = BytesIO(), BytesIO() db.copy_cover_to(1, d1, True) ndb.copy_cover_to(1, d2, True) From 37484fb02c8f6fcd17066088f60f2c4ba1bab3b4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 12:04:38 +0530 Subject: [PATCH 0477/1154] Fix refresh() test failling on OS X because of file system resolution --- src/calibre/db/tests/legacy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 55610f6401..03abdee26e 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' -import inspect +import inspect, time from io import BytesIO from repr import repr from functools import partial @@ -126,6 +126,9 @@ class LegacyTest(BaseTest): ' Test refreshing the view after a change to metadata.db ' db = self.init_legacy() db2 = self.init_legacy() + # Ensure that the following change will actually update the timestamp + # on filesystems with one second resolution (OS X) + time.sleep(1) self.assertEqual(db2.data.cache.set_field('title', {1:'xxx'}), set([1])) db2.close() del db2 From 17dcc39aff04ba8e3b8c44ccf61615ef12d62200 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 12:31:47 +0530 Subject: [PATCH 0478/1154] Fix test failures due to file locking on windows --- src/calibre/db/tests/legacy.py | 39 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 03abdee26e..4baf1a627e 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -10,7 +10,6 @@ import inspect, time from io import BytesIO from repr import repr from functools import partial -from tempfile import NamedTemporaryFile from operator import itemgetter from calibre.db.tests.base import BaseTest @@ -313,19 +312,20 @@ class LegacyTest(BaseTest): def test_legacy_adding_books(self): # {{{ 'Test various adding/deleting books methods' from calibre.ebooks.metadata.book.base import Metadata + from calibre.ptempfile import TemporaryFile legacy, old = self.init_legacy(self.cloned_library), self.init_old(self.cloned_library) mi = Metadata('Added Book0', authors=('Added Author',)) - with NamedTemporaryFile(suffix='.aff') as f: - f.write(b'xxx') - f.flush() - T = partial(ET, 'add_books', ([f.name], ['AFF'], [mi]), old=old, legacy=legacy) + with TemporaryFile(suffix='.aff') as name: + with open(name, 'wb') as f: + f.write(b'xxx') + T = partial(ET, 'add_books', ([name], ['AFF'], [mi]), old=old, legacy=legacy) T()(self) book_id = T(kwargs={'return_ids':True})(self)[1][0] self.assertEqual(legacy.new_api.formats(book_id), ('AFF',)) T(kwargs={'add_duplicates':False})(self) mi.title = 'Added Book1' mi.uuid = 'uuu' - T = partial(ET, 'import_book', (mi,[f.name]), old=old, legacy=legacy) + T = partial(ET, 'import_book', (mi,[name]), old=old, legacy=legacy) book_id = T()(self) self.assertNotEqual(legacy.uuid(book_id, index_is_id=True), old.uuid(book_id, index_is_id=True)) book_id = T(kwargs={'preserve_uuid':True})(self) @@ -336,10 +336,10 @@ class LegacyTest(BaseTest): T((0, 'AFF', BytesIO(b'fffff')))(self) T((0, 'AFF', BytesIO(b'fffff')))(self) T((0, 'AFF', BytesIO(b'fffff')), {'replace':True})(self) - with NamedTemporaryFile(suffix='.opf') as f: - f.write(b'zzzz') - f.flush() - T = partial(ET, 'import_book', (mi,[f.name]), old=old, legacy=legacy) + with TemporaryFile(suffix='.opf') as name: + with open(name, 'wb') as f: + f.write(b'zzzz') + T = partial(ET, 'import_book', (mi,[name]), old=old, legacy=legacy) book_id = T()(self) self.assertFalse(legacy.new_api.formats(book_id)) @@ -349,21 +349,21 @@ class LegacyTest(BaseTest): T({'add_duplicates':False}) T({'force_id':1000}) - with NamedTemporaryFile(suffix='.txt') as f: - f.write(b'tttttt') - f.seek(0) - bid = legacy.add_catalog(f.name, 'My Catalog') - self.assertEqual(old.add_catalog(f.name, 'My Catalog'), bid) + with TemporaryFile(suffix='.txt') as name: + with open(name, 'wb') as f: + f.write(b'tttttt') + bid = legacy.add_catalog(name, 'My Catalog') + self.assertEqual(old.add_catalog(name, 'My Catalog'), bid) cache = legacy.new_api self.assertEqual(cache.formats(bid), ('TXT',)) self.assertEqual(cache.field_for('title', bid), 'My Catalog') self.assertEqual(cache.field_for('authors', bid), ('calibre',)) self.assertEqual(cache.field_for('tags', bid), (_('Catalog'),)) - self.assertTrue(bid < legacy.add_catalog(f.name, 'Something else')) - self.assertEqual(legacy.add_catalog(f.name, 'My Catalog'), bid) - self.assertEqual(old.add_catalog(f.name, 'My Catalog'), bid) + self.assertTrue(bid < legacy.add_catalog(name, 'Something else')) + self.assertEqual(legacy.add_catalog(name, 'My Catalog'), bid) + self.assertEqual(old.add_catalog(name, 'My Catalog'), bid) - bid = legacy.add_news(f.name, {'title':'Events', 'add_title_tag':True, 'custom_tags':('one', 'two')}) + bid = legacy.add_news(name, {'title':'Events', 'add_title_tag':True, 'custom_tags':('one', 'two')}) self.assertEqual(cache.formats(bid), ('TXT',)) self.assertEqual(cache.field_for('authors', bid), ('calibre',)) self.assertEqual(cache.field_for('tags', bid), (_('News'), 'Events', 'one', 'two')) @@ -783,5 +783,6 @@ class LegacyTest(BaseTest): ('saved_search_names',), ('saved_search_lookup', 'n'), )) + db.close() # }}} From 2affa5bf4e17399732464d067959b086dc9ac172 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 12:37:00 +0530 Subject: [PATCH 0479/1154] Fix test failure on windows because of slowness --- src/calibre/db/tests/writing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 96e2bb0c34..19f6e70f48 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -343,12 +343,12 @@ class WritingTest(BaseTest): ae(sf('authors', {1:'author1 & author2', 2:'author1 & author2', 3:'author1 & author2'}), {1,2,3}) count = 6 while cache.dirty_queue_length() and count > 0: - mb.join(interval) + mb.join(2) count -= 1 af(cache.dirty_queue_length()) finally: mb.stop() - mb.join(interval) + mb.join(2) af(mb.is_alive()) from calibre.ebooks.metadata.opf2 import OPF for book_id in (1, 2, 3): From b458cf07c4d73b2e84c6c0c5c31d647a3cfe09c6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 12:41:47 +0530 Subject: [PATCH 0480/1154] Another test fix for windows --- src/calibre/db/tests/locking.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/db/tests/locking.py b/src/calibre/db/tests/locking.py index a9dcb43ae4..24d146fa3a 100644 --- a/src/calibre/db/tests/locking.py +++ b/src/calibre/db/tests/locking.py @@ -123,6 +123,7 @@ class TestLock(BaseTest): lock.release() self.assertFalse(lock.acquire(shared=False, blocking=False)) lock.acquire(shared=False) + shared.join(1) self.assertFalse(shared.is_alive()) lock.release() self.assertTrue(lock.acquire(shared=False, blocking=False)) From 44545b0c34e726958ca7cdf09f4f6c1ab07ef9b0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 12:47:48 +0530 Subject: [PATCH 0481/1154] ... --- src/calibre/db/tests/locking.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/db/tests/locking.py b/src/calibre/db/tests/locking.py index 24d146fa3a..405c8669b9 100644 --- a/src/calibre/db/tests/locking.py +++ b/src/calibre/db/tests/locking.py @@ -136,6 +136,7 @@ class TestLock(BaseTest): self.assertFalse(lock.acquire(shared=False, blocking=False)) self.assertFalse(lock.acquire(shared=True, blocking=False)) lock.acquire(shared=True) + exclusive.join(1) self.assertFalse(exclusive.is_alive()) lock.release() lock.acquire(shared=False) From cc7c53289f7c44fbc0d5b5fd9511381b81509352 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 13:08:40 +0530 Subject: [PATCH 0482/1154] Ensure the recycle bin is not used while running tests --- src/calibre/db/tests/base.py | 4 ++++ src/calibre/utils/recycle_bin.py | 14 +++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index 7a44897600..e541b410f2 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -27,10 +27,14 @@ class BaseTest(unittest.TestCase): reset_tweaks_to_default() def setUp(self): + from calibre.utils.recycle_bin import nuke_recycle + nuke_recycle() self.library_path = self.mkdtemp() self.create_db(self.library_path) def tearDown(self): + from calibre.utils.recycle_bin import restore_recyle + restore_recyle() gc.collect(), gc.collect() shutil.rmtree(self.library_path) diff --git a/src/calibre/utils/recycle_bin.py b/src/calibre/utils/recycle_bin.py index d1a63e8d01..7b1b38261b 100644 --- a/src/calibre/utils/recycle_bin.py +++ b/src/calibre/utils/recycle_bin.py @@ -36,8 +36,16 @@ elif islinux: can_recycle = callable(recycle) -def delete_file(path): - if callable(recycle): +def nuke_recycle(): + global can_recycle + can_recycle = False + +def restore_recyle(): + global can_recycle + can_recycle = callable(recycle) + +def delete_file(path, permanent=False): + if not permanent and can_recycle: try: recycle(path) return @@ -59,7 +67,7 @@ def delete_tree(path, permanent=False): time.sleep(1) shutil.rmtree(path) else: - if callable(recycle): + if can_recycle: try: recycle(path) return From 7b7f5c81dc62e4c5d36e379e14e5168a1a2840ae Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 13:13:29 +0530 Subject: [PATCH 0483/1154] newdb: More efficient deleteing of multiple books --- src/calibre/gui2/library/models.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 5f9e7bfdc6..df2afe5afc 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -315,9 +315,14 @@ class BooksModel(QAbstractTableModel): # {{{ return ids def delete_books_by_id(self, ids, permanent=False): - for id in ids: - self.db.delete_book(id, permanent=permanent, do_clean=False) - self.db.clean() + if hasattr(self.db, 'new_api'): + self.db.new_api.remove_books(ids, permanent=permanent) + self.db.data.books_deleted(tuple(ids)) + self.db.notify('delete', list(ids)) + else: + for id in ids: + self.db.delete_book(id, permanent=permanent, do_clean=False) + self.db.clean() self.books_deleted() def books_added(self, num): From 66c5a1072bee017de327951303751cda6e931211 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 15:52:27 +0530 Subject: [PATCH 0484/1154] newdb: Workaround windows spinning the event loop when deleting files Windows, being deigned by the geniuses that it is, spins the event loop while deleting files to the recycle bin, and there exists *no other way* to move files to the Recycle Bin, since the Recycle Bin format is not documented or stable. So we move only files out of the library in the thread calling delete_books(). The files are moved to the Recycle Bin in a worker thread. This has two advantages: 1) Faster deletes, since the main thread does not have to wait on the Recycle Bin (some windows' installs are so badly messed up that moving a single file to the Bin takes seconds) 2) Restoring deleted files from the bin will not restore them inside the calibre library folder, where they become orphed. They will be restored elsewhere. Disadvantages: 1) If the user deletes a lot of books and quits calibre, they might not be finished deleting on quit, this can probably be mitigated by popping up a warning at shutdown --- src/calibre/db/backend.py | 45 +++++++---- src/calibre/db/delete_service.py | 116 +++++++++++++++++++++++++++++ src/calibre/db/tests/add_remove.py | 17 ++++- src/calibre/gui2/ui.py | 2 + 4 files changed, 162 insertions(+), 18 deletions(-) create mode 100644 src/calibre/db/delete_service.py diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 05a8887576..871d704ce1 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -8,7 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' # Imports {{{ -import os, shutil, uuid, json, glob, time, cPickle, hashlib +import os, shutil, uuid, json, glob, time, cPickle, hashlib, errno from functools import partial import apsw @@ -19,6 +19,7 @@ from calibre.constants import (iswindows, filesystem_encoding, from calibre.ptempfile import PersistentTemporaryFile, TemporaryFile from calibre.db import SPOOL_SIZE from calibre.db.schema_upgrades import SchemaUpgrade +from calibre.db.delete_service import delete_service from calibre.db.errors import NoSuchFormat from calibre.library.field_metadata import FieldMetadata from calibre.ebooks.metadata import title_sort, author_to_author_sort @@ -28,7 +29,6 @@ from calibre.utils.date import utcfromtimestamp, parse_date from calibre.utils.filenames import ( is_case_sensitive, samefile, hardlink_file, ascii_filename, WindowsAtomicFolderMove, atomic_rename) from calibre.utils.magick.draw import save_cover_data_to -from calibre.utils.recycle_bin import delete_tree, delete_file from calibre.utils.formatter_functions import load_user_template_functions from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable, @@ -1035,9 +1035,18 @@ class DB(object): path = os.path.normcase(path).lower() return path - def rmtree(self, path, permanent=False): - if not self.normpath(self.library_path).startswith(self.normpath(path)): - delete_tree(path, permanent=permanent) + def is_deletable(self, path): + return path and not self.normpath(self.library_path).startswith(self.normpath(path)) + + def rmtree(self, path): + if self.is_deletable(path): + try: + shutil.rmtree(path) + except: + import traceback + traceback.print_exc() + time.sleep(1) # In case something has temporarily locked a file + shutil.rmtree(path) def construct_path_name(self, book_id, title, author): ''' @@ -1170,7 +1179,7 @@ class DB(object): path = self.format_abspath(book_id, fmt, fname, path) if path is not None: try: - delete_file(path) + delete_service().delete_files((path,), self.library_path) except: import traceback traceback.print_exc() @@ -1360,10 +1369,10 @@ class DB(object): if os.path.exists(spath) and not samefile(spath, tpath): if wam is not None: wam.delete_originals() - self.rmtree(spath, permanent=True) + self.rmtree(spath) parent = os.path.dirname(spath) if len(os.listdir(parent)) == 0: - self.rmtree(parent, permanent=True) + self.rmtree(parent) finally: if wam is not None: wam.close_handles() @@ -1404,16 +1413,20 @@ class DB(object): return f.read() def remove_books(self, path_map, permanent=False): - for book_id, path in path_map.iteritems(): - if path: - path = os.path.join(self.library_path, path) - if os.path.exists(path): - self.rmtree(path, permanent=permanent) - parent = os.path.dirname(path) - if len(os.listdir(parent)) == 0: - self.rmtree(parent, permanent=permanent) self.conn.executemany( 'DELETE FROM books WHERE id=?', [(x,) for x in path_map]) + paths = {os.path.join(self.library_path, x) for x in path_map.itervalues() if x} + paths = {x for x in paths if os.path.exists(x) and self.is_deletable(x)} + if permanent: + for path in paths: + self.rmtree(path) + try: + os.rmdir(os.path.dirname(path)) + except OSError as e: + if e.errno != errno.ENOTEMPTY: + raise + else: + delete_service().delete_books(paths, self.library_path) def add_custom_data(self, name, val_map, delete_first): if delete_first: diff --git a/src/calibre/db/delete_service.py b/src/calibre/db/delete_service.py new file mode 100644 index 0000000000..a79afc7d4b --- /dev/null +++ b/src/calibre/db/delete_service.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +import os, tempfile, shutil, errno, time +from threading import Thread +from Queue import Queue + +from calibre.utils.recycle_bin import delete_tree, delete_file + +class DeleteService(Thread): + + ''' Provide a blocking file delete implementation with support for the + recycle bin. On windows, deleting files to the recycle bin spins the event + loop, which can cause locking errors in the main thread. We get around this + by only moving the files/folders to be deleted out of the library in the + main thread, they are deleted to recycle bin in a separate worker thread. + + This has the added advantage that doing a restore from the recycle bin wont + cause metadata.db and the file system to get out of sync. Also, deleting + becomes much faster, since in the common case, the move is done by a simple + os.rename(). The downside is that if the user quits calibre while a long + move to recycle bin is happening, the files may not all be deleted.''' + + daemon = True + + def __init__(self): + Thread.__init__(self) + self.requests = Queue() + + def shutdown(self, timeout=20): + self.requests.put(None) + self.join(timeout) + + def create_staging(self, library_path): + base_path = os.path.dirname(library_path) + base = os.path.basename(library_path) + try: + return tempfile.mkdtemp(prefix=base+' deleted ', dir=base_path) + except OSError: + return tempfile.mkdtemp(prefix=base+' deleted ') + + def delete_books(self, paths, library_path): + tdir = self.create_staging(library_path) + self.queue_paths(tdir, paths, delete_empty_parent=True) + + def queue_paths(self, tdir, paths, delete_empty_parent=True): + for path in paths: + if os.path.exists(path): + try: + shutil.move(path, tdir) + except EnvironmentError: + # Wait a little in case something has locked a file + time.sleep(1) + shutil.move(path, tdir) + if delete_empty_parent: + parent = os.path.dirname(path) + try: + os.rmdir(parent) + except OSError as e: + if e.errno != errno.ENOTEMPTY: + raise + self.requests.put(os.path.join(tdir, os.path.basename(path))) + + def delete_files(self, paths, library_path): + tdir = self.create_staging(library_path) + self.queue_paths(tdir, paths, delete_empty_parent=False) + + def run(self): + while True: + x = self.requests.get() + try: + if x is None: + break + try: + self.do_delete(x) + except: + import traceback + traceback.print_exc() + finally: + self.requests.task_done() + + def wait(self): + 'Blocks until all pending deletes have completed' + self.requests.join() + + def do_delete(self, x): + if os.path.isdir(x): + delete_tree(x) + else: + delete_file(x) + try: + os.rmdir(os.path.dirname(x)) + except OSError as e: + if e.errno != errno.ENOTEMPTY: + raise + +__ds = None +def delete_service(): + global __ds + if __ds is None: + __ds = DeleteService() + __ds.start() + return __ds + +def shutdown(timeout=20): + global __ds + if __ds is not None: + __ds.shutdown(timeout) + __ds = None + + diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index 5cbac5deca..fb44d0e62c 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -209,6 +209,7 @@ class AddRemoveTest(BaseTest): def test_remove_books(self): # {{{ 'Test removal of books' + cl = self.cloned_library cache = self.init_cache() af, ae, at = self.assertFalse, self.assertEqual, self.assertTrue authors = cache.fields['authors'].table @@ -233,7 +234,7 @@ class AddRemoveTest(BaseTest): item_id = {v:k for k, v in cache.fields['#series'].table.id_map.iteritems()}['My Series Two'] cache.remove_books((1,), permanent=True) for x in (fmtpath, bookpath, authorpath): - af(os.path.exists(x)) + af(os.path.exists(x), 'The file %s exists, when it should not' % x) for c in (cache, self.init_cache()): table = c.fields['authors'].table self.assertNotIn(1, c.all_book_ids()) @@ -252,6 +253,19 @@ class AddRemoveTest(BaseTest): self.assertFalse(table.book_col_map) self.assertFalse(table.col_book_map) + # Test the delete service + from calibre.db.delete_service import delete_service + cache = self.init_cache(cl) + # Check that files are removed + fmtpath = cache.format_abspath(1, 'FMT1') + bookpath = os.path.dirname(fmtpath) + authorpath = os.path.dirname(bookpath) + item_id = {v:k for k, v in cache.fields['#series'].table.id_map.iteritems()}['My Series Two'] + cache.remove_books((1,)) + delete_service().wait() + for x in (fmtpath, bookpath, authorpath): + af(os.path.exists(x), 'The file %s exists, when it should not' % x) + # }}} def test_original_fmt(self): # {{{ @@ -271,4 +285,3 @@ class AddRemoveTest(BaseTest): af(db.has_format(1, 'ORIGINAL_FMT1')) ae(set(fmts), set(db.formats(1, verify_formats=False))) # }}} - diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 8e34c2a84f..90eac209b2 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -854,6 +854,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ pass except KeyboardInterrupt: pass + from calibre.db.delete_service import shutdown + shutdown() time.sleep(2) self.istores.join() self.hide_windows() From 5f518fba1d9280bc9945518c7b6897dff5e2e020 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 15:56:37 +0530 Subject: [PATCH 0485/1154] Jot Down by desUbiKado --- recipes/jot_down.recipe | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 recipes/jot_down.recipe diff --git a/recipes/jot_down.recipe b/recipes/jot_down.recipe new file mode 100644 index 0000000000..ac5a0b7010 --- /dev/null +++ b/recipes/jot_down.recipe @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import unicode_literals +__license__ = 'GPL v3' +__copyright__ = '23 June 2013, desUBIKado' +__author__ = 'desUBIKado' +__description__ = 'Contemporary Culture Magazine' +__version__ = 'v0.01' +__date__ = '23, June 2013' +''' +http://www.jotdown.es/ +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class jotdown(BasicNewsRecipe): + author = 'desUBIKado' + description = 'Revista digital con magníficos y extensos artículos' + title = u'Jot Down - Contemporary Culture Magazine' + publisher = 'Wabi Sabi Investments, S.C.' + category = 'Opinion, culture, science, movies, TV shows, music, blogs' + language = 'es' + timefmt = '[%a, %d %b, %Y]' + oldest_article = 7 + delay = 1 + max_articles_per_feed = 20 + masthead_url = 'http://www.jotdown.es/wp-content/uploads/2011/04/logoJotDown.png' + use_embedded_content = False + remove_javascript = True + no_stylesheets = True + + feeds = [ + (u'Portada', u'http://www.jotdown.es/feed/') + ] + + keep_only_tags = [dict(name='div', attrs={'class':['single']}), + dict(name='div', attrs={'id':['comments']}), + ] + + remove_tags = [dict(name='a', attrs={'href':['http://alternativaseconomicas.coop/']}), + dict(name='div', attrs={'class':['reply','after-meta','comment-author vcard']}), + dict(name='div', attrs={'align':['center']}), + dict(name='span', attrs={'class':['fbreplace']}), + dict(name='div', attrs={'id':'respond'}) + ] + + remove_tags_after = dict(name='div' , attrs={'id':'respond'}) + + extra_css = ''' + .comment-list {font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:12px;} + ''' + + preprocess_regexps = [ + # To present the image of the embedded video + (re.compile(r'', re.DOTALL|re.IGNORECASE), lambda match: '
    '), + (re.compile(r' —', re.DOTALL|re.IGNORECASE), lambda match: ''), + # To remove the link of the title + (re.compile(r'

    ', re.DOTALL|re.IGNORECASE), lambda match: '
    ') + + ] + From a33ce97c17986c6bf1681a8324a8106848ad5882 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 16:48:20 +0530 Subject: [PATCH 0486/1154] Search highlighting for the grid view --- src/calibre/gui2/library/alternate_views.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 0e88953426..bcbc99d57c 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -352,6 +352,7 @@ class CoverDelegate(QStyledItemDelegate): self.cover_cache = CoverCache(limit=gprefs['cover_grid_cache_size']) self.render_queue = Queue() self.animating = None + self.highlight_color = QColor(Qt.white) def set_dimensions(self): width = self.original_width = gprefs['cover_grid_width'] @@ -385,11 +386,20 @@ class CoverDelegate(QStyledItemDelegate): def paint(self, painter, option, index): QStyledItemDelegate.paint(self, painter, option, QModelIndex()) # draw the hover and selection highlights - db = index.model().db + m = index.model() + db = m.db try: book_id = db.id(index.row()) except (ValueError, IndexError, KeyError): return + if book_id in m.ids_to_highlight_set: + painter.save() + try: + painter.setPen(self.highlight_color) + painter.setRenderHint(QPainter.Antialiasing, True) + painter.drawRoundedRect(option.rect, 10, 10, Qt.RelativeSize) + finally: + painter.restore() db = db.new_api cdata = self.cover_cache[book_id] painter.save() @@ -435,7 +445,6 @@ class GridView(QListView): def __init__(self, parent): QListView.__init__(self, parent) setup_dnd_interface(self) - self.set_color() self.setUniformItemSizes(True) self.setWrapping(True) self.setFlow(self.LeftToRight) @@ -450,6 +459,7 @@ class GridView(QListView): self.delegate.animation.finished.connect(self.animation_done) self.setItemDelegate(self.delegate) self.setSpacing(self.delegate.spacing) + self.set_color() self.ignore_render_requests = Event() self.render_thread = None self.update_item.connect(self.re_render, type=Qt.QueuedConnection) @@ -478,9 +488,12 @@ class GridView(QListView): def set_color(self): r, g, b = gprefs['cover_grid_color'] pal = QPalette() - pal.setColor(pal.Base, QColor(r, g, b)) - pal.setColor(pal.Text, QColor(Qt.white if (r + g + b)/3.0 < 128 else Qt.black)) + col = QColor(r, g, b) + pal.setColor(pal.Base, col) + dark = (r + g + b)/3.0 < 128 + pal.setColor(pal.Text, QColor(Qt.white if dark else Qt.black)) self.setPalette(pal) + self.delegate.highlight_color = pal.color(pal.Text) def refresh_settings(self): if gprefs['cover_grid_width'] != self.delegate.original_width or gprefs['cover_grid_height'] != self.delegate.original_height: From 5c7124b999cd7ae704327fb511c599d7e49795a3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 16:52:42 +0530 Subject: [PATCH 0487/1154] Ensure that text is anti-aliased in the grid view --- src/calibre/gui2/library/alternate_views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index bcbc99d57c..4733518889 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -409,6 +409,7 @@ class CoverDelegate(QStyledItemDelegate): if cdata is None or cdata is False: title = db.field_for('title', book_id, default_value='') authors = ' & '.join(db.field_for('authors', book_id, default_value=())) + painter.setRenderHint(QPainter.TextAntialiasing, True) painter.drawText(rect, Qt.AlignCenter|Qt.TextWordWrap, '%s\n\n%s' % (title, authors)) if cdata is False: self.render_queue.put(book_id) From a0fb42865d37ac2b2fb399e7ddaac3740b05df4b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 21:46:46 +0530 Subject: [PATCH 0488/1154] Add an icon for the sort by context menu entry --- src/calibre/gui2/library/alternate_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 4733518889..80988f2296 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -16,7 +16,7 @@ from functools import wraps, partial from PyQt4.Qt import ( QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, - QPalette, QColor, QItemSelection, QPixmap, QMenu, QApplication, QMimeData, + QPalette, QColor, QItemSelection, QPixmap, QMenu, QApplication, QMimeData, QIcon, QUrl, QDrag, QPoint, QPainter, QRect, pyqtProperty, QPropertyAnimation, QEasingCurve) from calibre import fit_image @@ -621,7 +621,7 @@ class GridView(QListView): for ac in self.context_menu.actions(): menu.addAction(ac) - menu.addMenu(sm) + menu.addMenu(sm).setIcon(QIcon(I('arrow-up.png'))) menu.popup(event.globalPos()) event.accept() From 727944a4b5cbb9cfdd4339f639d8b59ba4259c01 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Aug 2013 21:51:56 +0530 Subject: [PATCH 0489/1154] Update Telegraph UK --- recipes/telegraph_uk.recipe | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/recipes/telegraph_uk.recipe b/recipes/telegraph_uk.recipe index 347352d424..57a2068b62 100644 --- a/recipes/telegraph_uk.recipe +++ b/recipes/telegraph_uk.recipe @@ -12,7 +12,8 @@ class TelegraphUK(BasicNewsRecipe): description = 'News from United Kingdom' oldest_article = 2 category = 'news, politics, UK' - publisher = 'Telegraph Media Group ltd.' + publisher = 'Telegraph Media Group ltd.' + compress_news_images = True max_articles_per_feed = 100 no_stylesheets = True language = 'en_GB' @@ -34,30 +35,30 @@ class TelegraphUK(BasicNewsRecipe): , 'publisher' : publisher , 'language' : language } - - + keep_only_tags = [ dict(name='div', attrs={'class':['storyHead','byline']}) - ,dict(name='div', attrs={'id':'mainBodyArea' }) + ,dict(name='div', attrs={'id':'mainBodyArea'}) ] - remove_tags = [dict(name='div', attrs={'class':['related_links_inline',"imgindex","next","prev","gutterUnder",'ssImgHide','imageExtras','ssImg hide','related_links_video']}) + remove_tags = [dict(name='div', attrs={ + 'class':['related_links_inline',"imgindex","next","prev","gutterUnder",'ssImgHide','imageExtras','ssImg hide','related_links_video']}) ,dict(name='ul' , attrs={'class':['shareThis shareBottom']}) ,dict(name='span', attrs={'class':['num','placeComment']}) ] feeds = [ - (u'UK News' , u'http://www.telegraph.co.uk/news/uknews/rss' ) - ,(u'World News' , u'http://www.telegraph.co.uk/news/worldnews/rss' ) - ,(u'Politics' , u'http://www.telegraph.co.uk/news/newstopics/politics/rss' ) - ,(u'Finance' , u'http://www.telegraph.co.uk/finance/rss' ) - ,(u'Technology News', u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologynews/rss' ) + (u'UK News' , u'http://www.telegraph.co.uk/news/uknews/rss') + ,(u'World News' , u'http://www.telegraph.co.uk/news/worldnews/rss') + ,(u'Politics' , u'http://www.telegraph.co.uk/news/newstopics/politics/rss') + ,(u'Finance' , u'http://www.telegraph.co.uk/finance/rss') + ,(u'Technology News', u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologynews/rss') ,(u'UK News' , u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologyreviews/rss') - ,(u'Science News' , u'http://www.telegraph.co.uk/scienceandtechnology/science/sciencenews/rss' ) - ,(u'Sport' , u'http://www.telegraph.co.uk/sport/rss' ) - ,(u'Earth News' , u'http://www.telegraph.co.uk/earth/earthnews/rss' ) - ,(u'Comment' , u'http://www.telegraph.co.uk/comment/rss' ) - ,(u'Travel' , u'http://www.telegraph.co.uk/travel/rss' ) - ,(u'How about that?', u'http://www.telegraph.co.uk/news/newstopics/howaboutthat/rss' ) + ,(u'Science News' , u'http://www.telegraph.co.uk/scienceandtechnology/science/sciencenews/rss') + ,(u'Sport' , u'http://www.telegraph.co.uk/sport/rss') + ,(u'Earth News' , u'http://www.telegraph.co.uk/earth/earthnews/rss') + ,(u'Comment' , u'http://www.telegraph.co.uk/comment/rss') + ,(u'Travel' , u'http://www.telegraph.co.uk/travel/rss') + ,(u'How about that?', u'http://www.telegraph.co.uk/news/newstopics/howaboutthat/rss') ] def populate_article_metadata(self, article, soup, first): if first and hasattr(self, 'add_toc_thumbnail'): @@ -70,3 +71,4 @@ class TelegraphUK(BasicNewsRecipe): if 'picture-galleries' in url or 'pictures' in url or 'picturegalleries' in url : url = None return url + From 1294635c25d43c4df1b14b8103ec15dc121377c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Tue, 6 Aug 2013 20:47:23 +0200 Subject: [PATCH 0490/1154] fix detection of discounted price in koobe --- src/calibre/gui2/store/stores/koobe_plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/store/stores/koobe_plugin.py b/src/calibre/gui2/store/stores/koobe_plugin.py index b8b7386593..0b8ee8f719 100644 --- a/src/calibre/gui2/store/stores/koobe_plugin.py +++ b/src/calibre/gui2/store/stores/koobe_plugin.py @@ -62,6 +62,8 @@ class KoobeStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//div[@class="cover"]/a/img/@src')) price = ''.join(data.xpath('.//span[@class="current_price"]/text()')) + if not price: + price = ''.join(data.xpath('.//div[@class="book_promo_price"]/span/text()')) title = ''.join(data.xpath('.//h2[@class="title"]/a/text()')) author = ', '.join(data.xpath('.//h3[@class="book_author"]/a/text()')) formats = ', '.join(data.xpath('.//div[@class="formats"]/div/div/@title')) From 7fb0bf9c0ffe57f9718a6966caa68e0f00ae255b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Tue, 6 Aug 2013 20:51:45 +0200 Subject: [PATCH 0491/1154] bump up koobe plugin version --- src/calibre/gui2/store/stores/koobe_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/stores/koobe_plugin.py b/src/calibre/gui2/store/stores/koobe_plugin.py index 0b8ee8f719..d767a832cb 100644 --- a/src/calibre/gui2/store/stores/koobe_plugin.py +++ b/src/calibre/gui2/store/stores/koobe_plugin.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 3 # Needed for dynamic plugin loading +store_version = 4 # Needed for dynamic plugin loading __license__ = 'GPL 3' __copyright__ = '2013, Tomasz Długosz ' From d9ab10dd1613a42c6422eab4b6a73f6ab7e0eac1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 Aug 2013 08:06:57 +0530 Subject: [PATCH 0492/1154] Make the sort sub menu a little clearer --- src/calibre/gui2/library/alternate_views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 80988f2296..d5c7472d84 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -616,8 +616,10 @@ class GridView(QListView): extra = '' if last[0] == col: ascending = not last[1] - extra = ' [%s]' % _('reverse') - sm.addAction('%s%s' % (m.get('name', col), extra), partial(self.do_sort, col, ascending)) + extra = ' [%s]' % _('reverse current sort') + ac = sm.addAction('%s%s' % (m.get('name', col), extra), partial(self.do_sort, col, ascending)) + if last[0] == col: + ac.setIcon(QIcon(I('ok.png'))) for ac in self.context_menu.actions(): menu.addAction(ac) From 6ce79b2d96eb58690e68266499f34b48ba1d489c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 Aug 2013 09:44:17 +0530 Subject: [PATCH 0493/1154] Grid View: Do not render covers while user is using slider on scrollbar --- src/calibre/gui2/library/alternate_views.py | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index d5c7472d84..193afb5c95 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -468,6 +468,39 @@ class GridView(QListView): self.setCursor(Qt.PointingHandCursor) self.gui = parent self.context_menu = None + self.verticalScrollBar().sliderPressed.connect(self.slider_pressed) + self.verticalScrollBar().sliderReleased.connect(self.slider_released) + + @property + def first_visible_row(self): + geom = self.viewport().geometry() + for y in xrange(geom.top(), (self.spacing()*2) + geom.top(), 5): + for x in xrange(geom.left(), (self.spacing()*2) + geom.left(), 5): + ans = self.indexAt(QPoint(x, y)).row() + if ans > -1: + return ans + + @property + def last_visible_row(self): + geom = self.viewport().geometry() + for y in xrange(geom.bottom(), geom.bottom() - 2 * self.spacing(), -5): + for x in xrange(geom.left(), (self.spacing()*2) + geom.left(), 5): + ans = self.indexAt(QPoint(x, y)).row() + if ans > -1: + item_width = self.delegate.item_size.width() + 2*self.spacing() + return ans + (geom.width() // item_width) + + def update_viewport(self): + m = self.model() + for r in xrange(self.first_visible_row or 0, self.last_visible_row or (m.count() - 1)): + self.update(m.index(r, 0)) + + def slider_pressed(self): + self.ignore_render_requests.set() + + def slider_released(self): + self.ignore_render_requests.clear() + self.update_viewport() def double_clicked(self, index): d = self.delegate From 798eb321d8a9b20e4c3ea145b0c4b288cdcb368f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 Aug 2013 09:54:25 +0530 Subject: [PATCH 0494/1154] Grid View: Delay rendering on wheel events Ignore continuous wheel events, only rendering after the "last" wheel event, where two wheel event are continuous if they occur within 200 msecs of each other. --- src/calibre/gui2/library/alternate_views.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 193afb5c95..656d8baaff 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -15,7 +15,7 @@ from Queue import Queue from functools import wraps, partial from PyQt4.Qt import ( - QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, + QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, QTimer, QPalette, QColor, QItemSelection, QPixmap, QMenu, QApplication, QMimeData, QIcon, QUrl, QDrag, QPoint, QPainter, QRect, pyqtProperty, QPropertyAnimation, QEasingCurve) @@ -470,6 +470,10 @@ class GridView(QListView): self.context_menu = None self.verticalScrollBar().sliderPressed.connect(self.slider_pressed) self.verticalScrollBar().sliderReleased.connect(self.slider_released) + self.update_timer = QTimer(self) + self.update_timer.setInterval(200) + self.update_timer.timeout.connect(self.update_viewport) + self.update_timer.setSingleShot(True) @property def first_visible_row(self): @@ -491,6 +495,7 @@ class GridView(QListView): return ans + (geom.width() // item_width) def update_viewport(self): + self.ignore_render_requests.clear() m = self.model() for r in xrange(self.first_visible_row or 0, self.last_visible_row or (m.count() - 1)): self.update(m.index(r, 0)) @@ -502,6 +507,11 @@ class GridView(QListView): self.ignore_render_requests.clear() self.update_viewport() + def wheelEvent(self, e): + self.ignore_render_requests.set() + QListView.wheelEvent(self, e) + self.update_timer.start() + def double_clicked(self, index): d = self.delegate if d.animating is None and not config['disable_animations']: From 05572d975af092ce3e9a2318ef3d41ad9d6c53e9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 Aug 2013 10:52:04 +0530 Subject: [PATCH 0495/1154] Allow skipping the confirm bulk reconvert dialog When bulk converting previously converted books, calibre asks for a confirmation. Make this confirmation dialog skippable. --- src/calibre/gui2/tools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 32e3174c8d..db15837227 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -373,9 +373,9 @@ def convert_existing(parent, db, book_ids, output_format): # {{{ if already_converted_ids: if not question_dialog(parent, _('Convert existing'), - _('The following books have already been converted to %s format. ' - 'Do you wish to reconvert them?') % output_format, - '\n'.join(already_converted_titles)): + _('The following books have already been converted to the %s format. ' + 'Do you wish to reconvert them?') % output_format.upper(), + det_msg='\n'.join(already_converted_titles), skip_dialog_name='confirm_bulk_reconvert'): book_ids = [x for x in book_ids if x not in already_converted_ids] return book_ids From c4a82de4c7ae655d1e2cf3bcb1eed08ad832e63f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 Aug 2013 12:50:24 +0530 Subject: [PATCH 0496/1154] Confirm quit while delete in progress --- src/calibre/db/delete_service.py | 5 +++++ src/calibre/gui2/ui.py | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/src/calibre/db/delete_service.py b/src/calibre/db/delete_service.py index a79afc7d4b..ce5578f888 100644 --- a/src/calibre/db/delete_service.py +++ b/src/calibre/db/delete_service.py @@ -113,4 +113,9 @@ def shutdown(timeout=20): __ds.shutdown(timeout) __ds = None +def has_jobs(): + global __ds + if __ds is not None: + return (not __ds.requests.empty()) or __ds.requests.unfinished_tasks + return False diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 90eac209b2..196de7cb6a 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -809,6 +809,14 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ if not question_dialog(self, _('Active jobs'), msg): return False + from calibre.db.delete_service import has_jobs + if has_jobs(): + msg = _('Some deleted books are still being moved to the Recycle ' + 'Bin, if you quit now, they will be left behind. Are you ' + 'sure you want to quit?') + if not question_dialog(self, _('Active jobs'), msg): + return False + return True def shutdown(self, write_settings=True): From 8b5e9ed035d1610a77e59cf283092eb244f986d3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 Aug 2013 16:33:43 +0530 Subject: [PATCH 0497/1154] Add Sort By context menu entry A new "Sort By" action for the right click menu. This allows sorting on all columns in the library, not just the visible columns. To use it go to Preferences->Toolbars and add it to "The context menu for books in the calibre library" --- src/calibre/customize/builtins.py | 6 +- src/calibre/gui2/actions/sort.py | 79 +++++++++++++++++++++ src/calibre/gui2/library/alternate_views.py | 33 +++------ src/calibre/gui2/library/views.py | 5 ++ 4 files changed, 98 insertions(+), 25 deletions(-) create mode 100644 src/calibre/gui2/actions/sort.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 3e573be026..0812325157 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -912,6 +912,10 @@ class ActionPickRandom(InterfaceActionBase): actual_plugin = 'calibre.gui2.actions.random:PickRandomAction' description = _('Choose a random book from your calibre library') +class ActionSortBy(InterfaceActionBase): + name = 'Sort By' + actual_plugin = 'calibre.gui2.actions.sort:SortByAction' + description = _('Sort the list of books') class ActionStore(InterfaceActionBase): name = 'Store' @@ -943,7 +947,7 @@ plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog, ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks, ActionAddToLibrary, ActionEditCollections, ActionMatchBooks, ActionChooseLibrary, ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore, - ActionPluginUpdater, ActionPickRandom, ActionEditToC] + ActionPluginUpdater, ActionPickRandom, ActionEditToC, ActionSortBy] # }}} diff --git a/src/calibre/gui2/actions/sort.py b/src/calibre/gui2/actions/sort.py new file mode 100644 index 0000000000..14bd641020 --- /dev/null +++ b/src/calibre/gui2/actions/sort.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +from PyQt4.Qt import QToolButton, QAction, pyqtSignal, QIcon + +from calibre.gui2.actions import InterfaceAction +from calibre.utils.icu import sort_key + +class SortAction(QAction): + + sort_requested = pyqtSignal(object, object) + + def __init__(self, text, key, ascending, parent): + QAction.__init__(self, text, parent) + self.key, self.ascending = key, ascending + self.triggered.connect(self) + + def __call__(self): + self.sort_requested(self.key, self.ascending) + +class SortByAction(InterfaceAction): + + name = 'Sort By' + action_spec = (_('Sort By'), 'arrow-up.png', _('Sort the list of books'), None) + action_type = 'current' + popup_type = QToolButton.InstantPopup + action_add_menu = True + dont_add_to = frozenset([ + 'toolbar', 'toolbar-device', 'context-menu-device', 'toolbar-child', + 'menubar', 'menubar-device', 'context-menu-cover-browser']) + + def genesis(self): + self.sorted_icon = QIcon(I('ok.png')) + + def location_selected(self, loc): + self.qaction.setEnabled(loc == 'library') + + def update_menu(self): + menu = self.qaction.menu() + for action in menu.actions(): + action.sort_requested.disconnect() + menu.clear() + lv = self.gui.library_view + m = lv.model() + db = m.db + try: + sort_col, order = m.sorted_on + except TypeError: + sort_col, order = 'date', True + fm = db.field_metadata + name_map = {fm[k]['name']:k for k in fm.sortable_field_keys() if fm[k]['name']} + self._sactions = [] + for name in sorted(name_map, key=sort_key): + key = name_map[name] + if key == 'title': + continue + if key == 'sort': + name = _('Title') + if key == 'ondevice' and self.gui.device_connected is None: + continue + ascending = True + if key == sort_col: + name = _('%s [reverse current sort]') % name + ascending = not order + sac = SortAction(name, key, ascending, menu) + if key == sort_col: + sac.setIcon(self.sorted_icon) + sac.sort_requested.connect(self.sort_requested) + menu.addAction(sac) + + def sort_requested(self, key, ascending): + self.gui.library_view.sort_by_named_field(key, ascending) + + diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 656d8baaff..ec3a788017 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -15,9 +15,10 @@ from Queue import Queue from functools import wraps, partial from PyQt4.Qt import ( - QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, QTimer, - QPalette, QColor, QItemSelection, QPixmap, QMenu, QApplication, QMimeData, QIcon, - QUrl, QDrag, QPoint, QPainter, QRect, pyqtProperty, QPropertyAnimation, QEasingCurve) + QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, + QTimer, QPalette, QColor, QItemSelection, QPixmap, QMenu, QApplication, + QMimeData, QUrl, QDrag, QPoint, QPainter, QRect, pyqtProperty, + QPropertyAnimation, QEasingCurve) from calibre import fit_image from calibre.gui2 import gprefs, config @@ -215,7 +216,6 @@ class AlternateViews(object): view.setModel(self.main_view._model) view.selectionModel().currentChanged.connect(self.slave_current_changed) view.selectionModel().selectionChanged.connect(self.slave_selection_changed) - view.sort_requested.connect(self.main_view.sort_by_named_field) view.files_dropped.connect(self.main_view.files_dropped) def show_view(self, key=None): @@ -440,7 +440,6 @@ def join_with_timeout(q, timeout=2): class GridView(QListView): update_item = pyqtSignal(object) - sort_requested = pyqtSignal(object, object) files_dropped = pyqtSignal(object) def __init__(self, parent): @@ -648,31 +647,17 @@ class GridView(QListView): def contextMenuEvent(self, event): if self.context_menu is not None: - lv = self.gui.library_view menu = self._temp_menu = QMenu(self) - sm = QMenu(_('Sort by'), menu) - db = self.model().db - for col in lv.visible_columns: - m = db.metadata_for_field(col) - last = self.model().sorted_on - ascending = True - extra = '' - if last[0] == col: - ascending = not last[1] - extra = ' [%s]' % _('reverse current sort') - ac = sm.addAction('%s%s' % (m.get('name', col), extra), partial(self.do_sort, col, ascending)) - if last[0] == col: - ac.setIcon(QIcon(I('ok.png'))) - + sac = self.gui.iactions['Sort By'] + sort_added = tuple(ac for ac in self.context_menu.actions() if ac is sac.qaction) + if not sort_added: + menu.addAction(sac.qaction) for ac in self.context_menu.actions(): menu.addAction(ac) - menu.addMenu(sm).setIcon(QIcon(I('arrow-up.png'))) + sac.update_menu() menu.popup(event.globalPos()) event.accept() - def do_sort(self, column, ascending): - self.sort_requested.emit(column, ascending) - def get_selected_ids(self): m = self.model() return [m.id(i) for i in self.selectionModel().selectedIndexes()] diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 0b22999b27..34716445cc 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -157,6 +157,7 @@ class BooksView(QTableView): # {{{ def __init__(self, parent, modelcls=BooksModel, use_edit_metadata_dialog=True): QTableView.__init__(self, parent) + self.gui = parent self.setProperty('highlight_current_item', 150) self.row_sizing_done = False self.alternate_views = AlternateViews(self) @@ -713,6 +714,10 @@ class BooksView(QTableView): # {{{ self.edit_collections_action = edit_collections_action def contextMenuEvent(self, event): + sac = self.gui.iactions['Sort By'] + sort_added = tuple(ac for ac in self.context_menu.actions() if ac is sac.qaction) + if sort_added: + sac.update_menu() self.context_menu.popup(event.globalPos()) event.accept() # }}} From 39e76dfedf77b0145315d53c38bb525422365014 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 Aug 2013 21:36:10 +0530 Subject: [PATCH 0498/1154] La Capital de Rosario by Darko Miletic Fixes #1209289 [New recipe for La Capital de Rosario](https://bugs.launchpad.net/calibre/+bug/1209289) --- recipes/icons/lacapital.png | Bin 0 -> 1120 bytes recipes/lacapital.recipe | 76 ++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 recipes/icons/lacapital.png create mode 100644 recipes/lacapital.recipe diff --git a/recipes/icons/lacapital.png b/recipes/icons/lacapital.png new file mode 100644 index 0000000000000000000000000000000000000000..1b245e6874cfef054be50a14e2df5885d1bbd550 GIT binary patch literal 1120 zcmcgq`Crp@9R7UeNT4aGX=>qtfh9&sQG%G_xQqxnr4o@cgzT`>3L>T;W1IsQ#wj3R z0)c=W0&W}|C=9lJ`A*pZY-9HgY%pNo=0DN<^}L_=>v?{8|L{Ecu+Tt^{Vsa|046vH z8*aId&v&Dhg%2Ly907n82n&hyw@4I4ZOai6YDVI87Yw#bH@=8<)UR8w`fy z&0OmIVg_BQRjJ%6r(vZou3I1wiahIhKJB81v${Pc>^`v}qHS(TtL~gpVOx3U2IZtl zC4NkXuC9osvVb083|Z+c=B;lb^9!a+6tQzHC$?*5eSHn!VY^J) ziFUz-UQyDJES)N`FM^j1`kf_guV#KwkJz;wZsTebX$rkYV^siqR8RZf<%4X7Z86N( zF1uGvd)4y-ItA85_O_Cz=LaNdLsF1Gl|WOc&}7DCqeiRSkOSKh5e#v1-yP1eHbG#Q zD4HTqc_L4vNkJCVA(!b|g?Kg2_}!Z+o>cg?3&UFG&h$&uX|gDaGF4>E9^~ZJi5$B#9!+P}FQjQIlzP$zagvbl?U9WHA8V0w%z- zKsFO(vjBk!2r$TDK?D|CWdyk}$g^My?7Bh z^1~a0s@=9^yt)Ip_TJleH~i!e@0V7+c53JKqu|H>{Rdyp;g)q>dibKpuJmAV;tPMr zSq;Qy})3ZTr~wPi{53Z~p5`9H`k_ImJ#xF4M18w!KHg;jDtMtX}U; QH9ij{*gq88>=%#w4^h%VE&u=k literal 0 HcmV?d00001 diff --git a/recipes/lacapital.recipe b/recipes/lacapital.recipe new file mode 100644 index 0000000000..0c9553903c --- /dev/null +++ b/recipes/lacapital.recipe @@ -0,0 +1,76 @@ +__license__ = 'GPL v3' +__copyright__ = '2013, Darko Miletic ' +''' +www.lacapital.com.ar +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class LaCapital(BasicNewsRecipe): + title = 'La Capital de Rosario' + __author__ = 'Darko Miletic' + description = 'Noticias, actualidad y toda la informacion de Rosario y la region' + publisher = 'Diario La Capital S. A.' + category = 'news, politics, Rosario, Santa Fe, Argentina' + oldest_article = 2 + max_articles_per_feed = 200 + no_stylesheets = True + encoding = 'utf8' + use_embedded_content = False + language = 'es_AR' + remove_empty_feeds = True + publication_type = 'newspaper' + masthead_url = 'http://www.lacapital.com.ar/system/modules/com.tfsla.diario.core/resources/images/logoLaCapital_noCom.png' + extra_css = """ + body{font-family: Georgia,"Times New Roman",Times,serif } + img{margin-bottom: 0.4em; display:block} + """ + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + } + + keep_only_tags=[dict(attrs={'class':'leer'})] + remove_tags_after=dict(attrs={'class':'notaA'}) + remove_tags = [ + dict(name=['meta','link','iframe','object']) + ,dict(name='div', attrs={'class':['herramientas','almargen','relacionadas']}) + ] + + + feeds = [ + (u'Portada' , u'http://www.lacapital.com.ar/rss/home.xml' ) + ,(u'La Ciudad' , u'http://www.lacapital.com.ar/rss/laciudad.xml' ) + ,(u'Politica' , u'http://www.lacapital.com.ar/rss/politica.xml' ) + ,(u'Economia' , u'http://www.lacapital.com.ar/rss/economia.xml' ) + ,(u'La Region' , u'http://www.lacapital.com.ar/rss/laregion.xml' ) + ,(u'Informacion General' , u'http://www.lacapital.com.ar/rss/informaciongral.xml' ) + ,(u'El Mundo' , u'http://www.lacapital.com.ar/rss/elmundo.xml' ) + ,(u'Opinion' , u'http://www.lacapital.com.ar/rss/opinion.xml' ) + ,(u'Cartas de lectores' , u'http://www.lacapital.com.ar/rss/cartasdelectores.xml') + ,(u'Escenario' , u'http://www.lacapital.com.ar/rss/escenario.xml' ) + ,(u'Policiales' , u'http://www.lacapital.com.ar/rss/policiales.xml' ) + ,(u'Ovacion' , u'http://www.lacapital.com.ar/rss/ovacion.xml' ) + ,(u'Turismo' , u'http://www.lacapital.com.ar/rss/turismo.xml' ) + ,(u'Economia' , u'http://www.lacapital.com.ar/rss/economia.xml' ) + ,(u'Señales' , u'http://www.lacapital.com.ar/rss/senales.xml' ) + ,(u'Educacion' , u'http://www.lacapital.com.ar/rss/educacion.xml' ) + ,(u'Estilo' , u'http://www.lacapital.com.ar/rss/estilo.xml' ) + ,(u'Salud' , u'http://www.lacapital.com.ar/rss/salud.xml' ) + ,(u'Tecnologia' , u'http://www.lacapital.com.ar/rss/tecnologia.xml' ) + ] + + def get_cover_url(self): + soup = self.index_to_soup('http://www.lacapital.com.ar/impresa/tapa.html') + for image in soup.findAll('img',alt=True): + if image['alt'].startswith('Tapa de papel'): + return image['src'] + return None + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + return soup From 6640ca524281a56c17c2bed1fc1f2e8127c71a66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Wed, 7 Aug 2013 21:24:57 +0200 Subject: [PATCH 0499/1154] ebookpoint plugin: fix detecting author for cases where the name is a link --- src/calibre/gui2/store/stores/ebookpoint_plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/store/stores/ebookpoint_plugin.py b/src/calibre/gui2/store/stores/ebookpoint_plugin.py index d0306a45ee..dd8f45d4ae 100644 --- a/src/calibre/gui2/store/stores/ebookpoint_plugin.py +++ b/src/calibre/gui2/store/stores/ebookpoint_plugin.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) -store_version = 1 # Needed for dynamic plugin loading +store_version = 2 # Needed for dynamic plugin loading __license__ = 'GPL 3' -__copyright__ = '2011-2012, Tomasz Długosz ' +__copyright__ = '2011-2013, Tomasz Długosz ' __docformat__ = 'restructuredtext en' import re @@ -62,7 +62,7 @@ class EbookpointStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//a[@class="cover"]/img/@src')) title = ''.join(data.xpath('.//h3/a/@title')) title = re.sub('eBook.', '', title) - author = ''.join(data.xpath('.//p[@class="author"]/text()')) + author = ''.join(data.xpath('.//p[@class="author"]//text()')) price = ''.join(data.xpath('.//p[@class="price"]/ins/text()')) formats = ', '.join(data.xpath('.//div[@class="ikony"]/span/text()')) From a8f29d9159ec5d04c202a0177b050238cda51788 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 8 Aug 2013 08:03:26 +0530 Subject: [PATCH 0500/1154] Update The Sunday Times UK and The Times Online Fixes #1208519 [Private bug](https://bugs.launchpad.net/calibre/+bug/1208519) --- recipes/sunday_times.recipe | 4 ++-- recipes/times_online.recipe | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/recipes/sunday_times.recipe b/recipes/sunday_times.recipe index 2ffb65423d..c4cdbcbd85 100644 --- a/recipes/sunday_times.recipe +++ b/recipes/sunday_times.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2010-2012, Darko Miletic ' +__copyright__ = '2010-2013, Darko Miletic ' ''' www.thesundaytimes.co.uk ''' @@ -50,7 +50,7 @@ class TimesOnline(BasicNewsRecipe): ,'username':self.username ,'password':self.password }) - br.open('https://acs.thetimes.co.uk/user/login',data) + br.open('https://login.thetimes.co.uk/',data) return br remove_tags = [ diff --git a/recipes/times_online.recipe b/recipes/times_online.recipe index 2213c3a116..318c1c7508 100644 --- a/recipes/times_online.recipe +++ b/recipes/times_online.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2009-2012, Darko Miletic ' +__copyright__ = '2009-2013, Darko Miletic ' ''' www.thetimes.co.uk ''' @@ -49,7 +49,7 @@ class TimesOnline(BasicNewsRecipe): ,'username':self.username ,'password':self.password }) - br.open('https://acs.thetimes.co.uk/user/login',data) + br.open('https://login.thetimes.co.uk/',data) return br remove_tags = [ From b29c4fa56d8846d835c1b91e7b9502a22be33b47 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 8 Aug 2013 08:46:19 +0530 Subject: [PATCH 0501/1154] Various Frech news sources by Malah --- recipes/gamekult.recipe | 36 ++++++++++++++++++++ recipes/jeuxvideo.recipe | 44 +++++++++++++++++++++++++ recipes/le_gorafi.recipe | 34 +++++++++++++++++++ recipes/le_nouvel_observateur.recipe | 49 ++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+) create mode 100644 recipes/gamekult.recipe create mode 100644 recipes/jeuxvideo.recipe create mode 100644 recipes/le_gorafi.recipe create mode 100644 recipes/le_nouvel_observateur.recipe diff --git a/recipes/gamekult.recipe b/recipes/gamekult.recipe new file mode 100644 index 0000000000..0b528fe196 --- /dev/null +++ b/recipes/gamekult.recipe @@ -0,0 +1,36 @@ +from __future__ import unicode_literals +__license__ = 'GPL v3' +__copyright__ = '2013, Malah ' +''' +Gamekult.com +''' + +__author__ = '2013, Malah ' +from calibre.web.feeds.news import BasicNewsRecipe + +class GamekultCom(BasicNewsRecipe): + title = u'Gamekult.com' + __author__ = 'Malah' + description = u'Toute l`actualité du jeu vidéo PC, consoles, mobiles.' + oldest_article = 1.5 + language = 'fr' + max_articles_per_feed = 100 + remove_empty_feeds = True + use_embedded_content = False + no_stylesheets = True + ignore_duplicate_articles = {'title', 'url'} + keep_only_tags = [dict(id=['story-page','story-body'])] + remove_tags = [ + dict(name='div', attrs={'class':'sharebar'}), + dict(name='object', attrs={'type':'application/x-shockwave-flash'}), + dict(name='span', attrs={'class':'share'}), + dict(name='div', attrs={'class':'story-pagination'}), + dict(name='div', attrs={'class':'pagination pagination-centered'}), + ] + + masthead_url = u'https://upload.wikimedia.org/wikipedia/fr/9/9c/Logo_-_GAMEKULT.png' + + feeds = [ + ('Test', u'http://www.gamekult.com/feeds/test.html'), + ('Actu', u'http://www.gamekult.com/feeds/actu.html'), + ] diff --git a/recipes/jeuxvideo.recipe b/recipes/jeuxvideo.recipe new file mode 100644 index 0000000000..a41fe94753 --- /dev/null +++ b/recipes/jeuxvideo.recipe @@ -0,0 +1,44 @@ +from __future__ import unicode_literals +__license__ = 'GPL v3' +__copyright__ = '2013, Malah ' +''' +JeuxVideo.com +''' + +__author__ = '2013, Malah ' +from calibre.web.feeds.news import BasicNewsRecipe + +class JeuxVideoCom(BasicNewsRecipe): + title = 'JeuxVideo.com' + __author__ = 'Malah' + description = 'La Référence des Jeux Vidéo sur PC et Consoles !' + oldest_article = 1.5 + language = 'fr' + max_articles_per_feed = 100 + remove_empty_feeds = True + use_embedded_content = False + no_stylesheets = True + ignore_duplicate_articles = {'title', 'url'} + keep_only_tags = [dict(id=['news_detail','test_txt','test_avis'])] + remove_tags = [ + dict(name='div', attrs={'id':'player_video_article'}), + dict(name='div', attrs={'class':'liste-fiches'}) + ] + masthead_url = u'https://upload.wikimedia.org/wikipedia/commons/3/39/Jeuxvideocom.png' + feeds = [ + (u'Section PC',u'http://www.jeuxvideo.com/rss/rss-pc.xml'), + (u'Section Xbox 360',u'http://www.jeuxvideo.com/rss/rss-360.xml'), + (u'Section PlayStation 3',u'http://www.jeuxvideo.com/rss/rss-ps3.xml'), + (u'Section Wii U',u'http://www.jeuxvideo.com/rss/rss-wiiu.xml'), + (u'Section Wii',u'http://www.jeuxvideo.com/rss/rss-wii.xml'), + (u'Section Nintendo 3DS',u'http://www.jeuxvideo.com/rss/rss-3ds.xml'), + (u'Section Nintendo DS',u'http://www.jeuxvideo.com/rss/rss-ds.xml'), + (u'Section PlayStation Vita',u'http://www.jeuxvideo.com/rss/rss-vita.xml'), + (u'Section PlayStation Protable',u'http://www.jeuxvideo.com/rss/rss-psp.xml'), + (u'Section Android',u'http://www.jeuxvideo.com/rss/rss-android.xml'), + (u'Section Iphone',u'http://www.jeuxvideo.com/rss/rss-iphone.xml'), + (u'Section Web',u'http://www.jeuxvideo.com/rss/rss-wb.xml'), + (u'Autres news', u'http://www.jeuxvideo.com/rss/rss-news.xml'), + (u'Autres vidéos', u'http://www.jeuxvideo.com/rss/rss-videos.xml'), + (u'Autres articles', u'http://www.jeuxvideo.com/rss/rss.xml'), + ] diff --git a/recipes/le_gorafi.recipe b/recipes/le_gorafi.recipe new file mode 100644 index 0000000000..d88d0affc2 --- /dev/null +++ b/recipes/le_gorafi.recipe @@ -0,0 +1,34 @@ +from __future__ import unicode_literals +__license__ = 'GPL v3' +__copyright__ = '2013, Malah ' +''' +Le GORAFI.fr +''' + +__author__ = '2013, Malah ' +from calibre.web.feeds.news import BasicNewsRecipe + +class legorafi(BasicNewsRecipe): + title = u'Le GORAFI.fr' + __author__ = 'Malah' + description = u'Depuis 1826, toute l\'information de sources contradictoires' + oldest_article = 7 + language = 'fr' + max_articles_per_feed = 100 + use_embedded_content = False + no_stylesheets = True + keep_only_tags = [ + dict(name='div', attrs={'class':'entry-content'}), + dict(name='h3', attrs={'id':'comments-title'}), + ] + remove_tags = [ + dict(name='div', attrs={'id':'soshake-sharebox'}), + dict(name='div', attrs={'class':'social-ring'}), + dict(name='div', attrs={'class':'entry-utility'}), + dict(name='div', attrs={'id':'respond'}), + ] + masthead_url = u'http://web.gweno.free.fr/img/logositeter.png' + couverture_url = u'http://www.legorafi.fr/wp-content/uploads/2013/02/iconegorafi.png' + feeds = [ + (u'Articles', u'http://www.legorafi.fr/feed/'), + ] diff --git a/recipes/le_nouvel_observateur.recipe b/recipes/le_nouvel_observateur.recipe new file mode 100644 index 0000000000..b3a0460423 --- /dev/null +++ b/recipes/le_nouvel_observateur.recipe @@ -0,0 +1,49 @@ +from __future__ import unicode_literals +__license__ = 'GPL v3' +__copyright__ = '2013, Malah ' +''' +Le Nouvel Observateur +''' + +__author__ = '2013, Malah ' + +from calibre.web.feeds.news import BasicNewsRecipe + +class LeNouvelObs(BasicNewsRecipe): + title = u'Le Nouvel Observateur' + __author__ = 'Malah' + description = u'Actualités en temps réel, Info à la Une' + oldest_article = 1 + language = 'fr' + max_articles_per_feed = 25 + use_embedded_content = False + ignore_duplicate_articles = ('title', 'url') + remove_empty_feeds = True + no_stylesheets = True + masthead_url = u'https://upload.wikimedia.org/wikipedia/fr/f/f9/Le_Nouvel_observateur.png' + feeds = [ + (u'Politique', u'http://tempsreel.nouvelobs.com/politique/rss.xml'), + (u'Société', u'http://tempsreel.nouvelobs.com/societe/rss.xml'), + (u'Monde', u'http://tempsreel.nouvelobs.com/monde/rss.xml'), + (u'Economie', u'http://tempsreel.nouvelobs.com/economie/rss.xml'), + (u'Culture', u'http://tempsreel.nouvelobs.com/culture/rss.xml'), + (u'High Tech', u'http://obsession.nouvelobs.com/high-tech/rss.xml'), + (u'Education', u'http://tempsreel.nouvelobs.com/education/rss.xml'), + (u'Services', u'http://tempsreel.nouvelobs.com/services/rss.xml'), + (u'Sport', u'http://tempsreel.nouvelobs.com/sport/rss.xml'), + (u'CinéObs', u'http://cinema.nouvelobs.com/articles.rss'), + (u'TéléObs', u'http://teleobs.nouvelobs.com/rss.xml'), + (u'Autres Actualités',u'http://tempsreel.nouvelobs.com/rss.xml'), + ] + keep_only_tags = [ + dict(name='h1', attrs={'id':'obs-article-title'}), + dict(name='div', attrs={'class':'obs-date'}), + dict(name='div', attrs={'class':'art-auteur'}), + dict(name='h2', attrs={'class':'obs-article-intro'}), + dict(name='div', attrs={'id':'obs-article-keywords'}), + dict(name='div', attrs={'id':'obs-article-mainpic'}), + dict(name='div', attrs={'itemprop':'articleBody'}), + dict(name='img', attrs={'id':'ObsImg'}), + dict(name='p', attrs={'class':'date-media'}), + dict(name='p', attrs={'id':'ObsDesc'}), + ] From faf7060c3bf6adce29ae7d0d6438b32a9b0032b2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 8 Aug 2013 13:05:36 +0530 Subject: [PATCH 0502/1154] Increase max cover grid cache size --- src/calibre/gui2/preferences/look_feel.ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index 2438c2f9c1..f10de2460f 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -364,7 +364,7 @@ - 3000 + 50000 From 560a09710796d461fbab720019ec5a7db4dc775d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 8 Aug 2013 15:35:56 +0530 Subject: [PATCH 0503/1154] ... --- src/calibre/gui2/preferences/look_feel.ui | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index f10de2460f..98296dd014 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -366,6 +366,9 @@ 50000 + + The maximum number of covers to keep in memory. Increasing this will make rendering faster, at the cost of more memory usage. + From e5ce1ca2a469156ebb3697903ba7fade25324cf2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 8 Aug 2013 16:01:19 +0530 Subject: [PATCH 0504/1154] Allow sending by email to combinations of recipients Sending by email: Allow sending by email to an arbitrary combination of email address. Access it via the "Select recipients" menu entry in the Email To menu. Fixes #1207818 [[Enhancement] - "Email to selected"](https://bugs.launchpad.net/calibre/+bug/1207818) --- src/calibre/gui2/actions/convert.py | 16 +++ src/calibre/gui2/actions/device.py | 12 +- src/calibre/gui2/device.py | 5 + src/calibre/gui2/email.py | 170 +++++++++++++++++++++++++++- 4 files changed, 200 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/actions/convert.py b/src/calibre/gui2/actions/convert.py index c5e1580c6d..aaadd1a452 100644 --- a/src/calibre/gui2/actions/convert.py +++ b/src/calibre/gui2/actions/convert.py @@ -106,6 +106,15 @@ class ConvertAction(InterfaceAction): self.book_auto_converted_mail, extra_job_args=[delete_from_library, to, fmts, subject]) + def auto_convert_multiple_mail(self, book_ids, data, ofmt, delete_from_library): + previous = self.gui.library_view.currentIndex() + rows = [x.row() for x in self.gui.library_view.selectionModel().selectedRows()] + jobs, changed, bad = convert_single_ebook(self.gui, self.gui.library_view.model().db, book_ids, True, ofmt) + if jobs == []: return + self.queue_convert_jobs(jobs, changed, bad, rows, previous, + self.book_auto_converted_multiple_mail, + extra_job_args=[delete_from_library, data]) + def auto_convert_news(self, book_ids, format): previous = self.gui.library_view.currentIndex() rows = [x.row() for x in \ @@ -207,6 +216,13 @@ class ConvertAction(InterfaceAction): self.gui.send_by_mail(to, fmts, delete_from_library, subject=subject, specific_format=fmt, send_ids=[book_id], do_auto_convert=False) + def book_auto_converted_multiple_mail(self, job): + temp_files, fmt, book_id, delete_from_library, data = self.conversion_jobs[job] + self.book_converted(job) + for to, subject in data: + self.gui.send_by_mail(to, (fmt,), delete_from_library, subject=subject, + specific_format=fmt, send_ids=[book_id], do_auto_convert=False) + def book_auto_converted_news(self, job): temp_files, fmt, book_id = self.conversion_jobs[job] self.book_converted(job) diff --git a/src/calibre/gui2/actions/device.py b/src/calibre/gui2/actions/device.py index 09cadceb9c..e6432f9334 100644 --- a/src/calibre/gui2/actions/device.py +++ b/src/calibre/gui2/actions/device.py @@ -19,7 +19,7 @@ from calibre.gui2.dialogs.smartdevice import SmartdeviceDialog from calibre.gui2 import info_dialog, question_dialog from calibre.library.server import server_config as content_server_config -class ShareConnMenu(QMenu): # {{{ +class ShareConnMenu(QMenu): # {{{ connect_to_folder = pyqtSignal() connect_to_itunes = pyqtSignal() @@ -138,8 +138,17 @@ class ShareConnMenu(QMenu): # {{{ ac.a_s.connect(sync_menu.action_triggered) action1.a_s.connect(sync_menu.action_triggered) action2.a_s.connect(sync_menu.action_triggered) + action1 = DeviceAction('choosemail:', False, False, I('mail.png'), + _('Select recipients')) + action2 = DeviceAction('choosemail:', True, False, I('mail.png'), + _('Select recipients') + ' ' + _('(delete from library)')) + self.email_to_menu.addAction(action1) + self.email_to_and_delete_menu.addAction(action2) + map(self.memory.append, (action1, action2)) ac = self.addMenu(self.email_to_and_delete_menu) self.email_actions.append(ac) + action1.a_s.connect(sync_menu.action_triggered) + action2.a_s.connect(sync_menu.action_triggered) else: ac = self.addAction(_('Setup email based sharing of books')) self.email_actions.append(ac) @@ -287,3 +296,4 @@ class ConnectShareAction(InterfaceAction): ac.setIcon(QIcon(I('dot_%s.png'%icon))) ac.setText(text) + diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index d3225e66e7..bddfa3efa9 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1214,6 +1214,11 @@ class DeviceMixin(object): # {{{ subject = ';'.join(sub_dest_parts[2:]) fmts = [x.strip().lower() for x in fmts.split(',')] self.send_by_mail(to, fmts, delete, subject=subject) + elif dest == 'choosemail': + from calibre.gui2.email import select_recipients + data = select_recipients(self) + if data: + self.send_multiple_by_mail(data, delete) def cover_to_thumbnail(self, data): if self.device_manager.device and \ diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 6645441158..5ecce8248f 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -11,6 +11,11 @@ from binascii import unhexlify from functools import partial from threading import Thread from itertools import repeat +from collections import defaultdict + +from PyQt4.Qt import ( + Qt, QDialog, QGridLayout, QIcon, QListWidget, QDialogButtonBox, + QListWidgetItem, QLabel, QLineEdit, QPushButton) from calibre.utils.smtp import (compose_mail, sendmail, extract_email_address, config as email_config) @@ -18,9 +23,10 @@ from calibre.utils.filenames import ascii_filename from calibre.customize.ui import available_input_formats, available_output_formats from calibre.ebooks.metadata import authors_to_string from calibre.constants import preferred_encoding -from calibre.gui2 import config, Dispatcher, warning_dialog +from calibre.gui2 import config, Dispatcher, warning_dialog, error_dialog from calibre.library.save_to_disk import get_components -from calibre.utils.config import tweaks +from calibre.utils.config import tweaks, prefs +from calibre.utils.icu import sort_key from calibre.gui2.threaded_jobs import ThreadedJob class Worker(Thread): @@ -166,8 +172,164 @@ def email_news(mi, remove, get_fmts, done, job_manager): plugboard_email_value = 'email' plugboard_email_formats = ['epub', 'mobi', 'azw3'] +class SelectRecipients(QDialog): # {{{ + + def __init__(self, parent=None): + QDialog.__init__(self, parent) + self._layout = l = QGridLayout(self) + self.setLayout(l) + self.setWindowIcon(QIcon(I('mail.png'))) + self.setWindowTitle(_('Select recipients')) + self.recipients = r = QListWidget(self) + l.addWidget(r, 0, 0, 1, -1) + self.la = la = QLabel(_('Add a new recipient:')) + la.setStyleSheet('QLabel { font-weight: bold }') + l.addWidget(la, l.rowCount(), 0, 1, -1) + + self.labels = tuple(map(QLabel, ( + _('&Address'), _('A&lias'), _('&Formats'), _('&Subject')))) + tooltips = ( + _('The email address of the recipient'), + _('The optional alias (simple name) of the recipient'), + _('Formats to email. The first matching one will be sent (comma separated list)'), + _('The optional subject for email sent to this recipient')) + + for i, name in enumerate(('address', 'alias', 'formats', 'subject')): + c = i % 2 + row = l.rowCount() - c + self.labels[i].setText(unicode(self.labels[i].text()) + ':') + l.addWidget(self.labels[i], row, (2*c)) + le = QLineEdit(self) + le.setToolTip(tooltips[i]) + setattr(self, name, le) + self.labels[i].setBuddy(le) + l.addWidget(le, row, (2*c) + 1) + self.formats.setText(prefs['output_format'].upper()) + self.add_button = b = QPushButton(QIcon(I('plus.png')), _('&Add recipient'), self) + b.clicked.connect(self.add_recipient) + l.addWidget(b, l.rowCount(), 0, 1, -1) + + self.bb = bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel) + l.addWidget(bb, l.rowCount(), 0, 1, -1) + bb.accepted.connect(self.accept) + bb.rejected.connect(self.reject) + self.setMinimumWidth(500) + self.setMinimumHeight(400) + self.resize(self.sizeHint()) + self.init_list() + + def add_recipient(self): + to = unicode(self.address.text()).strip() + if not to: + return error_dialog( + self, _('Need address'), _('You must specify an address'), show=True) + formats = ','.join([x.strip().upper() for x in unicode(self.formats.text()).strip().split(',') if x.strip()]) + if not formats: + return error_dialog( + self, _('Need formats'), _('You must specify at least one format to send'), show=True) + opts = email_config().parse() + if to in opts.accounts: + return error_dialog( + self, _('Already exists'), _('The recipient %s already exists') % to, show=True) + acc = opts.accounts + acc[to] = [formats, False, False] + c = email_config() + c.set('accounts', acc) + alias = unicode(self.alias.text()).strip() + if alias: + opts.aliases[to] = alias + c.set('aliases', opts.aliases) + subject = unicode(self.subject.text()).strip() + if subject: + opts.subjects[to] = subject + c.set('subjects', opts.subjects) + self.create_item(alias or to, to, checked=True) + + def create_item(self, alias, key, checked=False): + i = QListWidgetItem(alias, self.recipients) + i.setFlags(Qt.ItemIsEnabled | Qt.ItemIsUserCheckable) + i.setCheckState(Qt.Checked if checked else Qt.Unchecked) + i.setData(Qt.UserRole, key) + self.items.append(i) + + def init_list(self): + opts = email_config().parse() + self.items = [] + for key in sorted(opts.accounts or (), key=sort_key): + self.create_item(opts.aliases.get(key, key), key) + + @property + def ans(self): + opts = email_config().parse() + ans = [] + for i in self.items: + if i.checkState() == Qt.Checked: + to = unicode(i.data(Qt.UserRole).toString()) + fmts = tuple(x.strip().upper() for x in (opts.accounts[to][0] or '').split(',')) + subject = opts.subjects.get(to, '') + ans.append((to, fmts, subject)) + return ans + +def select_recipients(parent=None): + d = SelectRecipients(parent) + if d.exec_() == d.Accepted: + return d.ans + return () +# }}} + class EmailMixin(object): # {{{ + def send_multiple_by_mail(self, recipients, delete_from_library): + ids = set(self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()) + if not ids: + return + db = self.current_db + db_fmt_map = {book_id:set((db.formats(book_id, index_is_id=True) or '').upper().split(',')) for book_id in ids} + ofmts = {x.upper() for x in available_output_formats()} + ifmts = {x.upper() for x in available_input_formats()} + bad_recipients = {} + auto_convert_map = defaultdict(list) + + for to, fmts, subject in recipients: + rfmts = set(fmts) + ok_ids = {book_id for book_id, bfmts in db_fmt_map.iteritems() if bfmts.intersection(rfmts)} + convert_ids = ids - ok_ids + self.send_by_mail(to, fmts, delete_from_library, subject=subject, send_ids=ok_ids, do_auto_convert=False) + if not rfmts.intersection(ofmts): + bad_recipients[to] = (convert_ids, True) + continue + outfmt = tuple(f for f in fmts if f in ofmts)[0] + ok_ids = {book_id for book_id in convert_ids if db_fmt_map[book_id].intersection(ifmts)} + bad_ids = convert_ids - ok_ids + if bad_ids: + bad_recipients[to] = (bad_ids, False) + if ok_ids: + auto_convert_map[outfmt].append((to, subject, ok_ids)) + + if auto_convert_map: + titles = {book_id for x in auto_convert_map.itervalues() for data in x for book_id in data[2]} + titles = {db.title(book_id, index_is_id=True) for book_id in titles} + if self.auto_convert_question( + _('Auto convert the following books before sending via email?'), list(titles)): + for ofmt, data in auto_convert_map.iteritems(): + ids = {bid for x in data for bid in x[2]} + data = [(to, subject) for to, subject, x in data] + self.iactions['Convert Books'].auto_convert_multiple_mail(ids, data, ofmt, delete_from_library) + + if bad_recipients: + det_msg = [] + titles = {book_id for x in bad_recipients.itervalues() for book_id in x[0]} + titles = {book_id:db.title(book_id, index_is_id=True) for book_id in titles} + for to, (ids, nooutput) in bad_recipients.iteritems(): + msg = _('This recipient has no valid formats defined') if nooutput else \ + _('These books have no suitable input formats for conversion') + det_msg.append('%s - %s' % (to, msg)) + det_msg.extend('\t' + titles[bid] for bid in ids) + det_msg.append('\n') + warning_dialog(self, _('Could not send'), + _('Could not send books to some recipients. Click Show Details for more information'), + det_msg='\n'.join(det_msg), show=True) + def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None, do_auto_convert=True, specific_format=None): ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids @@ -307,4 +469,8 @@ class EmailMixin(object): # {{{ # }}} +if __name__ == '__main__': + from PyQt4.Qt import QApplication + app = QApplication([]) # noqa + print (select_recipients()) From ad1de3729eb830570bc580b0252451efbb920942 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 8 Aug 2013 16:43:33 +0530 Subject: [PATCH 0505/1154] Grid view: Render covers in pauses during scrolling or slow scrolling --- src/calibre/gui2/library/alternate_views.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index ec3a788017..bbb20527f1 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -495,16 +495,24 @@ class GridView(QListView): def update_viewport(self): self.ignore_render_requests.clear() + self.update_timer.stop() m = self.model() for r in xrange(self.first_visible_row or 0, self.last_visible_row or (m.count() - 1)): self.update(m.index(r, 0)) def slider_pressed(self): self.ignore_render_requests.set() + self.verticalScrollBar().valueChanged.connect(self.value_changed_during_scroll) def slider_released(self): - self.ignore_render_requests.clear() self.update_viewport() + self.verticalScrollBar().valueChanged.disconnect(self.value_changed_during_scroll) + + def value_changed_during_scroll(self): + if self.ignore_render_requests.is_set(): + self.update_timer.start() + else: + self.ignore_render_requests.set() def wheelEvent(self, e): self.ignore_render_requests.set() From 2c5ab05e9cc3bacd7be9c02fec3451e54a99e8db Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 8 Aug 2013 16:55:08 +0530 Subject: [PATCH 0506/1154] Increase update interval in slider drag mode --- src/calibre/gui2/library/alternate_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index bbb20527f1..09daa04ddc 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -503,10 +503,12 @@ class GridView(QListView): def slider_pressed(self): self.ignore_render_requests.set() self.verticalScrollBar().valueChanged.connect(self.value_changed_during_scroll) + self.update_timer.setInterval(500) def slider_released(self): self.update_viewport() self.verticalScrollBar().valueChanged.disconnect(self.value_changed_during_scroll) + self.update_timer.setInterval(200) def value_changed_during_scroll(self): if self.ignore_render_requests.is_set(): From 36cace1b483a4a7ac3033e3ffad5f2d6dfd3f221 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 8 Aug 2013 19:13:30 +0530 Subject: [PATCH 0507/1154] Grid view: Add tooltips --- src/calibre/gui2/library/alternate_views.py | 26 +++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 09daa04ddc..6a414d433b 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -13,12 +13,14 @@ from collections import OrderedDict from threading import Lock, Event, Thread, current_thread from Queue import Queue from functools import wraps, partial +from textwrap import wrap from PyQt4.Qt import ( QListView, QSize, QStyledItemDelegate, QModelIndex, Qt, QImage, pyqtSignal, QTimer, QPalette, QColor, QItemSelection, QPixmap, QMenu, QApplication, - QMimeData, QUrl, QDrag, QPoint, QPainter, QRect, pyqtProperty, - QPropertyAnimation, QEasingCurve) + QMimeData, QUrl, QDrag, QPoint, QPainter, QRect, pyqtProperty, QEvent, + QPropertyAnimation, QEasingCurve, pyqtSlot, QHelpEvent, QAbstractItemView, + QStyleOptionViewItem, QToolTip) from calibre import fit_image from calibre.gui2 import gprefs, config @@ -423,6 +425,26 @@ class CoverDelegate(QStyledItemDelegate): finally: painter.restore() + @pyqtSlot(QHelpEvent, QAbstractItemView, QStyleOptionViewItem, QModelIndex, result=bool) + def helpEvent(self, event, view, option, index): + if event is not None and view is not None and event.type() == QEvent.ToolTip: + try: + db = index.model().db + except AttributeError: + return False + try: + book_id = db.id(index.row()) + except (ValueError, IndexError, KeyError): + return False + title = db.new_api.field_for('title', book_id) + authors = db.new_api.field_for('authors', book_id) + if title and authors: + title = '%s' % ('\n'.join(wrap(title, 100))) + authors = '\n'.join(wrap(' & '.join(authors), 100)) + QToolTip.showText(event.globalPos(), '%s

    %s' % (title, authors), view) + return True + return False + def join_with_timeout(q, timeout=2): q.all_tasks_done.acquire() try: From 9996f3b88a3a05b9f0a042281a265acce14a30e7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 8 Aug 2013 21:32:35 +0530 Subject: [PATCH 0508/1154] pep8 --- recipes/le_monde_sub.recipe | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/recipes/le_monde_sub.recipe b/recipes/le_monde_sub.recipe index e55d71e0a7..e3fcda6f68 100644 --- a/recipes/le_monde_sub.recipe +++ b/recipes/le_monde_sub.recipe @@ -2,7 +2,7 @@ __author__ = 'Sylvain Durand ' __license__ = 'GPL v3' -import time +import time, re from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe @@ -79,18 +79,18 @@ class LeMonde(BasicNewsRecipe): art.insert(6,'
    ') if art.find(['lgd']) and art.find(['lgd']).string: art.insert(7,'
    '+art.find(['lgd']).string+'
    ') - + def guillemets(match): if match.group(1) == u"=": return match.group(0) return u'%s« %s »' % (match.group(1), match.group(2)) - + article = ""+unicode(art)+"" article = article.replace('','').replace(' oC ','°C ') article = article.replace('srttr>','h3>').replace('ssttr>','h2>').replace('ttr>','h1>') article = article.replace("'" , u'\u2019') article = re.sub('(.|^)"([^"]+)"', guillemets, article) - + f = PersistentTemporaryFile() f.write(article) articles.append({'title':art.ttr.string,'url':"file:///"+f.name}) @@ -99,9 +99,9 @@ class LeMonde(BasicNewsRecipe): self.log("Vos identifiants sont incorrects, ou votre abonnement LeMonde.fr ne vous permet pas de télécharger le journal.") return sections - def preprocess_html(self, soup): for lgd in soup.findAll(id="lgd"): lgd.contents[-1].extract() return soup + From 0a508d2ff56ce564302771343bb6eacaea56323e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 08:05:27 +0530 Subject: [PATCH 0509/1154] version 0.9.43 --- Changelog.yaml | 53 ++++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index afd0516e1a..c5c1ac3315 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,59 @@ # new recipes: # - title: +- version: 0.9.43 + date: 2013-08-09 + + new features: + - title: "TXT Input: Allow using various markdown extensions for more features when converting markdown formatted txt files. See http://pythonhosted.org/Markdown/extensions/index.html for details." + + - title: "Sending by email: Allow sending by email to an arbitrary combination of email address. Access it via the 'Select recipients' menu entry in the Email To menu." + tickets: [1207818] + + - title: "A new 'Sort By' action for the right click menu. This allows sorting on all columns in the library, not just the visible columns. To use it go to Preferences->Toolbars and add it to 'The context menu for books in the calibre library'" + + - title: "Allow adding images into the comments field, by clicking on the insert link button in the comments editor in the edit metadata dialog." + + - title: "Allow skipping the confirm bulk reconvert dialog" + + - title: "EPUB Input: If the EPUB file identifies an actual cover image in addition to the titlepage html file, use the cover image instead of rendering the titlepage. This is faster and has the advantage that an EPUB to EPUB conversion preserves internal cover structure." + + - title: "Get Books: Improve searching by removing punctuation from title/authors before matching." + + bug fixes: + - title: "Conversion: Fix empty inline tags that are the second child of a paragraph causing text change location." + tickets: [1207735] + + - title: "Fix book count in tooltip of choose library button not updating" + ticket: [1208217] + + - title: "Kobo driver: When deleting shelves that have been synced, the Activity entry for the shelf was not being deleted. This left a tile for the shelf on the home screen of the Glo and AuraHD." + tickets: [1208159] + + - title: "Comments editor: The Insert Link button has no effect until the user clicks inside the comments box, therefore disable it until it is ready, to prevent confusion." + tickets: [1208073] + + - title: "Get Books: Update various Polish store plugins" + + improved recipes: + - The Sunday Times UK and The Times Online + - Telegraph UK + - Le Monde: Edition abonnés + - The Scotsman + + new recipes: + - title: Various French news sources + author: Malah + + - title: La Capital de Rosario + author: Darko Miletic + + - title: Jot Down + author: desUbiKado + + - title: Private Eye + author: Martyn Pritchard + - version: 0.9.42 date: 2013-08-02 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index c1ea2b80df..e0c0f6b845 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 9, 42) +numeric_version = (0, 9, 43) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " From 83126c4bf5c198e0364432b5e3ee38750fcb6977 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 08:17:49 +0530 Subject: [PATCH 0510/1154] ... --- src/calibre/ebooks/oeb/polish/embed.py | 4 ++-- src/calibre/gui2/convert/page_setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/oeb/polish/embed.py b/src/calibre/ebooks/oeb/polish/embed.py index 1f5412bbc9..4d6b826de7 100644 --- a/src/calibre/ebooks/oeb/polish/embed.py +++ b/src/calibre/ebooks/oeb/polish/embed.py @@ -67,8 +67,8 @@ def embed_font(container, font, all_font_rules, report, warned): rule['src'] = 'url(%s)' % href rule['name'] = name return rule - msg = _('Failed to find font matching: family: %s; weight: %s; style: %s; stretch: %s') % ( - ff, font['font-weight'], font['font-style'], font['font-stretch']) + msg = _('Failed to find font matching: family: %(family)s; weight: %(weight)s; style: %(style)s; stretch: %(stretch)s') % dict( + family=ff, weight=font['font-weight'], style=font['font-style'], stretch=font['font-stretch']) if msg not in warned: warned.add(msg) report(msg) diff --git a/src/calibre/gui2/convert/page_setup.py b/src/calibre/gui2/convert/page_setup.py index ac93557dd1..16196990f2 100644 --- a/src/calibre/gui2/convert/page_setup.py +++ b/src/calibre/gui2/convert/page_setup.py @@ -31,7 +31,7 @@ class ProfileModel(QAbstractListModel): if w >= 10000: ss = _('unlimited') else: - ss = _('%d x %d pixels') % (w, h) + ss = _('%(width)d x %(height)d pixels') % dict(width=w, height=h) ss = _('Screen size: %s') % ss return QVariant('%s [%s]' % (profile.description, ss)) return NONE From 2ae75daf61317676f82a45f6a33b26e8181ba53d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 11:26:05 +0530 Subject: [PATCH 0511/1154] ... --- Changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index c5c1ac3315..fee1971d3f 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -57,7 +57,7 @@ improved recipes: - The Sunday Times UK and The Times Online - Telegraph UK - - Le Monde: Edition abonnés + - "Le Monde: Edition abonnés" - The Scotsman new recipes: From 618aa68822ba48d891630c8a83a3ba5deede039a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 13:12:30 +0530 Subject: [PATCH 0512/1154] oops --- src/calibre/gui2/actions/sort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/actions/sort.py b/src/calibre/gui2/actions/sort.py index 14bd641020..76d22d5da6 100644 --- a/src/calibre/gui2/actions/sort.py +++ b/src/calibre/gui2/actions/sort.py @@ -21,7 +21,7 @@ class SortAction(QAction): self.triggered.connect(self) def __call__(self): - self.sort_requested(self.key, self.ascending) + self.sort_requested.emit(self.key, self.ascending) class SortByAction(InterfaceAction): From 7fe8d334d421017ef364a0cc9094da05ac5f07c0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 13:44:36 +0530 Subject: [PATCH 0513/1154] Invalidate cover caches when deleting books --- src/calibre/db/cache.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 677c3168d5..0b58c23c1d 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1351,6 +1351,9 @@ class Cache(object): table.remove_books(book_ids, self.backend) self._search_api.discard_books(book_ids) self._clear_caches(book_ids=book_ids, template_cache=False, search_cache=False) + for cc in self.cover_caches: + for book_id in book_ids: + cc.invalidate(book_id) @read_api def author_sort_strings_for_books(self, book_ids): From e844dd117fce83d38fb9de46ccd388aec0af23bb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 13:52:47 +0530 Subject: [PATCH 0514/1154] Move the cover cache into its own module --- src/calibre/gui2/library/alternate_views.py | 71 ++----------------- src/calibre/gui2/library/caches.py | 77 +++++++++++++++++++++ 2 files changed, 82 insertions(+), 66 deletions(-) create mode 100644 src/calibre/gui2/library/caches.py diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index 6a414d433b..cce5ceb102 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -9,8 +9,7 @@ __copyright__ = '2013, Kovid Goyal ' import itertools, operator, os from types import MethodType from time import time -from collections import OrderedDict -from threading import Lock, Event, Thread, current_thread +from threading import Event, Thread from Queue import Queue from functools import wraps, partial from textwrap import wrap @@ -24,6 +23,7 @@ from PyQt4.Qt import ( from calibre import fit_image from calibre.gui2 import gprefs, config +from calibre.gui2.library.caches import CoverCache from calibre.utils.config import prefs CM_TO_INCH = 0.393701 @@ -271,69 +271,7 @@ class AlternateViews(object): view.set_context_menu(menu) # }}} -# Caching and rendering of covers {{{ -class CoverCache(dict): - - def __init__(self, limit=200): - self.items = OrderedDict() - self.lock = Lock() - self.limit = limit - self.pixmap_staging = [] - self.gui_thread = current_thread() - - def clear_staging(self): - ' Must be called in the GUI thread ' - self.pixmap_staging = [] - - def invalidate(self, book_id): - with self.lock: - self._pop(book_id) - - def _pop(self, book_id): - val = self.items.pop(book_id, None) - if type(val) is QPixmap and current_thread() is not self.gui_thread: - self.pixmap_staging.append(val) - - def __getitem__(self, key): - ' Must be called in the GUI thread ' - with self.lock: - self.clear_staging() - ans = self.items.pop(key, False) # pop() so that item is moved to the top - if ans is not False: - if type(ans) is QImage: - # Convert to QPixmap, since rendering QPixmap is much - # faster - ans = QPixmap.fromImage(ans) - self.items[key] = ans - - return ans - - def set(self, key, val): - with self.lock: - self._pop(key) # pop() so that item is moved to the top - self.items[key] = val - if len(self.items) > self.limit: - del self.items[next(self.items.iterkeys())] - - def clear(self): - with self.lock: - if current_thread() is not self.gui_thread: - pixmaps = (x for x in self.items.itervalues() if type(x) is QPixmap) - self.pixmap_staging.extend(pixmaps) - self.items.clear() - - def __hash__(self): - return id(self) - - def set_limit(self, limit): - with self.lock: - self.limit = limit - if len(self.items) > self.limit: - extra = len(self.items) - self.limit - remove = tuple(self.iterkeys())[:extra] - for k in remove: - self._pop(k) - +# Rendering of covers {{{ class CoverDelegate(QStyledItemDelegate): @pyqtProperty(float) @@ -458,6 +396,7 @@ def join_with_timeout(q, timeout=2): q.all_tasks_done.release() # }}} +# The View {{{ @setup_dnd_interface class GridView(QListView): @@ -699,4 +638,4 @@ class GridView(QListView): def restore_hpos(self, hpos): pass - +# }}} diff --git a/src/calibre/gui2/library/caches.py b/src/calibre/gui2/library/caches.py new file mode 100644 index 0000000000..45dc5cddd5 --- /dev/null +++ b/src/calibre/gui2/library/caches.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +from threading import Lock, current_thread +from collections import OrderedDict + +from PyQt4.Qt import QImage, QPixmap + +class CoverCache(dict): + + ' This is a RAM cache to speed up rendering of covers by storing them as QPixmaps ' + + def __init__(self, limit=200): + self.items = OrderedDict() + self.lock = Lock() + self.limit = limit + self.pixmap_staging = [] + self.gui_thread = current_thread() + + def clear_staging(self): + ' Must be called in the GUI thread ' + self.pixmap_staging = [] + + def invalidate(self, book_id): + with self.lock: + self._pop(book_id) + + def _pop(self, book_id): + val = self.items.pop(book_id, None) + if type(val) is QPixmap and current_thread() is not self.gui_thread: + self.pixmap_staging.append(val) + + def __getitem__(self, key): + ' Must be called in the GUI thread ' + with self.lock: + self.clear_staging() + ans = self.items.pop(key, False) # pop() so that item is moved to the top + if ans is not False: + if type(ans) is QImage: + # Convert to QPixmap, since rendering QPixmap is much + # faster + ans = QPixmap.fromImage(ans) + self.items[key] = ans + + return ans + + def set(self, key, val): + with self.lock: + self._pop(key) # pop() so that item is moved to the top + self.items[key] = val + if len(self.items) > self.limit: + del self.items[next(self.items.iterkeys())] + + def clear(self): + with self.lock: + if current_thread() is not self.gui_thread: + pixmaps = (x for x in self.items.itervalues() if type(x) is QPixmap) + self.pixmap_staging.extend(pixmaps) + self.items.clear() + + def __hash__(self): + return id(self) + + def set_limit(self, limit): + with self.lock: + self.limit = limit + if len(self.items) > self.limit: + extra = len(self.items) - self.limit + remove = tuple(self.iterkeys())[:extra] + for k in remove: + self._pop(k) + From eb9e9dfbaf43d989638aae8aec5b0b8b06918d5c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 14:46:16 +0530 Subject: [PATCH 0515/1154] Cover grid: Add option toshow book title below cover --- src/calibre/gui2/__init__.py | 1 + src/calibre/gui2/library/alternate_views.py | 34 ++++++++++++++++++--- src/calibre/gui2/preferences/look_feel.py | 1 + src/calibre/gui2/preferences/look_feel.ui | 21 ++++++++----- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index bec9a3177c..7d0f85094b 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -116,6 +116,7 @@ defs['cover_grid_height'] = 0 defs['cover_grid_color'] = (80, 80, 80) defs['cover_grid_cache_size'] = 200 defs['cover_grid_spacing'] = 0 +defs['cover_grid_show_title'] = False del defs # }}} diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py index cce5ceb102..2bd39ff0ac 100644 --- a/src/calibre/gui2/library/alternate_views.py +++ b/src/calibre/gui2/library/alternate_views.py @@ -274,6 +274,8 @@ class AlternateViews(object): # Rendering of covers {{{ class CoverDelegate(QStyledItemDelegate): + MARGIN = 4 + @pyqtProperty(float) def animated_size(self): return self._animated_size @@ -297,6 +299,7 @@ class CoverDelegate(QStyledItemDelegate): def set_dimensions(self): width = self.original_width = gprefs['cover_grid_width'] height = self.original_height = gprefs['cover_grid_height'] + self.original_show_title = show_title = gprefs['cover_grid_show_title'] if height < 0.1: height = max(185, QApplication.instance().desktop().availableGeometry(self.parent()).height() / 5.0) @@ -308,7 +311,14 @@ class CoverDelegate(QStyledItemDelegate): else: width *= self.parent().logicalDpiX() * CM_TO_INCH self.cover_size = QSize(width, height) - self.item_size = self.cover_size + QSize(8, 8) + self.title_height = 0 + if show_title: + f = self.parent().font() + sz = f.pixelSize() + if sz < 5: + sz = f.pointSize() * self.parent().logicalDpiY() / 72.0 + self.title_height = max(25, sz + 10) + self.item_size = self.cover_size + QSize(2 * self.MARGIN, (2 * self.MARGIN) + self.title_height) self.calculate_spacing() self.animation.setStartValue(1.0) self.animation.setKeyValueAt(0.5, 0.5) @@ -345,7 +355,7 @@ class CoverDelegate(QStyledItemDelegate): painter.save() try: rect = option.rect - rect.adjust(4, 4, -4, -4) + rect.adjust(self.MARGIN, self.MARGIN, -self.MARGIN, -self.MARGIN) if cdata is None or cdata is False: title = db.field_for('title', book_id, default_value='') authors = ' & '.join(db.field_for('authors', book_id, default_value=())) @@ -354,12 +364,23 @@ class CoverDelegate(QStyledItemDelegate): if cdata is False: self.render_queue.put(book_id) else: + if self.title_height != 0: + orect = QRect(rect) + rect.setBottom(rect.bottom() - self.title_height) if self.animating is not None and self.animating.row() == index.row(): cdata = cdata.scaled(cdata.size() * self._animated_size) dx = max(0, int((rect.width() - cdata.width())/2.0)) dy = max(0, rect.height() - cdata.height()) rect.adjust(dx, dy, -dx, 0) painter.drawPixmap(rect, cdata) + if self.title_height != 0: + rect = orect + rect.setTop(rect.bottom() - self.title_height + 5) + painter.setRenderHint(QPainter.TextAntialiasing, True) + title = db.field_for('title', book_id, default_value='') + metrics = painter.fontMetrics() + painter.drawText(rect, Qt.AlignCenter|Qt.TextSingleLine, + metrics.elidedText(title, Qt.ElideRight, rect.width())) finally: painter.restore() @@ -510,10 +531,15 @@ class GridView(QListView): self.delegate.highlight_color = pal.color(pal.Text) def refresh_settings(self): - if gprefs['cover_grid_width'] != self.delegate.original_width or gprefs['cover_grid_height'] != self.delegate.original_height: + size_changed = ( + gprefs['cover_grid_width'] != self.delegate.original_width or + gprefs['cover_grid_height'] != self.delegate.original_height + ) + if (size_changed or gprefs['cover_grid_show_title'] != self.delegate.original_show_title): self.delegate.set_dimensions() self.setSpacing(self.delegate.spacing) - self.delegate.cover_cache.clear() + if size_changed: + self.delegate.cover_cache.clear() if gprefs['cover_grid_spacing'] != self.delegate.original_spacing: self.delegate.calculate_spacing() self.setSpacing(self.delegate.spacing) diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index 80521abf9f..552f907ab2 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -114,6 +114,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('cover_grid_height', gprefs) r('cover_grid_cache_size', gprefs) r('cover_grid_spacing', gprefs) + r('cover_grid_show_title', gprefs) r('cover_flow_queue_length', config, restart_required=True) r('cover_browser_reflections', gprefs) diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index 98296dd014..ca9c83d3d3 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -351,7 +351,7 @@
    - + Number of covers to cache in &memory: @@ -361,17 +361,17 @@ - + - - 50000 - The maximum number of covers to keep in memory. Increasing this will make rendering faster, at the cost of more memory usage. + + 50000 + - + &Spacing between covers: @@ -381,7 +381,7 @@ - + The spacing between covers. A value of zero means calculate automatically based on cover size. @@ -397,6 +397,13 @@ + + + + Show the book &title below the cover + + + From 063bcabad4636729a7d8f2a9aad6524d81161819 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 15:06:30 +0530 Subject: [PATCH 0516/1154] Fix book list not scrolling when changing current book with book info dialog --- src/calibre/gui2/dialogs/book_info.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/dialogs/book_info.py b/src/calibre/gui2/dialogs/book_info.py index 284d17a349..92691fd7cd 100644 --- a/src/calibre/gui2/dialogs/book_info.py +++ b/src/calibre/gui2/dialogs/book_info.py @@ -31,7 +31,6 @@ class BookInfo(QDialog, Ui_BookInfo): palette.setBrush(QPalette.Base, Qt.transparent) self.details.page().setPalette(palette) - self.view = view self.current_row = None self.fit_cover.setChecked(dynamic.get('book_info_dialog_fit_cover', @@ -85,20 +84,25 @@ class BookInfo(QDialog, Ui_BookInfo): QTimer.singleShot(1, self.resize_cover) def slave(self, current, previous): - row = current.row() - self.refresh(row) + if current.row() != previous.row(): + row = current.row() + self.refresh(row) + + def move(self, delta=1): + idx = self.view.currentIndex() + if idx.isValid(): + m = self.view.model() + ni = m.index(idx.row() + delta, idx.column()) + if ni.isValid(): + self.view.setCurrentIndex(ni) + if self.view.isVisible(): + self.view.scrollTo(ni) def next(self): - row = self.view.currentIndex().row() - ni = self.view.model().index(row+1, 0) - if ni.isValid(): - self.view.setCurrentIndex(ni) + self.move() def previous(self): - row = self.view.currentIndex().row() - ni = self.view.model().index(row-1, 0) - if ni.isValid(): - self.view.setCurrentIndex(ni) + self.move(-1) def resize_cover(self): if self.cover_pixmap is None: @@ -138,4 +142,3 @@ class BookInfo(QDialog, Ui_BookInfo): self.resize_cover() html = render_html(mi, self.css, True, self, all_fields=True) self.details.setHtml(html) - From 89c712128e2b87fda81769630951cfe8865f9377 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 15:10:12 +0530 Subject: [PATCH 0517/1154] Remove deprecated use of SIGNAL --- src/calibre/gui2/dialogs/book_info.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/book_info.py b/src/calibre/gui2/dialogs/book_info.py index 92691fd7cd..cf79c224cd 100644 --- a/src/calibre/gui2/dialogs/book_info.py +++ b/src/calibre/gui2/dialogs/book_info.py @@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import (QCoreApplication, SIGNAL, QModelIndex, QTimer, Qt, +from PyQt4.Qt import (QCoreApplication, QModelIndex, QTimer, Qt, QDialog, QPixmap, QIcon, QSize, QPalette, QShortcut, QKeySequence) from calibre.gui2.dialogs.book_info_ui import Ui_BookInfo @@ -36,9 +36,9 @@ class BookInfo(QDialog, Ui_BookInfo): self.fit_cover.setChecked(dynamic.get('book_info_dialog_fit_cover', True)) self.refresh(row) - self.connect(self.view.selectionModel(), SIGNAL('currentChanged(QModelIndex,QModelIndex)'), self.slave) - self.connect(self.next_button, SIGNAL('clicked()'), self.next) - self.connect(self.previous_button, SIGNAL('clicked()'), self.previous) + self.view.selectionModel().currentChanged.connect(self.slave) + self.next_button.clicked.connect(self.next) + self.previous_button.clicked.connect(self.previous) self.fit_cover.stateChanged.connect(self.toggle_cover_fit) self.cover.resizeEvent = self.cover_view_resized self.cover.cover_changed.connect(self.cover_changed) From 4ce6bb4378f8ddaec3f4c81c2067981a7071a72f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 15:25:11 +0530 Subject: [PATCH 0518/1154] Keep references to BookInfo to prevent object lifetime issues --- src/calibre/gui2/actions/show_book_details.py | 16 ++++++++++++++-- src/calibre/gui2/dialogs/book_info.py | 9 ++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/actions/show_book_details.py b/src/calibre/gui2/actions/show_book_details.py index 83e570d7b6..520d0522c9 100644 --- a/src/calibre/gui2/actions/show_book_details.py +++ b/src/calibre/gui2/actions/show_book_details.py @@ -5,6 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' +from PyQt4.Qt import Qt from calibre.gui2.actions import InterfaceAction from calibre.gui2.dialogs.book_info import BookInfo @@ -20,6 +21,7 @@ class ShowBookDetailsAction(InterfaceAction): def genesis(self): self.qaction.triggered.connect(self.show_book_info) + self.memory = [] def show_book_info(self, *args): if self.gui.current_view() is not self.gui.library_view: @@ -29,6 +31,16 @@ class ShowBookDetailsAction(InterfaceAction): return index = self.gui.library_view.currentIndex() if index.isValid(): - BookInfo(self.gui, self.gui.library_view, index, - self.gui.book_details.handle_click).show() + d = BookInfo(self.gui, self.gui.library_view, index, + self.gui.book_details.handle_click) + self.memory.append(d) + d.closed.connect(self.closed, type=Qt.QueuedConnection) + d.show() + + def closed(self, d): + try: + d.closed.disconnect(self.closed) + self.memory.remove(d) + except ValueError: + pass diff --git a/src/calibre/gui2/dialogs/book_info.py b/src/calibre/gui2/dialogs/book_info.py index cf79c224cd..898058a74d 100644 --- a/src/calibre/gui2/dialogs/book_info.py +++ b/src/calibre/gui2/dialogs/book_info.py @@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import (QCoreApplication, QModelIndex, QTimer, Qt, +from PyQt4.Qt import (QCoreApplication, QModelIndex, QTimer, Qt, pyqtSignal, QDialog, QPixmap, QIcon, QSize, QPalette, QShortcut, QKeySequence) from calibre.gui2.dialogs.book_info_ui import Ui_BookInfo @@ -14,6 +14,8 @@ from calibre.gui2.book_details import render_html class BookInfo(QDialog, Ui_BookInfo): + closed = pyqtSignal(object) + def __init__(self, parent, view, row, link_delegate): QDialog.__init__(self, parent) Ui_BookInfo.__init__(self) @@ -59,6 +61,11 @@ class BookInfo(QDialog, Ui_BookInfo): link = unicode(qurl.toString()) self.link_delegate(link) + def done(self, r): + ret = QDialog.done(self, r) + self.closed.emit(self) + return ret + def cover_changed(self, data): if self.current_row is not None: id_ = self.view.model().id(self.current_row) From 840f316703e3f5da5ce86297448ac11f31f637fb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 9 Aug 2013 15:29:29 +0530 Subject: [PATCH 0519/1154] Update El Periodica de Aragon and El Correo --- recipes/el_correo.recipe | 47 +++++++++++++++++-------------------- recipes/el_periodico.recipe | 18 +++++++------- 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/recipes/el_correo.recipe b/recipes/el_correo.recipe index 110c19d7ba..235d5e0fc7 100644 --- a/recipes/el_correo.recipe +++ b/recipes/el_correo.recipe @@ -3,10 +3,10 @@ __license__ = 'GPL v3' __copyright__ = '08 Januery 2011, desUBIKado' __author__ = 'desUBIKado' __description__ = 'Daily newspaper from Biscay' -__version__ = 'v0.08' -__date__ = '08, Januery 2011' +__version__ = 'v0.10' +__date__ = '07, August 2013' ''' -[url]http://www.elcorreo.com/[/url] +http://www.elcorreo.com/ ''' import time @@ -24,6 +24,7 @@ class heraldo(BasicNewsRecipe): max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False + masthead_url = 'http://www.elcorreo.com/vizcaya/noticias/201002/02/Media/logo-elcorreo-nuevo.png' language = 'es' timefmt = '[%a, %d %b, %Y]' encoding = 'iso-8859-1' @@ -33,15 +34,15 @@ class heraldo(BasicNewsRecipe): feeds = [ (u'Portada', u'http://www.elcorreo.com/vizcaya/portada.xml'), (u'Local', u'http://www.elcorreo.com/vizcaya/rss/feeds/vizcaya.xml'), - (u'Internacional', u'hhttp://www.elcorreo.com/vizcaya/rss/feeds/internacional.xml'), - (u'Econom\xeda', u'http://www.elcorreo.com/vizcaya/rss/feeds/economia.xml'), + (u'Internacional', u'hhttp://www.elcorreo.com/vizcaya/rss/feeds/internacional.xml'), + (u'Econom\xeda', u'http://www.elcorreo.com/vizcaya/rss/feeds/economia.xml'), (u'Pol\xedtica', u'http://www.elcorreo.com/vizcaya/rss/feeds/politica.xml'), - (u'Opini\xf3n', u'http://www.elcorreo.com/vizcaya/rss/feeds/opinion.xml'), - (u'Deportes', u'http://www.elcorreo.com/vizcaya/rss/feeds/deportes.xml'), + (u'Opini\xf3n', u'http://www.elcorreo.com/vizcaya/rss/feeds/opinion.xml'), + (u'Deportes', u'http://www.elcorreo.com/vizcaya/rss/feeds/deportes.xml'), (u'Sociedad', u'http://www.elcorreo.com/vizcaya/rss/feeds/sociedad.xml'), - (u'Cultura', u'http://www.elcorreo.com/vizcaya/rss/feeds/cultura.xml'), - (u'Televisi\xf3n', u'http://www.elcorreo.com/vizcaya/rss/feeds/television.xml'), - (u'Gente', u'http://www.elcorreo.com/vizcaya/rss/feeds/gente.xml') + (u'Cultura', u'http://www.elcorreo.com/vizcaya/rss/feeds/cultura.xml'), + (u'Televisi\xf3n', u'http://www.elcorreo.com/vizcaya/rss/feeds/television.xml'), + (u'Gente', u'http://www.elcorreo.com/vizcaya/rss/feeds/gente.xml') ] keep_only_tags = [ @@ -54,14 +55,14 @@ class heraldo(BasicNewsRecipe): dict(name='div', attrs={'class':['mod_lomas','bloque_lomas','blm_header','link-app3','link-app4','botones_listado']}), dict(name='div', attrs={'class':['navegacion_galeria','modulocanalpromocion','separa','separacion','compartir','tags_relacionados']}), dict(name='div', attrs={'class':['moduloBuscadorDeportes','modulo-gente','moddestacadopeq','OpcArt','articulopiniones']}), - dict(name='div', attrs={'class':['modulo-especial','publiEspecial']}), - dict(name='div', attrs={'id':['articulopina']}), + dict(name='div', attrs={'class':['modulo-especial','publiEspecial','carruselNoticias','vj','modulocomun2']}), + dict(name='div', attrs={'id':['articulopina','webs_asociadas']}), dict(name='br', attrs={'class':'clear'}), dict(name='form', attrs={'name':'frm_conversor2'}) ] remove_tags_before = dict(name='div' , attrs={'class':'articulo '}) - remove_tags_after = dict(name='div' , attrs={'class':'comentarios'}) + remove_tags_after = dict(name='div' , attrs={'class':'robapaginas'}) def get_cover_url(self): cover = None @@ -69,10 +70,8 @@ class heraldo(BasicNewsRecipe): year = str(st.tm_year) month = "%.2d" % st.tm_mon day = "%.2d" % st.tm_mday - #[url]http://img.kiosko.net/2011/01/02/es/elcorreo.750.jpg[/url] - #[url]http://info.elcorreo.com/pdf/06012011-viz.pdf[/url] - cover='http://info.elcorreo.com/pdf/'+ day + month + year +'-viz.pdf' - + # http://info.elcorreo.com/pdf/07082013-viz.pdf + cover='http://info.elcorreo.com/pdf/'+ day + month + year +'-viz.pdf' br = BasicNewsRecipe.get_browser(self) try: br.open(cover) @@ -92,29 +91,27 @@ class heraldo(BasicNewsRecipe): img{margin-bottom: 0.4em} ''' - - preprocess_regexps = [ - # To present the image of the embedded video + # Para presentar la imagen de los video incrustados (re.compile(r'var RUTA_IMAGEN', re.DOTALL|re.IGNORECASE), lambda match: ''), (re.compile(r'var SITIO = "elcorreo";', re.DOTALL|re.IGNORECASE), lambda match: '