diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index adc66edea4..b84836c465 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -43,14 +43,16 @@ class StoreAction(InterfaceAction): icon.addFile(I('donate.png'), QSize(16, 16)) for n, p in sorted(self.gui.istores.items(), key=lambda x: x[0].lower()): if p.base_plugin.affiliate: - self.store_list_menu.addAction(icon, n, partial(self.open_store, p)) + self.store_list_menu.addAction(icon, n, + partial(self.open_store, n)) else: - self.store_list_menu.addAction(n, partial(self.open_store, p)) + self.store_list_menu.addAction(n, partial(self.open_store, n)) def do_search(self): return self.search() def search(self, query=''): + self.gui.istores.check_for_updates() self.show_disclaimer() from calibre.gui2.store.search.search import SearchDialog sd = SearchDialog(self.gui, self.gui, query) @@ -125,9 +127,13 @@ class StoreAction(InterfaceAction): self.gui.load_store_plugins() self.load_menu() - def open_store(self, store_plugin): + def open_store(self, store_plugin_name): + self.gui.istores.check_for_updates() self.show_disclaimer() - store_plugin.open(self.gui) + # It's not too important that the updated plugin have finished loading + # at this point + self.gui.istores.join(1.0) + self.gui.istores[store_plugin_name].open(self.gui) def show_disclaimer(self): confirm(('

' + diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index ae42d82032..3af0a14cda 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -49,13 +49,16 @@ class StorePlugin(object): # {{{ See declined.txt for a list of stores that do not want to be included. ''' - def __init__(self, gui, name): - from calibre.gui2 import JSONConfig + minimum_calibre_version = (0, 9, 14) + def __init__(self, gui, name, config=None, base_plugin=None): self.gui = gui self.name = name - self.base_plugin = None - self.config = JSONConfig('store/stores/' + ascii_filename(self.name)) + self.base_plugin = base_plugin + if config is None: + from calibre.gui2 import JSONConfig + config = JSONConfig('store/stores/' + ascii_filename(self.name)) + self.config = config def open(self, gui, parent=None, detail_item=None, external=False): ''' diff --git a/src/calibre/gui2/store/loader.py b/src/calibre/gui2/store/loader.py new file mode 100644 index 0000000000..c0769991dc --- /dev/null +++ b/src/calibre/gui2/store/loader.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import sys, time, io, re +from zlib import decompressobj +from collections import OrderedDict +from threading import Thread +from urllib import urlencode + +from calibre import prints, browser +from calibre.constants import numeric_version, DEBUG +from calibre.gui2.store import StorePlugin +from calibre.utils.config import JSONConfig + +class VersionMismatch(ValueError): + def __init__(self, ver): + ValueError.__init__(self, 'calibre too old') + self.ver = ver + +def download_updates(ver_map={}, server='http://status.calibre-ebook.com'): + data = {k:type(u'')(v) for k, v in ver_map.iteritems()} + data['ver'] = '1' + url = '%s/stores?%s'%(server, urlencode(data)) + br = browser() + # We use a timeout here to ensure the non-daemonic update thread does not + # cause calibre to hang indefinitely during shutdown + raw = br.open(url, timeout=4.0).read() + + while raw: + name, raw = raw.partition(b'\0')[0::2] + name = name.decode('utf-8') + d = decompressobj() + src = d.decompress(raw) + src = src.decode('utf-8') + # Python complains if there is a coding declaration in a unicode string + src = re.sub(r'^#.*coding\s*[:=]\s*([-\w.]+)', '#', src, flags=re.MULTILINE) + # Translate newlines to \n + src = io.StringIO(src, newline=None).getvalue() + yield name, src + raw = d.unused_data + +class Stores(OrderedDict): + + CHECK_INTERVAL = 24 * 60 * 60 + + def builtins_loaded(self): + self.last_check_time = 0 + self.version_map = {} + self.cached_version_map = {} + self.name_rmap = {} + for key, val in self.iteritems(): + prefix, name = val.__module__.rpartition('.')[0::2] + if prefix == 'calibre.gui2.store.stores' and name.endswith('_plugin'): + module = sys.modules[val.__module__] + sv = getattr(module, 'store_version', None) + if sv is not None: + name = name.rpartition('_')[0] + self.version_map[name] = sv + self.name_rmap[name] = key + self.cache_file = JSONConfig('store/plugin_cache') + self.load_cache() + + def load_cache(self): + # Load plugins from on disk cache + remove = set() + pat = re.compile(r'^store_version\s*=\s*(\d+)', re.M) + for name, src in self.cache_file.iteritems(): + try: + key = self.name_rmap[name] + except KeyError: + # Plugin has been disabled + m = pat.search(src[:512]) + if m is not None: + try: + self.cached_version_map[name] = int(m.group(1)) + except (TypeError, ValueError): + pass + continue + + try: + obj, ver = self.load_object(src, key) + except VersionMismatch as e: + self.cached_version_map[name] = e.ver + continue + except: + import traceback + prints('Failed to load cached store:', name) + traceback.print_exc() + else: + if not self.replace_plugin(ver, name, obj, 'cached'): + # Builtin plugin is newer than cached + remove.add(name) + + if remove: + with self.cache_file: + for name in remove: + del self.cache_file[name] + + def check_for_updates(self): + if hasattr(self, 'update_thread') and self.update_thread.is_alive(): + return + if time.time() - self.last_check_time < self.CHECK_INTERVAL: + return + try: + self.update_thread.start() + except (RuntimeError, AttributeError): + self.update_thread = Thread(target=self.do_update) + self.update_thread.start() + + def join(self, timeout=None): + hasattr(self, 'update_thread') and self.update_thread.join(timeout) + + def download_updates(self): + ver_map = {name:max(ver, self.cached_version_map.get(name, -1)) + for name, ver in self.version_map.iteritems()} + try: + updates = download_updates(ver_map) + except: + import traceback + traceback.print_exc() + else: + for name, code in updates: + yield name, code + + def do_update(self): + replacements = {} + + for name, src in self.download_updates(): + try: + key = self.name_rmap[name] + except KeyError: + # Plugin has been disabled + replacements[name] = src + continue + try: + obj, ver = self.load_object(src, key) + except VersionMismatch as e: + self.cached_version_map[name] = e.ver + replacements[name] = src + continue + except: + import traceback + prints('Failed to load downloaded store:', name) + traceback.print_exc() + else: + if self.replace_plugin(ver, name, obj, 'downloaded'): + replacements[name] = src + + if replacements: + with self.cache_file: + for name, src in replacements.iteritems(): + self.cache_file[name] = src + + def replace_plugin(self, ver, name, obj, source): + if ver > self.version_map[name]: + if DEBUG: + prints('Loaded', source, 'store plugin for:', + self.name_rmap[name], 'at version:', ver) + self[self.name_rmap[name]] = obj + self.version_map[name] = ver + return True + return False + + def load_object(self, src, key): + namespace = {} + builtin = self[key] + exec src in namespace + ver = namespace['store_version'] + cls = None + for x in namespace.itervalues(): + if (isinstance(x, type) and issubclass(x, StorePlugin) and x is not + StorePlugin): + cls = x + break + if cls is None: + raise ValueError('No store plugin found') + if cls.minimum_calibre_version > numeric_version: + raise VersionMismatch(ver) + return cls(builtin.gui, builtin.name, config=builtin.config, + base_plugin=builtin.base_plugin), ver + +if __name__ == '__main__': + st = time.time() + for name, code in download_updates(): + print(name) + print(code) + print('\n', '_'*80, '\n', sep='') + print ('Time to download all plugins: %.2f'%( time.time() - st)) + + diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index b4ae0bc943..20c6c09a03 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -194,6 +194,7 @@ class SearchDialog(QDialog, Ui_Dialog): query = self.clean_query(query) shuffle(store_names) # Add plugins that the user has checked to the search pool's work queue. + self.gui.istores.join(4.0) # Wait for updated plugins to load for n in store_names: if self.store_checks[n].isChecked(): self.search_pool.add_task(query, n, self.gui.istores[n], self.max_results, self.timeout) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index db64969179..65993ff31c 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -155,7 +155,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ acmap[ac.name] = ac def load_store_plugins(self): - self.istores = OrderedDict() + from calibre.gui2.store.loader import Stores + self.istores = Stores() for store in available_store_plugins(): if self.opts.ignore_plugins and store.plugin_path is not None: continue @@ -169,6 +170,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ if store.plugin_path is None: raise continue + self.istores.builtins_loaded() def init_istore(self, store): st = store.load_actual_plugin(self) @@ -790,6 +792,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ except KeyboardInterrupt: pass time.sleep(2) + self.istores.join() self.hide_windows() # Do not report any errors that happen after the shutdown sys.excepthook = sys.__excepthook__