Dynamic loading for get books plugins

src/calibre/gui2/actions/store.py

@@ -43,14 +43,16 @@ class StoreAction(InterfaceAction):
         icon.addFile(I('donate.png'), QSize(16, 16))
         for n, p in sorted(self.gui.istores.items(), key=lambda x: x[0].lower()):
             if p.base_plugin.affiliate:
-                self.store_list_menu.addAction(icon, n, partial(self.open_store, p))
+                self.store_list_menu.addAction(icon, n,
+                                               partial(self.open_store, n))
             else:
-                self.store_list_menu.addAction(n, partial(self.open_store, p))
+                self.store_list_menu.addAction(n, partial(self.open_store, n))
 
     def do_search(self):
         return self.search()
 
     def search(self, query=''):
+        self.gui.istores.check_for_updates()
         self.show_disclaimer()
         from calibre.gui2.store.search.search import SearchDialog
         sd = SearchDialog(self.gui, self.gui, query)
@@ -125,9 +127,13 @@ class StoreAction(InterfaceAction):
         self.gui.load_store_plugins()
         self.load_menu()
 
-    def open_store(self, store_plugin):
+    def open_store(self, store_plugin_name):
+        self.gui.istores.check_for_updates()
         self.show_disclaimer()
-        store_plugin.open(self.gui)
+        # It's not too important that the updated plugin have finished loading
+        # at this point
+        self.gui.istores.join(1.0)
+        self.gui.istores[store_plugin_name].open(self.gui)
 
     def show_disclaimer(self):
         confirm(('<p>' +

src/calibre/gui2/store/__init__.py

@@ -49,13 +49,16 @@ class StorePlugin(object): # {{{
     See declined.txt for a list of stores that do not want to be included.
     '''
 
-    def __init__(self, gui, name):
-        from calibre.gui2 import JSONConfig
+    minimum_calibre_version = (0, 9, 14)
 
+    def __init__(self, gui, name, config=None, base_plugin=None):
         self.gui = gui
         self.name = name
-        self.base_plugin = None
-        self.config = JSONConfig('store/stores/' + ascii_filename(self.name))
+        self.base_plugin = base_plugin
+        if config is None:
+            from calibre.gui2 import JSONConfig
+            config = JSONConfig('store/stores/' + ascii_filename(self.name))
+        self.config = config
 
     def open(self, gui, parent=None, detail_item=None, external=False):
         '''

src/calibre/gui2/store/loader.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import sys, time, io, re
+from zlib import decompressobj
+from collections import OrderedDict
+from threading import Thread
+from urllib import urlencode
+
+from calibre import prints, browser
+from calibre.constants import numeric_version, DEBUG
+from calibre.gui2.store import StorePlugin
+from calibre.utils.config import JSONConfig
+
+class VersionMismatch(ValueError):
+    def __init__(self, ver):
+        ValueError.__init__(self, 'calibre too old')
+        self.ver = ver
+
+def download_updates(ver_map={}, server='http://status.calibre-ebook.com'):
+    data = {k:type(u'')(v) for k, v in ver_map.iteritems()}
+    data['ver'] = '1'
+    url = '%s/stores?%s'%(server, urlencode(data))
+    br = browser()
+    # We use a timeout here to ensure the non-daemonic update thread does not
+    # cause calibre to hang indefinitely during shutdown
+    raw = br.open(url, timeout=4.0).read()
+
+    while raw:
+        name, raw = raw.partition(b'\0')[0::2]
+        name = name.decode('utf-8')
+        d = decompressobj()
+        src = d.decompress(raw)
+        src = src.decode('utf-8')
+        # Python complains if there is a coding declaration in a unicode string
+        src = re.sub(r'^#.*coding\s*[:=]\s*([-\w.]+)', '#', src, flags=re.MULTILINE)
+        # Translate newlines to \n
+        src = io.StringIO(src, newline=None).getvalue()
+        yield name, src
+        raw = d.unused_data
+
+class Stores(OrderedDict):
+
+    CHECK_INTERVAL = 24 * 60 * 60
+
+    def builtins_loaded(self):
+        self.last_check_time = 0
+        self.version_map = {}
+        self.cached_version_map = {}
+        self.name_rmap = {}
+        for key, val in self.iteritems():
+            prefix, name = val.__module__.rpartition('.')[0::2]
+            if prefix == 'calibre.gui2.store.stores' and name.endswith('_plugin'):
+                module = sys.modules[val.__module__]
+                sv = getattr(module, 'store_version', None)
+                if sv is not None:
+                    name = name.rpartition('_')[0]
+                    self.version_map[name] = sv
+                    self.name_rmap[name] = key
+        self.cache_file = JSONConfig('store/plugin_cache')
+        self.load_cache()
+
+    def load_cache(self):
+        # Load plugins from on disk cache
+        remove = set()
+        pat = re.compile(r'^store_version\s*=\s*(\d+)', re.M)
+        for name, src in self.cache_file.iteritems():
+            try:
+                key = self.name_rmap[name]
+            except KeyError:
+                # Plugin has been disabled
+                m = pat.search(src[:512])
+                if m is not None:
+                    try:
+                        self.cached_version_map[name] = int(m.group(1))
+                    except (TypeError, ValueError):
+                        pass
+                continue
+
+            try:
+                obj, ver = self.load_object(src, key)
+            except VersionMismatch as e:
+                self.cached_version_map[name] = e.ver
+                continue
+            except:
+                import traceback
+                prints('Failed to load cached store:', name)
+                traceback.print_exc()
+            else:
+                if not self.replace_plugin(ver, name, obj, 'cached'):
+                    # Builtin plugin is newer than cached
+                    remove.add(name)
+
+        if remove:
+            with self.cache_file:
+                for name in remove:
+                    del self.cache_file[name]
+
+    def check_for_updates(self):
+        if hasattr(self, 'update_thread') and self.update_thread.is_alive():
+            return
+        if time.time() - self.last_check_time < self.CHECK_INTERVAL:
+            return
+        try:
+            self.update_thread.start()
+        except (RuntimeError, AttributeError):
+            self.update_thread = Thread(target=self.do_update)
+            self.update_thread.start()
+
+    def join(self, timeout=None):
+        hasattr(self, 'update_thread') and self.update_thread.join(timeout)
+
+    def download_updates(self):
+        ver_map = {name:max(ver, self.cached_version_map.get(name, -1))
+            for name, ver in self.version_map.iteritems()}
+        try:
+            updates = download_updates(ver_map)
+        except:
+            import traceback
+            traceback.print_exc()
+        else:
+            for name, code in updates:
+                yield name, code
+
+    def do_update(self):
+        replacements = {}
+
+        for name, src in self.download_updates():
+            try:
+                key = self.name_rmap[name]
+            except KeyError:
+                # Plugin has been disabled
+                replacements[name] = src
+                continue
+            try:
+                obj, ver = self.load_object(src, key)
+            except VersionMismatch as e:
+                self.cached_version_map[name] = e.ver
+                replacements[name] = src
+                continue
+            except:
+                import traceback
+                prints('Failed to load downloaded store:', name)
+                traceback.print_exc()
+            else:
+                if self.replace_plugin(ver, name, obj, 'downloaded'):
+                    replacements[name] = src
+
+        if replacements:
+            with self.cache_file:
+                for name, src in replacements.iteritems():
+                    self.cache_file[name] = src
+
+    def replace_plugin(self, ver, name, obj, source):
+        if ver > self.version_map[name]:
+            if DEBUG:
+                prints('Loaded', source, 'store plugin for:',
+                       self.name_rmap[name], 'at version:', ver)
+            self[self.name_rmap[name]] = obj
+            self.version_map[name] = ver
+            return True
+        return False
+
+    def load_object(self, src, key):
+        namespace = {}
+        builtin = self[key]
+        exec src in namespace
+        ver = namespace['store_version']
+        cls = None
+        for x in namespace.itervalues():
+            if (isinstance(x, type) and issubclass(x, StorePlugin) and x is not
+                StorePlugin):
+                cls = x
+                break
+        if cls is None:
+            raise ValueError('No store plugin found')
+        if cls.minimum_calibre_version > numeric_version:
+            raise VersionMismatch(ver)
+        return cls(builtin.gui, builtin.name, config=builtin.config,
+                   base_plugin=builtin.base_plugin), ver
+
+if __name__ == '__main__':
+    st = time.time()
+    for name, code in download_updates():
+        print(name)
+        print(code)
+        print('\n', '_'*80, '\n', sep='')
+    print ('Time to download all plugins: %.2f'%( time.time() - st))
+
+

src/calibre/gui2/store/search/search.py

@@ -194,6 +194,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         query = self.clean_query(query)
         shuffle(store_names)
         # Add plugins that the user has checked to the search pool's work queue.
+        self.gui.istores.join(4.0) # Wait for updated plugins to load
         for n in store_names:
             if self.store_checks[n].isChecked():
                 self.search_pool.add_task(query, n, self.gui.istores[n], self.max_results, self.timeout)

src/calibre/gui2/ui.py

@@ -155,7 +155,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             acmap[ac.name] = ac
 
     def load_store_plugins(self):
-        self.istores = OrderedDict()
+        from calibre.gui2.store.loader import Stores
+        self.istores = Stores()
         for store in available_store_plugins():
             if self.opts.ignore_plugins and store.plugin_path is not None:
                 continue
@@ -169,6 +170,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                 if store.plugin_path is None:
                     raise
                 continue
+        self.istores.builtins_loaded()
 
     def init_istore(self, store):
         st = store.load_actual_plugin(self)
@@ -790,6 +792,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         except KeyboardInterrupt:
             pass
         time.sleep(2)
+        self.istores.join()
         self.hide_windows()
         # Do not report any errors that happen after the shutdown
         sys.excepthook = sys.__excepthook__