From f8c1dd46bba2a351a1b4b546a07b76a191c6f458 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Nov 2016 17:14:44 +0530 Subject: [PATCH] Fix detection of plugin data for the amazon multiple countries plugin --- setup/plugins_mirror.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/setup/plugins_mirror.py b/setup/plugins_mirror.py index b8e250cdb1..3ffc013121 100644 --- a/setup/plugins_mirror.py +++ b/setup/plugins_mirror.py @@ -29,6 +29,7 @@ u = HTMLParser.HTMLParser().unescape socket.setdefaulttimeout(60) + def read(url, get_info=False): # {{{ if url.startswith("file://"): return urllib2.urlopen(url).read() @@ -52,6 +53,7 @@ def read(url, get_info=False): # {{{ return raw # }}} + def url_to_plugin_id(url, deprecated): query = urlparse.parse_qs(urlparse.urlparse(url).query) ans = (query['t'] if 't' in query else query['p'])[0] @@ -59,6 +61,7 @@ def url_to_plugin_id(url, deprecated): ans += '-deprecated' return ans + def parse_index(raw=None): # {{{ raw = raw or read(INDEX).decode('utf-8', 'replace') @@ -90,6 +93,7 @@ def parse_index(raw=None): # {{{ yield entry # }}} + def parse_plugin_zip_url(raw): for m in re.finditer(r'''(?is)]*>([^<>]+?\.zip)\s*<''', raw): url, name = u(m.group(1)), u(m.group(2).strip()) @@ -97,6 +101,7 @@ def parse_plugin_zip_url(raw): return MR_URL + url, name return None, None + def load_plugins_index(): try: with open(PLUGINS, 'rb') as f: @@ -108,6 +113,8 @@ def load_plugins_index(): return json.loads(bz2.decompress(raw)) # Get metadata from plugin zip file {{{ + + def convert_node(fields, x, names={}, import_data=None): name = x.__class__.__name__ conv = lambda x:convert_node(fields, x, names=names, import_data=import_data) @@ -138,6 +145,7 @@ def convert_node(fields, x, names={}, import_data=None): Alias = namedtuple('Alias', 'name asname') + def get_import_data(name, mod, zf, names): mod = mod.split('.') if mod[0] == 'calibre_plugins': @@ -157,6 +165,7 @@ def get_import_data(name, mod, zf, names): else: raise ValueError('Failed to find module: %r' % mod) + def parse_metadata(raw, namelist, zf): module = ast.parse(raw, filename='__init__.py') top_level_imports = filter(lambda x:x.__class__.__name__ == 'ImportFrom', ast.iter_child_nodes(module)) @@ -176,7 +185,7 @@ def parse_metadata(raw, namelist, zf): names = [Alias(n.name, getattr(n, 'asname', None)) for n in names] if mod in { 'calibre.customize', 'calibre.customize.conversion', - 'calibre.ebooks.metadata.sources.base', 'calibre.ebooks.metadata.covers', + 'calibre.ebooks.metadata.sources.base', 'calibre.ebooks.metadata.sources.amazon', 'calibre.ebooks.metadata.covers', 'calibre.devices.interface', 'calibre.ebooks.metadata.fetch', 'calibre.customize.builtins', } or re.match(r'calibre\.devices\.[a-z0-9]+\.driver', mod) is not None: inames = {n.asname or n.name for n in names} @@ -232,6 +241,7 @@ def parse_metadata(raw, namelist, zf): raise ValueError('Could not find plugin class') + def check_qt5_compatibility(zf, names): uses_qt = False for name in names: @@ -243,6 +253,7 @@ def check_qt5_compatibility(zf, names): return False return True + def get_plugin_info(raw, check_for_qt5=False): metadata = None with zipfile.ZipFile(io.BytesIO(raw)) as zf: @@ -290,6 +301,7 @@ def update_plugin_from_entry(plugin, entry): for x in ('donate', 'history', 'deprecated', 'uninstall', 'thread_id'): plugin[x] = getattr(entry, x) + def fetch_plugin(old_index, entry): lm_map = {plugin['thread_id']:plugin for plugin in old_index.itervalues()} raw = read(entry.url) @@ -323,6 +335,7 @@ def fetch_plugin(old_index, entry): f.write(raw) return plugin + def parallel_fetch(old_index, entry): try: return fetch_plugin(old_index, entry) @@ -330,18 +343,21 @@ def parallel_fetch(old_index, entry): import traceback return traceback.format_exc() + def log(*args, **kwargs): print (*args, **kwargs) with open('log', 'a') as f: kwargs['file'] = f print (*args, **kwargs) + def atomic_write(raw, name): with tempfile.NamedTemporaryFile(dir=os.getcwdu(), delete=False) as f: f.write(raw) os.fchmod(f.fileno(), stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH) os.rename(f.name, name) + def fetch_plugins(old_index): ans = {} pool = ThreadPool(processes=10) @@ -371,6 +387,7 @@ def fetch_plugins(old_index): os.unlink(x) return ans + def plugin_to_index(plugin, count): title = '

%s

' % ( # noqa quoteattr(plugin['thread_url']), escape(plugin['name'])) @@ -401,6 +418,7 @@ def plugin_to_index(plugin, count): desc = '

%s

' % desc return '%s\n%s\n%s\n%s\n\n' % (title, desc, block, zipfile) + def create_index(index, raw_stats): plugins = [] stats = {} @@ -482,6 +500,8 @@ h1 { text-align: center } _singleinstance = None + + def singleinstance(): global _singleinstance s = _singleinstance = socket.socket(socket.AF_UNIX) @@ -493,6 +513,7 @@ def singleinstance(): raise return True + def update_stats(): log = olog = 'stats.log' if not os.path.exists(log): @@ -521,6 +542,7 @@ def update_stats(): json.dump(stats, f, indent=2) return stats + def check_for_qt5_incompatibility(): ok_plugins, bad_plugins = [], [] for name in os.listdir('.'): @@ -592,6 +614,7 @@ def main(): log(traceback.format_exc()) raise SystemExit(1) + def test_parse(): # {{{ raw = read(INDEX).decode('utf-8', 'replace') @@ -650,6 +673,7 @@ def test_parse(): # {{{ # }}} + def test_parse_metadata(): # {{{ raw = b'''\ import os