diff --git a/setup/plugins_mirror.py b/setup/plugins_mirror.py index 042d4940fd..ecc372f637 100644 --- a/setup/plugins_mirror.py +++ b/setup/plugins_mirror.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' -import urllib2, re, HTMLParser, zlib, gzip, io, sys, bz2, json, errno, urlparse, os, zipfile, ast, tempfile, glob, stat, socket +import urllib2, re, HTMLParser, zlib, gzip, io, sys, bz2, json, errno, urlparse, os, zipfile, ast, tempfile, glob, stat, socket, subprocess, atexit from future_builtins import map, zip, filter from collections import namedtuple from multiprocessing.pool import ThreadPool @@ -18,7 +18,8 @@ from xml.sax.saxutils import escape, quoteattr USER_AGENT = 'calibre mirror' MR_URL = 'http://www.mobileread.com/forums/' -WORKDIR = '/srv/plugins' if os.path.exists('/srv') else '/t/plugins' +IS_PRODUCTION = os.path.exists('/srv/plugins') +WORKDIR = '/srv/plugins' if IS_PRODUCTION else '/t/plugins' PLUGINS = 'plugins.json.bz2' INDEX = MR_URL + 'showpost.php?p=1362767&postcount=1' # INDEX = 'file:///t/raw.html' @@ -349,7 +350,7 @@ def fetch_plugins(old_index): os.unlink(x) return ans -def plugin_to_index(plugin): +def plugin_to_index(plugin, count): title = '

%s

' % ( # noqa quoteattr(plugin['thread_url']), escape(plugin['name'])) released = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]).strftime('%e %b, %Y').lstrip() @@ -371,20 +372,25 @@ def plugin_to_index(plugin): block.append('
') block.append('
  • %s
    • ' % li) block = '' % ('\n'.join(block)) - zipfile = '
    Download plugin \u2193
    ' % ( - quoteattr(plugin['file']), quoteattr(plugin['name'] + '.zip')) + downloads = ('\xa0[%d total downloads]' % count) if count else '' + zipfile = '
    Download plugin \u2193%s
    ' % ( + quoteattr(plugin['file']), quoteattr(plugin['name'] + '.zip'), downloads) desc = plugin['description'] or '' if desc: desc = '

    %s

    ' % desc return '%s\n%s\n%s\n%s\n\n' % (title, desc, block, zipfile) -def create_index(index): +def create_index(index, raw_stats): plugins = [] + stats = {} for name in sorted(index): plugin = index[name] if not plugin['deprecated']: + count = raw_stats.get(plugin['file'].rpartition('.')[0], 0) + if count > 0: + stats[plugin['name']] = count plugins.append( - plugin_to_index(plugin)) + plugin_to_index(plugin, count)) index = '''\ @@ -401,6 +407,7 @@ li+li:before { content: " - " } .end { border-bottom: solid 1pt black; padding-bottom: 0.5ex; margin-bottom: 4ex; } h1 img, h3 img { vertical-align: middle; margin-right: 0.5em; } h1 { text-align: center } +.download-count { color: gray; font-size: smaller } @@ -418,6 +425,40 @@ h1 { text-align: center } if raw != oraw: atomic_write(raw, 'index.html') + def plugin_stats(x): + name, count = x + return '%s%s\n' % (escape(name), count) + + pstats = map(plugin_stats, sorted(stats.iteritems(), reverse=True, key=lambda x:x[1])) + stats = '''\ + + +Stats for calibre plugins + + + + +

    Stats for calibre plugins

    + + +%s +
    PluginTotal downloads
    + + + ''' % ('\n'.join(pstats)) + raw = stats.encode('utf-8') + try: + with open('stats.html', 'rb') as f: + oraw = f.read() + except EnvironmentError: + oraw = None + if raw != oraw: + atomic_write(raw, 'stats.html') + _singleinstance = None def singleinstance(): @@ -431,6 +472,32 @@ def singleinstance(): raise return True +def update_stats(): + log = olog = 'stats.log' + if not os.path.exists(log): + return + stats = {} + if IS_PRODUCTION: + try: + with open('stats.json', 'rb') as f: + stats = json.load(f) + except EnvironmentError as err: + if err.errno != errno.ENOENT: + raise + log = 'rotated-' + log + os.rename(olog, log) + subprocess.check_call(['nginx', '-s', 'reopen']) + atexit.register(os.remove, log) + pat = re.compile(br'GET /(\d+)(?:-deprecated){0,1}\.zip') + for line in open(log, 'rb'): + m = pat.search(line) + if m is not None: + plugin = m.group(1).decode('utf-8') + stats[plugin] = stats.get(plugin, 0) + 1 + with open('stats.json', 'wb') as f: + json.dump(stats, f, indent=2) + return stats + def main(): try: os.chdir(WORKDIR) @@ -447,10 +514,11 @@ def main(): print('Another instance of plugins-mirror is running', file=sys.stderr) raise SystemExit(1) open('log', 'w').close() + stats = update_stats() try: plugins_index = load_plugins_index() plugins_index = fetch_plugins(plugins_index) - create_index(plugins_index) + create_index(plugins_index, stats) except: import traceback log('Failed to run at:', datetime.utcnow().isoformat())