Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Publish download counts for plugins

Browse Source
This commit is contained in:
Kovid Goyal 2014-04-08 14:03:46 +05:30
parent 460aceb329
commit 354589c7a8
1 changed files with 76 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
setup/plugins_mirror.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import urllib2, re, HTMLParser, zlib, gzip, io, sys, bz2, json, errno, urlparse, os, zipfile, ast, tempfile, glob, stat, socket import urllib2, re, HTMLParser, zlib, gzip, io, sys, bz2, json, errno, urlparse, os, zipfile, ast, tempfile, glob, stat, socket, subprocess, atexit
from future_builtins import map, zip, filter from future_builtins import map, zip, filter
from collections import namedtuple from collections import namedtuple
from multiprocessing.pool import ThreadPool from multiprocessing.pool import ThreadPool
@ -18,7 +18,8 @@ from xml.sax.saxutils import escape, quoteattr
USER_AGENT = 'calibre mirror' USER_AGENT = 'calibre mirror'
MR_URL = 'http://www.mobileread.com/forums/' MR_URL = 'http://www.mobileread.com/forums/'
WORKDIR = '/srv/plugins' if os.path.exists('/srv') else '/t/plugins' IS_PRODUCTION = os.path.exists('/srv/plugins')
WORKDIR = '/srv/plugins' if IS_PRODUCTION else '/t/plugins'
PLUGINS = 'plugins.json.bz2' PLUGINS = 'plugins.json.bz2'
INDEX = MR_URL + 'showpost.php?p=1362767&postcount=1' INDEX = MR_URL + 'showpost.php?p=1362767&postcount=1'
# INDEX = 'file:///t/raw.html' # INDEX = 'file:///t/raw.html'
@ -349,7 +350,7 @@ def fetch_plugins(old_index):
os.unlink(x) os.unlink(x)
return ans return ans
def plugin_to_index(plugin): def plugin_to_index(plugin, count):
title = '<h3><img src="http://icons.iconarchive.com/icons/oxygen-icons.org/oxygen/32/Apps-preferences-plugin-icon.png"><a href=%s title="Plugin forum thread">%s</a></h3>' % ( # noqa title = '<h3><img src="http://icons.iconarchive.com/icons/oxygen-icons.org/oxygen/32/Apps-preferences-plugin-icon.png"><a href=%s title="Plugin forum thread">%s</a></h3>' % ( # noqa
quoteattr(plugin['thread_url']), escape(plugin['name'])) quoteattr(plugin['thread_url']), escape(plugin['name']))
released = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]).strftime('%e %b, %Y').lstrip() released = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]).strftime('%e %b, %Y').lstrip()
@ -371,20 +372,25 @@ def plugin_to_index(plugin):
block.append('<br>') block.append('<br>')
block.append('<li>%s</li>' % li) block.append('<li>%s</li>' % li)
block = '<ul>%s</ul>' % ('\n'.join(block)) block = '<ul>%s</ul>' % ('\n'.join(block))
zipfile = '<div class="end"><a href=%s title="Download plugin" download=%s>Download plugin \u2193</a></div>' % ( downloads = ('\xa0<span class="download-count">[%d total downloads]</span>' % count) if count else ''
quoteattr(plugin['file']), quoteattr(plugin['name'] + '.zip')) zipfile = '<div class="end"><a href=%s title="Download plugin" download=%s>Download plugin \u2193</a>%s</div>' % (
quoteattr(plugin['file']), quoteattr(plugin['name'] + '.zip'), downloads)
desc = plugin['description'] or '' desc = plugin['description'] or ''
if desc: if desc:
desc = '<p>%s</p>' % desc desc = '<p>%s</p>' % desc
return '%s\n%s\n%s\n%s\n\n' % (title, desc, block, zipfile) return '%s\n%s\n%s\n%s\n\n' % (title, desc, block, zipfile)
def create_index(index): def create_index(index, raw_stats):
plugins = [] plugins = []
stats = {}
for name in sorted(index): for name in sorted(index):
plugin = index[name] plugin = index[name]
if not plugin['deprecated']: if not plugin['deprecated']:
count = raw_stats.get(plugin['file'].rpartition('.')[0], 0)
if count > 0:
stats[plugin['name']] = count
plugins.append( plugins.append(
plugin_to_index(plugin)) plugin_to_index(plugin, count))
index = '''\ index = '''\
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>
@ -401,6 +407,7 @@ li+li:before { content: " - " }
.end { border-bottom: solid 1pt black; padding-bottom: 0.5ex; margin-bottom: 4ex; } .end { border-bottom: solid 1pt black; padding-bottom: 0.5ex; margin-bottom: 4ex; }
h1 img, h3 img { vertical-align: middle; margin-right: 0.5em; } h1 img, h3 img { vertical-align: middle; margin-right: 0.5em; }
h1 { text-align: center } h1 { text-align: center }
.download-count { color: gray; font-size: smaller }
</style> </style>
</head> </head>
<body> <body>
@ -418,6 +425,40 @@ h1 { text-align: center }
if raw != oraw: if raw != oraw:
atomic_write(raw, 'index.html') atomic_write(raw, 'index.html')
def plugin_stats(x):
name, count = x
return '<tr><td>%s</td><td>%s</td></tr>\n' % (escape(name), count)
pstats = map(plugin_stats, sorted(stats.iteritems(), reverse=True, key=lambda x:x[1]))
stats = '''\
<!DOCTYPE html>
<html>
<head><meta charset="utf-8"><title>Stats for calibre plugins</title>
<link rel="icon" type="image/x-icon" href="http://calibre-ebook.com/favicon.ico" />
<style type="text/css">
body { background-color: #eee; }
h1 img, h3 img { vertical-align: middle; margin-right: 0.5em; }
h1 { text-align: center }
</style>
</head>
<body>
<h1><img src="http://manual.calibre-ebook.com/_static/logo.png">Stats for calibre plugins</h1>
<table>
<tr><th>Plugin</th><th>Total downloads</th></tr>
%s
</table>
</body>
</html>
''' % ('\n'.join(pstats))
raw = stats.encode('utf-8')
try:
with open('stats.html', 'rb') as f:
oraw = f.read()
except EnvironmentError:
oraw = None
if raw != oraw:
atomic_write(raw, 'stats.html')
_singleinstance = None _singleinstance = None
def singleinstance(): def singleinstance():
@ -431,6 +472,32 @@ def singleinstance():
raise raise
return True return True
def update_stats():
log = olog = 'stats.log'
if not os.path.exists(log):
return
stats = {}
if IS_PRODUCTION:
try:
with open('stats.json', 'rb') as f:
stats = json.load(f)
except EnvironmentError as err:
if err.errno != errno.ENOENT:
raise
log = 'rotated-' + log
os.rename(olog, log)
subprocess.check_call(['nginx', '-s', 'reopen'])
atexit.register(os.remove, log)
pat = re.compile(br'GET /(\d+)(?:-deprecated){0,1}\.zip')
for line in open(log, 'rb'):
m = pat.search(line)
if m is not None:
plugin = m.group(1).decode('utf-8')
stats[plugin] = stats.get(plugin, 0) + 1
with open('stats.json', 'wb') as f:
json.dump(stats, f, indent=2)
return stats
def main(): def main():
try: try:
os.chdir(WORKDIR) os.chdir(WORKDIR)
@ -447,10 +514,11 @@ def main():
print('Another instance of plugins-mirror is running', file=sys.stderr) print('Another instance of plugins-mirror is running', file=sys.stderr)
raise SystemExit(1) raise SystemExit(1)
open('log', 'w').close() open('log', 'w').close()
stats = update_stats()
try: try:
plugins_index = load_plugins_index() plugins_index = load_plugins_index()
plugins_index = fetch_plugins(plugins_index) plugins_index = fetch_plugins(plugins_index)
create_index(plugins_index) create_index(plugins_index, stats)
except: except:
import traceback import traceback
log('Failed to run at:', datetime.utcnow().isoformat()) log('Failed to run at:', datetime.utcnow().isoformat())