Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Handle partial re-uploads when uploading to fosshub

Browse Source
This commit is contained in:
Kovid Goyal 2014-08-22 13:01:45 +05:30
parent 4debe239b1
commit 8982e9d52c
1 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
setup/hosting.py
View File

@ -7,9 +7,9 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil, glob
from argparse import ArgumentParser, FileType from argparse import ArgumentParser, FileType
from subprocess import check_call, CalledProcessError from subprocess import check_call, CalledProcessError, check_output
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
from collections import OrderedDict from collections import OrderedDict
@ -466,8 +466,10 @@ def generate_index(): # {{{
# }}} # }}}
SERVER_BASE = '/srv/download/'
def upload_to_servers(files, version): # {{{ def upload_to_servers(files, version): # {{{
base = '/srv/download/' base = SERVER_BASE
dest = os.path.join(base, version) dest = os.path.join(base, version)
if not os.path.exists(dest): if not os.path.exists(dest):
os.mkdir(dest) os.mkdir(dest)
@ -511,11 +513,19 @@ def upload_to_dbs(files, version): # {{{
sys.stdout.flush() sys.stdout.flush()
server = 'mirror10.fosshub.com' server = 'mirror10.fosshub.com'
rdir = 'release/' rdir = 'release/'
check_call(['ssh', 'kovid@%s' % server, 'rm -f release/*']) old_files = set(check_output(['ssh', 'kovid@' + server, 'ls ' + rdir]).decode('utf-8').split())
if len(files) < 7:
existing = set(map(os.path.basename, files))
# fosshub does not support partial re-uploads
for f in glob.glob('%s/%s/calibre-*' % (SERVER_BASE, version)):
if os.path.basename(f) not in existing:
files[f] = None
for x in files: for x in files:
start = time.time() start = time.time()
print ('Uploading', x) print ('Uploading', x)
sys.stdout.flush() sys.stdout.flush()
old_files.discard(os.path.basename(x))
for i in range(5): for i in range(5):
try: try:
check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x, check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
@ -530,6 +540,8 @@ def upload_to_dbs(files, version): # {{{
break break
print ('Uploaded in', int(time.time() - start), 'seconds\n\n') print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
sys.stdout.flush() sys.stdout.flush()
if old_files:
check_call(['ssh', 'kovid@' + server, 'rm -f %s' % (' '.join(rdir + x for x in old_files))])
try: try:
check_call(['ssh', 'kovid@%s' % server, '/home/kovid/uploadFiles']) check_call(['ssh', 'kovid@%s' % server, '/home/kovid/uploadFiles'])
except CalledProcessError as err: except CalledProcessError as err: