IGN:Move to using mobileread for hosting the binaries

osx_installer.py

@@ -306,7 +306,7 @@ def main():
                          'iconfile' : 'icons/library.icns',
                          'frameworks': ['libusb.dylib', 'libunrar.dylib'],
                          'includes' : ['sip', 'pkg_resources', 'PyQt4.QtXml', 
-                                       'PyQt4.QtSvg', 
+                                       'PyQt4.QtSvg', 'PyQt4.QtWebKit',
                                        'mechanize', 'ClientForm', 'usbobserver', 
                                        'genshi', 'calibre.web.feeds.recipes.*',
                                        'keyword', 'codeop', 'pydoc'],

src/calibre/linux_installer.py

@@ -8,6 +8,8 @@ Download and install the linux binary.
 '''
 import sys, os, shutil, tarfile, subprocess, tempfile, urllib2, re, stat
 
+MOBILEREAD='https://dev.mobileread.com/dist/kovid/calibre/'
+
 class TerminalController:
     """
     A class that can be used to portably generate formatted output to
@@ -239,7 +241,7 @@ def do_postinstall(destdir):
 
 def download_tarball():
     pb  = ProgressBar(TerminalController(sys.stdout), 'Downloading calibre...')
-    src = urllib2.urlopen('http://calibre.kovidgoyal.net/downloads/latest-linux-binary.tar.bz2')
+    src = urllib2.urlopen(MOBILEREAD+'calibre-%version-i686.tar.bz2')
     size = int(src.info()['content-length'])
     f = tempfile.NamedTemporaryFile()
     while f.tell() < size:

src/calibre/trac/plugins/download.py

@@ -1,6 +1,6 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import re, glob
+import re
 from pkg_resources import resource_filename
 
 from trac.core import Component, implements
@@ -12,7 +12,7 @@ from trac.util import Markup
 __appname__ = 'calibre'
 DOWNLOAD_DIR = '/var/www/calibre.kovidgoyal.net/htdocs/downloads'
 LINUX_INSTALLER = '/var/www/calibre.kovidgoyal.net/calibre/src/calibre/linux_installer.py'
-
+MOBILEREAD = 'https://dev.mobileread.com/dist/kovid/calibre/'
 
 class OS(dict):
     """Dictionary with a default value for unknown keys."""
@@ -119,7 +119,7 @@ class Download(Component):
         if req.path_info == '/download':
             return self.top_level(req)
         elif req.path_info == '/download_linux_binary_installer':
-            req.send(open(LINUX_INSTALLER).read(), 'text/x-python')
+            req.send(open(LINUX_INSTALLER).read().replace('%version', self.version_from_filename()), 'text/x-python')
         else:
             match = re.match(r'\/download_(\S+)', req.path_info)
             if match:
@@ -153,8 +153,7 @@ class Download(Component):
     
     def version_from_filename(self):
         try:
-            file = glob.glob(DOWNLOAD_DIR+'/*.exe')[0]
+            return open(DOWNLOAD_DIR+'/latest_version', 'rb').read().strip()
-            return re.search(r'\S+-(\d+\.\d+\.\d+)\.', file).group(1)
         except:
             return '0.0.0'
     
@@ -165,7 +164,7 @@ class Download(Component):
             installer_name='Windows installer', 
             title='Download %s for windows'%(__appname__),
             compatibility='%s works on Windows XP and Windows Vista.'%(__appname__,),
-            path='/downloads/'+file, app=__appname__,
+            path=MOBILEREAD+file, app=__appname__,
             note=Markup(\
 '''
 <p>If you are using the <b>SONY PRS-500</b> and %(appname)s does not detect your reader, read on:</p>
@@ -203,7 +202,7 @@ You can uninstall a driver by right clicking on it and selecting uninstall.
             installer_name='OS X universal dmg', 
             title='Download %s for OS X'%(__appname__),
             compatibility='%s works on OS X Tiger and above.'%(__appname__,),
-            path='/downloads/'+file, app=__appname__,
+            path=MOBILEREAD+file, app=__appname__,
             note=Markup(\
 '''
 <ol>

upload.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-import sys, os, shutil, time, tempfile, socket, fcntl, struct
+import sys, os, shutil, time, tempfile, socket, fcntl, struct, cStringIO, pycurl, re
 sys.path.append('src')
 import subprocess
 from subprocess import check_call as _check_call
@@ -24,6 +24,7 @@ DOCS = PREFIX+"/htdocs/apidocs"
 USER_MANUAL = PREFIX+'/htdocs/user_manual'
 HTML2LRF = "src/calibre/ebooks/lrf/html/demo"
 TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
+MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
 BUILD_SCRIPT ='''\
 #!/bin/bash
 cd ~/build && \
@@ -110,19 +111,72 @@ def upload_demo():
     check_call('cd src/calibre/ebooks/lrf/txt/demo/ && zip -j /tmp/txt-demo.zip * /tmp/txt2lrf.lrf')
     check_call('''scp /tmp/txt-demo.zip divok:%s/'''%(DOWNLOADS,))
 
+def curl_list_dir(url=MOBILEREAD, listonly=1):
+    c = pycurl.Curl()
+    c.setopt(pycurl.URL, url)
+    c.setopt(c.FTP_USE_EPSV, 1)
+    c.setopt(c.NETRC, c.NETRC_REQUIRED)
+    c.setopt(c.FTPLISTONLY, listonly)
+    c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
+    b = cStringIO.StringIO()
+    c.setopt(c.WRITEFUNCTION, b.write)
+    c.perform()
+    c.close()
+    return b.getvalue().split() if listonly else b.getvalue().splitlines()
+
+def curl_delete_file(path, url=MOBILEREAD):
+    c = pycurl.Curl()
+    c.setopt(pycurl.URL, url)
+    c.setopt(c.FTP_USE_EPSV, 1)
+    c.setopt(c.NETRC, c.NETRC_REQUIRED)
+    print 'Deleting file %s on %s'%(path, url)
+    c.setopt(c.QUOTE, ['dele '+ path])
+    c.perform()
+    c.close()
+    
+
+def curl_upload_file(stream, url):
+    c = pycurl.Curl()
+    c.setopt(pycurl.URL, url)
+    c.setopt(pycurl.UPLOAD, 1)
+    c.setopt(c.NETRC, c.NETRC_REQUIRED)
+    c.setopt(pycurl.READFUNCTION, stream.read)
+    stream.seek(0, 2)
+    c.setopt(pycurl.INFILESIZE_LARGE, stream.tell())
+    stream.seek(0)
+    c.setopt(c.NOPROGRESS, 0)
+    c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
+    print 'Uploading file %s to url %s' % (getattr(stream, 'name', ''), url)
+    try:
+        c.perform()
+        c.close()
+    except:
+        pass
+    files = curl_list_dir(listonly=0)
+    for line in files:
+        line = line.split()
+        if url.endswith(line[-1]):
+            size = long(line[4])
+            stream.seek(0,2)
+            if size != stream.tell():
+                raise RuntimeError('curl failed to upload %s correctly'%getattr(stream, 'name', ''))
+                             
+        
+    
+def upload_installer(name):
+    bname = os.path.basename(name)
+    pat = re.compile(bname.replace(__version__, r'\d+\.\d+\.\d+'))
+    for f in curl_list_dir():
+        if pat.search(f):
+            curl_delete_file('/calibre/'+f)
+    curl_upload_file(open(name, 'rb'), MOBILEREAD+os.path.basename(name))
+
 def upload_installers():
-    exe, dmg, tbz2 = installer_name('exe'), installer_name('dmg'), installer_name('tar.bz2')
+    for i in ('dmg', 'exe', 'tar.bz2'):
-    if exe and os.path.exists(exe):
+        upload_installer(installer_name(i))
-        check_call('''ssh divok rm -f %s/calibre\*.exe'''%(DOWNLOADS,))
+        
-        check_call('''scp %s divok:%s/'''%(exe, DOWNLOADS))
+    check_call('''ssh divok echo %s \\> %s/latest_version'''%(__version__, DOWNLOADS))
-    if dmg and os.path.exists(dmg):
+        
-        check_call('''ssh divok rm -f %s/calibre\*.dmg'''%(DOWNLOADS,)) 
-        check_call('''scp %s divok:%s/'''%(dmg, DOWNLOADS))
-    if tbz2 and os.path.exists(tbz2):
-        check_call('''ssh divok rm -f %s/calibre-\*-i686.tar.bz2 %s/latest-linux-binary.tar.bz2'''%(DOWNLOADS,DOWNLOADS))
-        check_call('''scp %s divok:%s/'''%(tbz2, DOWNLOADS))
-        check_call('''ssh divok ln -s %s/calibre-\*-i686.tar.bz2 %s/latest-linux-binary.tar.bz2'''%(DOWNLOADS,DOWNLOADS))
-    check_call('''ssh divok chmod a+r %s/\*'''%(DOWNLOADS,))
         
 def upload_docs():
     check_call('''epydoc --config epydoc.conf''')

windows_installer.py

@@ -564,7 +564,7 @@ def main():
                                              'win32file', 'pythoncom', 'rtf2xml', 
                                              'lxml', 'lxml._elementpath', 'genshi',
                                              'path', 'pydoc', 'IPython.Extensions.*',
-                                             'calibre.web.feeds.recipes.*', 
+                                             'calibre.web.feeds.recipes.*', 'PyQt4.QtWebKit',
                                              ],                                
                                   'packages'  : ['PIL'],
                                   'excludes'  : ["Tkconstants", "Tkinter", "tcl",