IGN:Switch primary hosting of calibre download files from Mobileread to Google code

README

@@ -15,3 +15,5 @@ bzr branch lp:calibre
 To update your copy of the source code:
 bzr merge
 
+Tarballs of the source code for each release are now available \
+at http://code.google.com/p/calibre-ebook

setup/commands.py

@@ -16,7 +16,8 @@ __all__ = [
         'sdist',
         'manual', 'tag_release', 'upload_rss',
         'pypi_register', 'pypi_upload', 'upload_to_server',
-        'upload_user_manual', 'upload_installers', 'upload_demo',
+        'upload_user_manual', 'upload_to_mobileread', 'upload_demo',
+        'upload_to_sourceforge', 'upload_to_google_code',
         'linux32', 'linux64', 'linux', 'linux_freeze',
         'osx32_freeze', 'osx32', 'osx', 'rsync',
         'win32_freeze', 'win32', 'win',
@@ -59,11 +60,13 @@ stage3 = Stage3()
 publish = Publish()
 
 from setup.upload import UploadUserManual, UploadInstallers, UploadDemo, \
-        UploadToServer
+        UploadToServer, UploadToSourceForge, UploadToGoogleCode
 upload_user_manual = UploadUserManual()
-upload_installers = UploadInstallers()
+upload_to_mobileread = UploadInstallers()
 upload_demo = UploadDemo()
 upload_to_server = UploadToServer()
+upload_to_sourceforge = UploadToSourceForge()
+upload_to_google_code = UploadToGoogleCode()
 
 from setup.installer import Rsync
 rsync = Rsync()

setup/publish.py

@@ -44,8 +44,9 @@ class Stage3(Command):
 
    description = 'Stage 3 of the publish process'
    sub_commands = ['upload_rss', 'upload_user_manual', 'upload_demo',
-            'pypi_upload', 'tag_release', 'upload_installers',
+            'upload_to_sourceforge', 'upload_to_google_code', 'tag_release',
-            'upload_to_server']
+            'upload_to_server', 'upload_to_mobileread',
+]
 
 class Publish(Command):
 

setup/pypi.py

@@ -124,8 +124,6 @@ class PyPIRegister(Command):
             auth)
         self.info('Server response (%s): %s' % (code, result))
 
-
-
     def verify_metadata(self):
         ''' Send the metadata to the package index server to be checked.
         '''

setup/upload.py

@@ -6,10 +6,11 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, re, cStringIO
+import os, re, cStringIO, base64, httplib, subprocess
 from subprocess import check_call
+from tempfile import NamedTemporaryFile
 
-from setup import Command, __version__, installer_name
+from setup import Command, __version__, installer_name, __appname__
 
 PREFIX = "/var/www/calibre.kovidgoyal.net"
 DOWNLOADS = PREFIX+"/htdocs/downloads"
@@ -21,6 +22,214 @@ TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
 MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
 
 
+def installers():
+    installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2')))
+    installers.append(installer_name('tar.bz2', is64bit=True))
+    installers.insert(0, 'dist/%s-%s.tar.gz'%(__appname__, __version__))
+    return installers
+
+def installer_description(fname):
+    if fname.endswith('.tar.gz'):
+        return 'Source code'
+    if fname.endswith('.tar.bz2'):
+        bits = '32' if 'i686' in fname else '64'
+        return bits + 'bit Linux binary'
+    if fname.endswith('.msi'):
+        return 'Windows installer'
+    if fname.endswith('.dmg'):
+        return 'OS X dmg'
+    return 'Unknown file'
+
+
+class UploadToGoogleCode(Command):
+
+    USERNAME = 'kovidgoyal'
+    # Password can be gotten by going to
+    # http://code.google.com/hosting/settings
+    # while logged into gmail
+    PASSWORD_FILE = os.path.expanduser('~/.googlecodecalibre')
+    OFFLINEIMAP   = os.path.expanduser('~/work/kde/conf/offlineimap/rc')
+    GPATHS = '/var/www/status.calibre-ebook.com/googlepaths'
+    UPLOAD_HOST = 'calibre-ebook.googlecode.com'
+    FILES_LIST = 'http://code.google.com/p/calibre-ebook/downloads/list'
+
+    def run(self, opts):
+        self.opts = opts
+        self.password = open(self.PASSWORD_FILE).read().strip()
+        self.paths = {}
+        self.old_files = self.get_files_hosted_by_google_code()
+
+        for fname in installers():
+            self.info('Uploading', fname)
+            typ = 'Type-Source' if fname.endswith('.gz') else 'Type-Installer'
+            ext = os.path.splitext(fname)[1][1:]
+            op  = 'OpSys-'+{'msi':'Windows','dmg':'OSX','bz2':'Linux','gz':'All'}[ext]
+            desc = installer_description(fname)
+            path = self.upload(os.path.abspath(fname), desc,
+                    labels=[typ, op, 'Featured'])
+            self.info('\tUploaded to:', path)
+            self.paths[os.path.basename(fname)] = path
+        self.info('Updating path map')
+        self.info(repr(self.paths))
+        raw = subprocess.Popen(['ssh', 'divok', 'cat', self.GPATHS],
+                stdout=subprocess.PIPE).stdout.read()
+        paths = eval(raw)
+        paths.update(self.paths)
+        rem = [x for x in paths if __version__ not in x]
+        for x in rem: paths.pop(x)
+        raw = ['%r : %r,'%(k, v) for k, v in paths.items()]
+        raw = '{\n\n%s\n\n}\n'%('\n'.join(raw))
+        t = NamedTemporaryFile()
+        t.write(raw)
+        t.flush()
+        check_call(['scp', t.name, 'divok:'+self.GPATHS])
+        self.br = self.login_to_gmail()
+        self.delete_old_files()
+        if len(self.get_files_hosted_by_google_code()) > len(installers()):
+            self.warn('Some old files were not deleted from Google Code')
+
+    def login_to_gmail(self):
+        import mechanize
+        self.info('Logging into Gmail')
+        raw = open(self.OFFLINEIMAP).read()
+        pw = re.search(r'(?s)remoteuser = .*@gmail.com.*?remotepass = (\S+)',
+                raw).group(1).strip()
+        br = mechanize.Browser()
+        br.open('http://gmail.com')
+        br.select_form(nr=0)
+        br.form['Email'] = self.USERNAME
+        br.form['Passwd'] = pw
+        res = br.submit()
+        return br
+
+    def get_files_hosted_by_google_code(self):
+        import urllib2
+        from lxml import html
+        self.info('Getting existing files in google code')
+        raw = urllib2.urlopen(self.FILES_LIST).read()
+        root = html.fromstring(raw)
+        ans = {}
+        for a in root.xpath('//td[@class="vt id col_0"]/a[@href]'):
+            ans[a.text.strip()] = a.get('href')
+        return ans
+
+    def delete_old_files(self):
+        self.info('Deleting old files from Google Code...')
+        for fname in self.old_files:
+            self.info('\tDeleting', fname)
+            self.br.open('http://code.google.com/p/calibre-ebook/downloads/delete?name=%s'%fname)
+            self.br.select_form(predicate=lambda x: 'delete.do' in x.action)
+            submit = self.br.form.find_control(name='delete')
+            res = self.br.submit(name='delete')
+            #from calibre import ipython
+            #ipython({'br':self.br, 'res':res})
+            #return
+
+
+
+    def encode_upload_request(self, fields, file_path):
+        BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla'
+        CRLF = '\r\n'
+
+        body = []
+
+        # Add the metadata about the upload first
+        for key, value in fields:
+            body.extend(
+            ['--' + BOUNDARY,
+            'Content-Disposition: form-data; name="%s"' % key,
+            '',
+            value,
+            ])
+
+        # Now add the file itself
+        file_name = os.path.basename(file_path)
+        f = open(file_path, 'rb')
+        file_content = f.read()
+        f.close()
+
+        body.extend(
+            ['--' + BOUNDARY,
+            'Content-Disposition: form-data; name="filename"; filename="%s"'
+            % file_name,
+            # The upload server determines the mime-type, no need to set it.
+            'Content-Type: application/octet-stream',
+            '',
+            file_content,
+            ])
+
+        # Finalize the form body
+        body.extend(['--' + BOUNDARY + '--', ''])
+
+        return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body)
+
+    def upload(self, fname, desc, labels=[]):
+        form_fields = [('summary', desc)]
+        form_fields.extend([('label', l.strip()) for l in labels])
+
+        content_type, body = self.encode_upload_request(form_fields, fname)
+        upload_uri = '/files'
+        auth_token = base64.b64encode('%s:%s'% (self.USERNAME, self.password))
+        headers = {
+            'Authorization': 'Basic %s' % auth_token,
+            'User-Agent': 'Calibre googlecode.com uploader v0.1.0',
+            'Content-Type': content_type,
+            }
+
+        server = httplib.HTTPSConnection(self.UPLOAD_HOST)
+        server.request('POST', upload_uri, body, headers)
+        resp = server.getresponse()
+        server.close()
+
+        if resp.status == 201:
+            return resp.getheader('Location')
+
+        print 'Failed to upload with code %d and reason: %s'%(resp.status,
+                resp.reason)
+        raise Exception('Failed to upload '+fname)
+
+
+
+
+
+class UploadToSourceForge(Command):
+
+    description = 'Upload release files to sourceforge'
+
+    USERNAME = 'kovidgoyal'
+    PROJECT  = 'calibre'
+    BASE     = '/home/frs/project/c/ca/'+PROJECT
+
+    def create(self):
+        self.info('Creating shell...')
+        check_call(['ssh', '-x',
+            '%s,%s@shell.sourceforge.net'%(self.USERNAME, self.PROJECT),
+            'create'])
+
+    @property
+    def rdir(self):
+        return self.BASE+'/'+__version__
+
+    def mk_release_dir(self):
+        self.info('Creating release directory...')
+        check_call(['ssh', '-x',
+            '%s,%s@shell.sourceforge.net'%(self.USERNAME, self.PROJECT),
+            'mkdir', '-p', self.rdir])
+
+    def upload_installers(self):
+        for x in installers():
+            if not os.path.exists(x): continue
+            self.info('Uploading', x)
+            check_call(['rsync', '-v', '-e', 'ssh -x', x,
+                '%s,%s@frs.sourceforge.net:%s'%(self.USERNAME, self.PROJECT,
+                    self.rdir+'/')])
+
+    def run(self, opts):
+        self.opts = opts
+        self.create()
+        self.mk_release_dir()
+        self.upload_installers()
+
 
 class UploadInstallers(Command):
     description = 'Upload any installers present in dist/'

src/calibre/__init__.py

@@ -419,3 +419,15 @@ if isosx:
     except:
         import traceback
         traceback.print_exc()
+
+def ipython(user_ns=None):
+    if user_ns is None:
+        user_ns = locals()
+    from calibre.utils.config import config_dir
+    ipydir = os.path.join(config_dir, ('_' if iswindows else '.')+'ipython')
+    os.environ['IPYTHONDIR'] = ipydir
+    from IPython.Shell import IPShellEmbed
+    ipshell = IPShellEmbed(user_ns=user_ns)
+    ipshell()
+
+

src/calibre/debug.py

@@ -190,14 +190,8 @@ def main(args=sys.argv):
     elif opts.develop_from is not None:
         develop_from(opts.develop_from)
     else:
-        from calibre.utils.config import config_dir
+        from calibre import ipython
-        ipydir = os.path.join(config_dir, ('_' if iswindows else '.')+'ipython')
+        ipython()
-        os.environ['IPYTHONDIR'] = ipydir
-        from IPython.Shell import IPShellEmbed
-        ipshell = IPShellEmbed()
-        ipshell()
-
-
 
     return 0
 

src/calibre/trac/plugins/download.py

@@ -6,7 +6,6 @@ import re, textwrap
 DEPENDENCIES = [
             #(Generic, version, gentoo, ubuntu, fedora)
             ('python', '2.6', None, None, None),
-            ('setuptools', '0.6c5', 'setuptools', 'python-setuptools', 'python-setuptools-devel'),
             ('Python Imaging Library', '1.1.6', 'imaging', 'python-imaging', 'python-imaging'),
             ('libusb', '0.1.12', None, None, None),
             ('Qt', '4.5.1', 'qt', 'libqt4-core libqt4-gui', 'qt4'),
@@ -23,7 +22,7 @@ DEPENDENCIES = [
             ('podofo', '0.7', 'podofo', 'podofo', 'podofo', 'podofo'),
             ('libwmf', '0.2.8', 'libwmf', 'libwmf', 'libwmf', 'libwmf'),
             ]
-
+STATUS = 'http://status.calibre-ebook.com/dist'
 
 class CoolDistro:
 
@@ -148,7 +147,7 @@ else:
                 compatibility=('%(a)s works on Windows XP, Vista and 7.'
                     'If you are upgrading from a version older than 0.6.17, '
                     'please uninstall %(a)s first.')%dict(a=__appname__,),
-                path=MOBILEREAD+file, app=__appname__,
+                path=STATUS+'/win32', app=__appname__,
                 note=Markup(\
     '''
     <p>If you are updating from a version of calibre older than 0.6.12 on
@@ -189,7 +188,7 @@ else:
                 installer_name='OS X universal dmg',
                 title='Download %s for OS X'%(__appname__),
                 compatibility='%s works on OS X Tiger, Leopard, and Snow Leopard.'%(__appname__,),
-                path=MOBILEREAD+file, app=__appname__,
+                path=STATUS+'/osx32', app=__appname__,
                 note=Markup(\
     u'''
     <ol>

src/calibre/trac/plugins/templates/linux.html

@@ -106,7 +106,7 @@ sudo python -c "import urllib2; exec urllib2.urlopen('http://status.calibre-eboo
                                     </li>
                                 </ol>
                                 <pre class="wiki">
-wget -O- http://calibre.kovidgoyal.net/downloads/${app}-${version}.tar.gz | tar xvz 
+wget -O- http://status.calibre-ebook.com/dist/src | tar xvz 
 cd calibre*
 sudo python setup.py install
                                 </pre>