IGN:Initial implementation of the calibre content server

src/calibre/__init__.py

@@ -2,7 +2,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
-import sys, os, re, logging, time, subprocess, atexit
+import sys, os, re, logging, time, subprocess, atexit, mimetypes
 from htmlentitydefs import name2codepoint
 from math import floor
 from logging import Formatter
@@ -16,6 +16,10 @@ from calibre.constants import iswindows, isosx, islinux, isfrozen, \
                               win32event, win32api, winerror, fcntl
 import mechanize
 
+mimetypes.add_type('application/epub+zip', '.epub')
+mimetypes.add_type('text/x-sony-bbeb+xml', '.lrs')
+mimetypes.add_type('application/x-sony-bbeb', '.lrf')
+
 def unicode_path(path, abs=False):
     if not isinstance(path, unicode):
         path = path.decode(sys.getfilesystemencoding())

src/calibre/library/database2.py

@@ -589,13 +589,22 @@ class LibraryDatabase2(LibraryDatabase):
                 ans.append(format)
         return ','.join(ans)
                 
+    def has_format(self, index, format, index_is_id=False):
+        id = index if index_is_id else self.id(index)
+        name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False)
+        if name:
+            path = os.path.join(self.library_path, self.path(id, index_is_id=True))
+            format = ('.' + format.lower()) if format else ''
+            path = os.path.join(path, name+format)
+            return os.access(path, os.R_OK|os.W_OK)
+        return False
     
     def format_abspath(self, index, format, index_is_id=False):
         'Return absolute path to the ebook file of format `format`'
         id = index if index_is_id else self.id(index)
-        path = os.path.join(self.library_path, self.path(id, index_is_id=True))
         name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False)
         if name:
+            path = os.path.join(self.library_path, self.path(id, index_is_id=True))
             format = ('.' + format.lower()) if format else ''
             path = os.path.join(path, name+format)
             if os.access(path, os.R_OK|os.W_OK):
@@ -612,6 +621,7 @@ class LibraryDatabase2(LibraryDatabase):
         if path is not None:
             f = open(path, mode)
             return f if as_file else f.read()
+        if self.has_format(index, format, index_is_id):
             self.remove_format(id, format, index_is_id=True)
         
     def add_format(self, index, format, stream, index_is_id=False, path=None):
@@ -982,6 +992,14 @@ class LibraryDatabase2(LibraryDatabase):
             progress.hide()
             
     
+    def __iter__(self):
+        if len(self.data) == 0:
+            self.refresh('timestamp', True)
+        for record in self.data:
+            if record is not None:
+                yield record
+        
+    
     def get_data_as_dict(self, prefix=None, authors_as_string=False):
         '''
         Return all metadata stored in the database as a dict. Includes paths to
@@ -994,11 +1012,7 @@ class LibraryDatabase2(LibraryDatabase):
             prefix = self.library_path
         FIELDS = set(['title', 'authors', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'isbn'])
         data = []
-        if len(self.data) == 0:
+        for record in iter(self):
-            self.refresh('timestamp', True)
-        for record in self.data:
-            if record is None:
-                continue
             x = {}
             for field in FIELDS:
                 x[field] = record[field]

src/calibre/library/server.py

@@ -7,96 +7,182 @@ __docformat__ = 'restructuredtext en'
 HTTP server for remote access to the calibre database.
 '''
 
-import sys, logging, re, SocketServer, gzip, cStringIO
+import sys, textwrap, cStringIO, mimetypes
-from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
+import cherrypy
+from PIL import Image
 
-from calibre.constants import __version__
+from calibre.constants import __version__, __appname__
 from calibre.utils.config import StringConfig, Config
-from calibre import ColoredFormatter
+from calibre.utils.genshi.template import MarkupTemplate
+from calibre import fit_image
 
-class Server(SocketServer.ThreadingMixIn, HTTPServer):
+def expose(func):
-    def __init__(self, opts, db):
+    
+    def do(self, *args, **kwargs):
+        dict.update(cherrypy.response.headers, {'Server':self.server_name})
+        return func(self, *args, **kwargs)
+    
+    return cherrypy.expose(do)
+
+class LibraryServer(object):
+    
+    server_name = __appname__ + '/' + __version__
+
+    BOOK = textwrap.dedent('''\
+        <book xmlns:py="http://genshi.edgewall.org/" 
+            id="${r[0]}" 
+            title="${r[1]}"
+            sort="${r[11]}"
+            author_sort="${r[12]}"
+            authors="${authors}" 
+            rating="${r[4]}"
+            timestamp="${timestamp.ctime()}" 
+            size="${r[6]}" 
+            isbn="${r[14] if r[14] else ''}"
+            formats="${r[13] if r[13] else ''}"
+            series = "${r[9] if r[9] else ''}"
+            series_index="${r[10]}"
+            tags="${r[7] if r[7] else ''}"
+            publisher="${r[3] if r[3] else ''}">${r[8] if r[8] else ''}</book>
+        ''')
+    
+    LIBRARY = MarkupTemplate(textwrap.dedent('''\
+    <?xml version="1.0" encoding="utf-8"?>
+    <library xmlns:py="http://genshi.edgewall.org/" size="${len(books)}">
+    <py:for each="book in books">
+    ${Markup(book)}
+    </py:for>
+    </library>
+    '''))
+    
+    STANZA_ENTRY=MarkupTemplate(textwrap.dedent('''\
+    <entry xmlns:py="http://genshi.edgewall.org/">
+        <title>${record['title']}</title>
+        <id>urn:calibre:${record['id']}</id>
+        <author><name>${authors}</name></author>
+        <updated>${record['timestamp'].strftime('%Y-%m-%dT%H:%M:%S+0000')}</updated>
+        <link type="application/epub+zip" href="http://${server}:${port}/get/epub/${record['id']}" />
+        <link rel="x-stanza-cover-image" type="image/jpeg" href="http://${server}:${port}/get/cover/${record['id']}" />
+        <link rel="x-stanza-cover-image-thumbnail" type="image/jpeg" href="http://${server}:${port}/get/thumb/${record['id']}" />
+        <content py:if="record['comments']" type="xhtml">
+          <pre>${record['comments']}</pre>
+        </content>
+    </entry>
+    '''))
+    
+    STANZA = MarkupTemplate(textwrap.dedent('''\
+    <?xml version="1.0" encoding="utf-8"?>
+    <feed xmlns="http://www.w3.org/2005/Atom" xmlns:py="http://genshi.edgewall.org/">
+      <title>calibre Library</title>
+      <author>
+        <name>calibre</name>
+        <uri>http://calibre.kovidgoyal.net</uri>
+      </author>
+      <subtitle>
+            ${subtitle}
+      </subtitle>
+      <py:for each="entry in data">
+      ${Markup(entry)}
+      </py:for>
+    </feed>
+    '''))
+
+    
+    def __init__(self, db, opts):
         self.db = db
-        HTTPServer.__init__(self, ('', opts.port), DBHandler)
+        for item in self.db:
+            item
+            break
+        self.opts = opts
+        cherrypy.config.update({
+                                'server.socket_port': opts.port,
+                                'server.socket_timeout': opts.timeout, #seconds
+                                'server.thread_pool': opts.thread_pool, # number of threads
+                               })
+        self.config = textwrap.dedent('''\
+        [global]
+        engine.autoreload_on = %(autoreload)s
+        tools.gzip.on = True
+        tools.gzip.mime_types = ['text/html', 'text/plain', 'text/xml']
+        ''')%dict(autoreload=opts.develop)
         
-    def serve_forever(self):
+    def to_xml(self):
-        logging.getLogger('calibre.server').info('calibre-server starting...')
+        books = []
-        HTTPServer.serve_forever(self)
+        book = MarkupTemplate(self.BOOK)
+        for record in iter(self.db):
+            authors = ' & '.join([i.replace('|', ',') for i in record[2].split(',')])
+            books.append(book.generate(r=record, authors=authors).render('xml').decode('utf-8'))
+        return self.LIBRARY.generate(books=books).render('xml')
     
+    def start(self):
+        cherrypy.quickstart(self, config=cStringIO.StringIO(self.config))
     
-class DBHandler(BaseHTTPRequestHandler):
+    def get_cover(self, id, thumbnail=False):
+        cover = self.db.cover(id, index_is_id=True, as_file=True)
+        if cover is None:
+            raise cherrypy.HTTPError(404, 'no cover available for id: %d'%id)
+        cherrypy.response.headers['Content-Type'] = 'image/jpeg'
+        if not thumbnail:
+            return cover.read()
+        try:
+            im = Image.open(cover)
+            width, height = im.size
+            scaled, width, height = fit_image(width, height, 80, 60)
+            if not scaled:
+                return cover.read()
+            im.thumbnail((width, height))
+            o = cStringIO.StringIO()
+            im.save(o, 'JPEG')
+            return o.getvalue()
+        except Exception, err:
+            raise cherrypy.HTTPError(404, 'failed to generate thumbnail: %s'%err)
         
-    server_version = 'calibre/'+__version__
+    def get_format(self, id, format):
-    protocol_version = 'HTTP/1.0'
+        format = format.upper()
-    cover_request = re.compile(r'/(\d+)/cover', re.IGNORECASE)
+        fmt = self.db.format(id, format, index_is_id=True)
-    thumbnail_request = re.compile(r'/(\d+)/thumb', re.IGNORECASE)
+        if fmt is None:
-    fmt_request   = re.compile(r'/(\d+)/([a-z0-9]+)', re.IGNORECASE)
+            raise cherrypy.HTTPError(404, 'book: %d does not have format: %s'%(id, format))
+        mt = mimetypes.guess_type('dummy.'+format.lower())[0]
+        if mt is None:
+            mt = 'application/octet-stream'
+        cherrypy.response.headers['Content-Type'] = mt
+        return fmt
         
+    @expose
+    def stanza(self):
+        cherrypy.response.headers['Content-Type'] = 'text/xml'
+        books = []
+        for record in iter(self.db):
+            authors = ' & '.join([i.replace('|', ',') for i in record[2].split(',')])
+            books.append(self.STANZA_ENTRY.generate(authors=authors, 
+                                                    record=record,
+                                                    port=self.opts.port, 
+                                                    server=self.opts.hostname,
+                                                    ).render('xml').decode('utf8'))
+        return self.STANZA.generate(subtitle='', data=books).render('xml')
     
-    def __init__(self, request, client_address, server, *args, **kwargs):
+    @expose
-        self.l = logging.getLogger('calibre.server')
+    def library(self):
-        self.db = server.db
+        cherrypy.response.headers['Content-Type'] = 'text/xml'
-        BaseHTTPRequestHandler.__init__(self, request, client_address, server, *args, **kwargs)
+        return self.to_xml()
     
+    @expose
+    def index(self):
+        return 'Hello, World!'
     
-    def log_message(self, fmt, *args):
+    @expose
-        self.l.info("%s - - [%s] %s\n" %
+    def get(self, what, id):
-                         (self.address_string(),
+        try:
-                          self.log_date_time_string(),
+            id = int(id)
-                          fmt%args))
+        except ValueError:
-        
+            raise cherrypy.HTTPError(400, 'id:%s not an integer'%id)
-    def log_error(self, fmt, *args):
+        if not self.db.has_id(id):
-        self.l.error("%s - - [%s] %s\n" %
+            raise cherrypy.HTTPError(400, 'id:%d does not exist in database'%id)
-                         (self.address_string(),
+        if what == 'thumb':
-                          self.log_date_time_string(),
+            return self.get_cover(id, thumbnail=True)
-                          fmt%args))
+        if what == 'cover':
-        
+            return self.get_cover(id)
-    def do_GET(self):
+        return self.get_format(id, what)
-        cover = self.cover_request.match(self.path)
-        thumb = self.thumbnail_request.match(self.path)
-        fmt   = self.fmt_request.match(self.path) 
-        if self.path == '/':
-            self.send_index()
-        elif self.path == '/stanza.atom':
-            self.send_stanza_index()
-        elif self.path == '/library':
-            self.send_library()
-        elif thumb:
-            self.send_cover(int(thumb.group(1)), thumbnail=True) 
-        elif cover:
-            self.send_cover(int(cover.group(1)))
-        elif fmt:
-            self.send_format(int(fmt.group(1)), fmt.group(2).upper())
-        elif self.path == '/help':
-            self.send_help()
-            
-        self.send_error(400, 'Bad request. Try /help for usage.')
-        
-    def compress(buf):
-        zbuf = cStringIO.StringIO()
-        zfile = gzip.GzipFile(mode = 'wb',  fileobj = zbuf, compresslevel = 9)
-        zfile.write(buf)
-        zfile.close()
-        return zbuf.getvalue()
-
-    
-    def send_help(self):
-        self.send_error(501, 'Not Implemented')
-    
-    def send_index(self):
-        self.send_error(501, 'Not Implemented')
-        
-    def send_stanza_index(self):
-        self.send_error(501, 'Not Implemented')
-        
-    def send_library(self):
-        self.send_error(501, 'Not Implemented')
-    
-    def send_cover(self, id, thumbnail=False):
-        self.send_error(501, 'Not Implemented')
-        
-    def send_format(self, id, fmt):
-        self.send_error(501, 'Not Implemented')
     
 def config(defaults=None):
     desc=_('Settings to control the calibre content server')
@@ -104,8 +190,15 @@ def config(defaults=None):
     
     c.add_opt('port', ['-p', '--port'], default=8080, 
               help=_('The port on which to listen. Default is %default'))
-    c.add_opt('debug', ['--debug'], default=False,
+    c.add_opt('timeout', ['-t', '--timeout'], default=120, 
-              help=_('Detailed logging'))
+              help=_('The server timeout in seconds. Default is %default'))
+    c.add_opt('thread_pool', ['--thread-pool'], default=30, 
+              help=_('The max number of worker threads to use. Default is %default'))
+    c.add_opt('hostname', ['--hostname'], default='localhost', 
+              help=_('The hostname of the machine the server is running on. Used when generating the stanza feeds. Default is %default'))
+    
+    c.add_opt('develop', ['--develop'], default=False,
+              help='Development mode. Server automatically restarts on file changes.')
     return c
 
 def option_parser():
@@ -114,23 +207,12 @@ def option_parser():
 def main(args=sys.argv):
     parser = option_parser()
     opts, args = parser.parse_args(args)
-    l = logging.getLogger('calibre.server')
+    cherrypy.log.screen = True
-    l.setLevel(logging.DEBUG if opts.debug else logging.INFO)
-    l.addHandler(logging.StreamHandler(sys.stdout))
-    l.handlers[-1].setLevel(logging.DEBUG if opts.debug else logging.INFO)
-    formatter = ColoredFormatter('%(levelname)s: %(message)s')
-    l.handlers[-1].setFormatter(formatter)
-    
     from calibre.utils.config import prefs
     from calibre.library.database2 import LibraryDatabase2
-    db = LibraryDatabase2(prefs['library_path'])
+    db = LibraryDatabase2(prefs['library_path'], row_factory=True)
-    try:
+    server = LibraryServer(db, opts)
-        print 'Starting server...'
+    server.start()
-        s = Server(opts, db)
-        s.serve_forever()
-    except KeyboardInterrupt:
-        print 'Server interrupted'
-        s.socket.close()
     return 0
 
 if __name__ == '__main__':