diff --git a/resources/content_server/gui.js b/resources/content_server/gui.js index ba2b0af940..9c20037207 100644 --- a/resources/content_server/gui.js +++ b/resources/content_server/gui.js @@ -123,7 +123,7 @@ function fetch_library_books(start, num, timeout, sort, order, search) { current_library_request = $.ajax({ type: "GET", - url: "library", + url: "xml", data: data, cache: false, timeout: timeout, //milliseconds diff --git a/src/calibre/library/server/base.py b/src/calibre/library/server/base.py index 666ce52ffc..a8d4ae899c 100644 --- a/src/calibre/library/server/base.py +++ b/src/calibre/library/server/base.py @@ -14,14 +14,46 @@ import cherrypy from calibre.constants import __appname__, __version__ from calibre.utils.date import fromtimestamp from calibre.library.server import listen_on, log_access_file, log_error_file +from calibre.library.server.utils import expose from calibre.utils.mdns import publish as publish_zeroconf, \ stop_server as stop_zeroconf, get_external_ip from calibre.library.server.content import ContentServer from calibre.library.server.mobile import MobileServer from calibre.library.server.xml import XMLServer from calibre.library.server.opds import OPDSServer +from calibre.library.server.cache import Cache -class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer): + +class DispatchController(object): # {{{ + + def __init__(self): + self.dispatcher = cherrypy.dispatch.RoutesDispatcher() + self.funcs = [] + self.seen = set([]) + + def __call__(self, name, route, func, **kwargs): + if name in self.seen: + raise NameError('Route name: '+ repr(name) + ' already used') + self.seen.add(name) + kwargs['action'] = 'f_%d'%len(self.funcs) + self.dispatcher.connect(name, route, self, **kwargs) + self.funcs.append(expose(func)) + + def __getattr__(self, attr): + if not attr.startswith('f_'): + raise AttributeError(attr + ' not found') + num = attr.rpartition('_')[-1] + try: + num = int(num) + except: + raise AttributeError(attr + ' not found') + if num < 0 or num >= len(self.funcs): + raise AttributeError(attr + ' not found') + return self.funcs[num] + +# }}} + +class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache): server_name = __appname__ + '/' + __version__ @@ -88,8 +120,16 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer): def start(self): self.is_running = False + d = DispatchController() + for x in self.__class__.__bases__: + if hasattr(x, 'add_routes'): + x.add_routes(self, d) + root_conf = self.config.get('/', {}) + root_conf['request.dispatch'] = d.dispatcher + self.config['/'] = root_conf + self.setup_loggers() - cherrypy.tree.mount(self, '', config=self.config) + cherrypy.tree.mount(root=None, config=self.config) try: try: cherrypy.engine.start() diff --git a/src/calibre/library/server/cache.py b/src/calibre/library/server/cache.py new file mode 100644 index 0000000000..89dc140434 --- /dev/null +++ b/src/calibre/library/server/cache.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from calibre.utils.date import utcnow + +class Cache(object): + + @property + def categories_cache(self): + old = getattr(self, '_category_cache', None) + if old is None or old[0] <= self.db.last_modified(): + categories = self.db.get_categories() + self._category_cache = (utcnow(), categories) + return self._category_cache[1] diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index d1a695cee1..8638035c88 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -16,7 +16,7 @@ except ImportError: from calibre import fit_image, guess_type from calibre.utils.date import fromtimestamp -from calibre.library.server.utils import expose + class ContentServer(object): @@ -25,6 +25,13 @@ class ContentServer(object): a few utility methods. ''' + def add_routes(self, connect): + connect('root', '/', self.index) + connect('get', '/get/{what}/{id}', self.get, + conditions=dict(method=["GET", "HEAD"])) + connect('static', '/static/{name}', self.static, + conditions=dict(method=["GET", "HEAD"])) + # Utility methods {{{ def last_modified(self, updated): ''' @@ -68,8 +75,7 @@ class ContentServer(object): # }}} - @expose - def get(self, what, id, *args, **kwargs): + def get(self, what, id): 'Serves files, covers, thumbnails from the calibre database' try: id = int(id) @@ -87,7 +93,6 @@ class ContentServer(object): return self.get_cover(id) return self.get_format(id, what) - @expose def static(self, name): 'Serves static content' name = name.lower() @@ -108,7 +113,6 @@ class ContentServer(object): cherrypy.response.headers['Last-Modified'] = self.last_modified(lm) return open(path, 'rb').read() - @expose def index(self, **kwargs): 'The / URL' ua = cherrypy.request.headers.get('User-Agent', '').strip() diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py index 9bec6cce35..afb31815d5 100644 --- a/src/calibre/library/server/mobile.py +++ b/src/calibre/library/server/mobile.py @@ -11,7 +11,7 @@ import __builtin__ import cherrypy from calibre.utils.genshi.template import MarkupTemplate -from calibre.library.server.utils import strftime, expose +from calibre.library.server.utils import strftime from calibre.ebooks.metadata import fmt_sidx # Templates {{{ @@ -173,7 +173,9 @@ class MobileServer(object): MOBILE_UA = re.compile('(?i)(?:iPhone|Opera Mini|NetFront|webOS|Mobile|Android|imode|DoCoMo|Minimo|Blackberry|MIDP|Symbian|HD2)') - @expose + def add_routes(self, connect): + connect('mobile', '/mobile', self.mobile) + def mobile(self, start='1', num='25', sort='date', search='', _=None, order='descending'): ''' diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index f7a7679813..359449a838 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -5,15 +5,102 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re +import re, hashlib from itertools import repeat +from functools import partial import cherrypy +from lxml import etree +from lxml.builder import ElementMaker from calibre.utils.genshi.template import MarkupTemplate from calibre.library.server.utils import strftime, expose from calibre.ebooks.metadata import fmt_sidx, title_sort from calibre import guess_type, prepare_string_for_xml +from calibre.constants import __appname__ + +# Vocabulary for building OPDS feeds {{{ +E = ElementMaker(namespace='http://www.w3.org/2005/Atom', + nsmap={ + None : 'http://www.w3.org/2005/Atom', + 'dc' : 'http://purl.org/dc/terms/', + 'opds' : 'http://opds-spec.org/2010/catalog', + }) + + +FEED = E.feed +TITLE = E.title +ID = E.id + +def UPDATED(dt, *args, **kwargs): + return E.updated(dt.strftime('%Y-%m-%dT%H:%M:%S+00:00'), *args, **kwargs) + +LINK = partial(E.link, type='application/atom+xml') +NAVLINK = partial(E.link, + type='application/atom+xml;type=feed;profile=opds-catalog') + +def SEARCH(base_href, *args, **kwargs): + kwargs['rel'] = 'search' + kwargs['title'] = 'Search' + kwargs['href'] = base_href+'/?search={searchTerms}' + return LINK(*args, **kwargs) + +def AUTHOR(name, uri=None): + args = [E.name(name)] + if uri is not None: + args.append(E.uri(uri)) + return E.author(*args) + +SUBTITLE = E.subtitle + +def NAVCATALOG_ENTRY(base_href, updated, title, description, query_data): + data = [u'%s=%s'%(key, val) for key, val in query_data.items()] + data = '&'.join(data) + href = base_href+'/?'+data + id_ = 'calibre-subcatalog:'+str(hashlib.sha1(href).hexdigest()) + return E.entry( + TITLE(title), + ID(id_), + UPDATED(updated), + E.content(description, type='text'), + NAVLINK(href=href) + ) + +# }}} + +class Feed(object): + + def __str__(self): + return etree.tostring(self.root, pretty_print=True, encoding='utf-8', + xml_declaration=True) + +class TopLevel(Feed): + + def __init__(self, + updated, # datetime object in UTC + categories, + id_ = 'urn:calibre:main', + base_href = '/stanza' + ): + self.base_href = base_href + subc = partial(NAVCATALOG_ENTRY, base_href, updated) + + subcatalogs = [subc('By '+title, + 'Books sorted by '+desc, {'sortby':q}) for title, desc, q in + categories] + + self.root = \ + FEED( + TITLE(__appname__ + ' ' + _('Library')), + ID(id_), + UPDATED(updated), + SEARCH(base_href), + AUTHOR(__appname__, uri='http://calibre-ebook.com'), + SUBTITLE(_('Books in your library')), + *subcatalogs + ) + + # Templates {{{ @@ -42,6 +129,7 @@ STANZA_SUBCATALOG_ENTRY=MarkupTemplate('''\ ''') +# Feed of books STANZA = MarkupTemplate('''\ @@ -63,62 +151,20 @@ STANZA = MarkupTemplate('''\ ''') -STANZA_MAIN = MarkupTemplate('''\ - - - calibre Library - $id - ${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')} - - - calibre - http://calibre-ebook.com - - - ${subtitle} - - - By Author - urn:uuid:fc000fa0-8c23-11de-a31d-0002a5d5c51b - ${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')} - - Books sorted by Author - - - By Title - urn:uuid:1df4fe40-8c24-11de-b4c6-0002a5d5c51b - ${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')} - - Books sorted by Title - - - By Newest - urn:uuid:3c6d4940-8c24-11de-a4d7-0002a5d5c51b - ${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')} - - Books sorted by Date - - - By Tag - urn:uuid:824921e8-db8a-4e61-7d38-f1ce41502853 - ${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')} - - Books sorted by Tags - - - By Series - urn:uuid:512a5e50-a88f-f6b8-82aa-8f129c719f61 - ${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')} - - Books sorted by Series - - -''') # }}} class OPDSServer(object): + def build_top_level(self, updated, base_href='/stanza'): + categories = self.categories_cache + categories = [(x.capitalize(), x.capitalize(), x) for x in + categories.keys()] + categories.append(('Title', 'Title', '|title|')) + categories.append(('Newest', 'Newest', '|newest|')) + + return TopLevel(updated, categories, base_href=base_href) + def get_matches(self, location, query): base = self.db.data.get_matches(location, query) epub = self.db.data.get_matches('format', '=epub') @@ -173,10 +219,6 @@ class OPDSServer(object): return STANZA.generate(subtitle=subtitle, data=entries, FM=self.db.FIELD_MAP, updated=updated, id='urn:calibre:main', next_link=next_link).render('xml') - def stanza_main(self, updated): - return STANZA_MAIN.generate(subtitle='', data=[], FM=self.db.FIELD_MAP, - updated=updated, id='urn:calibre:main').render('xml') - @expose def stanza(self, search=None, sortby=None, authorid=None, tagid=None, seriesid=None, offset=0): @@ -186,9 +228,11 @@ class OPDSServer(object): offset = int(offset) cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) cherrypy.response.headers['Content-Type'] = 'text/xml' - # Main feed + + # Top Level feed if not sortby and not search and not authorid and not tagid and not seriesid: - return self.stanza_main(updated) + return str(self.build_top_level(updated)) + if sortby in ('byseries', 'byauthor', 'bytag'): return self.stanza_sortby_subcategory(updated, sortby, offset) @@ -296,5 +340,8 @@ class OPDSServer(object): next_link=next_link, updated=updated, id='urn:calibre:main').render('xml') - +if __name__ == '__main__': + from datetime import datetime + f = TopLevel(datetime.utcnow()) + print f diff --git a/src/calibre/library/server/utils.py b/src/calibre/library/server/utils.py index 7dc0884e1a..ad5aaac169 100644 --- a/src/calibre/library/server/utils.py +++ b/src/calibre/library/server/utils.py @@ -7,34 +7,33 @@ __docformat__ = 'restructuredtext en' import time +import cherrypy + from calibre import strftime as _strftime, prints from calibre.utils.date import now as nowf def expose(func): - import cherrypy - def do(self, *args, **kwargs): + def do(*args, **kwargs): + self = func.im_self + if self.opts.develop: + start = time.time() + dict.update(cherrypy.response.headers, {'Server':self.server_name}) if not self.embedded: self.db.check_if_modified() - return func(self, *args, **kwargs) - - return cherrypy.expose(do) - -def timeit(func): - - def do(self, *args, **kwargs): - if self.opts.develop: - start = time.time() - ans = func(self, *args, **kwargs) + ans = func(*args, **kwargs) if self.opts.develop: prints('Function', func.__name__, 'called with args:', args, kwargs) prints('\tTime:', func.__name__, time.time()-start) return ans + do.__name__ = func.__name__ + return do + def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None): if not hasattr(dt, 'timetuple'): dt = nowf() diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py index e9f9a02548..036a2051bf 100644 --- a/src/calibre/library/server/xml.py +++ b/src/calibre/library/server/xml.py @@ -5,52 +5,26 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import copy, __builtin__ +import __builtin__ import cherrypy +from lxml.builder import ElementMaker +from lxml import etree -from calibre.utils.genshi.template import MarkupTemplate -from calibre.library.server.utils import strftime, expose +from calibre.library.server.utils import strftime from calibre.ebooks.metadata import fmt_sidx +from calibre.constants import preferred_encoding +from calibre import isbytestring -# Templates {{{ -BOOK = '''\ -${r[FM['comments']] if r[FM['comments']] else ''} - -''' - - -LIBRARY = MarkupTemplate('''\ - - - - ${Markup(book)} - - -''') - -# }}} +E = ElementMaker() class XMLServer(object): 'Serves XML and the Ajax based HTML frontend' - @expose - def library(self, start='0', num='50', sort=None, search=None, + def add_routes(self, connect): + connect('xml', '/xml', self.xml) + + def xml(self, start='0', num='50', sort=None, search=None, _=None, order='ascending'): ''' Serves metadata from the calibre database as XML. @@ -68,30 +42,63 @@ class XMLServer(object): num = int(num) except ValueError: raise cherrypy.HTTPError(400, 'num: %s is not an integer'%num) + order = order.lower().strip() == 'ascending' + ids = self.db.data.parse(search) if search and search.strip() else self.db.data.universal_set() - ids = sorted(ids) + FM = self.db.FIELD_MAP - items = copy.deepcopy([r for r in iter(self.db) if r[FM['id']] in ids]) + + items = [r for r in iter(self.db) if r[FM['id']] in ids] if sort is not None: self.sort(items, sort, order) - book, books = MarkupTemplate(BOOK), [] + + books = [] + + def serialize(x): + if isinstance(x, unicode): + return x + if isbytestring(x): + return x.decode(preferred_encoding, 'replace') + return unicode(x) + for record in items[start:start+num]: + kwargs = {} aus = record[FM['authors']] if record[FM['authors']] else __builtin__._('Unknown') authors = '|'.join([i.replace('|', ',') for i in aus.split(',')]) - record[FM['series_index']] = \ + kwargs['authors'] = authors + + kwargs['series_index'] = \ fmt_sidx(float(record[FM['series_index']])) - ts, pd = strftime('%Y/%m/%d %H:%M:%S', record[FM['timestamp']]), \ - strftime('%Y/%m/%d %H:%M:%S', record[FM['pubdate']]) - books.append(book.generate(r=record, authors=authors, timestamp=ts, - pubdate=pd, FM=FM).render('xml').decode('utf-8')) + + for x in ('timestamp', 'pubdate'): + kwargs[x] = strftime('%Y/%m/%d %H:%M:%S', record[FM[x]]) + + for x in ('id', 'title', 'sort', 'author_sort', 'rating', 'size'): + kwargs[x] = serialize(record[FM[x]]) + + for x in ('isbn', 'formats', 'series', 'tags', 'publisher', + 'comments'): + y = record[FM[x]] + kwargs[x] = serialize(y) if y else '' + + c = kwargs.pop('comments') + books.append(E.book(c, **kwargs)) + updated = self.db.last_modified() + kwargs = dict( + start = str(start), + updated=updated.strftime('%Y-%m-%dT%H:%M:%S+00:00'), + total=str(len(ids)), + num=str(len(books))) + ans = E.library(*books, **kwargs) cherrypy.response.headers['Content-Type'] = 'text/xml' cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) - return LIBRARY.generate(books=books, start=start, updated=updated, - total=len(ids), FM=FM).render('xml') + + return etree.tostring(ans, encoding='utf-8', pretty_print=True, + xml_declaration=True) diff --git a/src/routes/__init__.py b/src/routes/__init__.py new file mode 100644 index 0000000000..d252c700e4 --- /dev/null +++ b/src/routes/__init__.py @@ -0,0 +1,142 @@ +"""Provides common classes and functions most users will want access to.""" +import threading, sys + +class _RequestConfig(object): + """ + RequestConfig thread-local singleton + + The Routes RequestConfig object is a thread-local singleton that should + be initialized by the web framework that is utilizing Routes. + """ + __shared_state = threading.local() + + def __getattr__(self, name): + return getattr(self.__shared_state, name) + + def __setattr__(self, name, value): + """ + If the name is environ, load the wsgi envion with load_wsgi_environ + and set the environ + """ + if name == 'environ': + self.load_wsgi_environ(value) + return self.__shared_state.__setattr__(name, value) + return self.__shared_state.__setattr__(name, value) + + def __delattr__(self, name): + delattr(self.__shared_state, name) + + def load_wsgi_environ(self, environ): + """ + Load the protocol/server info from the environ and store it. + Also, match the incoming URL if there's already a mapper, and + store the resulting match dict in mapper_dict. + """ + if 'HTTPS' in environ or environ.get('wsgi.url_scheme') == 'https' \ + or environ.get('HTTP_X_FORWARDED_PROTO') == 'https': + self.__shared_state.protocol = 'https' + else: + self.__shared_state.protocol = 'http' + try: + self.mapper.environ = environ + except AttributeError: + pass + + # Wrap in try/except as common case is that there is a mapper + # attached to self + try: + if 'PATH_INFO' in environ: + mapper = self.mapper + path = environ['PATH_INFO'] + result = mapper.routematch(path) + if result is not None: + self.__shared_state.mapper_dict = result[0] + self.__shared_state.route = result[1] + else: + self.__shared_state.mapper_dict = None + self.__shared_state.route = None + except AttributeError: + pass + + if 'HTTP_X_FORWARDED_HOST' in environ: + self.__shared_state.host = environ['HTTP_X_FORWARDED_HOST'] + elif 'HTTP_HOST' in environ: + self.__shared_state.host = environ['HTTP_HOST'] + else: + self.__shared_state.host = environ['SERVER_NAME'] + if environ['wsgi.url_scheme'] == 'https': + if environ['SERVER_PORT'] != '443': + self.__shared_state.host += ':' + environ['SERVER_PORT'] + else: + if environ['SERVER_PORT'] != '80': + self.__shared_state.host += ':' + environ['SERVER_PORT'] + +def request_config(original=False): + """ + Returns the Routes RequestConfig object. + + To get the Routes RequestConfig: + + >>> from routes import * + >>> config = request_config() + + The following attributes must be set on the config object every request: + + mapper + mapper should be a Mapper instance thats ready for use + host + host is the hostname of the webapp + protocol + protocol is the protocol of the current request + mapper_dict + mapper_dict should be the dict returned by mapper.match() + redirect + redirect should be a function that issues a redirect, + and takes a url as the sole argument + prefix (optional) + Set if the application is moved under a URL prefix. Prefix + will be stripped before matching, and prepended on generation + environ (optional) + Set to the WSGI environ for automatic prefix support if the + webapp is underneath a 'SCRIPT_NAME' + + Setting the environ will use information in environ to try and + populate the host/protocol/mapper_dict options if you've already + set a mapper. + + **Using your own requst local** + + If you have your own request local object that you'd like to use instead + of the default thread local provided by Routes, you can configure Routes + to use it:: + + from routes import request_config() + config = request_config() + if hasattr(config, 'using_request_local'): + config.request_local = YourLocalCallable + config = request_config() + + Once you have configured request_config, its advisable you retrieve it + again to get the object you wanted. The variable you assign to + request_local is assumed to be a callable that will get the local config + object you wish. + + This example tests for the presence of the 'using_request_local' attribute + which will be present if you haven't assigned it yet. This way you can + avoid repeat assignments of the request specific callable. + + Should you want the original object, perhaps to change the callable its + using or stop this behavior, call request_config(original=True). + """ + obj = _RequestConfig() + try: + if obj.request_local and original is False: + return getattr(obj, 'request_local')() + except AttributeError: + obj.request_local = False + obj.using_request_local = False + return _RequestConfig() + +from routes.mapper import Mapper +from routes.util import redirect_to, url_for, URLGenerator +__all__=['Mapper', 'url_for', 'URLGenerator', 'redirect_to', 'request_config'] diff --git a/src/routes/base.py b/src/routes/base.py new file mode 100644 index 0000000000..f9e2f64973 --- /dev/null +++ b/src/routes/base.py @@ -0,0 +1,4 @@ +"""Route and Mapper core classes""" +from routes import request_config +from routes.mapper import Mapper +from routes.route import Route diff --git a/src/routes/lru.py b/src/routes/lru.py new file mode 100644 index 0000000000..9fb2329e44 --- /dev/null +++ b/src/routes/lru.py @@ -0,0 +1,70 @@ +"""LRU caching class and decorator""" +import threading + +_marker = object() + +class LRUCache(object): + def __init__(self, size): + """ Implements a psueudo-LRU algorithm (CLOCK) """ + if size < 1: + raise ValueError('size must be >1') + self.clock = [] + for i in xrange(0, size): + self.clock.append({'key':_marker, 'ref':False}) + self.size = size + self.maxpos = size - 1 + self.hand = 0 + self.data = {} + self.lock = threading.Lock() + + def __contains__(self, key): + return key in self.data + + def __getitem__(self, key, default=None): + try: + datum = self.data[key] + except KeyError: + return default + pos, val = datum + self.clock[pos]['ref'] = True + hand = pos + 1 + if hand > self.maxpos: + hand = 0 + self.hand = hand + return val + + def __setitem__(self, key, val, _marker=_marker): + hand = self.hand + maxpos = self.maxpos + clock = self.clock + data = self.data + lock = self.lock + + end = hand - 1 + if end < 0: + end = maxpos + + while 1: + current = clock[hand] + ref = current['ref'] + if ref is True: + current['ref'] = False + hand = hand + 1 + if hand > maxpos: + hand = 0 + elif ref is False or hand == end: + lock.acquire() + try: + oldkey = current['key'] + if oldkey in data: + del data[oldkey] + current['key'] = key + current['ref'] = True + data[key] = (hand, val) + hand += 1 + if hand > maxpos: + hand = 0 + self.hand = hand + finally: + lock.release() + break \ No newline at end of file diff --git a/src/routes/mapper.py b/src/routes/mapper.py new file mode 100644 index 0000000000..50f7482580 --- /dev/null +++ b/src/routes/mapper.py @@ -0,0 +1,1161 @@ +"""Mapper and Sub-Mapper""" +import re +import sys +import threading + +from routes import request_config +from routes.lru import LRUCache +from routes.util import controller_scan, MatchException, RoutesException +from routes.route import Route + + +COLLECTION_ACTIONS = ['index', 'create', 'new'] +MEMBER_ACTIONS = ['show', 'update', 'delete', 'edit'] + + +def strip_slashes(name): + """Remove slashes from the beginning and end of a part/URL.""" + if name.startswith('/'): + name = name[1:] + if name.endswith('/'): + name = name[:-1] + return name + + +class SubMapperParent(object): + """Base class for Mapper and SubMapper, both of which may be the parent + of SubMapper objects + """ + + def submapper(self, **kargs): + """Create a partial version of the Mapper with the designated + options set + + This results in a :class:`routes.mapper.SubMapper` object. + + If keyword arguments provided to this method also exist in the + keyword arguments provided to the submapper, their values will + be merged with the saved options going first. + + In addition to :class:`routes.route.Route` arguments, submapper + can also take a ``path_prefix`` argument which will be + prepended to the path of all routes that are connected. + + Example:: + + >>> map = Mapper(controller_scan=None) + >>> map.connect('home', '/', controller='home', action='splash') + >>> map.matchlist[0].name == 'home' + True + >>> m = map.submapper(controller='home') + >>> m.connect('index', '/index', action='index') + >>> map.matchlist[1].name == 'index' + True + >>> map.matchlist[1].defaults['controller'] == 'home' + True + + Optional ``collection_name`` and ``resource_name`` arguments are + used in the generation of route names by the ``action`` and + ``link`` methods. These in turn are used by the ``index``, + ``new``, ``create``, ``show``, ``edit``, ``update`` and + ``delete`` methods which may be invoked indirectly by listing + them in the ``actions`` argument. If the ``formatted`` argument + is set to ``True`` (the default), generated paths are given the + suffix '{.format}' which matches or generates an optional format + extension. + + Example:: + + >>> from routes.util import url_for + >>> map = Mapper(controller_scan=None) + >>> m = map.submapper(path_prefix='/entries', collection_name='entries', resource_name='entry', actions=['index', 'new']) + >>> url_for('entries') == '/entries' + True + >>> url_for('new_entry', format='xml') == '/entries/new.xml' + True + + """ + return SubMapper(self, **kargs) + + def collection(self, collection_name, resource_name, path_prefix=None, + member_prefix='/{id}', controller=None, + collection_actions=COLLECTION_ACTIONS, + member_actions = MEMBER_ACTIONS, member_options=None, + **kwargs): + """Create a submapper that represents a collection. + + This results in a :class:`routes.mapper.SubMapper` object, with a + ``member`` property of the same type that represents the collection's + member resources. + + Its interface is the same as the ``submapper`` together with + ``member_prefix``, ``member_actions`` and ``member_options`` + which are passed to the ``member` submatter as ``path_prefix``, + ``actions`` and keyword arguments respectively. + + Example:: + + >>> from routes.util import url_for + >>> map = Mapper(controller_scan=None) + >>> c = map.collection('entries', 'entry') + >>> c.member.link('ping', method='POST') + >>> url_for('entries') == '/entries' + True + >>> url_for('edit_entry', id=1) == '/entries/1/edit' + True + >>> url_for('ping_entry', id=1) == '/entries/1/ping' + True + + """ + if controller is None: + controller = resource_name or collection_name + + if path_prefix is None: + path_prefix = '/' + collection_name + + collection = SubMapper(self, collection_name=collection_name, + resource_name=resource_name, + path_prefix=path_prefix, controller=controller, + actions=collection_actions, **kwargs) + + collection.member = SubMapper(collection, path_prefix=member_prefix, + actions=member_actions, + **(member_options or {})) + + return collection + + +class SubMapper(SubMapperParent): + """Partial mapper for use with_options""" + def __init__(self, obj, resource_name=None, collection_name=None, + actions=None, formatted=None, **kwargs): + self.kwargs = kwargs + self.obj = obj + self.collection_name = collection_name + self.member = None + self.resource_name = resource_name \ + or getattr(obj, 'resource_name', None) \ + or kwargs.get('controller', None) \ + or getattr(obj, 'controller', None) + if formatted is not None: + self.formatted = formatted + else: + self.formatted = getattr(obj, 'formatted', None) + if self.formatted is None: + self.formatted = True + + self.add_actions(actions or []) + + def connect(self, *args, **kwargs): + newkargs = {} + newargs = args + for key, value in self.kwargs.items(): + if key == 'path_prefix': + if len(args) > 1: + newargs = (args[0], self.kwargs[key] + args[1]) + else: + newargs = (self.kwargs[key] + args[0],) + elif key in kwargs: + if isinstance(value, dict): + newkargs[key] = dict(value, **kwargs[key]) # merge dicts + else: + newkargs[key] = value + kwargs[key] + else: + newkargs[key] = self.kwargs[key] + for key in kwargs: + if key not in self.kwargs: + newkargs[key] = kwargs[key] + return self.obj.connect(*newargs, **newkargs) + + def link(self, rel=None, name=None, action=None, method='GET', + formatted=None, **kwargs): + """Generates a named route for a subresource. + + Example:: + + >>> from routes.util import url_for + >>> map = Mapper(controller_scan=None) + >>> c = map.collection('entries', 'entry') + >>> c.link('recent', name='recent_entries') + >>> c.member.link('ping', method='POST', formatted=True) + >>> url_for('entries') == '/entries' + True + >>> url_for('recent_entries') == '/entries/recent' + True + >>> url_for('ping_entry', id=1) == '/entries/1/ping' + True + >>> url_for('ping_entry', id=1, format='xml') == '/entries/1/ping.xml' + True + + """ + if formatted or (formatted is None and self.formatted): + suffix = '{.format}' + else: + suffix = '' + + return self.connect(name or (rel + '_' + self.resource_name), + '/' + (rel or name) + suffix, + action=action or rel or name, + **_kwargs_with_conditions(kwargs, method)) + + def new(self, **kwargs): + """Generates the "new" link for a collection submapper.""" + return self.link(rel='new', **kwargs) + + def edit(self, **kwargs): + """Generates the "edit" link for a collection member submapper.""" + return self.link(rel='edit', **kwargs) + + def action(self, name=None, action=None, method='GET', formatted=None, + **kwargs): + """Generates a named route at the base path of a submapper. + + Example:: + + >>> from routes import url_for + >>> map = Mapper(controller_scan=None) + >>> c = map.submapper(path_prefix='/entries', controller='entry') + >>> c.action(action='index', name='entries', formatted=True) + >>> c.action(action='create', method='POST') + >>> url_for(controller='entry', action='index', method='GET') == '/entries' + True + >>> url_for(controller='entry', action='index', method='GET', format='xml') == '/entries.xml' + True + >>> url_for(controller='entry', action='create', method='POST') == '/entries' + True + + """ + if formatted or (formatted is None and self.formatted): + suffix = '{.format}' + else: + suffix = '' + return self.connect(name or (action + '_' + self.resource_name), + suffix, + action=action or name, + **_kwargs_with_conditions(kwargs, method)) + + def index(self, name=None, **kwargs): + """Generates the "index" action for a collection submapper.""" + return self.action(name=name or self.collection_name, + action='index', method='GET', **kwargs) + + def show(self, name = None, **kwargs): + """Generates the "show" action for a collection member submapper.""" + return self.action(name=name or self.resource_name, + action='show', method='GET', **kwargs) + + def create(self, **kwargs): + """Generates the "create" action for a collection submapper.""" + return self.action(action='create', method='POST', **kwargs) + + def update(self, **kwargs): + """Generates the "update" action for a collection member submapper.""" + return self.action(action='update', method='PUT', **kwargs) + + def delete(self, **kwargs): + """Generates the "delete" action for a collection member submapper.""" + return self.action(action='delete', method='DELETE', **kwargs) + + def add_actions(self, actions): + [getattr(self, action)() for action in actions] + + # Provided for those who prefer using the 'with' syntax in Python 2.5+ + def __enter__(self): + return self + + def __exit__(self, type, value, tb): + pass + +# Create kwargs with a 'conditions' member generated for the given method +def _kwargs_with_conditions(kwargs, method): + if method and 'conditions' not in kwargs: + newkwargs = kwargs.copy() + newkwargs['conditions'] = {'method': method} + return newkwargs + else: + return kwargs + + + +class Mapper(SubMapperParent): + """Mapper handles URL generation and URL recognition in a web + application. + + Mapper is built handling dictionary's. It is assumed that the web + application will handle the dictionary returned by URL recognition + to dispatch appropriately. + + URL generation is done by passing keyword parameters into the + generate function, a URL is then returned. + + """ + def __init__(self, controller_scan=controller_scan, directory=None, + always_scan=False, register=True, explicit=True): + """Create a new Mapper instance + + All keyword arguments are optional. + + ``controller_scan`` + Function reference that will be used to return a list of + valid controllers used during URL matching. If + ``directory`` keyword arg is present, it will be passed + into the function during its call. This option defaults to + a function that will scan a directory for controllers. + + Alternatively, a list of controllers or None can be passed + in which are assumed to be the definitive list of + controller names valid when matching 'controller'. + + ``directory`` + Passed into controller_scan for the directory to scan. It + should be an absolute path if using the default + ``controller_scan`` function. + + ``always_scan`` + Whether or not the ``controller_scan`` function should be + run during every URL match. This is typically a good idea + during development so the server won't need to be restarted + anytime a controller is added. + + ``register`` + Boolean used to determine if the Mapper should use + ``request_config`` to register itself as the mapper. Since + it's done on a thread-local basis, this is typically best + used during testing though it won't hurt in other cases. + + ``explicit`` + Boolean used to determine if routes should be connected + with implicit defaults of:: + + {'controller':'content','action':'index','id':None} + + When set to True, these defaults will not be added to route + connections and ``url_for`` will not use Route memory. + + Additional attributes that may be set after mapper + initialization (ie, map.ATTRIBUTE = 'something'): + + ``encoding`` + Used to indicate alternative encoding/decoding systems to + use with both incoming URL's, and during Route generation + when passed a Unicode string. Defaults to 'utf-8'. + + ``decode_errors`` + How to handle errors in the encoding, generally ignoring + any chars that don't convert should be sufficient. Defaults + to 'ignore'. + + ``minimization`` + Boolean used to indicate whether or not Routes should + minimize URL's and the generated URL's, or require every + part where it appears in the path. Defaults to True. + + ``hardcode_names`` + Whether or not Named Routes result in the default options + for the route being used *or* if they actually force url + generation to use the route. Defaults to False. + + """ + self.matchlist = [] + self.maxkeys = {} + self.minkeys = {} + self.urlcache = LRUCache(1600) + self._created_regs = False + self._created_gens = False + self._master_regexp = None + self.prefix = None + self.req_data = threading.local() + self.directory = directory + self.always_scan = always_scan + self.controller_scan = controller_scan + self._regprefix = None + self._routenames = {} + self.debug = False + self.append_slash = False + self.sub_domains = False + self.sub_domains_ignore = [] + self.domain_match = '[^\.\/]+?\.[^\.\/]+' + self.explicit = explicit + self.encoding = 'utf-8' + self.decode_errors = 'ignore' + self.hardcode_names = True + self.minimization = False + self.create_regs_lock = threading.Lock() + if register: + config = request_config() + config.mapper = self + + def __str__(self): + """Generates a tabular string representation.""" + def format_methods(r): + if r.conditions: + method = r.conditions.get('method', '') + return type(method) is str and method or ', '.join(method) + else: + return '' + + table = [('Route name', 'Methods', 'Path')] + \ + [(r.name or '', format_methods(r), r.routepath or '') + for r in self.matchlist] + + widths = [max(len(row[col]) for row in table) + for col in range(len(table[0]))] + + return '\n'.join( + ' '.join(row[col].ljust(widths[col]) + for col in range(len(widths))) + for row in table) + + def _envget(self): + try: + return self.req_data.environ + except AttributeError: + return None + def _envset(self, env): + self.req_data.environ = env + def _envdel(self): + del self.req_data.environ + environ = property(_envget, _envset, _envdel) + + def extend(self, routes, path_prefix=''): + """Extends the mapper routes with a list of Route objects + + If a path_prefix is provided, all the routes will have their + path prepended with the path_prefix. + + Example:: + + >>> map = Mapper(controller_scan=None) + >>> map.connect('home', '/', controller='home', action='splash') + >>> map.matchlist[0].name == 'home' + True + >>> routes = [Route('index', '/index.htm', controller='home', + ... action='index')] + >>> map.extend(routes) + >>> len(map.matchlist) == 2 + True + >>> map.extend(routes, path_prefix='/subapp') + >>> len(map.matchlist) == 3 + True + >>> map.matchlist[2].routepath == '/subapp/index.htm' + True + + .. note:: + + This function does not merely extend the mapper with the + given list of routes, it actually creates new routes with + identical calling arguments. + + """ + for route in routes: + if path_prefix and route.minimization: + routepath = '/'.join([path_prefix, route.routepath]) + elif path_prefix: + routepath = path_prefix + route.routepath + else: + routepath = route.routepath + self.connect(route.name, routepath, **route._kargs) + + def connect(self, *args, **kargs): + """Create and connect a new Route to the Mapper. + + Usage: + + .. code-block:: python + + m = Mapper() + m.connect(':controller/:action/:id') + m.connect('date/:year/:month/:day', controller="blog", action="view") + m.connect('archives/:page', controller="blog", action="by_page", + requirements = { 'page':'\d{1,2}' }) + m.connect('category_list', 'archives/category/:section', controller='blog', action='category', + section='home', type='list') + m.connect('home', '', controller='blog', action='view', section='home') + + """ + routename = None + if len(args) > 1: + routename = args[0] + else: + args = (None,) + args + if '_explicit' not in kargs: + kargs['_explicit'] = self.explicit + if '_minimize' not in kargs: + kargs['_minimize'] = self.minimization + route = Route(*args, **kargs) + + # Apply encoding and errors if its not the defaults and the route + # didn't have one passed in. + if (self.encoding != 'utf-8' or self.decode_errors != 'ignore') and \ + '_encoding' not in kargs: + route.encoding = self.encoding + route.decode_errors = self.decode_errors + + if not route.static: + self.matchlist.append(route) + + if routename: + self._routenames[routename] = route + route.name = routename + if route.static: + return + exists = False + for key in self.maxkeys: + if key == route.maxkeys: + self.maxkeys[key].append(route) + exists = True + break + if not exists: + self.maxkeys[route.maxkeys] = [route] + self._created_gens = False + + def _create_gens(self): + """Create the generation hashes for route lookups""" + # Use keys temporailly to assemble the list to avoid excessive + # list iteration testing with "in" + controllerlist = {} + actionlist = {} + + # Assemble all the hardcoded/defaulted actions/controllers used + for route in self.matchlist: + if route.static: + continue + if route.defaults.has_key('controller'): + controllerlist[route.defaults['controller']] = True + if route.defaults.has_key('action'): + actionlist[route.defaults['action']] = True + + # Setup the lists of all controllers/actions we'll add each route + # to. We include the '*' in the case that a generate contains a + # controller/action that has no hardcodes + controllerlist = controllerlist.keys() + ['*'] + actionlist = actionlist.keys() + ['*'] + + # Go through our list again, assemble the controllers/actions we'll + # add each route to. If its hardcoded, we only add it to that dict key. + # Otherwise we add it to every hardcode since it can be changed. + gendict = {} # Our generated two-deep hash + for route in self.matchlist: + if route.static: + continue + clist = controllerlist + alist = actionlist + if 'controller' in route.hardcoded: + clist = [route.defaults['controller']] + if 'action' in route.hardcoded: + alist = [unicode(route.defaults['action'])] + for controller in clist: + for action in alist: + actiondict = gendict.setdefault(controller, {}) + actiondict.setdefault(action, ([], {}))[0].append(route) + self._gendict = gendict + self._created_gens = True + + def create_regs(self, *args, **kwargs): + """Atomically creates regular expressions for all connected + routes + """ + self.create_regs_lock.acquire() + try: + self._create_regs(*args, **kwargs) + finally: + self.create_regs_lock.release() + + def _create_regs(self, clist=None): + """Creates regular expressions for all connected routes""" + if clist is None: + if self.directory: + clist = self.controller_scan(self.directory) + elif callable(self.controller_scan): + clist = self.controller_scan() + elif not self.controller_scan: + clist = [] + else: + clist = self.controller_scan + + for key, val in self.maxkeys.iteritems(): + for route in val: + route.makeregexp(clist) + + regexps = [] + routematches = [] + for route in self.matchlist: + if not route.static: + routematches.append(route) + regexps.append(route.makeregexp(clist, include_names=False)) + self._routematches = routematches + + # Create our regexp to strip the prefix + if self.prefix: + self._regprefix = re.compile(self.prefix + '(.*)') + + # Save the master regexp + regexp = '|'.join(['(?:%s)' % x for x in regexps]) + self._master_reg = regexp + self._master_regexp = re.compile(regexp) + self._created_regs = True + + def _match(self, url, environ): + """Internal Route matcher + + Matches a URL against a route, and returns a tuple of the match + dict and the route object if a match is successfull, otherwise + it returns empty. + + For internal use only. + + """ + if not self._created_regs and self.controller_scan: + self.create_regs() + elif not self._created_regs: + raise RoutesException("You must generate the regular expressions" + " before matching.") + + if self.always_scan: + self.create_regs() + + matchlog = [] + if self.prefix: + if re.match(self._regprefix, url): + url = re.sub(self._regprefix, r'\1', url) + if not url: + url = '/' + else: + return (None, None, matchlog) + + environ = environ or self.environ + sub_domains = self.sub_domains + sub_domains_ignore = self.sub_domains_ignore + domain_match = self.domain_match + debug = self.debug + + # Check to see if its a valid url against the main regexp + # Done for faster invalid URL elimination + valid_url = re.match(self._master_regexp, url) + if not valid_url: + return (None, None, matchlog) + + for route in self.matchlist: + if route.static: + if debug: + matchlog.append(dict(route=route, static=True)) + continue + match = route.match(url, environ, sub_domains, sub_domains_ignore, + domain_match) + if debug: + matchlog.append(dict(route=route, regexp=bool(match))) + if isinstance(match, dict) or match: + return (match, route, matchlog) + return (None, None, matchlog) + + def match(self, url=None, environ=None): + """Match a URL against against one of the routes contained. + + Will return None if no valid match is found. + + .. code-block:: python + + resultdict = m.match('/joe/sixpack') + + """ + if not url and not environ: + raise RoutesException('URL or environ must be provided') + + if not url: + url = environ['PATH_INFO'] + + result = self._match(url, environ) + if self.debug: + return result[0], result[1], result[2] + if isinstance(result[0], dict) or result[0]: + return result[0] + return None + + def routematch(self, url=None, environ=None): + """Match a URL against against one of the routes contained. + + Will return None if no valid match is found, otherwise a + result dict and a route object is returned. + + .. code-block:: python + + resultdict, route_obj = m.match('/joe/sixpack') + + """ + if not url and not environ: + raise RoutesException('URL or environ must be provided') + + if not url: + url = environ['PATH_INFO'] + result = self._match(url, environ) + if self.debug: + return result[0], result[1], result[2] + if isinstance(result[0], dict) or result[0]: + return result[0], result[1] + return None + + def generate(self, *args, **kargs): + """Generate a route from a set of keywords + + Returns the url text, or None if no URL could be generated. + + .. code-block:: python + + m.generate(controller='content',action='view',id=10) + + """ + # Generate ourself if we haven't already + if not self._created_gens: + self._create_gens() + + if self.append_slash: + kargs['_append_slash'] = True + + if not self.explicit: + if 'controller' not in kargs: + kargs['controller'] = 'content' + if 'action' not in kargs: + kargs['action'] = 'index' + + controller = kargs.get('controller', None) + action = kargs.get('action', None) + + # If the URL didn't depend on the SCRIPT_NAME, we'll cache it + # keyed by just by kargs; otherwise we need to cache it with + # both SCRIPT_NAME and kargs: + cache_key = unicode(args).encode('utf8') + \ + unicode(kargs).encode('utf8') + + if self.urlcache is not None: + if self.environ: + cache_key_script_name = '%s:%s' % ( + self.environ.get('SCRIPT_NAME', ''), cache_key) + else: + cache_key_script_name = cache_key + + # Check the url cache to see if it exists, use it if it does + for key in [cache_key, cache_key_script_name]: + if key in self.urlcache: + return self.urlcache[key] + + actionlist = self._gendict.get(controller) or self._gendict.get('*', {}) + if not actionlist and not args: + return None + (keylist, sortcache) = actionlist.get(action) or \ + actionlist.get('*', (None, {})) + if not keylist and not args: + return None + + keys = frozenset(kargs.keys()) + cacheset = False + cachekey = unicode(keys) + cachelist = sortcache.get(cachekey) + if args: + keylist = args + elif cachelist: + keylist = cachelist + else: + cacheset = True + newlist = [] + for route in keylist: + if len(route.minkeys - route.dotkeys - keys) == 0: + newlist.append(route) + keylist = newlist + + def keysort(a, b): + """Sorts two sets of sets, to order them ideally for + matching.""" + am = a.minkeys + a = a.maxkeys + b = b.maxkeys + + lendiffa = len(keys^a) + lendiffb = len(keys^b) + # If they both match, don't switch them + if lendiffa == 0 and lendiffb == 0: + return 0 + + # First, if a matches exactly, use it + if lendiffa == 0: + return -1 + + # Or b matches exactly, use it + if lendiffb == 0: + return 1 + + # Neither matches exactly, return the one with the most in + # common + if cmp(lendiffa, lendiffb) != 0: + return cmp(lendiffa, lendiffb) + + # Neither matches exactly, but if they both have just as much + # in common + if len(keys&b) == len(keys&a): + # Then we return the shortest of the two + return cmp(len(a), len(b)) + + # Otherwise, we return the one that has the most in common + else: + return cmp(len(keys&b), len(keys&a)) + + keylist.sort(keysort) + if cacheset: + sortcache[cachekey] = keylist + + # Iterate through the keylist of sorted routes (or a single route if + # it was passed in explicitly for hardcoded named routes) + for route in keylist: + fail = False + for key in route.hardcoded: + kval = kargs.get(key) + if not kval: + continue + if isinstance(kval, str): + kval = kval.decode(self.encoding) + else: + kval = unicode(kval) + if kval != route.defaults[key] and not callable(route.defaults[key]): + fail = True + break + if fail: + continue + path = route.generate(**kargs) + if path: + if self.prefix: + path = self.prefix + path + external_static = route.static and route.external + if self.environ and self.environ.get('SCRIPT_NAME', '') != ''\ + and not route.absolute and not external_static: + path = self.environ['SCRIPT_NAME'] + path + key = cache_key_script_name + else: + key = cache_key + if self.urlcache is not None: + self.urlcache[key] = str(path) + return str(path) + else: + continue + return None + + def resource(self, member_name, collection_name, **kwargs): + """Generate routes for a controller resource + + The member_name name should be the appropriate singular version + of the resource given your locale and used with members of the + collection. The collection_name name will be used to refer to + the resource collection methods and should be a plural version + of the member_name argument. By default, the member_name name + will also be assumed to map to a controller you create. + + The concept of a web resource maps somewhat directly to 'CRUD' + operations. The overlying things to keep in mind is that + mapping a resource is about handling creating, viewing, and + editing that resource. + + All keyword arguments are optional. + + ``controller`` + If specified in the keyword args, the controller will be + the actual controller used, but the rest of the naming + conventions used for the route names and URL paths are + unchanged. + + ``collection`` + Additional action mappings used to manipulate/view the + entire set of resources provided by the controller. + + Example:: + + map.resource('message', 'messages', collection={'rss':'GET'}) + # GET /message/rss (maps to the rss action) + # also adds named route "rss_message" + + ``member`` + Additional action mappings used to access an individual + 'member' of this controllers resources. + + Example:: + + map.resource('message', 'messages', member={'mark':'POST'}) + # POST /message/1/mark (maps to the mark action) + # also adds named route "mark_message" + + ``new`` + Action mappings that involve dealing with a new member in + the controller resources. + + Example:: + + map.resource('message', 'messages', new={'preview':'POST'}) + # POST /message/new/preview (maps to the preview action) + # also adds a url named "preview_new_message" + + ``path_prefix`` + Prepends the URL path for the Route with the path_prefix + given. This is most useful for cases where you want to mix + resources or relations between resources. + + ``name_prefix`` + Perpends the route names that are generated with the + name_prefix given. Combined with the path_prefix option, + it's easy to generate route names and paths that represent + resources that are in relations. + + Example:: + + map.resource('message', 'messages', controller='categories', + path_prefix='/category/:category_id', + name_prefix="category_") + # GET /category/7/message/1 + # has named route "category_message" + + ``parent_resource`` + A ``dict`` containing information about the parent + resource, for creating a nested resource. It should contain + the ``member_name`` and ``collection_name`` of the parent + resource. This ``dict`` will + be available via the associated ``Route`` object which can + be accessed during a request via + ``request.environ['routes.route']`` + + If ``parent_resource`` is supplied and ``path_prefix`` + isn't, ``path_prefix`` will be generated from + ``parent_resource`` as + "/:_id". + + If ``parent_resource`` is supplied and ``name_prefix`` + isn't, ``name_prefix`` will be generated from + ``parent_resource`` as "_". + + Example:: + + >>> from routes.util import url_for + >>> m = Mapper() + >>> m.resource('location', 'locations', + ... parent_resource=dict(member_name='region', + ... collection_name='regions')) + >>> # path_prefix is "regions/:region_id" + >>> # name prefix is "region_" + >>> url_for('region_locations', region_id=13) + '/regions/13/locations' + >>> url_for('region_new_location', region_id=13) + '/regions/13/locations/new' + >>> url_for('region_location', region_id=13, id=60) + '/regions/13/locations/60' + >>> url_for('region_edit_location', region_id=13, id=60) + '/regions/13/locations/60/edit' + + Overriding generated ``path_prefix``:: + + >>> m = Mapper() + >>> m.resource('location', 'locations', + ... parent_resource=dict(member_name='region', + ... collection_name='regions'), + ... path_prefix='areas/:area_id') + >>> # name prefix is "region_" + >>> url_for('region_locations', area_id=51) + '/areas/51/locations' + + Overriding generated ``name_prefix``:: + + >>> m = Mapper() + >>> m.resource('location', 'locations', + ... parent_resource=dict(member_name='region', + ... collection_name='regions'), + ... name_prefix='') + >>> # path_prefix is "regions/:region_id" + >>> url_for('locations', region_id=51) + '/regions/51/locations' + + """ + collection = kwargs.pop('collection', {}) + member = kwargs.pop('member', {}) + new = kwargs.pop('new', {}) + path_prefix = kwargs.pop('path_prefix', None) + name_prefix = kwargs.pop('name_prefix', None) + parent_resource = kwargs.pop('parent_resource', None) + + # Generate ``path_prefix`` if ``path_prefix`` wasn't specified and + # ``parent_resource`` was. Likewise for ``name_prefix``. Make sure + # that ``path_prefix`` and ``name_prefix`` *always* take precedence if + # they are specified--in particular, we need to be careful when they + # are explicitly set to "". + if parent_resource is not None: + if path_prefix is None: + path_prefix = '%s/:%s_id' % (parent_resource['collection_name'], + parent_resource['member_name']) + if name_prefix is None: + name_prefix = '%s_' % parent_resource['member_name'] + else: + if path_prefix is None: path_prefix = '' + if name_prefix is None: name_prefix = '' + + # Ensure the edit and new actions are in and GET + member['edit'] = 'GET' + new.update({'new': 'GET'}) + + # Make new dict's based off the old, except the old values become keys, + # and the old keys become items in a list as the value + def swap(dct, newdct): + """Swap the keys and values in the dict, and uppercase the values + from the dict during the swap.""" + for key, val in dct.iteritems(): + newdct.setdefault(val.upper(), []).append(key) + return newdct + collection_methods = swap(collection, {}) + member_methods = swap(member, {}) + new_methods = swap(new, {}) + + # Insert create, update, and destroy methods + collection_methods.setdefault('POST', []).insert(0, 'create') + member_methods.setdefault('PUT', []).insert(0, 'update') + member_methods.setdefault('DELETE', []).insert(0, 'delete') + + # If there's a path prefix option, use it with the controller + controller = strip_slashes(collection_name) + path_prefix = strip_slashes(path_prefix) + path_prefix = '/' + path_prefix + if path_prefix and path_prefix != '/': + path = path_prefix + '/' + controller + else: + path = '/' + controller + collection_path = path + new_path = path + "/new" + member_path = path + "/:(id)" + + options = { + 'controller': kwargs.get('controller', controller), + '_member_name': member_name, + '_collection_name': collection_name, + '_parent_resource': parent_resource, + '_filter': kwargs.get('_filter') + } + + def requirements_for(meth): + """Returns a new dict to be used for all route creation as the + route options""" + opts = options.copy() + if method != 'any': + opts['conditions'] = {'method':[meth.upper()]} + return opts + + # Add the routes for handling collection methods + for method, lst in collection_methods.iteritems(): + primary = (method != 'GET' and lst.pop(0)) or None + route_options = requirements_for(method) + for action in lst: + route_options['action'] = action + route_name = "%s%s_%s" % (name_prefix, action, collection_name) + self.connect("formatted_" + route_name, "%s/%s.:(format)" % \ + (collection_path, action), **route_options) + self.connect(route_name, "%s/%s" % (collection_path, action), + **route_options) + if primary: + route_options['action'] = primary + self.connect("%s.:(format)" % collection_path, **route_options) + self.connect(collection_path, **route_options) + + # Specifically add in the built-in 'index' collection method and its + # formatted version + self.connect("formatted_" + name_prefix + collection_name, + collection_path + ".:(format)", action='index', + conditions={'method':['GET']}, **options) + self.connect(name_prefix + collection_name, collection_path, + action='index', conditions={'method':['GET']}, **options) + + # Add the routes that deal with new resource methods + for method, lst in new_methods.iteritems(): + route_options = requirements_for(method) + for action in lst: + path = (action == 'new' and new_path) or "%s/%s" % (new_path, + action) + name = "new_" + member_name + if action != 'new': + name = action + "_" + name + route_options['action'] = action + formatted_path = (action == 'new' and new_path + '.:(format)') or \ + "%s/%s.:(format)" % (new_path, action) + self.connect("formatted_" + name_prefix + name, formatted_path, + **route_options) + self.connect(name_prefix + name, path, **route_options) + + requirements_regexp = '[^\/]+' + + # Add the routes that deal with member methods of a resource + for method, lst in member_methods.iteritems(): + route_options = requirements_for(method) + route_options['requirements'] = {'id':requirements_regexp} + if method not in ['POST', 'GET', 'any']: + primary = lst.pop(0) + else: + primary = None + for action in lst: + route_options['action'] = action + self.connect("formatted_%s%s_%s" % (name_prefix, action, + member_name), + "%s/%s.:(format)" % (member_path, action), **route_options) + self.connect("%s%s_%s" % (name_prefix, action, member_name), + "%s/%s" % (member_path, action), **route_options) + if primary: + route_options['action'] = primary + self.connect("%s.:(format)" % member_path, **route_options) + self.connect(member_path, **route_options) + + # Specifically add the member 'show' method + route_options = requirements_for('GET') + route_options['action'] = 'show' + route_options['requirements'] = {'id':requirements_regexp} + self.connect("formatted_" + name_prefix + member_name, + member_path + ".:(format)", **route_options) + self.connect(name_prefix + member_name, member_path, **route_options) + + def redirect(self, match_path, destination_path, *args, **kwargs): + """Add a redirect route to the mapper + + Redirect routes bypass the wrapped WSGI application and instead + result in a redirect being issued by the RoutesMiddleware. As + such, this method is only meaningful when using + RoutesMiddleware. + + By default, a 302 Found status code is used, this can be + changed by providing a ``_redirect_code`` keyword argument + which will then be used instead. Note that the entire status + code string needs to be present. + + When using keyword arguments, all arguments that apply to + matching will be used for the match, while generation specific + options will be used during generation. Thus all options + normally available to connected Routes may be used with + redirect routes as well. + + Example:: + + map = Mapper() + map.redirect('/legacyapp/archives/{url:.*}, '/archives/{url}) + map.redirect('/home/index', '/', _redirect_code='301 Moved Permanently') + + """ + both_args = ['_encoding', '_explicit', '_minimize'] + gen_args = ['_filter'] + + status_code = kwargs.pop('_redirect_code', '302 Found') + gen_dict, match_dict = {}, {} + + # Create the dict of args for the generation route + for key in both_args + gen_args: + if key in kwargs: + gen_dict[key] = kwargs[key] + gen_dict['_static'] = True + + # Create the dict of args for the matching route + for key in kwargs: + if key not in gen_args: + match_dict[key] = kwargs[key] + + self.connect(match_path, **match_dict) + match_route = self.matchlist[-1] + + self.connect('_redirect_%s' % id(match_route), destination_path, + **gen_dict) + match_route.redirect = True + match_route.redirect_status = status_code diff --git a/src/routes/middleware.py b/src/routes/middleware.py new file mode 100644 index 0000000000..d4c005ee78 --- /dev/null +++ b/src/routes/middleware.py @@ -0,0 +1,146 @@ +"""Routes WSGI Middleware""" +import re +import logging + +from webob import Request + +from routes.base import request_config +from routes.util import URLGenerator, url_for + +log = logging.getLogger('routes.middleware') + +class RoutesMiddleware(object): + """Routing middleware that handles resolving the PATH_INFO in + addition to optionally recognizing method overriding.""" + def __init__(self, wsgi_app, mapper, use_method_override=True, + path_info=True, singleton=True): + """Create a Route middleware object + + Using the use_method_override keyword will require Paste to be + installed, and your application should use Paste's WSGIRequest + object as it will properly handle POST issues with wsgi.input + should Routes check it. + + If path_info is True, then should a route var contain + path_info, the SCRIPT_NAME and PATH_INFO will be altered + accordingly. This should be used with routes like: + + .. code-block:: python + + map.connect('blog/*path_info', controller='blog', path_info='') + + """ + self.app = wsgi_app + self.mapper = mapper + self.singleton = singleton + self.use_method_override = use_method_override + self.path_info = path_info + log_debug = self.log_debug = logging.DEBUG >= log.getEffectiveLevel() + if self.log_debug: + log.debug("Initialized with method overriding = %s, and path " + "info altering = %s", use_method_override, path_info) + + def __call__(self, environ, start_response): + """Resolves the URL in PATH_INFO, and uses wsgi.routing_args + to pass on URL resolver results.""" + old_method = None + if self.use_method_override: + req = None + + # In some odd cases, there's no query string + try: + qs = environ['QUERY_STRING'] + except KeyError: + qs = '' + if '_method' in qs: + req = Request(environ) + req.errors = 'ignore' + if '_method' in req.GET: + old_method = environ['REQUEST_METHOD'] + environ['REQUEST_METHOD'] = req.GET['_method'].upper() + if self.log_debug: + log.debug("_method found in QUERY_STRING, altering request" + " method to %s", environ['REQUEST_METHOD']) + elif environ['REQUEST_METHOD'] == 'POST' and is_form_post(environ): + if req is None: + req = Request(environ) + req.errors = 'ignore' + if '_method' in req.POST: + old_method = environ['REQUEST_METHOD'] + environ['REQUEST_METHOD'] = req.POST['_method'].upper() + if self.log_debug: + log.debug("_method found in POST data, altering request " + "method to %s", environ['REQUEST_METHOD']) + + # Run the actual route matching + # -- Assignment of environ to config triggers route matching + if self.singleton: + config = request_config() + config.mapper = self.mapper + config.environ = environ + match = config.mapper_dict + route = config.route + else: + results = self.mapper.routematch(environ=environ) + if results: + match, route = results[0], results[1] + else: + match = route = None + + if old_method: + environ['REQUEST_METHOD'] = old_method + + if not match: + match = {} + if self.log_debug: + urlinfo = "%s %s" % (environ['REQUEST_METHOD'], environ['PATH_INFO']) + log.debug("No route matched for %s", urlinfo) + elif self.log_debug: + urlinfo = "%s %s" % (environ['REQUEST_METHOD'], environ['PATH_INFO']) + log.debug("Matched %s", urlinfo) + log.debug("Route path: '%s', defaults: %s", route.routepath, + route.defaults) + log.debug("Match dict: %s", match) + + url = URLGenerator(self.mapper, environ) + environ['wsgiorg.routing_args'] = ((url), match) + environ['routes.route'] = route + environ['routes.url'] = url + + if route and route.redirect: + route_name = '_redirect_%s' % id(route) + location = url(route_name, **match) + log.debug("Using redirect route, redirect to '%s' with status" + "code: %s", location, route.redirect_status) + start_response(route.redirect_status, + [('Content-Type', 'text/plain; charset=utf8'), + ('Location', location)]) + return [] + + # If the route included a path_info attribute and it should be used to + # alter the environ, we'll pull it out + if self.path_info and 'path_info' in match: + oldpath = environ['PATH_INFO'] + newpath = match.get('path_info') or '' + environ['PATH_INFO'] = newpath + if not environ['PATH_INFO'].startswith('/'): + environ['PATH_INFO'] = '/' + environ['PATH_INFO'] + environ['SCRIPT_NAME'] += re.sub(r'^(.*?)/' + re.escape(newpath) + '$', + r'\1', oldpath) + + response = self.app(environ, start_response) + + # Wrapped in try as in rare cases the attribute will be gone already + try: + del self.mapper.environ + except AttributeError: + pass + return response + +def is_form_post(environ): + """Determine whether the request is a POSTed html form""" + content_type = environ.get('CONTENT_TYPE', '').lower() + if ';' in content_type: + content_type = content_type.split(';', 1)[0] + return content_type in ('application/x-www-form-urlencoded', + 'multipart/form-data') diff --git a/src/routes/route.py b/src/routes/route.py new file mode 100644 index 0000000000..688d6e4cb9 --- /dev/null +++ b/src/routes/route.py @@ -0,0 +1,742 @@ +import re +import sys +import urllib + +if sys.version < '2.4': + from sets import ImmutableSet as frozenset + +from routes.util import _url_quote as url_quote, _str_encode + + +class Route(object): + """The Route object holds a route recognition and generation + routine. + + See Route.__init__ docs for usage. + + """ + # reserved keys that don't count + reserved_keys = ['requirements'] + + # special chars to indicate a natural split in the URL + done_chars = ('/', ',', ';', '.', '#') + + def __init__(self, name, routepath, **kargs): + """Initialize a route, with a given routepath for + matching/generation + + The set of keyword args will be used as defaults. + + Usage:: + + >>> from routes.base import Route + >>> newroute = Route(None, ':controller/:action/:id') + >>> sorted(newroute.defaults.items()) + [('action', 'index'), ('id', None)] + >>> newroute = Route(None, 'date/:year/:month/:day', + ... controller="blog", action="view") + >>> newroute = Route(None, 'archives/:page', controller="blog", + ... action="by_page", requirements = { 'page':'\d{1,2}' }) + >>> newroute.reqs + {'page': '\\\d{1,2}'} + + .. Note:: + Route is generally not called directly, a Mapper instance + connect method should be used to add routes. + + """ + self.routepath = routepath + self.sub_domains = False + self.prior = None + self.redirect = False + self.name = name + self._kargs = kargs + self.minimization = kargs.pop('_minimize', False) + self.encoding = kargs.pop('_encoding', 'utf-8') + self.reqs = kargs.get('requirements', {}) + self.decode_errors = 'replace' + + # Don't bother forming stuff we don't need if its a static route + self.static = kargs.pop('_static', False) + self.filter = kargs.pop('_filter', None) + self.absolute = kargs.pop('_absolute', False) + + # Pull out the member/collection name if present, this applies only to + # map.resource + self.member_name = kargs.pop('_member_name', None) + self.collection_name = kargs.pop('_collection_name', None) + self.parent_resource = kargs.pop('_parent_resource', None) + + # Pull out route conditions + self.conditions = kargs.pop('conditions', None) + + # Determine if explicit behavior should be used + self.explicit = kargs.pop('_explicit', False) + + # Since static need to be generated exactly, treat them as + # non-minimized + if self.static: + self.external = '://' in self.routepath + self.minimization = False + + # Strip preceding '/' if present, and not minimizing + if routepath.startswith('/') and self.minimization: + self.routepath = routepath[1:] + self._setup_route() + + def _setup_route(self): + # Build our routelist, and the keys used in the route + self.routelist = routelist = self._pathkeys(self.routepath) + routekeys = frozenset([key['name'] for key in routelist + if isinstance(key, dict)]) + self.dotkeys = frozenset([key['name'] for key in routelist + if isinstance(key, dict) and + key['type'] == '.']) + + if not self.minimization: + self.make_full_route() + + # Build a req list with all the regexp requirements for our args + self.req_regs = {} + for key, val in self.reqs.iteritems(): + self.req_regs[key] = re.compile('^' + val + '$') + # Update our defaults and set new default keys if needed. defaults + # needs to be saved + (self.defaults, defaultkeys) = self._defaults(routekeys, + self.reserved_keys, + self._kargs.copy()) + # Save the maximum keys we could utilize + self.maxkeys = defaultkeys | routekeys + + # Populate our minimum keys, and save a copy of our backward keys for + # quicker generation later + (self.minkeys, self.routebackwards) = self._minkeys(routelist[:]) + + # Populate our hardcoded keys, these are ones that are set and don't + # exist in the route + self.hardcoded = frozenset([key for key in self.maxkeys \ + if key not in routekeys and self.defaults[key] is not None]) + + # Cache our default keys + self._default_keys = frozenset(self.defaults.keys()) + + def make_full_route(self): + """Make a full routelist string for use with non-minimized + generation""" + regpath = '' + for part in self.routelist: + if isinstance(part, dict): + regpath += '%(' + part['name'] + ')s' + else: + regpath += part + self.regpath = regpath + + def make_unicode(self, s): + """Transform the given argument into a unicode string.""" + if isinstance(s, unicode): + return s + elif isinstance(s, str): + return s.decode(self.encoding) + elif callable(s): + return s + else: + return unicode(s) + + def _pathkeys(self, routepath): + """Utility function to walk the route, and pull out the valid + dynamic/wildcard keys.""" + collecting = False + current = '' + done_on = '' + var_type = '' + just_started = False + routelist = [] + for char in routepath: + if char in [':', '*', '{'] and not collecting and not self.static \ + or char in ['{'] and not collecting: + just_started = True + collecting = True + var_type = char + if char == '{': + done_on = '}' + just_started = False + if len(current) > 0: + routelist.append(current) + current = '' + elif collecting and just_started: + just_started = False + if char == '(': + done_on = ')' + else: + current = char + done_on = self.done_chars + ('-',) + elif collecting and char not in done_on: + current += char + elif collecting: + collecting = False + if var_type == '{': + if current[0] == '.': + var_type = '.' + current = current[1:] + else: + var_type = ':' + opts = current.split(':') + if len(opts) > 1: + current = opts[0] + self.reqs[current] = opts[1] + routelist.append(dict(type=var_type, name=current)) + if char in self.done_chars: + routelist.append(char) + done_on = var_type = current = '' + else: + current += char + if collecting: + routelist.append(dict(type=var_type, name=current)) + elif current: + routelist.append(current) + return routelist + + def _minkeys(self, routelist): + """Utility function to walk the route backwards + + Will also determine the minimum keys we can handle to generate + a working route. + + routelist is a list of the '/' split route path + defaults is a dict of all the defaults provided for the route + + """ + minkeys = [] + backcheck = routelist[:] + + # If we don't honor minimization, we need all the keys in the + # route path + if not self.minimization: + for part in backcheck: + if isinstance(part, dict): + minkeys.append(part['name']) + return (frozenset(minkeys), backcheck) + + gaps = False + backcheck.reverse() + for part in backcheck: + if not isinstance(part, dict) and part not in self.done_chars: + gaps = True + continue + elif not isinstance(part, dict): + continue + key = part['name'] + if self.defaults.has_key(key) and not gaps: + continue + minkeys.append(key) + gaps = True + return (frozenset(minkeys), backcheck) + + def _defaults(self, routekeys, reserved_keys, kargs): + """Creates default set with values stringified + + Put together our list of defaults, stringify non-None values + and add in our action/id default if they use it and didn't + specify it. + + defaultkeys is a list of the currently assumed default keys + routekeys is a list of the keys found in the route path + reserved_keys is a list of keys that are not + + """ + defaults = {} + # Add in a controller/action default if they don't exist + if 'controller' not in routekeys and 'controller' not in kargs \ + and not self.explicit: + kargs['controller'] = 'content' + if 'action' not in routekeys and 'action' not in kargs \ + and not self.explicit: + kargs['action'] = 'index' + defaultkeys = frozenset([key for key in kargs.keys() \ + if key not in reserved_keys]) + for key in defaultkeys: + if kargs[key] is not None: + defaults[key] = self.make_unicode(kargs[key]) + else: + defaults[key] = None + if 'action' in routekeys and not defaults.has_key('action') \ + and not self.explicit: + defaults['action'] = 'index' + if 'id' in routekeys and not defaults.has_key('id') \ + and not self.explicit: + defaults['id'] = None + newdefaultkeys = frozenset([key for key in defaults.keys() \ + if key not in reserved_keys]) + + return (defaults, newdefaultkeys) + + def makeregexp(self, clist, include_names=True): + """Create a regular expression for matching purposes + + Note: This MUST be called before match can function properly. + + clist should be a list of valid controller strings that can be + matched, for this reason makeregexp should be called by the web + framework after it knows all available controllers that can be + utilized. + + include_names indicates whether this should be a match regexp + assigned to itself using regexp grouping names, or if names + should be excluded for use in a single larger regexp to + determine if any routes match + + """ + if self.minimization: + reg = self.buildnextreg(self.routelist, clist, include_names)[0] + if not reg: + reg = '/' + reg = reg + '/?' + '$' + + if not reg.startswith('/'): + reg = '/' + reg + else: + reg = self.buildfullreg(clist, include_names) + + reg = '^' + reg + + if not include_names: + return reg + + self.regexp = reg + self.regmatch = re.compile(reg) + + def buildfullreg(self, clist, include_names=True): + """Build the regexp by iterating through the routelist and + replacing dicts with the appropriate regexp match""" + regparts = [] + for part in self.routelist: + if isinstance(part, dict): + var = part['name'] + if var == 'controller': + partmatch = '|'.join(map(re.escape, clist)) + elif part['type'] == ':': + partmatch = self.reqs.get(var) or '[^/]+?' + elif part['type'] == '.': + partmatch = self.reqs.get(var) or '[^/.]+?' + else: + partmatch = self.reqs.get(var) or '.+?' + if include_names: + regpart = '(?P<%s>%s)' % (var, partmatch) + else: + regpart = '(?:%s)' % partmatch + if part['type'] == '.': + regparts.append('(?:\.%s)??' % regpart) + else: + regparts.append(regpart) + else: + regparts.append(re.escape(part)) + regexp = ''.join(regparts) + '$' + return regexp + + def buildnextreg(self, path, clist, include_names=True): + """Recursively build our regexp given a path, and a controller + list. + + Returns the regular expression string, and two booleans that + can be ignored as they're only used internally by buildnextreg. + + """ + if path: + part = path[0] + else: + part = '' + reg = '' + + # noreqs will remember whether the remainder has either a string + # match, or a non-defaulted regexp match on a key, allblank remembers + # if the rest could possible be completely empty + (rest, noreqs, allblank) = ('', True, True) + if len(path[1:]) > 0: + self.prior = part + (rest, noreqs, allblank) = self.buildnextreg(path[1:], clist, include_names) + + if isinstance(part, dict) and part['type'] in (':', '.'): + var = part['name'] + typ = part['type'] + partreg = '' + + # First we plug in the proper part matcher + if self.reqs.has_key(var): + if include_names: + partreg = '(?P<%s>%s)' % (var, self.reqs[var]) + else: + partreg = '(?:%s)' % self.reqs[var] + if typ == '.': + partreg = '(?:\.%s)??' % partreg + elif var == 'controller': + if include_names: + partreg = '(?P<%s>%s)' % (var, '|'.join(map(re.escape, clist))) + else: + partreg = '(?:%s)' % '|'.join(map(re.escape, clist)) + elif self.prior in ['/', '#']: + if include_names: + partreg = '(?P<' + var + '>[^' + self.prior + ']+?)' + else: + partreg = '(?:[^' + self.prior + ']+?)' + else: + if not rest: + if typ == '.': + exclude_chars = '/.' + else: + exclude_chars = '/' + if include_names: + partreg = '(?P<%s>[^%s]+?)' % (var, exclude_chars) + else: + partreg = '(?:[^%s]+?)' % exclude_chars + if typ == '.': + partreg = '(?:\.%s)??' % partreg + else: + end = ''.join(self.done_chars) + rem = rest + if rem[0] == '\\' and len(rem) > 1: + rem = rem[1] + elif rem.startswith('(\\') and len(rem) > 2: + rem = rem[2] + else: + rem = end + rem = frozenset(rem) | frozenset(['/']) + if include_names: + partreg = '(?P<%s>[^%s]+?)' % (var, ''.join(rem)) + else: + partreg = '(?:[^%s]+?)' % ''.join(rem) + + if self.reqs.has_key(var): + noreqs = False + if not self.defaults.has_key(var): + allblank = False + noreqs = False + + # Now we determine if its optional, or required. This changes + # depending on what is in the rest of the match. If noreqs is + # true, then its possible the entire thing is optional as there's + # no reqs or string matches. + if noreqs: + # The rest is optional, but now we have an optional with a + # regexp. Wrap to ensure that if we match anything, we match + # our regexp first. It's still possible we could be completely + # blank as we have a default + if self.reqs.has_key(var) and self.defaults.has_key(var): + reg = '(' + partreg + rest + ')?' + + # Or we have a regexp match with no default, so now being + # completely blank form here on out isn't possible + elif self.reqs.has_key(var): + allblank = False + reg = partreg + rest + + # If the character before this is a special char, it has to be + # followed by this + elif self.defaults.has_key(var) and \ + self.prior in (',', ';', '.'): + reg = partreg + rest + + # Or we have a default with no regexp, don't touch the allblank + elif self.defaults.has_key(var): + reg = partreg + '?' + rest + + # Or we have a key with no default, and no reqs. Not possible + # to be all blank from here + else: + allblank = False + reg = partreg + rest + # In this case, we have something dangling that might need to be + # matched + else: + # If they can all be blank, and we have a default here, we know + # its safe to make everything from here optional. Since + # something else in the chain does have req's though, we have + # to make the partreg here required to continue matching + if allblank and self.defaults.has_key(var): + reg = '(' + partreg + rest + ')?' + + # Same as before, but they can't all be blank, so we have to + # require it all to ensure our matches line up right + else: + reg = partreg + rest + elif isinstance(part, dict) and part['type'] == '*': + var = part['name'] + if noreqs: + if include_names: + reg = '(?P<%s>.*)' % var + rest + else: + reg = '(?:.*)' + rest + if not self.defaults.has_key(var): + allblank = False + noreqs = False + else: + if allblank and self.defaults.has_key(var): + if include_names: + reg = '(?P<%s>.*)' % var + rest + else: + reg = '(?:.*)' + rest + elif self.defaults.has_key(var): + if include_names: + reg = '(?P<%s>.*)' % var + rest + else: + reg = '(?:.*)' + rest + else: + if include_names: + reg = '(?P<%s>.*)' % var + rest + else: + reg = '(?:.*)' + rest + allblank = False + noreqs = False + elif part and part[-1] in self.done_chars: + if allblank: + reg = re.escape(part[:-1]) + '(' + re.escape(part[-1]) + rest + reg += ')?' + else: + allblank = False + reg = re.escape(part) + rest + + # We have a normal string here, this is a req, and it prevents us from + # being all blank + else: + noreqs = False + allblank = False + reg = re.escape(part) + rest + + return (reg, noreqs, allblank) + + def match(self, url, environ=None, sub_domains=False, + sub_domains_ignore=None, domain_match=''): + """Match a url to our regexp. + + While the regexp might match, this operation isn't + guaranteed as there's other factors that can cause a match to + fail even though the regexp succeeds (Default that was relied + on wasn't given, requirement regexp doesn't pass, etc.). + + Therefore the calling function shouldn't assume this will + return a valid dict, the other possible return is False if a + match doesn't work out. + + """ + # Static routes don't match, they generate only + if self.static: + return False + + match = self.regmatch.match(url) + + if not match: + return False + + sub_domain = None + + if sub_domains and environ and 'HTTP_HOST' in environ: + host = environ['HTTP_HOST'].split(':')[0] + sub_match = re.compile('^(.+?)\.%s$' % domain_match) + subdomain = re.sub(sub_match, r'\1', host) + if subdomain not in sub_domains_ignore and host != subdomain: + sub_domain = subdomain + + if self.conditions: + if 'method' in self.conditions and environ and \ + environ['REQUEST_METHOD'] not in self.conditions['method']: + return False + + # Check sub-domains? + use_sd = self.conditions.get('sub_domain') + if use_sd and not sub_domain: + return False + elif not use_sd and 'sub_domain' in self.conditions and sub_domain: + return False + if isinstance(use_sd, list) and sub_domain not in use_sd: + return False + + matchdict = match.groupdict() + result = {} + extras = self._default_keys - frozenset(matchdict.keys()) + for key, val in matchdict.iteritems(): + if key != 'path_info' and self.encoding: + # change back into python unicode objects from the URL + # representation + try: + val = val and val.decode(self.encoding, self.decode_errors) + except UnicodeDecodeError: + return False + + if not val and key in self.defaults and self.defaults[key]: + result[key] = self.defaults[key] + else: + result[key] = val + for key in extras: + result[key] = self.defaults[key] + + # Add the sub-domain if there is one + if sub_domains: + result['sub_domain'] = sub_domain + + # If there's a function, call it with environ and expire if it + # returns False + if self.conditions and 'function' in self.conditions and \ + not self.conditions['function'](environ, result): + return False + + return result + + def generate_non_minimized(self, kargs): + """Generate a non-minimal version of the URL""" + # Iterate through the keys that are defaults, and NOT in the route + # path. If its not in kargs, or doesn't match, or is None, this + # route won't work + for k in self.maxkeys - self.minkeys: + if k not in kargs: + return False + elif self.make_unicode(kargs[k]) != \ + self.make_unicode(self.defaults[k]): + return False + + # Ensure that all the args in the route path are present and not None + for arg in self.minkeys: + if arg not in kargs or kargs[arg] is None: + if arg in self.dotkeys: + kargs[arg] = '' + else: + return False + + # Encode all the argument that the regpath can use + for k in kargs: + if k in self.maxkeys: + if k in self.dotkeys: + if kargs[k]: + kargs[k] = url_quote('.' + kargs[k], self.encoding) + else: + kargs[k] = url_quote(kargs[k], self.encoding) + + return self.regpath % kargs + + def generate_minimized(self, kargs): + """Generate a minimized version of the URL""" + routelist = self.routebackwards + urllist = [] + gaps = False + for part in routelist: + if isinstance(part, dict) and part['type'] in (':', '.'): + arg = part['name'] + + # For efficiency, check these just once + has_arg = kargs.has_key(arg) + has_default = self.defaults.has_key(arg) + + # Determine if we can leave this part off + # First check if the default exists and wasn't provided in the + # call (also no gaps) + if has_default and not has_arg and not gaps: + continue + + # Now check to see if there's a default and it matches the + # incoming call arg + if (has_default and has_arg) and self.make_unicode(kargs[arg]) == \ + self.make_unicode(self.defaults[arg]) and not gaps: + continue + + # We need to pull the value to append, if the arg is None and + # we have a default, use that + if has_arg and kargs[arg] is None and has_default and not gaps: + continue + + # Otherwise if we do have an arg, use that + elif has_arg: + val = kargs[arg] + + elif has_default and self.defaults[arg] is not None: + val = self.defaults[arg] + # Optional format parameter? + elif part['type'] == '.': + continue + # No arg at all? This won't work + else: + return False + + urllist.append(url_quote(val, self.encoding)) + if part['type'] == '.': + urllist.append('.') + + if has_arg: + del kargs[arg] + gaps = True + elif isinstance(part, dict) and part['type'] == '*': + arg = part['name'] + kar = kargs.get(arg) + if kar is not None: + urllist.append(url_quote(kar, self.encoding)) + gaps = True + elif part and part[-1] in self.done_chars: + if not gaps and part in self.done_chars: + continue + elif not gaps: + urllist.append(part[:-1]) + gaps = True + else: + gaps = True + urllist.append(part) + else: + gaps = True + urllist.append(part) + urllist.reverse() + url = ''.join(urllist) + return url + + def generate(self, _ignore_req_list=False, _append_slash=False, **kargs): + """Generate a URL from ourself given a set of keyword arguments + + Toss an exception if this + set of keywords would cause a gap in the url. + + """ + # Verify that our args pass any regexp requirements + if not _ignore_req_list: + for key in self.reqs.keys(): + val = kargs.get(key) + if val and not self.req_regs[key].match(self.make_unicode(val)): + return False + + # Verify that if we have a method arg, its in the method accept list. + # Also, method will be changed to _method for route generation + meth = kargs.get('method') + if meth: + if self.conditions and 'method' in self.conditions \ + and meth.upper() not in self.conditions['method']: + return False + kargs.pop('method') + + if self.minimization: + url = self.generate_minimized(kargs) + else: + url = self.generate_non_minimized(kargs) + + if url is False: + return url + + if not url.startswith('/') and not self.static: + url = '/' + url + extras = frozenset(kargs.keys()) - self.maxkeys + if extras: + if _append_slash and not url.endswith('/'): + url += '/' + fragments = [] + # don't assume the 'extras' set preserves order: iterate + # through the ordered kargs instead + for key in kargs: + if key not in extras: + continue + if key == 'action' or key == 'controller': + continue + val = kargs[key] + if isinstance(val, (tuple, list)): + for value in val: + fragments.append((key, _str_encode(value, self.encoding))) + else: + fragments.append((key, _str_encode(val, self.encoding))) + if fragments: + url += '?' + url += urllib.urlencode(fragments) + elif _append_slash and not url.endswith('/'): + url += '/' + return url diff --git a/src/routes/util.py b/src/routes/util.py new file mode 100644 index 0000000000..6c3f845015 --- /dev/null +++ b/src/routes/util.py @@ -0,0 +1,503 @@ +"""Utility functions for use in templates / controllers + +*PLEASE NOTE*: Many of these functions expect an initialized RequestConfig +object. This is expected to have been initialized for EACH REQUEST by the web +framework. + +""" +import os +import re +import urllib +from routes import request_config + + +class RoutesException(Exception): + """Tossed during Route exceptions""" + + +class MatchException(RoutesException): + """Tossed during URL matching exceptions""" + + +class GenerationException(RoutesException): + """Tossed during URL generation exceptions""" + + +def _screenargs(kargs, mapper, environ, force_explicit=False): + """ + Private function that takes a dict, and screens it against the current + request dict to determine what the dict should look like that is used. + This is responsible for the requests "memory" of the current. + """ + # Coerce any unicode args with the encoding + encoding = mapper.encoding + for key, val in kargs.iteritems(): + if isinstance(val, unicode): + kargs[key] = val.encode(encoding) + + if mapper.explicit and mapper.sub_domains and not force_explicit: + return _subdomain_check(kargs, mapper, environ) + elif mapper.explicit and not force_explicit: + return kargs + + controller_name = kargs.get('controller') + + if controller_name and controller_name.startswith('/'): + # If the controller name starts with '/', ignore route memory + kargs['controller'] = kargs['controller'][1:] + return kargs + elif controller_name and not kargs.has_key('action'): + # Fill in an action if we don't have one, but have a controller + kargs['action'] = 'index' + + route_args = environ.get('wsgiorg.routing_args') + if route_args: + memory_kargs = route_args[1].copy() + else: + memory_kargs = {} + + # Remove keys from memory and kargs if kargs has them as None + for key in [key for key in kargs.keys() if kargs[key] is None]: + del kargs[key] + if memory_kargs.has_key(key): + del memory_kargs[key] + + # Merge the new args on top of the memory args + memory_kargs.update(kargs) + + # Setup a sub-domain if applicable + if mapper.sub_domains: + memory_kargs = _subdomain_check(memory_kargs, mapper, environ) + return memory_kargs + + +def _subdomain_check(kargs, mapper, environ): + """Screen the kargs for a subdomain and alter it appropriately depending + on the current subdomain or lack therof.""" + if mapper.sub_domains: + subdomain = kargs.pop('sub_domain', None) + if isinstance(subdomain, unicode): + subdomain = str(subdomain) + + fullhost = environ.get('HTTP_HOST') or environ.get('SERVER_NAME') + + # In case environ defaulted to {} + if not fullhost: + return kargs + + hostmatch = fullhost.split(':') + host = hostmatch[0] + port = '' + if len(hostmatch) > 1: + port += ':' + hostmatch[1] + sub_match = re.compile('^.+?\.(%s)$' % mapper.domain_match) + domain = re.sub(sub_match, r'\1', host) + if subdomain and not host.startswith(subdomain) and \ + subdomain not in mapper.sub_domains_ignore: + kargs['_host'] = subdomain + '.' + domain + port + elif (subdomain in mapper.sub_domains_ignore or \ + subdomain is None) and domain != host: + kargs['_host'] = domain + port + return kargs + else: + return kargs + + +def _url_quote(string, encoding): + """A Unicode handling version of urllib.quote.""" + if encoding: + if isinstance(string, unicode): + s = string.encode(encoding) + elif isinstance(string, str): + # assume the encoding is already correct + s = string + else: + s = unicode(string).encode(encoding) + else: + s = str(string) + return urllib.quote(s, '/') + + +def _str_encode(string, encoding): + if encoding: + if isinstance(string, unicode): + s = string.encode(encoding) + elif isinstance(string, str): + # assume the encoding is already correct + s = string + else: + s = unicode(string).encode(encoding) + return s + + +def url_for(*args, **kargs): + """Generates a URL + + All keys given to url_for are sent to the Routes Mapper instance for + generation except for:: + + anchor specified the anchor name to be appened to the path + host overrides the default (current) host if provided + protocol overrides the default (current) protocol if provided + qualified creates the URL with the host/port information as + needed + + The URL is generated based on the rest of the keys. When generating a new + URL, values will be used from the current request's parameters (if + present). The following rules are used to determine when and how to keep + the current requests parameters: + + * If the controller is present and begins with '/', no defaults are used + * If the controller is changed, action is set to 'index' unless otherwise + specified + + For example, if the current request yielded a dict of + {'controller': 'blog', 'action': 'view', 'id': 2}, with the standard + ':controller/:action/:id' route, you'd get the following results:: + + url_for(id=4) => '/blog/view/4', + url_for(controller='/admin') => '/admin', + url_for(controller='admin') => '/admin/view/2' + url_for(action='edit') => '/blog/edit/2', + url_for(action='list', id=None) => '/blog/list' + + **Static and Named Routes** + + If there is a string present as the first argument, a lookup is done + against the named routes table to see if there's any matching routes. The + keyword defaults used with static routes will be sent in as GET query + arg's if a route matches. + + If no route by that name is found, the string is assumed to be a raw URL. + Should the raw URL begin with ``/`` then appropriate SCRIPT_NAME data will + be added if present, otherwise the string will be used as the url with + keyword args becoming GET query args. + + """ + anchor = kargs.get('anchor') + host = kargs.get('host') + protocol = kargs.get('protocol') + qualified = kargs.pop('qualified', None) + + # Remove special words from kargs, convert placeholders + for key in ['anchor', 'host', 'protocol']: + if kargs.get(key): + del kargs[key] + config = request_config() + route = None + static = False + encoding = config.mapper.encoding + url = '' + if len(args) > 0: + route = config.mapper._routenames.get(args[0]) + + # No named route found, assume the argument is a relative path + if not route: + static = True + url = args[0] + + if url.startswith('/') and hasattr(config, 'environ') \ + and config.environ.get('SCRIPT_NAME'): + url = config.environ.get('SCRIPT_NAME') + url + + if static: + if kargs: + url += '?' + query_args = [] + for key, val in kargs.iteritems(): + if isinstance(val, (list, tuple)): + for value in val: + query_args.append("%s=%s" % ( + urllib.quote(unicode(key).encode(encoding)), + urllib.quote(unicode(value).encode(encoding)))) + else: + query_args.append("%s=%s" % ( + urllib.quote(unicode(key).encode(encoding)), + urllib.quote(unicode(val).encode(encoding)))) + url += '&'.join(query_args) + environ = getattr(config, 'environ', {}) + if 'wsgiorg.routing_args' not in environ: + environ = environ.copy() + mapper_dict = getattr(config, 'mapper_dict', None) + if mapper_dict is not None: + match_dict = mapper_dict.copy() + else: + match_dict = {} + environ['wsgiorg.routing_args'] = ((), match_dict) + + if not static: + route_args = [] + if route: + if config.mapper.hardcode_names: + route_args.append(route) + newargs = route.defaults.copy() + newargs.update(kargs) + + # If this route has a filter, apply it + if route.filter: + newargs = route.filter(newargs) + + if not route.static: + # Handle sub-domains + newargs = _subdomain_check(newargs, config.mapper, environ) + else: + newargs = _screenargs(kargs, config.mapper, environ) + anchor = newargs.pop('_anchor', None) or anchor + host = newargs.pop('_host', None) or host + protocol = newargs.pop('_protocol', None) or protocol + url = config.mapper.generate(*route_args, **newargs) + if anchor is not None: + url += '#' + _url_quote(anchor, encoding) + if host or protocol or qualified: + if not host and not qualified: + # Ensure we don't use a specific port, as changing the protocol + # means that we most likely need a new port + host = config.host.split(':')[0] + elif not host: + host = config.host + if not protocol: + protocol = config.protocol + if url is not None: + url = protocol + '://' + host + url + + if not isinstance(url, str) and url is not None: + raise GenerationException("url_for can only return a string, got " + "unicode instead: %s" % url) + if url is None: + raise GenerationException( + "url_for could not generate URL. Called with args: %s %s" % \ + (args, kargs)) + return url + + +class URLGenerator(object): + """The URL Generator generates URL's + + It is automatically instantiated by the RoutesMiddleware and put + into the ``wsgiorg.routing_args`` tuple accessible as:: + + url = environ['wsgiorg.routing_args'][0][0] + + Or via the ``routes.url`` key:: + + url = environ['routes.url'] + + The url object may be instantiated outside of a web context for use + in testing, however sub_domain support and fully qualified URL's + cannot be generated without supplying a dict that must contain the + key ``HTTP_HOST``. + + """ + def __init__(self, mapper, environ): + """Instantiate the URLGenerator + + ``mapper`` + The mapper object to use when generating routes. + ``environ`` + The environment dict used in WSGI, alternately, any dict + that contains at least an ``HTTP_HOST`` value. + + """ + self.mapper = mapper + if 'SCRIPT_NAME' not in environ: + environ['SCRIPT_NAME'] = '' + self.environ = environ + + def __call__(self, *args, **kargs): + """Generates a URL + + All keys given to url_for are sent to the Routes Mapper instance for + generation except for:: + + anchor specified the anchor name to be appened to the path + host overrides the default (current) host if provided + protocol overrides the default (current) protocol if provided + qualified creates the URL with the host/port information as + needed + + """ + anchor = kargs.get('anchor') + host = kargs.get('host') + protocol = kargs.get('protocol') + qualified = kargs.pop('qualified', None) + + # Remove special words from kargs, convert placeholders + for key in ['anchor', 'host', 'protocol']: + if kargs.get(key): + del kargs[key] + + route = None + use_current = '_use_current' in kargs and kargs.pop('_use_current') + + static = False + encoding = self.mapper.encoding + url = '' + + more_args = len(args) > 0 + if more_args: + route = self.mapper._routenames.get(args[0]) + + if not route and more_args: + static = True + url = args[0] + if url.startswith('/') and self.environ.get('SCRIPT_NAME'): + url = self.environ.get('SCRIPT_NAME') + url + + if static: + if kargs: + url += '?' + query_args = [] + for key, val in kargs.iteritems(): + if isinstance(val, (list, tuple)): + for value in val: + query_args.append("%s=%s" % ( + urllib.quote(unicode(key).encode(encoding)), + urllib.quote(unicode(value).encode(encoding)))) + else: + query_args.append("%s=%s" % ( + urllib.quote(unicode(key).encode(encoding)), + urllib.quote(unicode(val).encode(encoding)))) + url += '&'.join(query_args) + if not static: + route_args = [] + if route: + if self.mapper.hardcode_names: + route_args.append(route) + newargs = route.defaults.copy() + newargs.update(kargs) + + # If this route has a filter, apply it + if route.filter: + newargs = route.filter(newargs) + if not route.static or (route.static and not route.external): + # Handle sub-domains, retain sub_domain if there is one + sub = newargs.get('sub_domain', None) + newargs = _subdomain_check(newargs, self.mapper, + self.environ) + # If the route requires a sub-domain, and we have it, restore + # it + if 'sub_domain' in route.defaults: + newargs['sub_domain'] = sub + + elif use_current: + newargs = _screenargs(kargs, self.mapper, self.environ, force_explicit=True) + elif 'sub_domain' in kargs: + newargs = _subdomain_check(kargs, self.mapper, self.environ) + else: + newargs = kargs + + anchor = anchor or newargs.pop('_anchor', None) + host = host or newargs.pop('_host', None) + protocol = protocol or newargs.pop('_protocol', None) + url = self.mapper.generate(*route_args, **newargs) + if anchor is not None: + url += '#' + _url_quote(anchor, encoding) + if host or protocol or qualified: + if 'routes.cached_hostinfo' not in self.environ: + cache_hostinfo(self.environ) + hostinfo = self.environ['routes.cached_hostinfo'] + + if not host and not qualified: + # Ensure we don't use a specific port, as changing the protocol + # means that we most likely need a new port + host = hostinfo['host'].split(':')[0] + elif not host: + host = hostinfo['host'] + if not protocol: + protocol = hostinfo['protocol'] + if url is not None: + if host[-1] != '/': + host += '/' + url = protocol + '://' + host + url.lstrip('/') + + if not isinstance(url, str) and url is not None: + raise GenerationException("Can only return a string, got " + "unicode instead: %s" % url) + if url is None: + raise GenerationException( + "Could not generate URL. Called with args: %s %s" % \ + (args, kargs)) + return url + + def current(self, *args, **kwargs): + """Generate a route that includes params used on the current + request + + The arguments for this method are identical to ``__call__`` + except that arguments set to None will remove existing route + matches of the same name from the set of arguments used to + construct a URL. + """ + return self(_use_current=True, *args, **kwargs) + + +def redirect_to(*args, **kargs): + """Issues a redirect based on the arguments. + + Redirect's *should* occur as a "302 Moved" header, however the web + framework may utilize a different method. + + All arguments are passed to url_for to retrieve the appropriate URL, then + the resulting URL it sent to the redirect function as the URL. + """ + target = url_for(*args, **kargs) + config = request_config() + return config.redirect(target) + + +def cache_hostinfo(environ): + """Processes the host information and stores a copy + + This work was previously done but wasn't stored in environ, nor is + it guaranteed to be setup in the future (Routes 2 and beyond). + + cache_hostinfo processes environ keys that may be present to + determine the proper host, protocol, and port information to use + when generating routes. + + """ + hostinfo = {} + if environ.get('HTTPS') or environ.get('wsgi.url_scheme') == 'https' \ + or environ.get('HTTP_X_FORWARDED_PROTO') == 'https': + hostinfo['protocol'] = 'https' + else: + hostinfo['protocol'] = 'http' + if environ.get('HTTP_X_FORWARDED_HOST'): + hostinfo['host'] = environ['HTTP_X_FORWARDED_HOST'] + elif environ.get('HTTP_HOST'): + hostinfo['host'] = environ['HTTP_HOST'] + else: + hostinfo['host'] = environ['SERVER_NAME'] + if environ.get('wsgi.url_scheme') == 'https': + if environ['SERVER_PORT'] != '443': + hostinfo['host'] += ':' + environ['SERVER_PORT'] + else: + if environ['SERVER_PORT'] != '80': + hostinfo['host'] += ':' + environ['SERVER_PORT'] + environ['routes.cached_hostinfo'] = hostinfo + return hostinfo + + +def controller_scan(directory=None): + """Scan a directory for python files and use them as controllers""" + if directory is None: + return [] + + def find_controllers(dirname, prefix=''): + """Locate controllers in a directory""" + controllers = [] + for fname in os.listdir(dirname): + filename = os.path.join(dirname, fname) + if os.path.isfile(filename) and \ + re.match('^[^_]{1,1}.*\.py$', fname): + controllers.append(prefix + fname[:-3]) + elif os.path.isdir(filename): + controllers.extend(find_controllers(filename, + prefix=prefix+fname+'/')) + return controllers + def longest_first(fst, lst): + """Compare the length of one string to another, shortest goes first""" + return cmp(len(lst), len(fst)) + controllers = find_controllers(directory) + controllers.sort(longest_first) + return controllers