merge from main branch

resources/content_server/browse/browse.css

@@ -259,4 +259,15 @@ h2.library_name {
 
 /* }}} */
 
+/* Booklist {{{ */
+
+#booklist .page { 
+    display: none; 
+}
+
+.loading img {
+    vertical-align: middle;
+}
+
+/* }}} */
 

resources/recipes/globe_and_mail.recipe

@@ -26,30 +26,11 @@ class GlobeAndMail(BasicNewsRecipe):
     #credit {margin-top:0px;}
     .tag {font-size: 22pt;}'''
     description = 'Canada\'s national newspaper'
-    remove_tags_before = dict(id="article-top")
-    remove_tags = [
-		{'id':['util', 'article-tabs', 'comments', 'article-relations',
-		'gallery-controls', 'video', 'galleryLoading','deck','header',
-        'toolsBottom'] },
-		{'class':['credit','inline-img-caption','tab-pointer'] },
-		dict(name='div', attrs={'id':['lead-photo', 'most-popular-story']}),
-		dict(name='div', attrs={'class':'right'}),
-		dict(name='div', attrs={'id':'footer'}),
-		dict(name='div', attrs={'id':'beta-msg'}),
-		dict(name='img', attrs={'class':'headshot'}),
-		dict(name='div', attrs={'class':'brand'}),
-		dict(name='div', attrs={'id':'nav-wrap'}),
-		dict(name='div', attrs={'id':'featureTopics'}),
-		dict(name='div', attrs={'id':'videoNav'}),
-		dict(name='div', attrs={'id':'blog-header'}),
-		dict(name='div', attrs={'id':'right-rail'}),
-		dict(name='div', attrs={'id':'group-footer-container'}),
-		dict(name=['iframe', 'style'])
-		]
-    remove_attributes = ['style']
-    remove_tags_after = [{'id':['article-content']},
-		{'class':['pull','inline-img'] },
-		dict(name='img', attrs={'class':'inline-media-embed'}),
+    keep_only_tags = [dict(name='article')]
+    remove_tags = [dict(name='aside'),
+                   dict(name='footer'),
+                   dict(name='div', attrs={'class':(lambda x: isinstance(x, (str,unicode)) and 'articlecommentcountholder' in x.split(' '))}),
+                   dict(name='ul', attrs={'class':(lambda x: isinstance(x, (str,unicode)) and 'articletoolbar' in x.split(' '))}),
                   ]
     feeds = [
             (u'Latest headlines', u'http://www.theglobeandmail.com/?service=rss'),

resources/recipes/orsai.recipe

@@ -0,0 +1,37 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+orsai.bitacoras.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Orsai(BasicNewsRecipe):
+    title                 = 'Orsai'
+    __author__            = 'Darko Miletic'
+    language = 'es'
+    oldest_article        = 35
+    max_articles_per_feed = 100
+    encoding              = 'utf-8'
+    no_stylesheets        = True
+    use_embedded_content  = False
+    publication_type      = 'blog'
+    masthead_url          = 'http://orsai.bitacoras.com/wp-content/themes/orsai/images/logo_orsai.png'
+
+    conversion_options = {
+                          'comment'  : 'Blog literario de Hernán Casciari'
+                        , 'tags'     : 'blog, Argentina, España, literatura, Casciari'
+                        , 'publisher': 'Editorial Orsai S.L.'
+                        , 'language' : 'es'
+                        }
+
+    keep_only_tags=[dict(attrs={'class':['entry-title','entry-meta','entry-content','commentlist']})]
+    remove_tags=[dict(name='img',attrs={'class':'avatar avatar-40 photo'})]
+    feeds = [(u'Articulos', u'http://orsai.bitacoras.com/feed')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return self.adeify_images(soup)
+

resources/recipes/volksrant.recipe

@@ -11,6 +11,7 @@ __docformat__ = 'restructuredtext en'
  on 10/10/10 to include function to grab print version of articles
 '''
 
+from datetime import date
 from calibre.web.feeds.news import BasicNewsRecipe
 '''
 added by Tony Stegall
@@ -27,7 +28,6 @@ class AdvancedUserRecipe1249039563(BasicNewsRecipe):
     no_stylesheets = True
     language = 'nl'
 
-
     extra_css      = '''
                         body{font-family:Arial,Helvetica,sans-serif; font-size:small;}
                         h1{font-size:large;}
@@ -43,10 +43,12 @@ class AdvancedUserRecipe1249039563(BasicNewsRecipe):
 
     def get_obfuscated_article(self, url):
         br = self.get_browser()
+        print 'THE CURRENT URL IS: ', url
         br.open(url)
+        year = date.today().year
 
         try:
-         response = br.follow_link(url_regex='.*?(2010)(\\/)(article)(\\/)(print)(\\/)', nr = 0)
+          response = br.follow_link(url_regex='.*?(%d)(\\/)(article)(\\/)(print)(\\/)'%year, nr = 0)
           html = response.read()
         except:
           response = br.open(url)
@@ -59,19 +61,22 @@ class AdvancedUserRecipe1249039563(BasicNewsRecipe):
 
    ###############################################################################################################
 
-    feeds          = [
-                      (u'Laatste Nieuws', u'http://volkskrant.nl/rss/laatstenieuws.rss'),
-                      (u'Binnenlands nieuws', u'http://volkskrant.nl/rss/nederland.rss'),
-                      (u'Buitenlands nieuws', u'http://volkskrant.nl/rss/internationaal.rss'),
-                      (u'Economisch nieuws', u'http://volkskrant.nl/rss/economie.rss'),
-                      (u'Sportnieuws', u'http://volkskrant.nl/rss/sport.rss'),
-                      (u'Kunstnieuws', u'http://volkskrant.nl/rss/kunst.rss'),
+    '''
+      Change Log:
+       Date: 10/15/2010
+       Feeds updated by Martin Tarenskeen
+    '''
+
+    feeds          = [
+                      (u'Laatste Nieuws', u'http://www.volkskrant.nl/rss/laatstenieuws.rss'),
+                      (u'Binnenland', u'http://www.volkskrant.nl/rss/nederland.rss'),
+                      (u'Buitenland', u'http://www.volkskrant.nl/rss/internationaal.rss'),
+                      (u'Economie', u'http://www.volkskrant.nl/rss/economie.rss'),
+                      (u'Sport', u'http://www.volkskrant.nl/rss/sport.rss'),
+                      (u'Cultuur', u'http://www.volkskrant.nl/rss/kunst.rss'),
+                      (u'Gezondheid & Wetenschap', u'http://www.volkskrant.nl/rss/wetenschap.rss'),
+                      (u'Internet & Media', u'http://www.volkskrant.nl/rss/media.rss') ]
 
-                        #both of these rss feeds link back to the main volksrant.nl url a.k.a Broken
-                        #If someone happens to know the correct paths then they can put them in here
-                      #(u'Wetenschapsnieuws', u'http://feeds.feedburner.com/DeVolkskrantWetenschap'),
-                      #(u'Technologienieuws', u'http://feeds.feedburner.com/vkmedia')
-                      ]
 
 '''
 example for formating

src/calibre/library/server/browse.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 
 import operator, os, json
 from urllib import quote
-from binascii import hexlify
+from binascii import hexlify, unhexlify
 
 import cherrypy
 
@@ -15,64 +15,33 @@ from calibre.constants import filesystem_encoding
 from calibre import isbytestring, force_unicode, prepare_string_for_xml as xml
 from calibre.utils.ordered_dict import OrderedDict
 
-def paginate(offsets, content, base_url, up_url=None): # {{{
-    'Create markup for pagination'
-
-    if '?' not in base_url:
-        base_url += '?'
-
-    if base_url[-1] != '?':
-        base_url += '&'
-
-    def navlink(decoration, name, cls, offset):
-        label = xml(name)
-        if cls in ('next', 'last'):
-            label += ' ' + decoration
-        else:
-            label = decoration + ' ' + label
-        return (u'<a class="{cls}" href="{base_url}&amp;offset={offset}" title={name}>'
-                u'{label}</a>').format(cls=cls, decoration=decoration,
-                        name=xml(name, True), offset=offset,
-                        base_url=xml(base_url, True), label=label)
-    left = ''
-    if offsets.offset > 0 and offsets.previous_offset > 0:
-        left += navlink(u'\u219e', _('First'), 'first', 0)
-    if offsets.offset > 0:
-        left += ' ' + navlink('&larr;', _('Previous'), 'previous',
-                offsets.previous_offset)
-
-    middle = ''
-    if up_url:
-        middle = '<a href="{0}" title="{1}">[{1} &uarr;]</a>'.format(xml(up_url, True),
-                xml(_('Up')))
-
-    right = ''
-    if offsets.next_offset > -1:
-        right += navlink('&rarr', _('Next'), 'next', offsets.next_offset)
-    if offsets.last_offset > offsets.next_offset and offsets.last_offset > 0:
-        right += ' ' + navlink(u'\u21A0', _('Last'), 'last', offsets.last_offset)
-
-    navbar = u'''
-            <table class="navbar">
-                <tr>
-                    <td class="left">{left}</td>
-                    <td class="middle">{middle}</td>
-                    <td class="right">{right}</td>
-                </tr>
-            <table>
-    '''.format(left=left, right=right, middle=middle)
-
-    templ = u'''
-        <div class="page">
-            {navbar}
-            <div class="page-contents">
-            {content}
-            </div>
-            {navbar}
+def render_book_list(ids):
+    pages = []
+    while ids:
+        page = list(ids[:25])
+        pages.append(page)
+        ids = ids[25:]
+    page_template = u'''\
+            <div class="page" id="page{0}">
+                <div class="load_data" title="{1}"></div>
+                <div class="loading"><img src="/static/loading.gif" /> {2}</div>
+                <div class="loaded"></div>
             </div>
             '''
-    return templ.format(navbar=navbar, content=content)
-# }}}
+    rpages = []
+    for i, pg in enumerate(pages):
+        ld = xml(json.dumps(pg), True)
+        rpages.append(page_template.format(i, ld,
+            xml(_('Loading, please wait')) + '&hellip;'))
+    rpages = u'\n\n'.join(rpages)
+
+    templ = u'''\
+            <h3>{0}</h3>
+            <div id="booklist">
+                {pages}
+            </div>
+            '''
+    return templ.format(_('Browsing %d books')%len(ids), pages=rpages)
 
 def utf8(x): # {{{
     if isinstance(x, unicode):
@@ -171,10 +140,15 @@ class BrowseServer(object):
         connect('browse_category_group',
                 base_href+'/category_group/{category}/{group}',
                 self.browse_category_group)
-        connect('browse_list', base_href+'/list/{query}', self.browse_list)
+        connect('browse_matches',
+                base_href+'/matches/{category}/{cid}',
+                self.browse_matches)
+        connect('browse_booklist_page',
+                base_href+'/booklist_page',
+                self.browse_booklist_page)
+
         connect('browse_search', base_href+'/search/{query}',
                 self.browse_search)
-        connect('browse_book', base_href+'/book/{uuid}', self.browse_book)
 
     def browse_template(self, sort, category=True):
 
@@ -267,12 +241,12 @@ class BrowseServer(object):
 
     def browse_category(self, category, sort):
         categories = self.categories_cache()
+        if category not in categories:
+            raise cherrypy.HTTPError(404, 'category not found')
         category_meta = self.db.field_metadata
         category_name = category_meta[category]['name']
         datatype = category_meta[category]['datatype']
 
-        if category not in categories:
-            raise cherrypy.HTTPError(404, 'category not found')
 
         items = categories[category]
         sort = self.browse_sort_categories(items, sort)
@@ -331,11 +305,12 @@ class BrowseServer(object):
         if sort not in ('rating', 'name', 'popularity'):
             sort = 'name'
         categories = self.categories_cache()
+        if category not in categories:
+            raise cherrypy.HTTPError(404, 'category not found')
+
         category_meta = self.db.field_metadata
         datatype = category_meta[category]['datatype']
 
-        if category not in categories:
-            raise cherrypy.HTTPError(404, 'category not found')
         if not group:
             raise cherrypy.HTTPError(404, 'invalid group')
 
@@ -360,6 +335,8 @@ class BrowseServer(object):
         'Entry point for top-level, categories and sub-categories'
         if category == None:
             ans = self.browse_toplevel()
+        elif category == 'newest':
+            raise cherrypy.InternalRedirect('/browse/matches/newest/dummy')
         else:
             ans = self.browse_category(category, category_sort)
 
@@ -368,8 +345,63 @@ class BrowseServer(object):
     # }}}
 
     # Book Lists {{{
-    def browse_list(self, query=None, offset=0, sort=None):
-        raise NotImplementedError()
+
+    def browse_sort_book_list(self, items, sort):
+        fm = self.db.field_metadata
+        keys = frozenset(fm.sortable_field_keys())
+        if sort not in keys:
+            sort = 'title'
+        self.sort(items, 'title', True)
+        if sort != 'title':
+            ascending = fm[sort]['datatype'] not in ('rating', 'datetime')
+            self.sort(items, sort, ascending)
+        return sort
+
+    @Endpoint(sort_type='list')
+    def browse_matches(self, category=None, cid=None, list_sort=None):
+        if not cid:
+            raise cherrypy.HTTPError(404, 'invalid category id: %r'%cid)
+        categories = self.categories_cache()
+
+        if category not in categories and category != 'newest':
+            raise cherrypy.HTTPError(404, 'category not found')
+        try:
+            category_name = self.db.field_metadata[category]['name']
+        except:
+            if category != 'newest':
+                raise
+            category_name = _('Newest')
+
+        if category == 'search':
+            which = unhexlify(cid)
+            try:
+                ids = self.search_cache('search:"%s"'%which)
+            except:
+                raise cherrypy.HTTPError(404, 'Search: %r not understood'%which)
+        elif category == 'newest':
+            ids = list(self.db.data.iterallids())
+        else:
+            ids = self.db.get_books_for_category(category, cid)
+
+        items = [self.db.data._data[x] for x in ids]
+        if category == 'newest':
+            list_sort = 'timestamp'
+        sort = self.browse_sort_book_list(items, list_sort)
+        ids = [x[0] for x in items]
+        html = render_book_list(ids)
+        return self.browse_template(sort).format(
+                title=_('Books in') + " " +category_name,
+                script='booklist();', main=html)
+
+    @Endpoint(mimetype='application/json; charset=utf-8', sort_type='list')
+    def browse_booklist_page(self, ids=None, list_sort=None):
+        if ids is None:
+            ids = json.dumps('[]')
+        try:
+            ids = json.loads(ids)
+        except:
+            raise cherrypy.HTTPError(404, 'invalid ids')
+
     # }}}
 
     # Search {{{

src/calibre/library/server/content.py

@@ -5,18 +5,15 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re, os, cStringIO
+import re, os
 
 import cherrypy
-try:
-    from PIL import Image as PILImage
-    PILImage
-except ImportError:
-    import Image as PILImage
 
 from calibre import fit_image, guess_type
 from calibre.utils.date import fromtimestamp
 from calibre.library.caches import SortKeyGenerator
+from calibre.utils.magick.draw import save_cover_data_to, Image, \
+        thumbnail as generate_thumbnail
 
 class CSSortKeyGenerator(SortKeyGenerator):
 
@@ -77,8 +74,13 @@ class ContentServer(object):
             id = int(match.group())
         if not self.db.has_id(id):
             raise cherrypy.HTTPError(400, 'id:%d does not exist in database'%id)
-        if what == 'thumb':
-            return self.get_cover(id, thumbnail=True)
+        if what == 'thumb' or what.startswith('thumb_'):
+            try:
+                width, height = map(int, what.split('_')[1:])
+            except:
+                width, height = 60, 80
+            return self.get_cover(id, thumbnail=True, thumb_width=width,
+                    thumb_height=height)
         if what == 'cover':
             return self.get_cover(id)
         return self.get_format(id, what)
@@ -128,37 +130,39 @@ class ContentServer(object):
         return self.static('index.html')
 
     # Actually get content from the database {{{
-    def get_cover(self, id, thumbnail=False):
-        cover = self.db.cover(id, index_is_id=True, as_file=False)
-        if cover is None:
-            cover = self.default_cover
+    def get_cover(self, id, thumbnail=False, thumb_width=60, thumb_height=80):
+        try:
             cherrypy.response.headers['Content-Type'] = 'image/jpeg'
             cherrypy.response.timeout = 3600
-        path = getattr(cover, 'name', False)
-        updated = fromtimestamp(os.stat(path).st_mtime) if path and \
-            os.access(path, os.R_OK) else self.build_time
+            cover = self.db.cover(id, index_is_id=True, as_file=True)
+            if cover is None:
+                cover = self.default_cover
+                updated = self.build_time
+            else:
+                with cover as f:
+                    updated = fromtimestamp(os.stat(f.name).st_mtime)
+                    cover = f.read()
             cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
-        try:
-            f = cStringIO.StringIO(cover)
-            try:
-                im = PILImage.open(f)
-            except IOError:
-                raise cherrypy.HTTPError(404, 'No valid cover found')
-            width, height = im.size
+
+            if thumbnail:
+                return generate_thumbnail(cover,
+                        width=thumb_width, height=thumb_height)[-1]
+
+            img = Image()
+            img.load(cover)
+            width, height = img.size
             scaled, width, height = fit_image(width, height,
-                60 if thumbnail else self.max_cover_width,
-                80 if thumbnail else self.max_cover_height)
+                thumb_width if thumbnail else self.max_cover_width,
+                thumb_height if thumbnail else self.max_cover_height)
             if not scaled:
                 return cover
-            im = im.resize((int(width), int(height)), PILImage.ANTIALIAS)
-            of = cStringIO.StringIO()
-            im.convert('RGB').save(of, 'JPEG')
-            return of.getvalue()
+            return save_cover_data_to(img, 'img.jpg', return_data=True,
+                    resize_to=(width, height))
         except Exception, err:
             import traceback
             cherrypy.log.error('Failed to generate cover:')
             cherrypy.log.error(traceback.print_exc())
-            raise cherrypy.HTTPError(404, 'Failed to generate cover: %s'%err)
+            raise cherrypy.HTTPError(404, 'Failed to generate cover: %r'%err)
 
     def get_format(self, id, format):
         format = format.upper()

src/calibre/manual/faq.rst

@@ -387,6 +387,12 @@ solve it, look for a corrupted font file on your system, in ~/Library/Fonts or t
 check for corrupted fonts in OS X is to start the "Font Book" application, select all fonts and then in the File
 menu, choose "Validate fonts".
 
+
+I downloaded the installer, but it is not working?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Downloading from the internet can sometimes result in a corrupted download. If the |app| installer you downloaded is not opening, try downloading it again. If re-downloading it does not work, download it from `an alternate location <http://sourceforge.net/projects/calibre/files/>`_. If the installer still doesn't work, then something on your computer is preventing it from running. Best place to ask for more help is in the `forums <http://www.mobileread.com/forums/usercp.php>`_.
+
 My antivirus program claims |app| is a virus/trojan?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

src/calibre/utils/magick/draw.py

@@ -25,6 +25,7 @@ def save_cover_data_to(data, path, bgcolor='#ffffff', resize_to=None,
     resize and the input and output image formats are the same, no changes are
     made.
 
+    :param data: Image data as bytestring or Image object
     :param compression_quality: The quality of the image after compression.
         Number between 1 and 100. 1 means highest compression, 100 means no
         compression (lossless).
@@ -33,6 +34,9 @@ def save_cover_data_to(data, path, bgcolor='#ffffff', resize_to=None,
 
     '''
     changed = False
+    if isinstance(data, Image):
+        img = data
+    else:
         img = Image()
         img.load(data)
     orig_fmt = normalize_format_name(img.format)