KG updates

resources/recipes/nytimes.recipe

@@ -10,7 +10,7 @@ import time
 from calibre import entity_to_unicode
 from calibre.web.feeds.recipes import BasicNewsRecipe
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString, \
-    Comment, BeautifulStoneSoup
+Comment, BeautifulStoneSoup
 
 class NYTimes(BasicNewsRecipe):
 
@@ -86,6 +86,7 @@ class NYTimes(BasicNewsRecipe):
                             'relatedSearchesModule',
                             'side_tool',
                             'singleAd',
+                            'subNavigation tabContent active',
                             'subNavigation tabContent active clearfix',
                             ]}),
                    dict(id=[
@@ -94,6 +95,7 @@ class NYTimes(BasicNewsRecipe):
                             'articleExtras',
                             'articleInline',
                             'blog_sidebar',
+                            'businessSearchBar',
                             'cCol',
                             'entertainmentSearchBar',
                             'footer',
@@ -101,6 +103,7 @@ class NYTimes(BasicNewsRecipe):
                             'header_search',
                             'login',
                             'masthead',
+                            'masthead-nav',
                             'memberTools',
                             'navigation',
                             'portfolioInline',

resources/recipes/nytimes_sub.recipe

@@ -74,6 +74,7 @@ class NYTimes(BasicNewsRecipe):
                             'relatedSearchesModule',
                             'side_tool',
                             'singleAd',
+                            'subNavigation tabContent active',
                             'subNavigation tabContent active clearfix',
                             ]}),
                    dict(id=[

src/calibre/devices/android/driver.py

@@ -44,6 +44,7 @@ class ANDROID(USBMS):
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER', 'GT-I5700']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD']
+    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE']
 
     OSX_MAIN_MEM = 'HTC Android Phone Media'
 

src/calibre/devices/apple/driver.py

@@ -54,15 +54,14 @@ class ITUNES(DevicePlugin):
     driver_version = '0.2'
 
     OPEN_FEEDBACK_MESSAGE = _(
-        'Apple device detected, launching iTunes, please wait...')
+        'Apple device detected, launching iTunes, please wait ...')
 
     FORMATS = ['epub']
 
-    VENDOR_ID = [0x05ac]
     # Product IDs:
-    #  0x129a:iPad
     #  0x1292:iPhone 3G
-    #PRODUCT_ID = [0x129a,0x1292]
+    #  0x129a:iPad
+    VENDOR_ID = [0x05ac]
     PRODUCT_ID = [0x129a]
     BCD = [0x01]
 

src/calibre/devices/binatone/driver.py

@@ -42,3 +42,13 @@ class README(USBMS):
         drives[0] = drives[1]
         drives[1] = t
         return tuple(drives)
+
+    def windows_sort_drives(self, drives):
+        if len(drives) < 2: return drives
+        main = drives.get('main', None)
+        carda = drives.get('carda', None)
+        if main and carda:
+            drives['main'] = carda
+            drives['carda'] = main
+        return drives
+

src/calibre/devices/hanlin/driver.py

@@ -71,6 +71,18 @@ class HANLINV3(USBMS):
         drives[1] = t
         return tuple(drives)
 
+    def windows_sort_drives(self, drives):
+        if len(drives) < 2: return drives
+        main = drives.get('main', None)
+        carda = drives.get('carda', None)
+        if main and carda:
+            drives['main'] = carda
+            drives['carda'] = main
+        return drives
+
+
+
+
 
 class HANLINV5(HANLINV3):
     name           = 'Hanlin V5 driver'

src/calibre/ebooks/chm/reader.py

@@ -132,6 +132,9 @@ class CHMReader(CHMFile):
             lpath = os.path.join(output_dir, path)
             self._ensure_dir(lpath)
             data = self.GetFile(path)
+            if lpath.find(';') != -1:
+                # fix file names with ";<junk>" at the end, see _reformat()
+                lpath = lpath.split(';')[0]
             with open(lpath, 'wb') as f:
                 if guess_mimetype(path)[0] == ('text/html'):
                     data = self._reformat(data)
@@ -158,14 +161,26 @@ class CHMReader(CHMFile):
         # cos they really fuck with the flow of things and generally waste space
         # since we can't use [a,b] syntax to select arbitrary items from a list
         # we'll have to do this manually...
+        # only remove the tables, if they have an image with an alt attribute
+        # containing prev, next or team
         t = soup('table')
         if t:
             if (t[0].previousSibling is None
               or t[0].previousSibling.previousSibling is None):
-                t[0].extract()
+                try:
+                    alt = t[0].img['alt'].lower()
+                    if alt.find('prev') != -1 or alt.find('next') != -1 or alt.find('team') != -1:
+                        t[0].extract()
+                except:
+                    pass
             if (t[-1].nextSibling is None
               or t[-1].nextSibling.nextSibling is None):
-                t[-1].extract()
+                try:
+                    alt = t[-1].img['alt'].lower()
+                    if alt.find('prev') != -1 or alt.find('next') != -1 or alt.find('team') != -1:
+                        t[-1].extract()
+                except:
+                    pass
         # for some very odd reason each page's content appears to be in a table
         # too. and this table has sub-tables for random asides... grr.
 
@@ -185,8 +200,24 @@ class CHMReader(CHMFile):
             except KeyError:
                 # and some don't even have a src= ?!
                 pass
-        # now give back some pretty html.
-        return soup.prettify('utf-8')
+        try:
+            # if there is only a single table with a single element
+            # in the body, replace it by the contents of this single element
+            tables = soup.body.findAll('table', recursive=False)
+            if tables and len(tables) == 1:
+                trs = tables[0].findAll('tr', recursive=False)
+                if trs and len(trs) == 1:
+                    tds = trs[0].findAll('td', recursive=False)
+                    if tds and len(tds) == 1:
+                        tdContents = tds[0].contents
+                        tableIdx = soup.body.contents.index(tables[0])
+                        tables[0].extract()
+                        while tdContents:
+                            soup.body.insert(tableIdx, tdContents.pop())
+        except:
+            pass
+        # do not prettify, it would reformat the <pre> tags!
+        return str(soup)
 
     def Contents(self):
         if self._contents is not None:

src/calibre/ebooks/oeb/transforms/cover.py

@@ -15,7 +15,7 @@ try:
 except ImportError:
     import Image as PILImage
 
-from calibre import __appname__, __version__, guess_type
+from calibre import guess_type
 
 class CoverManager(object):
 
@@ -89,7 +89,6 @@ class CoverManager(object):
         '''
         Create a generic cover for books that dont have a cover
         '''
-        from calibre.utils.pil_draw import draw_centered_text
         from calibre.ebooks.metadata import authors_to_string
         if self.no_default_cover:
             return None
@@ -98,46 +97,15 @@ class CoverManager(object):
         title = unicode(m.title[0])
         authors = [unicode(x) for x in m.creator if x.role == 'aut']
 
-        cover_file = cStringIO.StringIO()
         try:
-            try:
-                from PIL import Image, ImageDraw, ImageFont
-                Image, ImageDraw, ImageFont
-            except ImportError:
-                import Image, ImageDraw, ImageFont
-            font_path = P('fonts/liberation/LiberationSerif-Bold.ttf')
-            app = '['+__appname__ +' '+__version__+']'
-
-            COVER_WIDTH, COVER_HEIGHT = 590, 750
-            img = Image.new('RGB', (COVER_WIDTH, COVER_HEIGHT), 'white')
-            draw = ImageDraw.Draw(img)
-            # Title
-            font = ImageFont.truetype(font_path, 44)
-            bottom = draw_centered_text(img, draw, font, title, 15, ysep=9)
-            # Authors
-            bottom += 14
-            font = ImageFont.truetype(font_path, 32)
-            authors = authors_to_string(authors)
-            bottom = draw_centered_text(img, draw, font, authors, bottom, ysep=7)
-            # Vanity
-            font = ImageFont.truetype(font_path, 28)
-            width, height = draw.textsize(app, font=font)
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            top = COVER_HEIGHT - height - 15
-            draw.text((left, top), app, fill=(0,0,0), font=font)
-            # Logo
-            logo = Image.open(I('library.png'), 'r')
-            width, height = logo.size
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            top = max(int((COVER_HEIGHT - height)/2.), 0)
-            img.paste(logo, (left, max(bottom, top)))
-            img = img.convert('RGB').convert('P', palette=Image.ADAPTIVE)
-
-            img.convert('RGB').save(cover_file, 'JPEG')
-            cover_file.flush()
-            id, href = self.oeb.manifest.generate('cover_image', 'cover_image.jpg')
-            item = self.oeb.manifest.add(id, href, guess_type('t.jpg')[0],
-                        data=cover_file.getvalue())
+            from calibre.utils.magick_draw import create_cover_page, TextLine
+            lines = [TextLine(title, 44), TextLine(authors_to_string(authors),
+                32)]
+            img_data = create_cover_page(lines, I('library.png'))
+            id, href = self.oeb.manifest.generate('cover_image',
+                    'cover_image.png')
+            item = self.oeb.manifest.add(id, href, guess_type('t.png')[0],
+                        data=img_data)
             m.clear('cover')
             m.add('cover', item.id)
 

src/calibre/ebooks/rtf/rtfml.py

@@ -19,6 +19,8 @@ except ImportError:
 
 import cStringIO
 
+from lxml import etree
+
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace, \
     OEB_RASTER_IMAGES
 from calibre.ebooks.oeb.stylizer import Stylizer
@@ -118,13 +120,23 @@ class RTFMLizer(object):
         for item in self.oeb_book.spine:
             self.log.debug('Converting %s to RTF markup...' % item.href)
             stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts, self.opts.output_profile)
-            output += self.dump_text(item.data.find(XHTML('body')), stylizer)
+            content = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
+            content = self.remove_newlines(content)
+            output += self.dump_text(etree.fromstring(content), stylizer)
         output += self.footer()
         output = self.insert_images(output)
         output = self.clean_text(output)
 
         return output
 
+    def remove_newlines(self, text):
+        self.log.debug('\tRemove newlines for processing...')
+        text = text.replace('\r\n', ' ')
+        text = text.replace('\n', ' ')
+        text = text.replace('\r', ' ')
+
+        return text
+
     def header(self):
         return u'{\\rtf1{\\info{\\title %s}{\\author %s}}\\ansi\\ansicpg1252\\deff0\\deflang1033' % (self.oeb_book.metadata.title[0].value, authors_to_string([x.value for x in self.oeb_book.metadata.creator]))
 

src/calibre/ebooks/txt/input.py

@@ -8,7 +8,8 @@ import os
 
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
 from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \
-    separate_paragraphs_single_line, separate_paragraphs_print_formatted
+    separate_paragraphs_single_line, separate_paragraphs_print_formatted, \
+    preserve_spaces
 
 class TXTInput(InputFormatPlugin):
 
@@ -28,6 +29,9 @@ class TXTInput(InputFormatPlugin):
                 'an indent (either a tab or 2+ spaces) represents a paragraph. '
                 'Paragraphs end when the next line that starts with an indent '
                 'is reached.')),
+        OptionRecommendation(name='preserve_spaces', recommended_value=False,
+            help=_('Normally extra spaces are condensed into a single space. '
+                'With this option all spaces will be displayed.')),
         OptionRecommendation(name='markdown', recommended_value=False,
             help=_('Run the text input through the markdown pre-processor. To '
                 'learn more about markdown see')+' http://daringfireball.net/projects/markdown/'),
@@ -48,6 +52,8 @@ class TXTInput(InputFormatPlugin):
             txt = separate_paragraphs_single_line(txt)
         if options.print_formatted_paras:
             txt = separate_paragraphs_print_formatted(txt)
+        if options.preserve_spaces:
+            txt = preserve_spaces(txt)
 
         if options.markdown:
             log.debug('Running text though markdown conversion...')

src/calibre/ebooks/txt/processor.py

@@ -24,6 +24,9 @@ def convert_basic(txt, title=''):
     for line in txt.splitlines():
         lines.append(line.strip())
     txt = '\n'.join(lines)
+    
+    # Condense redundant spaces
+    txt = re.sub('[ ]{2,}', ' ', txt)
 
     # Remove blank lines from the beginning and end of the document.
     txt = re.sub('^\s+(?=.)', '', txt)
@@ -56,6 +59,11 @@ def separate_paragraphs_print_formatted(txt):
     txt = re.sub('(?miu)^(\t+|[ ]{2,})(?=.)', '\n\t', txt)
     return txt
 
+def preserve_spaces(txt):
+    txt = txt.replace(' ', ' ')
+    txt = txt.replace('\t', '	')
+    return txt
+
 def opf_writer(path, opf_name, manifest, spine, mi):
     opf = OPFCreator(path, mi)
     opf.create_manifest(manifest)

src/calibre/gui2/convert/txt_input.py

@@ -14,6 +14,7 @@ class PluginWidget(Widget, Ui_Form):
 
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
         Widget.__init__(self, parent, 'txt_input',
-            ['single_line_paras', 'print_formatted_paras', 'markdown', 'markdown_disable_toc'])
+            ['single_line_paras', 'print_formatted_paras', 'markdown',
+                'markdown_disable_toc', 'preserve_spaces'])
         self.db, self.book_id = db, book_id
         self.initialize_options(get_option, get_help, db, book_id)

src/calibre/gui2/convert/txt_input.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>400</width>
+    <width>470</width>
     <height>300</height>
    </rect>
   </property>
@@ -52,7 +52,7 @@
      </property>
     </widget>
    </item>
-   <item row="5" column="0">
+   <item row="6" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -65,10 +65,17 @@
      </property>
     </spacer>
    </item>
+   <item row="5" column="0">
+    <widget class="QCheckBox" name="opt_preserve_spaces">
+     <property name="text">
+      <string>Preserve &amp;spaces</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <resources/>
-<connections>
+ <connections>
   <connection>
    <sender>opt_markdown</sender>
    <signal>toggled(bool)</signal>

src/calibre/gui2/dialogs/config/__init__.py

@@ -486,7 +486,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
         if port < 1025:
             warning_dialog(self, _('System port selected'), '<p>'+
                     _('The value <b>%d</b> you have chosen for the content '
-                        'server port is a system port. You operating '
+                        'server port is a system port. Your operating '
                         'system <b>may</b> not allow the server to run on this '
                         'port. To be safe choose a port number larger than '
                         '1024.')%port, show=True)

src/calibre/gui2/search_box.py

@@ -297,9 +297,11 @@ class SavedSearchBox(QComboBox):
         if idx < 0:
             return
         ss = self.saved_searches.lookup(unicode(self.currentText()))
+        if ss is None:
+            return
         self.saved_searches.delete(unicode(self.currentText()))
         self.clear_to_help()
-        self.search_box.set_search_string(ss)
+        self.search_box.clear_to_help()
         self.emit(SIGNAL('changed()'))
 
     # SIGNALed from the main UI

src/calibre/library/database2.py

@@ -657,6 +657,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def get_recipe(self, id):
         return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False)
 
+    def get_books_for_category(self, category, id_):
+        ans = set([])
+
+        if category not in self.field_metadata:
+            return ans
+
+        field = self.field_metadata[category]
+        ans = self.conn.get(
+                'SELECT book FROM books_{tn}_link WHERE {col}=?'.format(
+                    tn=field['table'], col=field['link_column']), (id_,))
+        return set(x[0] for x in ans)
+
     def get_categories(self, sort_on_count=False, ids=None, icon_map=None):
         self.books_list_filter.change([] if not ids else ids)
 

src/calibre/library/field_metadata.py

@@ -374,6 +374,7 @@ class FieldMetadata(dict):
                              'search_terms':[key], 'label':label,
                              'colnum':colnum,      'display':display,
                              'is_custom':True,     'is_category':is_category,
+                             'link_column':'value',
                              'is_editable': is_editable,}
         self._add_search_terms_to_map(key, [key])
         self.custom_label_to_key_map[label] = key

src/calibre/library/server/cache.py

@@ -15,23 +15,27 @@ class Cache(object):
         self._search_cache = OrderedDict()
 
     def search_cache(self, search):
-        old = self._search_cache.get(search, None)
+        old = self._search_cache.pop(search, None)
         if old is None or old[0] <= self.db.last_modified():
             matches = self.db.data.search(search, return_matches=True,
                     ignore_search_restriction=True)
             if not matches:
                 matches = []
             self._search_cache[search] = (utcnow(), frozenset(matches))
-            if len(self._search_cache) > 10:
+            if len(self._search_cache) > 50:
                 self._search_cache.popitem(last=False)
+        else:
+            self._search_cache[search] = old
         return self._search_cache[search][1]
 
 
     def categories_cache(self, restrict_to=frozenset([])):
-        old = self._category_cache.get(frozenset(restrict_to), None)
+        old = self._category_cache.pop(frozenset(restrict_to), None)
         if old is None or old[0] <= self.db.last_modified():
             categories = self.db.get_categories(ids=restrict_to)
             self._category_cache[restrict_to] = (utcnow(), categories)
-            if len(self._category_cache) > 10:
+            if len(self._category_cache) > 20:
                 self._category_cache.popitem(last=False)
+        else:
+            self._category_cache[frozenset(restrict_to)] = old
         return self._category_cache[restrict_to][1]

src/calibre/library/server/opds.py

@@ -18,6 +18,7 @@ from calibre.constants import __appname__
 from calibre.ebooks.metadata import fmt_sidx
 from calibre.library.comments import comments_to_html
 from calibre import guess_type
+from calibre.utils.ordered_dict import OrderedDict
 
 BASE_HREFS = {
         0 : '/stanza',
@@ -31,6 +32,14 @@ def url_for(name, version, **kwargs):
         name += '_'
     return routes.url_for(name+str(version), **kwargs)
 
+def hexlify(x):
+    if isinstance(x, unicode):
+        x = x.encode('utf-8')
+    return binascii.hexlify(x)
+
+def unhexlify(x):
+    return binascii.unhexlify(x).decode('utf-8')
+
 # Vocabulary for building OPDS feeds {{{
 E = ElementMaker(namespace='http://www.w3.org/2005/Atom',
                  nsmap={
@@ -66,7 +75,7 @@ def AUTHOR(name, uri=None):
 SUBTITLE = E.subtitle
 
 def NAVCATALOG_ENTRY(base_href, updated, title, description, query, version=0):
-    href = base_href+'/navcatalog/'+binascii.hexlify(query)
+    href = base_href+'/navcatalog/'+hexlify(query)
     id_ = 'calibre-navcatalog:'+str(hashlib.sha1(href).hexdigest())
     return E.entry(
         TITLE(title),
@@ -90,6 +99,32 @@ def html_to_lxml(raw):
     raw = etree.tostring(root, encoding=None)
     return etree.fromstring(raw)
 
+def CATALOG_ENTRY(item, base_href, version, updated):
+    id_ = 'calibre:category:'+item.name
+    iid = 'N' + item.name
+    if item.id is not None:
+        iid = 'I' + str(item.id)
+    link = NAVLINK(href = base_href + '/' + hexlify(iid))
+    return E.entry(
+            TITLE(item.name),
+            ID(id_),
+            UPDATED(updated),
+            E.content(_('%d books')%item.count, type='text'),
+            link
+            )
+
+def CATALOG_GROUP_ENTRY(item, category, base_href, version, updated):
+    id_ = 'calibre:category-group:'+category+':'+item.text
+    iid = item.text
+    link = NAVLINK(href = base_href + '/' + hexlify(iid))
+    return E.entry(
+            TITLE(item.text),
+            ID(id_),
+            UPDATED(updated),
+            E.content(_('%d books')%item.count, type='text'),
+            link
+            )
+
 def ACQUISITION_ENTRY(item, version, FM, updated):
     title = item[FM['title']]
     if not title:
@@ -225,6 +260,22 @@ class AcquisitionFeed(NavFeed):
         for item in items:
             self.root.append(ACQUISITION_ENTRY(item, version, FM, updated))
 
+class CategoryFeed(NavFeed):
+
+    def __init__(self, items, which, id_, updated, version, offsets, page_url, up_url):
+        NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url)
+        base_href = self.base_href + '/category/' + hexlify(which)
+        for item in items:
+            self.root.append(CATALOG_ENTRY(item, base_href, version, updated))
+
+class CategoryGroupFeed(NavFeed):
+
+    def __init__(self, items, which, id_, updated, version, offsets, page_url, up_url):
+        NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url)
+        base_href = self.base_href + '/categorygroup/' + hexlify(which)
+        for item in items:
+            self.root.append(CATALOG_GROUP_ENTRY(item, which, base_href, version, updated))
+
 
 class OPDSOffsets(object):
 
@@ -254,8 +305,13 @@ class OPDSServer(object):
             base_href = BASE_HREFS[version]
             ver = str(version)
             connect('opds_'+ver, base_href, self.opds, version=version)
+            connect('opdst_'+ver, base_href+'/', self.opds, version=version)
             connect('opdsnavcatalog_'+ver, base_href+'/navcatalog/{which}',
                     self.opds_navcatalog, version=version)
+            connect('opdscategory_'+ver, base_href+'/category/{category}/{which}',
+                    self.opds_category, version=version)
+            connect('opdscategorygroup_'+ver, base_href+'/categorygroup/{category}/{which}',
+                    self.opds_category_group, version=version)
             connect('opdssearch_'+ver, base_href+'/search/{query}',
                     self.opds_search, version=version)
 
@@ -269,12 +325,17 @@ class OPDSServer(object):
             sort_by='title', ascending=True, version=0):
         idx = self.db.FIELD_MAP['id']
         ids &= self.get_opds_allowed_ids_for_version(version)
+        if not ids:
+            raise cherrypy.HTTPError(404, 'No books found')
         items = [x for x in self.db.data.iterall() if x[idx] in ids]
         self.sort(items, sort_by, ascending)
         max_items = self.opts.max_opds_items
         offsets = OPDSOffsets(offset, max_items, len(items))
         items = items[offsets.offset:offsets.offset+max_items]
-        return str(AcquisitionFeed(self.db.last_modified(), id_, items, offsets,
+        updated = self.db.last_modified()
+        cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
+        cherrypy.response.headers['Content-Type'] = 'text/xml'
+        return str(AcquisitionFeed(updated, id_, items, offsets,
             page_url, up_url, version, self.db.FIELD_MAP))
 
     def opds_search(self, query=None, version=0, offset=0):
@@ -309,28 +370,160 @@ class OPDSServer(object):
                 id_='calibre-all:'+sort, sort_by=sort, ascending=ascending,
                 version=version)
 
+    # Categories {{{
+
+    def opds_category_group(self, category=None, which=None, version=0, offset=0):
+        try:
+            offset = int(offset)
+            version = int(version)
+        except:
+            raise cherrypy.HTTPError(404, 'Not found')
+
+        if not which or not category or version not in BASE_HREFS:
+            raise cherrypy.HTTPError(404, 'Not found')
+
+        categories = self.categories_cache(
+                self.get_opds_allowed_ids_for_version(version))
+        page_url = url_for('opdscategorygroup', version, category=category, which=which)
+
+        category = unhexlify(category)
+        if category not in categories:
+            raise cherrypy.HTTPError(404, 'Category %r not found'%which)
+        which = unhexlify(which)
+        owhich = hexlify('N'+which)
+        up_url = url_for('opdsnavcatalog', version, which=owhich)
+        items = categories[category]
+        items = [x for x in items if x.name.startswith(which)]
+        if not items:
+            raise cherrypy.HTTPError(404, 'No items in group %r:%r'%(category,
+                which))
+        updated = self.db.last_modified()
+
+        id_ = 'calibre-category-group-feed:'+category+':'+which
+
+        max_items = self.opts.max_opds_items
+        offsets = OPDSOffsets(offset, max_items, len(items))
+        items = list(items)[offsets.offset:offsets.offset+max_items]
+
+        cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
+        cherrypy.response.headers['Content-Type'] = 'text/xml'
+
+        return str(CategoryFeed(items, category, id_, updated, version, offsets,
+            page_url, up_url))
+
+
     def opds_navcatalog(self, which=None, version=0, offset=0):
-        version = int(version)
+        try:
+            offset = int(offset)
+            version = int(version)
+        except:
+            raise cherrypy.HTTPError(404, 'Not found')
+
         if not which or version not in BASE_HREFS:
             raise cherrypy.HTTPError(404, 'Not found')
         page_url = url_for('opdsnavcatalog', version, which=which)
         up_url = url_for('opds', version)
-        which = binascii.unhexlify(which)
+        which = unhexlify(which)
         type_ = which[0]
         which = which[1:]
         if type_ == 'O':
             return self.get_opds_all_books(which, page_url, up_url,
                     version=version, offset=offset)
         elif type_ == 'N':
-            return self.get_opds_navcatalog(which, version=version, offset=offset)
+            return self.get_opds_navcatalog(which, page_url, up_url,
+                    version=version, offset=offset)
         raise cherrypy.HTTPError(404, 'Not found')
 
-    def get_opds_navcatalog(self, which, version=0, offset=0):
+    def get_opds_navcatalog(self, which, page_url, up_url, version=0, offset=0):
         categories = self.categories_cache(
                 self.get_opds_allowed_ids_for_version(version))
         if which not in categories:
             raise cherrypy.HTTPError(404, 'Category %r not found'%which)
 
+        items = categories[which]
+        updated = self.db.last_modified()
+
+        id_ = 'calibre-category-feed:'+which
+
+        MAX_ITEMS = 50
+
+        if len(items) <= MAX_ITEMS:
+            max_items = self.opts.max_opds_items
+            offsets = OPDSOffsets(offset, max_items, len(items))
+            items = list(items)[offsets.offset:offsets.offset+max_items]
+            ans = CategoryFeed(items, which, id_, updated, version, offsets,
+                page_url, up_url)
+        else:
+            class Group:
+                def __init__(self, text, count):
+                    self.text, self.count = text, count
+
+            starts = set([x.name[0] for x in items])
+            if len(starts) > MAX_ITEMS:
+                starts = set([x.name[:2] for x in items])
+            category_groups = OrderedDict()
+            for x in sorted(starts, cmp=lambda x,y:cmp(x.lower(), y.lower())):
+                category_groups[x] = len([y for y in items if
+                    y.name.startswith(x)])
+            items = [Group(x, y) for x, y in category_groups.items()]
+            max_items = self.opts.max_opds_items
+            offsets = OPDSOffsets(offset, max_items, len(items))
+            items = items[offsets.offset:offsets.offset+max_items]
+            ans = CategoryGroupFeed(items, which, id_, updated, version, offsets,
+                page_url, up_url)
+
+        cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
+        cherrypy.response.headers['Content-Type'] = 'text/xml'
+
+        return str(ans)
+
+    def opds_category(self, category=None, which=None, version=0, offset=0):
+        try:
+            offset = int(offset)
+            version = int(version)
+        except:
+            raise cherrypy.HTTPError(404, 'Not found')
+
+        if not which or not category or version not in BASE_HREFS:
+            raise cherrypy.HTTPError(404, 'Not found')
+        page_url = url_for('opdscategory', version, which=which,
+                category=category)
+        up_url = url_for('opdsnavcatalog', version, which=category)
+
+        which, category = unhexlify(which), unhexlify(category)
+        type_ = which[0]
+        which = which[1:]
+        if type_ == 'I':
+            try:
+                which = int(which)
+            except:
+                raise cherrypy.HTTPError(404, 'Tag %r not found'%which)
+
+        categories = self.categories_cache(
+                self.get_opds_allowed_ids_for_version(version))
+        if category not in categories:
+            raise cherrypy.HTTPError(404, 'Category %r not found'%which)
+
+        if category == 'search':
+            try:
+                ids = self.search_cache(which)
+            except:
+                raise cherrypy.HTTPError(404, 'Search: %r not understood'%which)
+            return self.get_opds_acquisition_feed(ids, offset, page_url,
+                    up_url, 'calibre-search:'+which,
+                    version=version)
+
+        if type_ != 'I':
+            raise cherrypy.HTTPError(404, 'Non id categories not supported')
+
+        ids = self.db.get_books_for_category(category, which)
+        sort_by = 'series' if category == 'series' else 'title'
+
+        return self.get_opds_acquisition_feed(ids, offset, page_url,
+                up_url, 'calibre-category:'+category+':'+str(which),
+                version=version, sort_by=sort_by)
+
+    # }}}
 
 
     def opds(self, version=0):

src/calibre/utils/PythonMagickWand.py

@@ -596,15 +596,22 @@ IndexChannel = ChannelType(32)
 AllChannels = ChannelType(255)
 DefaultChannels = ChannelType(247)
 
-class DistortImageMethod(ctypes.c_int): pass
-UndefinedDistortion = DistortImageMethod(0)
-AffineDistortion = DistortImageMethod(1)
-AffineProjectionDistortion = DistortImageMethod(2)
-ArcDistortion = DistortImageMethod(3)
-BilinearDistortion = DistortImageMethod(4)
-PerspectiveDistortion = DistortImageMethod(5)
-PerspectiveProjectionDistortion = DistortImageMethod(6)
-ScaleRotateTranslateDistortion = DistortImageMethod(7)
+UndefinedDistortion = 0
+AffineDistortion = 1
+AffineProjectionDistortion = 2
+ScaleRotateTranslateDistortion = 3
+PerspectiveDistortion = 4
+BilinearForwardDistortion = 5
+BilinearDistortion = 6
+BilinearReverseDistortion = 7
+PolynomialDistortion = 8
+ArcDistortion = 9
+PolarDistortion = 10
+DePolarDistortion = 11
+BarrelDistortion = 12
+BarrelInverseDistortion = 13
+ShepardsDistortion = 14
+SentinelDistortion = 15
 
 class FillRule(ctypes.c_int): pass
 UndefinedRule = FillRule(0)
@@ -2254,7 +2261,7 @@ else:
 #   MagickDistortImage
 try:
     _magick.MagickDistortImage.restype = MagickBooleanType
-    _magick.MagickDistortImage.argtypes = (MagickWand,DistortImageMethod,ctypes.c_ulong,ctypes.POINTER(ctypes.c_double),MagickBooleanType)
+    _magick.MagickDistortImage.argtypes = (MagickWand,ctypes.c_int,ctypes.c_ulong,ctypes.POINTER(ctypes.c_double),MagickBooleanType)
 except AttributeError,e:
     pass
 else:

src/calibre/utils/magick_draw.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from ctypes import byref, c_double
+
+import calibre.utils.PythonMagickWand as p
+from calibre.ptempfile import TemporaryFile
+from calibre.constants import filesystem_encoding, __appname__, __version__
+
+# Font metrics {{{
+class Rect(object):
+
+    def __init__(self, left, top, right, bottom):
+        self.left, self.top, self.right, self.bottom = left, top, right, bottom
+
+    def __str__(self):
+        return '(%s, %s) -- (%s, %s)'%(self.left, self.top, self.right,
+                self.bottom)
+
+class FontMetrics(object):
+
+    def __init__(self, ret):
+        self._attrs = []
+        for i, x in enumerate(('char_width', 'char_height', 'ascender',
+            'descender', 'text_width', 'text_height',
+            'max_horizontal_advance')):
+            setattr(self, x, ret[i])
+            self._attrs.append(x)
+        self.bounding_box = Rect(ret[7], ret[8], ret[9], ret[10])
+        self.x, self.y = ret[11], ret[12]
+        self._attrs.extend(['bounding_box', 'x', 'y'])
+        self._attrs = tuple(self._attrs)
+
+    def __str__(self):
+        return '''FontMetrics:
+            char_width: %s
+            char_height: %s
+            ascender: %s
+            descender: %s
+            text_width: %s
+            text_height: %s
+            max_horizontal_advance: %s
+            bounding_box: %s
+            x: %s
+            y: %s
+            '''%tuple([getattr(self, x) for x in self._attrs])
+
+
+def get_font_metrics(image, d_wand, text):
+    ret = p.MagickQueryFontMetrics(image, d_wand, text)
+    return FontMetrics(ret)
+
+# }}}
+
+class TextLine(object):
+
+    def __init__(self, text, font_size, bottom_margin=30, font_path=None):
+        self.text, self.font_size, = text, font_size
+        self.bottom_margin = bottom_margin
+        self.font_path = font_path
+
+def alloc_wand(name):
+    ans = getattr(p, name)()
+    if ans < 0:
+        raise RuntimeError('Cannot create wand')
+    return ans
+
+def create_text_wand(font_size, font_path=None):
+    if font_path is None:
+        font_path = P('fonts/liberation/LiberationSerif-Bold.ttf')
+    if isinstance(font_path, unicode):
+        font_path = font_path.encode(filesystem_encoding)
+    ans = alloc_wand('NewDrawingWand')
+    if not p.DrawSetFont(ans, font_path):
+        raise ValueError('Failed to set font to: '+font_path)
+    p.DrawSetFontSize(ans, font_size)
+    p.DrawSetGravity(ans, p.CenterGravity)
+    p.DrawSetTextAntialias(ans, p.MagickTrue)
+    return ans
+
+
+def _get_line(img, dw, tokens, line_width):
+    line, rest = tokens, []
+    while True:
+        m = get_font_metrics(img, dw, ' '.join(line))
+        width, height = m.text_width, m.text_height
+        if width < line_width:
+            return line, rest
+        rest = line[-1:] + rest
+        line = line[:-1]
+
+def annotate_img(img, dw, left, top, rotate, text,
+        translate_from_top_left=True):
+    if isinstance(text, unicode):
+        text = text.encode('utf-8')
+    if translate_from_top_left:
+        m = get_font_metrics(img, dw, text)
+        img_width = p.MagickGetImageWidth(img)
+        img_height = p.MagickGetImageHeight(img)
+        left = left - img_width/2. + m.text_width/2.
+        top  = top - img_height/2. + m.text_height/2.
+    p.MagickAnnotateImage(img, dw, left, top, rotate, text)
+
+def draw_centered_line(img, dw, line, top):
+    m = get_font_metrics(img, dw, line)
+    width, height = m.text_width, m.text_height
+    img_width = p.MagickGetImageWidth(img)
+    left = max(int((img_width - width)/2.), 0)
+    annotate_img(img, dw, left, top, 0, line)
+    return top + height
+
+def draw_centered_text(img, dw, text, top, margin=10):
+    img_width = p.MagickGetImageWidth(img)
+    tokens = text.split(' ')
+    while tokens:
+        line, tokens = _get_line(img, dw, tokens, img_width-2*margin)
+        bottom = draw_centered_line(img, dw, ' '.join(line), top)
+        top = bottom
+    return top
+
+def create_canvas(width, height, bgcolor):
+    canvas = alloc_wand('NewMagickWand')
+    p_wand = alloc_wand('NewPixelWand')
+    p.PixelSetColor(p_wand, bgcolor)
+    p.MagickNewImage(canvas, width, height, p_wand)
+    p.DestroyPixelWand(p_wand)
+    return canvas
+
+def compose_image(canvas, image, left, top):
+    p.MagickCompositeImage(canvas, image, p.OverCompositeOp, int(left),
+            int(top))
+
+def load_image(path):
+    img = alloc_wand('NewMagickWand')
+    if not p.MagickReadImage(img, path):
+        severity = p.ExceptionType(0)
+        msg = p.MagickGetException(img, byref(severity))
+        raise IOError('Failed to read image from: %s: %s'
+                %(path, msg))
+    return img
+
+def create_text_arc(text, font_size, font=None, bgcolor='white'):
+    if isinstance(text, unicode):
+        text = text.encode('utf-8')
+
+    canvas = create_canvas(300, 300, bgcolor)
+    tw = create_text_wand(font_size, font_path=font)
+    m = get_font_metrics(canvas, tw, text)
+    p.DestroyMagickWand(canvas)
+    canvas = create_canvas(int(m.text_width)+20, int(m.text_height*3.5), bgcolor)
+    p.MagickAnnotateImage(canvas, tw, 0, 0, 0, text)
+    angle = c_double(120.)
+    p.MagickDistortImage(canvas, 9, 1, byref(angle),
+            p.MagickTrue)
+    p.MagickTrimImage(canvas, 0)
+    return canvas
+
+
+def create_cover_page(top_lines, logo_path, width=590, height=750,
+        bgcolor='white', output_format='png'):
+    ans = None
+    with p.ImageMagick():
+        canvas = create_canvas(width, height, bgcolor)
+
+        bottom = 10
+        for line in top_lines:
+            twand = create_text_wand(line.font_size, font_path=line.font_path)
+            bottom = draw_centered_text(canvas, twand, line.text, bottom)
+            bottom += line.bottom_margin
+            p.DestroyDrawingWand(twand)
+        bottom -= top_lines[-1].bottom_margin
+
+        vanity = create_text_arc(__appname__ + ' ' + __version__, 24,
+                font=P('fonts/liberation/LiberationMono-Regular.ttf'))
+        lwidth = p.MagickGetImageWidth(vanity)
+        lheight = p.MagickGetImageHeight(vanity)
+        left = int(max(0, (width - lwidth)/2.))
+        top  = height - lheight - 10
+        compose_image(canvas, vanity, left, top)
+
+        logo = load_image(logo_path)
+        lwidth = p.MagickGetImageWidth(logo)
+        lheight = p.MagickGetImageHeight(logo)
+        left = int(max(0, (width - lwidth)/2.))
+        top  = max(int((height - lheight)/2.), bottom+20)
+        compose_image(canvas, logo, left, top)
+        p.DestroyMagickWand(logo)
+
+        with TemporaryFile('.'+output_format) as f:
+            p.MagickWriteImage(canvas, f)
+            with open(f, 'rb') as f:
+                ans = f.read()
+        p.DestroyMagickWand(canvas)
+    return ans
+
+def test():
+    import subprocess
+    with TemporaryFile('.png') as f:
+        data = create_cover_page(
+                [TextLine('A very long title indeed, don\'t you agree?', 42),
+                TextLine('Mad Max & Mixy poo', 32)], I('library.png'))
+        with open(f, 'wb') as g:
+            g.write(data)
+        subprocess.check_call(['gwenview', f])
+
+if __name__ == '__main__':
+    test()

src/calibre/utils/pil_draw.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
-
-__license__   = 'GPL v3'
-__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-def _get_line(draw, font, tokens, line_width):
-    line, rest = tokens, []
-    while True:
-        width, height = draw.textsize(' '.join(line), font=font)
-        if width < line_width:
-            return line, rest
-        rest = line[-1:] + rest
-        line = line[:-1]
-
-def draw_centered_line(img, draw, font, line, top):
-    width, height = draw.textsize(line, font=font)
-    left = max(int((img.size[0] - width)/2.), 0)
-    draw.text((left, top), line, fill=(0,0,0), font=font)
-    return top + height
-
-def draw_centered_text(img, draw, font, text, top, margin=10, ysep=5):
-    img_width, img_height = img.size
-    tokens = text.split(' ')
-    while tokens:
-        line, tokens = _get_line(draw, font, tokens, img_width-2*margin)
-        bottom = draw_centered_line(img, draw, font, ' '.join(line), top)
-        top = bottom + ysep
-    return top - ysep
-
-

src/calibre/web/feeds/news.py

@@ -14,7 +14,7 @@ from contextlib import nested, closing
 
 
 from calibre import browser, __appname__, iswindows, \
-                    strftime, __version__, preferred_encoding
+                    strftime, preferred_encoding
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
 from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre import entity_to_unicode
@@ -949,47 +949,13 @@ class BasicNewsRecipe(Recipe):
         Create a generic cover for recipes that dont have a cover
         '''
         try:
-            try:
-                from PIL import Image, ImageDraw, ImageFont
-                Image, ImageDraw, ImageFont
-            except ImportError:
-                import Image, ImageDraw, ImageFont
-            font_path = P('fonts/liberation/LiberationSerif-Bold.ttf')
+            from calibre.utils.magick_draw import create_cover_page, TextLine
             title = self.title if isinstance(self.title, unicode) else \
                     self.title.decode(preferred_encoding, 'replace')
             date = strftime(self.timefmt)
-            app = '['+__appname__ +' '+__version__+']'
-
-            COVER_WIDTH, COVER_HEIGHT = 590, 750
-            img = Image.new('RGB', (COVER_WIDTH, COVER_HEIGHT), 'white')
-            draw = ImageDraw.Draw(img)
-            # Title
-            font = ImageFont.truetype(font_path, 44)
-            width, height = draw.textsize(title, font=font)
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            top = 15
-            draw.text((left, top), title, fill=(0,0,0), font=font)
-            bottom = top + height
-            # Date
-            font = ImageFont.truetype(font_path, 32)
-            width, height = draw.textsize(date, font=font)
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            draw.text((left, bottom+15), date, fill=(0,0,0), font=font)
-            # Vanity
-            font = ImageFont.truetype(font_path, 28)
-            width, height = draw.textsize(app, font=font)
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            top = COVER_HEIGHT - height - 15
-            draw.text((left, top), app, fill=(0,0,0), font=font)
-            # Logo
-            logo = Image.open(I('library.png'), 'r')
-            width, height = logo.size
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            top = max(int((COVER_HEIGHT - height)/2.), 0)
-            img.paste(logo, (left, top))
-            img = img.convert('RGB').convert('P', palette=Image.ADAPTIVE)
-
-            img.convert('RGB').save(cover_file, 'JPEG')
+            lines = [TextLine(title, 44), TextLine(date, 32)]
+            img_data = create_cover_page(lines, I('library.png'), output_format='jpg')
+            cover_file.write(img_data)
             cover_file.flush()
         except:
             self.log.exception('Failed to generate default cover')

src/calibre/web/fetch/simple.py

@@ -148,6 +148,9 @@ class RecursiveFetcher(object):
         nmassage = copy.copy(BeautifulSoup.MARKUP_MASSAGE)
         nmassage.extend(self.preprocess_regexps)
         nmassage += [(re.compile(r'<!DOCTYPE .+?>', re.DOTALL), lambda m: '')] # Some websites have buggy doctype declarations that mess up beautifulsoup
+        # Remove comments as they can leave detritus when extracting tags leaves
+        # multiple nested comments
+        nmassage.append((re.compile(r'<!--.*?-->', re.DOTALL), lambda m: ''))
         soup = BeautifulSoup(xml_to_unicode(src, self.verbose, strip_encoding_pats=True)[0], markupMassage=nmassage)
 
         if self.keep_only_tags: