Sync to trunk.

resources/recipes/nytimes.recipe

@@ -10,7 +10,7 @@ import time
 from calibre import entity_to_unicode
 from calibre.web.feeds.recipes import BasicNewsRecipe
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString, \
-    Comment, BeautifulStoneSoup
+Comment, BeautifulStoneSoup
 
 class NYTimes(BasicNewsRecipe):
 
@@ -86,6 +86,7 @@ class NYTimes(BasicNewsRecipe):
                             'relatedSearchesModule',
                             'side_tool',
                             'singleAd',
+                            'subNavigation tabContent active',
                             'subNavigation tabContent active clearfix',
                             ]}),
                    dict(id=[
@@ -94,6 +95,7 @@ class NYTimes(BasicNewsRecipe):
                             'articleExtras',
                             'articleInline',
                             'blog_sidebar',
+                            'businessSearchBar',
                             'cCol',
                             'entertainmentSearchBar',
                             'footer',
@@ -101,6 +103,7 @@ class NYTimes(BasicNewsRecipe):
                             'header_search',
                             'login',
                             'masthead',
+                            'masthead-nav',
                             'memberTools',
                             'navigation',
                             'portfolioInline',

resources/recipes/nytimes_sub.recipe

@@ -74,6 +74,7 @@ class NYTimes(BasicNewsRecipe):
                             'relatedSearchesModule',
                             'side_tool',
                             'singleAd',
+                            'subNavigation tabContent active',
                             'subNavigation tabContent active clearfix',
                             ]}),
                    dict(id=[

src/calibre/devices/android/driver.py

@@ -44,6 +44,7 @@ class ANDROID(USBMS):
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER', 'GT-I5700']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD']
+    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE']
 
     OSX_MAIN_MEM = 'HTC Android Phone Media'
 

src/calibre/devices/binatone/driver.py

@@ -42,3 +42,13 @@ class README(USBMS):
         drives[0] = drives[1]
         drives[1] = t
         return tuple(drives)
+
+    def windows_sort_drives(self, drives):
+        if len(drives) < 2: return drives
+        main = drives.get('main', None)
+        carda = drives.get('carda', None)
+        if main and carda:
+            drives['main'] = carda
+            drives['carda'] = main
+        return drives
+

src/calibre/devices/hanlin/driver.py

@@ -71,6 +71,18 @@ class HANLINV3(USBMS):
         drives[1] = t
         return tuple(drives)
 
+    def windows_sort_drives(self, drives):
+        if len(drives) < 2: return drives
+        main = drives.get('main', None)
+        carda = drives.get('carda', None)
+        if main and carda:
+            drives['main'] = carda
+            drives['carda'] = main
+        return drives
+
+
+
+
 
 class HANLINV5(HANLINV3):
     name           = 'Hanlin V5 driver'

src/calibre/ebooks/chm/reader.py

@@ -132,6 +132,9 @@ class CHMReader(CHMFile):
             lpath = os.path.join(output_dir, path)
             self._ensure_dir(lpath)
             data = self.GetFile(path)
+            if lpath.find(';') != -1:
+                # fix file names with ";<junk>" at the end, see _reformat()
+                lpath = lpath.split(';')[0]
             with open(lpath, 'wb') as f:
                 if guess_mimetype(path)[0] == ('text/html'):
                     data = self._reformat(data)
@@ -158,14 +161,26 @@ class CHMReader(CHMFile):
         # cos they really fuck with the flow of things and generally waste space
         # since we can't use [a,b] syntax to select arbitrary items from a list
         # we'll have to do this manually...
+        # only remove the tables, if they have an image with an alt attribute
+        # containing prev, next or team
         t = soup('table')
         if t:
             if (t[0].previousSibling is None
               or t[0].previousSibling.previousSibling is None):
-                t[0].extract()
+                try:
+                    alt = t[0].img['alt'].lower()
+                    if alt.find('prev') != -1 or alt.find('next') != -1 or alt.find('team') != -1:
+                        t[0].extract()
+                except:
+                    pass
             if (t[-1].nextSibling is None
               or t[-1].nextSibling.nextSibling is None):
-                t[-1].extract()
+                try:
+                    alt = t[-1].img['alt'].lower()
+                    if alt.find('prev') != -1 or alt.find('next') != -1 or alt.find('team') != -1:
+                        t[-1].extract()
+                except:
+                    pass
         # for some very odd reason each page's content appears to be in a table
         # too. and this table has sub-tables for random asides... grr.
 
@@ -185,8 +200,24 @@ class CHMReader(CHMFile):
             except KeyError:
                 # and some don't even have a src= ?!
                 pass
-        # now give back some pretty html.
+        try:
-        return soup.prettify('utf-8')
+            # if there is only a single table with a single element
+            # in the body, replace it by the contents of this single element
+            tables = soup.body.findAll('table', recursive=False)
+            if tables and len(tables) == 1:
+                trs = tables[0].findAll('tr', recursive=False)
+                if trs and len(trs) == 1:
+                    tds = trs[0].findAll('td', recursive=False)
+                    if tds and len(tds) == 1:
+                        tdContents = tds[0].contents
+                        tableIdx = soup.body.contents.index(tables[0])
+                        tables[0].extract()
+                        while tdContents:
+                            soup.body.insert(tableIdx, tdContents.pop())
+        except:
+            pass
+        # do not prettify, it would reformat the <pre> tags!
+        return str(soup)
 
     def Contents(self):
         if self._contents is not None:

src/calibre/ebooks/oeb/transforms/cover.py

@@ -15,7 +15,7 @@ try:
 except ImportError:
     import Image as PILImage
 
-from calibre import __appname__, __version__, guess_type
+from calibre import guess_type
 
 class CoverManager(object):
 
@@ -89,7 +89,6 @@ class CoverManager(object):
         '''
         Create a generic cover for books that dont have a cover
         '''
-        from calibre.utils.pil_draw import draw_centered_text
         from calibre.ebooks.metadata import authors_to_string
         if self.no_default_cover:
             return None
@@ -98,46 +97,15 @@ class CoverManager(object):
         title = unicode(m.title[0])
         authors = [unicode(x) for x in m.creator if x.role == 'aut']
 
-        cover_file = cStringIO.StringIO()
         try:
-            try:
+            from calibre.utils.magick_draw import create_cover_page, TextLine
-                from PIL import Image, ImageDraw, ImageFont
+            lines = [TextLine(title, 44), TextLine(authors_to_string(authors),
-                Image, ImageDraw, ImageFont
+                32)]
-            except ImportError:
+            img_data = create_cover_page(lines, I('library.png'))
-                import Image, ImageDraw, ImageFont
+            id, href = self.oeb.manifest.generate('cover_image',
-            font_path = P('fonts/liberation/LiberationSerif-Bold.ttf')
+                    'cover_image.png')
-            app = '['+__appname__ +' '+__version__+']'
+            item = self.oeb.manifest.add(id, href, guess_type('t.png')[0],
-
+                        data=img_data)
-            COVER_WIDTH, COVER_HEIGHT = 590, 750
-            img = Image.new('RGB', (COVER_WIDTH, COVER_HEIGHT), 'white')
-            draw = ImageDraw.Draw(img)
-            # Title
-            font = ImageFont.truetype(font_path, 44)
-            bottom = draw_centered_text(img, draw, font, title, 15, ysep=9)
-            # Authors
-            bottom += 14
-            font = ImageFont.truetype(font_path, 32)
-            authors = authors_to_string(authors)
-            bottom = draw_centered_text(img, draw, font, authors, bottom, ysep=7)
-            # Vanity
-            font = ImageFont.truetype(font_path, 28)
-            width, height = draw.textsize(app, font=font)
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            top = COVER_HEIGHT - height - 15
-            draw.text((left, top), app, fill=(0,0,0), font=font)
-            # Logo
-            logo = Image.open(I('library.png'), 'r')
-            width, height = logo.size
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            top = max(int((COVER_HEIGHT - height)/2.), 0)
-            img.paste(logo, (left, max(bottom, top)))
-            img = img.convert('RGB').convert('P', palette=Image.ADAPTIVE)
-
-            img.convert('RGB').save(cover_file, 'JPEG')
-            cover_file.flush()
-            id, href = self.oeb.manifest.generate('cover_image', 'cover_image.jpg')
-            item = self.oeb.manifest.add(id, href, guess_type('t.jpg')[0],
-                        data=cover_file.getvalue())
             m.clear('cover')
             m.add('cover', item.id)
 

src/calibre/gui2/dialogs/config/__init__.py

@@ -471,7 +471,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
         if port < 1025:
             warning_dialog(self, _('System port selected'), '<p>'+
                     _('The value <b>%d</b> you have chosen for the content '
-                        'server port is a system port. You operating '
+                        'server port is a system port. Your operating '
                         'system <b>may</b> not allow the server to run on this '
                         'port. To be safe choose a port number larger than '
                         '1024.')%port, show=True)

src/calibre/utils/PythonMagickWand.py

@@ -596,15 +596,22 @@ IndexChannel = ChannelType(32)
 AllChannels = ChannelType(255)
 DefaultChannels = ChannelType(247)
 
-class DistortImageMethod(ctypes.c_int): pass
+UndefinedDistortion = 0
-UndefinedDistortion = DistortImageMethod(0)
+AffineDistortion = 1
-AffineDistortion = DistortImageMethod(1)
+AffineProjectionDistortion = 2
-AffineProjectionDistortion = DistortImageMethod(2)
+ScaleRotateTranslateDistortion = 3
-ArcDistortion = DistortImageMethod(3)
+PerspectiveDistortion = 4
-BilinearDistortion = DistortImageMethod(4)
+BilinearForwardDistortion = 5
-PerspectiveDistortion = DistortImageMethod(5)
+BilinearDistortion = 6
-PerspectiveProjectionDistortion = DistortImageMethod(6)
+BilinearReverseDistortion = 7
-ScaleRotateTranslateDistortion = DistortImageMethod(7)
+PolynomialDistortion = 8
+ArcDistortion = 9
+PolarDistortion = 10
+DePolarDistortion = 11
+BarrelDistortion = 12
+BarrelInverseDistortion = 13
+ShepardsDistortion = 14
+SentinelDistortion = 15
 
 class FillRule(ctypes.c_int): pass
 UndefinedRule = FillRule(0)
@@ -2254,7 +2261,7 @@ else:
 #   MagickDistortImage
 try:
     _magick.MagickDistortImage.restype = MagickBooleanType
-    _magick.MagickDistortImage.argtypes = (MagickWand,DistortImageMethod,ctypes.c_ulong,ctypes.POINTER(ctypes.c_double),MagickBooleanType)
+    _magick.MagickDistortImage.argtypes = (MagickWand,ctypes.c_int,ctypes.c_ulong,ctypes.POINTER(ctypes.c_double),MagickBooleanType)
 except AttributeError,e:
     pass
 else:

src/calibre/utils/magick_draw.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from ctypes import byref, c_double
+
+import calibre.utils.PythonMagickWand as p
+from calibre.ptempfile import TemporaryFile
+from calibre.constants import filesystem_encoding, __appname__, __version__
+
+# Font metrics {{{
+class Rect(object):
+
+    def __init__(self, left, top, right, bottom):
+        self.left, self.top, self.right, self.bottom = left, top, right, bottom
+
+    def __str__(self):
+        return '(%s, %s) -- (%s, %s)'%(self.left, self.top, self.right,
+                self.bottom)
+
+class FontMetrics(object):
+
+    def __init__(self, ret):
+        self._attrs = []
+        for i, x in enumerate(('char_width', 'char_height', 'ascender',
+            'descender', 'text_width', 'text_height',
+            'max_horizontal_advance')):
+            setattr(self, x, ret[i])
+            self._attrs.append(x)
+        self.bounding_box = Rect(ret[7], ret[8], ret[9], ret[10])
+        self.x, self.y = ret[11], ret[12]
+        self._attrs.extend(['bounding_box', 'x', 'y'])
+        self._attrs = tuple(self._attrs)
+
+    def __str__(self):
+        return '''FontMetrics:
+            char_width: %s
+            char_height: %s
+            ascender: %s
+            descender: %s
+            text_width: %s
+            text_height: %s
+            max_horizontal_advance: %s
+            bounding_box: %s
+            x: %s
+            y: %s
+            '''%tuple([getattr(self, x) for x in self._attrs])
+
+
+def get_font_metrics(image, d_wand, text):
+    ret = p.MagickQueryFontMetrics(image, d_wand, text)
+    return FontMetrics(ret)
+
+# }}}
+
+class TextLine(object):
+
+    def __init__(self, text, font_size, bottom_margin=30, font_path=None):
+        self.text, self.font_size, = text, font_size
+        self.bottom_margin = bottom_margin
+        self.font_path = font_path
+
+def alloc_wand(name):
+    ans = getattr(p, name)()
+    if ans < 0:
+        raise RuntimeError('Cannot create wand')
+    return ans
+
+def create_text_wand(font_size, font_path=None):
+    if font_path is None:
+        font_path = P('fonts/liberation/LiberationSerif-Bold.ttf')
+    if isinstance(font_path, unicode):
+        font_path = font_path.encode(filesystem_encoding)
+    ans = alloc_wand('NewDrawingWand')
+    if not p.DrawSetFont(ans, font_path):
+        raise ValueError('Failed to set font to: '+font_path)
+    p.DrawSetFontSize(ans, font_size)
+    p.DrawSetGravity(ans, p.CenterGravity)
+    p.DrawSetTextAntialias(ans, p.MagickTrue)
+    return ans
+
+
+def _get_line(img, dw, tokens, line_width):
+    line, rest = tokens, []
+    while True:
+        m = get_font_metrics(img, dw, ' '.join(line))
+        width, height = m.text_width, m.text_height
+        if width < line_width:
+            return line, rest
+        rest = line[-1:] + rest
+        line = line[:-1]
+
+def annotate_img(img, dw, left, top, rotate, text,
+        translate_from_top_left=True):
+    if isinstance(text, unicode):
+        text = text.encode('utf-8')
+    if translate_from_top_left:
+        m = get_font_metrics(img, dw, text)
+        img_width = p.MagickGetImageWidth(img)
+        img_height = p.MagickGetImageHeight(img)
+        left = left - img_width/2. + m.text_width/2.
+        top  = top - img_height/2. + m.text_height/2.
+    p.MagickAnnotateImage(img, dw, left, top, rotate, text)
+
+def draw_centered_line(img, dw, line, top):
+    m = get_font_metrics(img, dw, line)
+    width, height = m.text_width, m.text_height
+    img_width = p.MagickGetImageWidth(img)
+    left = max(int((img_width - width)/2.), 0)
+    annotate_img(img, dw, left, top, 0, line)
+    return top + height
+
+def draw_centered_text(img, dw, text, top, margin=10):
+    img_width = p.MagickGetImageWidth(img)
+    tokens = text.split(' ')
+    while tokens:
+        line, tokens = _get_line(img, dw, tokens, img_width-2*margin)
+        bottom = draw_centered_line(img, dw, ' '.join(line), top)
+        top = bottom
+    return top
+
+def create_canvas(width, height, bgcolor):
+    canvas = alloc_wand('NewMagickWand')
+    p_wand = alloc_wand('NewPixelWand')
+    p.PixelSetColor(p_wand, bgcolor)
+    p.MagickNewImage(canvas, width, height, p_wand)
+    p.DestroyPixelWand(p_wand)
+    return canvas
+
+def compose_image(canvas, image, left, top):
+    p.MagickCompositeImage(canvas, image, p.OverCompositeOp, int(left),
+            int(top))
+
+def load_image(path):
+    img = alloc_wand('NewMagickWand')
+    if not p.MagickReadImage(img, path):
+        severity = p.ExceptionType(0)
+        msg = p.MagickGetException(img, byref(severity))
+        raise IOError('Failed to read image from: %s: %s'
+                %(path, msg))
+    return img
+
+def create_text_arc(text, font_size, font=None, bgcolor='white'):
+    if isinstance(text, unicode):
+        text = text.encode('utf-8')
+
+    canvas = create_canvas(300, 300, bgcolor)
+    tw = create_text_wand(font_size, font_path=font)
+    m = get_font_metrics(canvas, tw, text)
+    p.DestroyMagickWand(canvas)
+    canvas = create_canvas(int(m.text_width)+20, int(m.text_height*3.5), bgcolor)
+    p.MagickAnnotateImage(canvas, tw, 0, 0, 0, text)
+    angle = c_double(120.)
+    p.MagickDistortImage(canvas, 9, 1, byref(angle),
+            p.MagickTrue)
+    p.MagickTrimImage(canvas, 0)
+    return canvas
+
+
+def create_cover_page(top_lines, logo_path, width=590, height=750,
+        bgcolor='white', output_format='png'):
+    ans = None
+    with p.ImageMagick():
+        canvas = create_canvas(width, height, bgcolor)
+
+        bottom = 10
+        for line in top_lines:
+            twand = create_text_wand(line.font_size, font_path=line.font_path)
+            bottom = draw_centered_text(canvas, twand, line.text, bottom)
+            bottom += line.bottom_margin
+            p.DestroyDrawingWand(twand)
+        bottom -= top_lines[-1].bottom_margin
+
+        vanity = create_text_arc(__appname__ + ' ' + __version__, 24,
+                font=P('fonts/liberation/LiberationMono-Regular.ttf'))
+        lwidth = p.MagickGetImageWidth(vanity)
+        lheight = p.MagickGetImageHeight(vanity)
+        left = int(max(0, (width - lwidth)/2.))
+        top  = height - lheight - 10
+        compose_image(canvas, vanity, left, top)
+
+        logo = load_image(logo_path)
+        lwidth = p.MagickGetImageWidth(logo)
+        lheight = p.MagickGetImageHeight(logo)
+        left = int(max(0, (width - lwidth)/2.))
+        top  = max(int((height - lheight)/2.), bottom+20)
+        compose_image(canvas, logo, left, top)
+        p.DestroyMagickWand(logo)
+
+        with TemporaryFile('.'+output_format) as f:
+            p.MagickWriteImage(canvas, f)
+            with open(f, 'rb') as f:
+                ans = f.read()
+        p.DestroyMagickWand(canvas)
+    return ans
+
+def test():
+    import subprocess
+    with TemporaryFile('.png') as f:
+        data = create_cover_page(
+                [TextLine('A very long title indeed, don\'t you agree?', 42),
+                TextLine('Mad Max & Mixy poo', 32)], I('library.png'))
+        with open(f, 'wb') as g:
+            g.write(data)
+        subprocess.check_call(['gwenview', f])
+
+if __name__ == '__main__':
+    test()

src/calibre/utils/pil_draw.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
-
-__license__   = 'GPL v3'
-__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-def _get_line(draw, font, tokens, line_width):
-    line, rest = tokens, []
-    while True:
-        width, height = draw.textsize(' '.join(line), font=font)
-        if width < line_width:
-            return line, rest
-        rest = line[-1:] + rest
-        line = line[:-1]
-
-def draw_centered_line(img, draw, font, line, top):
-    width, height = draw.textsize(line, font=font)
-    left = max(int((img.size[0] - width)/2.), 0)
-    draw.text((left, top), line, fill=(0,0,0), font=font)
-    return top + height
-
-def draw_centered_text(img, draw, font, text, top, margin=10, ysep=5):
-    img_width, img_height = img.size
-    tokens = text.split(' ')
-    while tokens:
-        line, tokens = _get_line(draw, font, tokens, img_width-2*margin)
-        bottom = draw_centered_line(img, draw, font, ' '.join(line), top)
-        top = bottom + ysep
-    return top - ysep
-
-

src/calibre/web/feeds/news.py

@@ -14,7 +14,7 @@ from contextlib import nested, closing
 
 
 from calibre import browser, __appname__, iswindows, \
-                    strftime, __version__, preferred_encoding
+                    strftime, preferred_encoding
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
 from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre import entity_to_unicode
@@ -949,47 +949,13 @@ class BasicNewsRecipe(Recipe):
         Create a generic cover for recipes that dont have a cover
         '''
         try:
-            try:
+            from calibre.utils.magick_draw import create_cover_page, TextLine
-                from PIL import Image, ImageDraw, ImageFont
-                Image, ImageDraw, ImageFont
-            except ImportError:
-                import Image, ImageDraw, ImageFont
-            font_path = P('fonts/liberation/LiberationSerif-Bold.ttf')
             title = self.title if isinstance(self.title, unicode) else \
                     self.title.decode(preferred_encoding, 'replace')
             date = strftime(self.timefmt)
-            app = '['+__appname__ +' '+__version__+']'
+            lines = [TextLine(title, 44), TextLine(date, 32)]
-
+            img_data = create_cover_page(lines, I('library.png'), output_format='jpg')
-            COVER_WIDTH, COVER_HEIGHT = 590, 750
+            cover_file.write(img_data)
-            img = Image.new('RGB', (COVER_WIDTH, COVER_HEIGHT), 'white')
-            draw = ImageDraw.Draw(img)
-            # Title
-            font = ImageFont.truetype(font_path, 44)
-            width, height = draw.textsize(title, font=font)
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            top = 15
-            draw.text((left, top), title, fill=(0,0,0), font=font)
-            bottom = top + height
-            # Date
-            font = ImageFont.truetype(font_path, 32)
-            width, height = draw.textsize(date, font=font)
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            draw.text((left, bottom+15), date, fill=(0,0,0), font=font)
-            # Vanity
-            font = ImageFont.truetype(font_path, 28)
-            width, height = draw.textsize(app, font=font)
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            top = COVER_HEIGHT - height - 15
-            draw.text((left, top), app, fill=(0,0,0), font=font)
-            # Logo
-            logo = Image.open(I('library.png'), 'r')
-            width, height = logo.size
-            left = max(int((COVER_WIDTH - width)/2.), 0)
-            top = max(int((COVER_HEIGHT - height)/2.), 0)
-            img.paste(logo, (left, top))
-            img = img.convert('RGB').convert('P', palette=Image.ADAPTIVE)
-
-            img.convert('RGB').save(cover_file, 'JPEG')
             cover_file.flush()
         except:
             self.log.exception('Failed to generate default cover')

src/calibre/web/fetch/simple.py

@@ -148,6 +148,9 @@ class RecursiveFetcher(object):
         nmassage = copy.copy(BeautifulSoup.MARKUP_MASSAGE)
         nmassage.extend(self.preprocess_regexps)
         nmassage += [(re.compile(r'<!DOCTYPE .+?>', re.DOTALL), lambda m: '')] # Some websites have buggy doctype declarations that mess up beautifulsoup
+        # Remove comments as they can leave detritus when extracting tags leaves
+        # multiple nested comments
+        nmassage.append((re.compile(r'<!--.*?-->', re.DOTALL), lambda m: ''))
         soup = BeautifulSoup(xml_to_unicode(src, self.verbose, strip_encoding_pats=True)[0], markupMassage=nmassage)
 
         if self.keep_only_tags: