Merge branch 'master' of https://github.com/kovidgoyal/calibre

Changelog.yaml

@@ -20,6 +20,50 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.9.37
+  date: 2013-06-28
+
+  new features:
+    - title: "Conversion: Add option to embed all referenced fonts"
+      type: major
+      description: "Add an option to embed all fonts that are referenced in the input document but are not already embedded. This will search your system for the referenced font, and if found, the font will be embedded.  Only works if the output format supports font embedding (for example: EPUB or AZW3). The option is under the Look & Feel section of the conversion dialog."
+
+    - title: "ToC Editor: When generating a ToC from files, if the file has no text, do not skip it. Instead create an entry using the filename of the file."
+
+    - title: "AZW3 Input: Add support for the page-progression-direction that is used to indicate page turns should happen from right to left. The attribute is passed into EPUB when converting."
+      tickets: [1194766]
+
+    - title: "ebook-convert: Add a --from-opf option to read metadata from OPF files directly, instead of having to run ebook-meta --from-opf after conversion"
+ 
+  bug fixes:
+    - title: "PDF Output: Fix Table of Contents being added to the end of the PDF even without the Add Table of Contents option being enabled."
+      tickets: [1194836]
+
+    - title: "When auto-merging books on add, also merge identifiers."
+
+    - title: "Fix an error when using the Template Editor to create a template that uses custom columns."
+      tickets: [1193763]
+
+    - title: "LRF Output: Fix " entities in attribute values causing problems"
+
+    - title: "News download: Apply the default page margin conversion settings. Also, when converting to PDF, apply the pdf conversion defaults."
+      tickets: [1193912]
+
+    - title: "Fix a regression that broke scanning for books on all devices that used the Aluratek Color driver."
+      tickets: [1192940]
+
+    - title: "fetch-ebbok-metadata: Fix --opf argument erroneously requiring a value"
+
+    - title: "When waiting before sending email, log the wait."
+      tickets: [1195173]
+
+  improved recipes:
+    - taz.de (RSS)
+    - Miradas al sur
+    - Frontline
+    - La Nacion (Costa Rica)
+
+
 - version: 0.9.36
   date: 2013-06-21
 

manual/templates/layout.html

@@ -62,7 +62,7 @@
         <form action="https://www.paypal.com/cgi-bin/webscr" method="post" title="Contribute to support calibre development">
             <input type="hidden" name="cmd" value="_s-xclick" />
             <input type="hidden" name="hosted_button_id" value="AF4H3B8QVDG6N" />
-            <input type="image" src="http://manual.calibre-ebook.com/simple_donate_button.gif" border="0" name="submit" alt="Contribute to support calibre development" style="border:0pt" />
+            <input type="image" src="_static/simple_donate_button.gif" border="0" name="submit" alt="Contribute to support calibre development" style="border:0pt" />
             <img alt="" border="0" src="https://www.paypalobjects.com/en_GB/i/scr/pixel.gif" width="1" height="1" />
         </form>
         <hr/>

recipes/frontlineonnet.recipe

@@ -46,23 +46,24 @@ class Frontlineonnet(BasicNewsRecipe):
 
     keep_only_tags= [
                       dict(name='div', attrs={'id':'content'})
-                     #,dict(attrs={'class':'byline'})
                     ]
-    #remove_attributes=['size','noshade','border']
+    remove_attributes=['size','noshade','border']
-
-    #def preprocess_html(self, soup):
-        #for item in soup.findAll(style=True):
-            #del item['style']
-        #for item in soup.findAll('img'):
-            #if not item.has_key('alt'):
-               #item['alt'] = 'image'
-        #return soup
 
     def parse_index(self):
         articles = []
+        current_section = None
+        feeds = []
         soup = self.index_to_soup(self.INDEX)
-        for feed_link in soup.findAll('div', id='headseccol'):
+        for h3 in soup.findAll('h3'):
-            a = feed_link.find('a', href=True)
+            if h3.get('class', None) == 'artListSec':
+                if articles:
+                    feeds.append((current_section, articles))
+                articles = []
+                current_section = self.tag_to_string(h3).strip()
+                self.log(current_section)
+            elif h3.get('id', None) in {'headseccol', 'headsec'}:
+                a = h3.find('a', href=True)
+                if a is not None:
                     title = self.tag_to_string(a)
                     url = a['href']
                     articles.append({
@@ -71,10 +72,8 @@ class Frontlineonnet(BasicNewsRecipe):
                                     ,'url'        :url
                                     ,'description':''
                                     })
-        return [('Frontline', articles)]
+                    self.log('\t', title, url)
+        if articles:
+            feeds.append((current_section, articles))
+        return feeds
 
-    #def print_version(self, url):
-        #return "http://www.hinduonnet.com/thehindu/thscrip/print.pl?prd=fline&file=" + url.rpartition('/')[2]
-
-    #def image_url_processor(self, baseurl, url):
-        #return url.replace('../images/', self.INDEX + 'images/').strip()

recipes/la_nacion_cr.recipe

@@ -20,7 +20,7 @@ class crnews(BasicNewsRecipe):
     no_stylesheets = True
 
 
-    feeds          = [(u'Portada', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=portada'), (u'Ultima Hora', u'http://www.nacion.com/Generales/RSS/UltimaHoraRss.aspx'), (u'Nacionales', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=elpais'), (u'Entretenimiento', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=entretenimiento'), (u'Sucesos', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=sucesos'), (u'Deportes', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=deportes'), (u'Internacionales', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=mundo'), (u'Economia', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=economia'), (u'Aldea Global', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=aldeaglobal'), (u'Tecnologia', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=tecnologia'), (u'Opinion', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=opinion')]
+    feeds          = [(u'Portada', u'http://www.nacion.com/rss/'), (u'Ultima Hora', u'http://www.nacion.com/rss/latest/'), (u'Nacionales', u'http://www.nacion.com/rss/nacional/'), (u'Entretenimiento', u'http://www.nacion.com/rss/ocio/'), (u'Sucesos', u'http://www.nacion.com/rss/sucesos/'), (u'Deportes', u'http://www.nacion.com/rss/deportes/'), (u'Internacionales', u'http://www.nacion.com/rss/mundo/'), (u'Economia', u'http://www.nacion.com/rss/economia/'), (u'Vivir', u'http://www.nacion.com/rss/vivir/'), (u'Tecnologia', u'http://www.nacion.com/rss/tecnologia/'), (u'Opinion', u'http://www.nacion.com/rss/opinion/')]
 
     def get_cover_url(self):
         index = 'http://kiosko.net/cr/np/cr_nacion.html'

recipes/living_digital.recipe

@@ -1,16 +0,0 @@
-from calibre.web.feeds.news import CalibrePeriodical
-
-class LivingDigital(CalibrePeriodical):
-
-    title = 'Living Digital'
-    calibre_periodicals_slug = 'living-digital'
-
-    description = '''
-    Catch the latest buzz in the digital world with Living Digital. Enjoy
-    reviews, news, features and recommendations on a wide range of consumer
-    technology products - from smartphones to flat panel TVs, netbooks to
-    cameras, and many more consumer lifestyle gadgets. To subscribe, visit
-    <a href="http://news.calibre-ebook.com/periodical/living-digital">calibre
-    Periodicals</a>.
-    '''
-    language = 'en_IN'

recipes/miradasalsur.recipe

@@ -1,18 +1,15 @@
-#!/usr/bin/env  python
+__copyright__ = '2009-2013, Darko Miletic <darko.miletic at gmail.com>'
-
-__license__   = 'GPL v3'
-__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
-elargentino.com
+sur.infonews.com
 '''
 
+import datetime
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import Tag
 
 class MiradasAlSur(BasicNewsRecipe):
     title                 = 'Miradas al Sur'
     __author__            = 'Darko Miletic'
-    description           = 'Revista Argentina'
+    description           = 'Semanario Argentino'
     publisher             = 'ElArgentino.com'
     category              = 'news, politics, Argentina'
     oldest_article        = 7
@@ -21,52 +18,50 @@ class MiradasAlSur(BasicNewsRecipe):
     use_embedded_content  = False
     encoding              = 'utf-8'
     language              = 'es_AR'
+    remove_empty_feeds    = True
+    masthead_url          = 'http://sur.infonews.com/sites/default/files/www_miradas_al_sur_com_logo.gif'
+    extra_css             = """
+                             body{font-family: Arial,Helvetica,sans-serif}
+                             h1{font-family: Georgia,Times,serif}
+                             .field-field-story-author{color: gray; font-size: small}
+                            """
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        , 'series'    : title
+                        }
 
-    lang                  = 'es-AR'
+    keep_only_tags = [dict(name='div', attrs={'id':['content-header', 'content-area']})]
-    direction             = 'ltr'
+    remove_tags    = [
-    INDEX                 = 'http://www.elargentino.com/medios/123/Miradas-al-Sur.html'
+                      dict(name=['link','meta','iframe','embed','object']),
-    extra_css             = ' .titulo{font-size: x-large; font-weight: bold} .volantaImp{font-size: small; font-weight: bold} '
+                      dict(name='form', attrs={'class':'fivestar-widget'}),
-
+                      dict(attrs={'class':lambda x: x and 'terms-inline' in x.split()})
-    html2lrf_options = [
-                          '--comment'  , description
-                        , '--category' , category
-                        , '--publisher', publisher
                      ]
 
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0cm; margin-top: 0em; margin-bottom: 0.5em} "'
+    feeds = [
-
+             (u'Politica'           , u'http://sur.infonews.com/taxonomy/term/1/0/feed'),
-    keep_only_tags = [dict(name='div', attrs={'class':'ContainerPop'})]
+             (u'Internacional'      , u'http://sur.infonews.com/taxonomy/term/2/0/feed'),
-
+             (u'Informe Especial'   , u'http://sur.infonews.com/taxonomy/term/14/0/feed'),
-    remove_tags = [dict(name='link')]
+             (u'Delitos y pesquisas', u'http://sur.infonews.com/taxonomy/term/6/0/feed'),
-
+             (u'Lesa Humanidad'     , u'http://sur.infonews.com/taxonomy/term/7/0/feed'),
-    feeds = [(u'Articulos', u'http://www.elargentino.com/Highlights.aspx?ParentType=Section&ParentId=123&Content-Type=text/xml&ChannelDesc=Miradas%20al%20Sur')]
+             (u'Cultura'            , u'http://sur.infonews.com/taxonomy/term/8/0/feed'),
-
+             (u'Deportes'           , u'http://sur.infonews.com/taxonomy/term/9/0/feed'),
-    def print_version(self, url):
+             (u'Contratapa'         , u'http://sur.infonews.com/taxonomy/term/10/0/feed'),
-        main, sep, article_part = url.partition('/nota-')
+            ]
-        article_id, rsep, rrest = article_part.partition('-')
-        return u'http://www.elargentino.com/Impresion.aspx?Id=' + article_id
-
-    def preprocess_html(self, soup):
-        for item in soup.findAll(style=True):
-            del item['style']
-        soup.html['lang'] = self.lang
-        soup.html['dir' ] = self.direction
-        mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)])
-        mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=utf-8")])
-        soup.head.insert(0,mlang)
-        soup.head.insert(1,mcharset)
-        return soup
 
     def get_cover_url(self):
+        # determine the series number, unfortunately not gonna happen now
+        #self.conversion_options.update({'series_index':seriesnr})
         cover_url = None
-        soup = self.index_to_soup(self.INDEX)
+        cdate = datetime.date.today()
-        cover_item = soup.find('div',attrs={'class':'colder'})
+        todayweekday = cdate.isoweekday()
+        if (todayweekday != 7):
+            cdate -= datetime.timedelta(days=todayweekday)
+        cover_page_url = cdate.strftime('http://sur.infonews.com/ediciones/%Y-%m-%d/tapa')
+        soup = self.index_to_soup(cover_page_url)
+        cover_item = soup.find('img', attrs={'class':lambda x: x and 'imagecache-tapa_edicion_full' in x.split()})
         if cover_item:
-           clean_url = self.image_url_processor(None,cover_item.div.img['src'])
+            cover_url = cover_item['src']
-           cover_url = 'http://www.elargentino.com' + clean_url + '&height=600'
         return cover_url
-
-    def image_url_processor(self, baseurl, url):
-        base, sep, rest = url.rpartition('?Id=')
-        img, sep2, rrest = rest.partition('&')
-        return base + sep + img

recipes/pc_quest_india.recipe

@@ -1,16 +0,0 @@
-from calibre.web.feeds.news import CalibrePeriodical
-
-class PCQ(CalibrePeriodical):
-
-    title = 'PCQuest'
-    calibre_periodicals_slug = 'pc-quest-india'
-
-    description = '''
-    Buying a tech product? Seeking a tech solution? Consult PCQuest, India's
-    market-leading selection and implementation guide for the latest
-    technologies: servers, business apps, security, open source, gadgets and
-    more. To subscribe visit, <a
-    href="http://news.calibre-ebook.com/periodical/pc-quest-india">calibre
-    Periodicals</a>.
-    '''
-    language = 'en_IN'

recipes/taz_rss.recipe

@@ -18,7 +18,6 @@ class TazRSSRecipe(BasicNewsRecipe):
 
     feeds          = [(u'TAZ main feed', u'http://www.taz.de/rss.xml')]
     keep_only_tags = [dict(name='div', attrs={'class': 'sect sect_article'})]
-    remove_tags_after = dict(name='div', attrs={'class': 'rack'})
     remove_tags = [
          dict(name=['div'], attrs={'class': 'artikelwerbung'}),
          dict(name=['ul'], attrs={'class': 'toolbar'}),]

setup/file_hosting_servers.rst

@@ -0,0 +1,47 @@
+Provisioning a file hosting server
+====================================
+
+Create the ssh authorized keys file.
+
+Edit /etc/ssh/sshd_config and change PermitRootLogin to without-password.
+Restart sshd.
+
+hostname whatever
+Edit /etc/hosts and put in FQDN in the appropriate places, for example::
+    27.0.1.1       download.calibre-ebook.com download
+    46.28.49.116 download.calibre-ebook.com download
+
+apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools mosh
+chsh -s /bin/zsh
+
+mkdir -p /root/staging /root/work/vim /srv/download /srv/manual
+
+scp .zshrc .vimrc  server:
+scp -r ~/work/vim/zsh-syntax-highlighting server:work/vim
+
+If the server has a backup hard-disk, mount it at /mnt/backup and edit /etc/fstab so that it is auto-mounted.
+Then, add the following to crontab::
+    @daily     /usr/bin/rsync -ha /srv /mnt/backup
+    @daily     /usr/bin/rsync -ha /etc /mnt/backup
+
+Nginx
+------
+
+Copy over /etc/nginx/sites-available/default from another file server. When
+copying, remember to use cat instead of cp to preserve hardlinks (the file is a
+hardlink to /etc/nginx/sites-enabled/default)
+
+rsync /srv from another file server
+
+service nginx start
+
+Services
+---------
+
+SSH into sourceforge and downloadbestsoftware so that their host keys are
+stored.
+
+   ssh -oStrictHostKeyChecking=no kovid@www.downloadbestsoft-mirror1.com
+   ssh -oStrictHostKeyChecking=no kovidgoyal,calibre@frs.sourceforge.net
+   ssh -oStrictHostKeyChecking=no files.calibre-ebook.com (and whatever other mirrors are present)
+

setup/hosting.py

@@ -7,16 +7,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib
+import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil
 from argparse import ArgumentParser, FileType
 from subprocess import check_call
-from tempfile import NamedTemporaryFile#, mkdtemp
+from tempfile import NamedTemporaryFile
 from collections import OrderedDict
 
+def login_to_google(username, password):  # {{{
     import mechanize
-from lxml import html
-
-def login_to_google(username, password):
     br = mechanize.Browser()
     br.addheaders = [('User-agent',
         'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')]
@@ -37,6 +35,7 @@ def login_to_google(username, password):
             'at: lynx -accept_all_cookies https://accounts.google.com/ServiceLogin?service=code')
                 %(username, password))
     return br
+# }}}
 
 class ReadFileWithProgressReporting(file):  # {{{
 
@@ -141,7 +140,7 @@ class GoogleCode(Base):# {{{
             # The pattern to match filenames for the files being uploaded and
             # extract version information from them. Must have a named group
             # named version
-            filename_pattern=r'{appname}-(?:portable-installer-)?(?P<version>.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)'
+            filename_pattern=r'{appname}-(?:portable-installer-)?(?P<version>.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)'  # noqa
 
             ):
         self.username, self.password, = username, password
@@ -227,7 +226,8 @@ class GoogleCode(Base):# {{{
             paths = eval(raw) if raw else {}
             paths.update(self.paths)
             rem = [x for x in paths if self.version not in x]
-            for x in rem: paths.pop(x)
+            for x in rem:
+                paths.pop(x)
             raw = ['%r : %r,'%(k, v) for k, v in paths.items()]
             raw = '{\n\n%s\n\n}\n'%('\n'.join(raw))
             with NamedTemporaryFile() as t:
@@ -244,6 +244,7 @@ class GoogleCode(Base):# {{{
         return login_to_google(self.username, self.gmail_password)
 
     def get_files_hosted_by_google_code(self):
+        from lxml import html
         self.info('Getting existing files in google code:', self.gc_project)
         raw = urllib2.urlopen(self.files_list).read()
         root = html.fromstring(raw)
@@ -378,6 +379,156 @@ class SourceForge(Base): # {{{
 
 # }}}
 
+def generate_index():  # {{{
+    os.chdir('/srv/download')
+    releases = set()
+    for x in os.listdir('.'):
+        if os.path.isdir(x) and '.' in x:
+            releases.add(tuple((int(y) for y in x.split('.'))))
+    rmap = OrderedDict()
+    for rnum in sorted(releases, reverse=True):
+        series = rnum[:2] if rnum[0] == 0 else rnum[:1]
+        if series not in rmap:
+            rmap[series] = []
+        rmap[series].append(rnum)
+
+    template = '''<!DOCTYPE html>\n<html lang="en"> <head> <meta charset="utf-8"> <title>{title}</title> <style type="text/css"> {style} </style> </head> <body> <h1>{title}</h1> <p>{msg}</p> {body} </body> </html> '''  # noqa
+    style = '''
+    body { font-family: sans-serif; background-color: #eee; }
+    a { text-decoration: none; }
+    a:visited { color: blue }
+    a:hover { color: red }
+    ul { list-style-type: none }
+    li { padding-bottom: 1ex }
+    dd li { text-indent: 0; margin: 0 }
+    dd ul { padding: 0; margin: 0 }
+    dt { font-weight: bold }
+    dd { margin-bottom: 2ex }
+    '''
+    body = []
+    for series in rmap:
+        body.append('<li><a href="{0}.html" title="Releases in the {0}.x series">{0}.x</a>\xa0\xa0\xa0<span style="font-size:smaller">[{1} releases]</span></li>'.format(  # noqa
+                '.'.join(map(type(''), series)), len(rmap[series])))
+    body = '<ul>{0}</ul>'.format(' '.join(body))
+    index = template.format(title='Previous calibre releases', style=style, msg='Choose a series of calibre releases', body=body)
+    with open('index.html', 'wb') as f:
+        f.write(index.encode('utf-8'))
+
+    for series, releases in rmap.iteritems():
+        sname = '.'.join(map(type(''), series))
+        body = [
+            '<li><a href="{0}/" title="Release {0}">{0}</a></li>'.format('.'.join(map(type(''), r)))
+            for r in releases]
+        body = '<ul class="release-list">{0}</ul>'.format(' '.join(body))
+        index = template.format(title='Previous calibre releases (%s.x)' % sname, style=style,
+                                msg='Choose a calibre release', body=body)
+        with open('%s.html' % sname, 'wb') as f:
+            f.write(index.encode('utf-8'))
+
+        for r in releases:
+            rname = '.'.join(map(type(''), r))
+            os.chdir(rname)
+            try:
+                body = []
+                files = os.listdir('.')
+                windows = [x for x in files if x.endswith('.msi')]
+                if windows:
+                    windows = ['<li><a href="{0}" title="{1}">{1}</a></li>'.format(
+                        x, 'Windows 64-bit Installer' if '64bit' in x else 'Windows 32-bit Installer')
+                        for x in windows]
+                    body.append('<dt>Windows</dt><dd><ul>{0}</ul></dd>'.format(' '.join(windows)))
+                portable = [x for x in files if '-portable-' in x]
+                if portable:
+                    body.append('<dt>Calibre Portable</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(
+                        portable[0], 'Calibre Portable Installer'))
+                osx = [x for x in files if x.endswith('.dmg')]
+                if osx:
+                    body.append('<dt>Apple Mac</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(
+                        osx[0], 'OS X Disk Image (.dmg)'))
+                linux = [x for x in files if x.endswith('.bz2')]
+                if linux:
+                    linux = ['<li><a href="{0}" title="{1}">{1}</a></li>'.format(
+                        x, 'Linux 64-bit binary' if 'x86_64' in x else 'Linux 32-bit binary')
+                        for x in linux]
+                    body.append('<dt>Linux</dt><dd><ul>{0}</ul></dd>'.format(' '.join(linux)))
+                source = [x for x in files if x.endswith('.xz') or x.endswith('.gz')]
+                if source:
+                    body.append('<dt>Source Code</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(
+                        source[0], 'Source code (all platforms)'))
+
+                body = '<dl>{0}</dl>'.format(''.join(body))
+                index = template.format(title='calibre release (%s)' % rname, style=style,
+                                msg='', body=body)
+                with open('index.html', 'wb') as f:
+                    f.write(index.encode('utf-8'))
+            finally:
+                os.chdir('..')
+
+# }}}
+
+def upload_to_servers(files, version):  # {{{
+    base = '/srv/download/'
+    dest = os.path.join(base, version)
+    if not os.path.exists(dest):
+        os.mkdir(dest)
+    for src in files:
+        shutil.copyfile(src, os.path.join(dest, os.path.basename(src)))
+    cwd = os.getcwd()
+    try:
+        generate_index()
+    finally:
+        os.chdir(cwd)
+
+    for server, rdir in {'files':'/srv/download/'}.iteritems():
+        print('Uploading to server:', server)
+        server = '%s.calibre-ebook.com' % server
+        # Copy the generated index files
+        print ('Copying generated index')
+        check_call(['rsync', '-hza', '-e', 'ssh -x', '--include', '*.html',
+                    '--filter', '-! */', base, 'root@%s:%s' % (server, rdir)])
+        # Copy the release files
+        rdir = '%s%s/' % (rdir, version)
+        for x in files:
+            start = time.time()
+            print ('Uploading', x)
+            for i in range(5):
+                try:
+                    check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
+                    'root@%s:%s'%(server, rdir)])
+                except KeyboardInterrupt:
+                    raise SystemExit(1)
+                except:
+                    print ('\nUpload failed, trying again in 30 seconds')
+                    time.sleep(30)
+                else:
+                    break
+            print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
+
+# }}}
+
+def upload_to_dbs(files, version):  # {{{
+    print('Uploading to downloadbestsoftware.com')
+    server = 'www.downloadbestsoft-mirror1.com'
+    rdir = 'release/'
+    check_call(['ssh', 'kovid@%s' % server, 'rm -f release/*'])
+    for x in files:
+        start = time.time()
+        print ('Uploading', x)
+        for i in range(5):
+            try:
+                check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
+                'kovid@%s:%s'%(server, rdir)])
+            except KeyboardInterrupt:
+                raise SystemExit(1)
+            except:
+                print ('\nUpload failed, trying again in 30 seconds')
+                time.sleep(30)
+            else:
+                break
+        print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
+    check_call(['ssh', 'kovid@%s' % server, '/home/kovid/uploadFiles'])
+# }}}
+
 # CLI {{{
 def cli_parser():
     epilog='Copyright Kovid Goyal 2012'
@@ -409,6 +560,8 @@ def cli_parser():
     sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge',
             epilog=epilog)
     cron = subparsers.add_parser('cron', help='Call script from cron')
+    subparsers.add_parser('calibre', help='Upload to calibre file servers')
+    subparsers.add_parser('dbs', help='Upload to downloadbestsoftware.com')
 
     a = gc.add_argument
 
@@ -471,8 +624,14 @@ def main(args=None):
         sf()
     elif args.service == 'cron':
         login_to_google(args.username, args.password)
+    elif args.service == 'calibre':
+        upload_to_servers(ofiles, args.version)
+    elif args.service == 'dbs':
+        upload_to_dbs(ofiles, args.version)
 
 if __name__ == '__main__':
     main()
 # }}}
 
+
+

setup/upload.py

@@ -19,10 +19,9 @@ from setup import Command, __version__, installer_name, __appname__
 PREFIX = "/var/www/calibre-ebook.com"
 DOWNLOADS = PREFIX+"/htdocs/downloads"
 BETAS = DOWNLOADS +'/betas'
-USER_MANUAL = '/var/www/localhost/htdocs/'
 HTML2LRF = "calibre/ebooks/lrf/html/demo"
 TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
-STAGING_HOST = '67.207.135.179'
+STAGING_HOST = 'download.calibre-ebook.com'
 STAGING_USER = 'root'
 STAGING_DIR = '/root/staging'
 
@@ -81,7 +80,7 @@ class ReUpload(Command):  # {{{
 
 # Data {{{
 def get_google_data():
-    with open(os.path.expanduser('~/work/kde/conf/googlecodecalibre'), 'rb') as f:
+    with open(os.path.expanduser('~/work/env/private/googlecodecalibre'), 'rb') as f:
         gc_password, ga_un, pw = f.read().strip().split('|')
 
     return {
@@ -111,6 +110,12 @@ def sf_cmdline(ver, sdata):
     return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'],
             sdata['username']]
 
+def calibre_cmdline(ver):
+    return [__appname__, ver, 'fmap', 'calibre']
+
+def dbs_cmdline(ver):
+    return [__appname__, ver, 'fmap', 'dbs']
+
 def run_remote_upload(args):
     print 'Running remotely:', ' '.join(args)
     subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST),
@@ -129,22 +134,37 @@ class UploadInstallers(Command):  # {{{
         available = set(glob.glob('dist/*'))
         files = {x:installer_description(x) for x in
                 all_possible.intersection(available)}
+        sizes = {os.path.basename(x):os.path.getsize(x) for x in files}
+        self.record_sizes(sizes)
         tdir = mkdtemp()
+        backup = os.path.join('/mnt/external/calibre/%s' % __version__)
+        if not os.path.exists(backup):
+            os.mkdir(backup)
         try:
-            self.upload_to_staging(tdir, files)
+            self.upload_to_staging(tdir, backup, files)
+            self.upload_to_calibre()
             self.upload_to_sourceforge()
-            self.upload_to_google(opts.replace)
+            self.upload_to_dbs()
+            # self.upload_to_google(opts.replace)
         finally:
             shutil.rmtree(tdir, ignore_errors=True)
 
-    def upload_to_staging(self, tdir, files):
+    def record_sizes(self, sizes):
+        print ('\nRecording dist sizes')
+        args = ['%s:%s:%s' % (__version__, fname, size) for fname, size in sizes.iteritems()]
+        check_call(['ssh', 'divok', 'dist_sizes'] + args)
+
+    def upload_to_staging(self, tdir, backup, files):
         os.mkdir(tdir+'/dist')
         hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)),
             'hosting.py')
         shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py'))
 
         for f in files:
-            shutil.copyfile(f, os.path.join(tdir, f))
+            for x in (tdir+'/dist', backup):
+                dest = os.path.join(x, os.path.basename(f))
+                shutil.copy2(f, x)
+                os.chmod(dest, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH)
 
         with open(os.path.join(tdir, 'fmap'), 'wb') as fo:
             for f, desc in files.iteritems():
@@ -170,6 +190,12 @@ class UploadInstallers(Command):  # {{{
         sdata = get_sourceforge_data()
         args = sf_cmdline(__version__, sdata)
         run_remote_upload(args)
+
+    def upload_to_calibre(self):
+        run_remote_upload(calibre_cmdline(__version__))
+
+    def upload_to_dbs(self):
+        run_remote_upload(dbs_cmdline(__version__))
 # }}}
 
 class UploadUserManual(Command):  # {{{
@@ -199,9 +225,9 @@ class UploadUserManual(Command):  # {{{
         for x in glob.glob(self.j(path, '*')):
             self.build_plugin_example(x)
 
+        for host in ('download', 'files'):
             check_call(' '.join(['rsync', '-z', '-r', '--progress',
-            'manual/.build/html/',
+                'manual/.build/html/', '%s:/srv/manual/' % host]), shell=True)
-                    'bugs:%s'%USER_MANUAL]), shell=True)
 # }}}
 
 class UploadDemo(Command):  # {{{
@@ -229,8 +255,6 @@ class UploadToServer(Command):  # {{{
     description = 'Upload miscellaneous data to calibre server'
 
     def run(self, opts):
-        check_call('ssh divok rm -f %s/calibre-\*.tar.xz'%DOWNLOADS, shell=True)
-        # check_call('scp dist/calibre-*.tar.xz divok:%s/'%DOWNLOADS, shell=True)
         check_call('gpg --armor --detach-sign dist/calibre-*.tar.xz',
                 shell=True)
         check_call('scp dist/calibre-*.tar.xz.asc divok:%s/signatures/'%DOWNLOADS,

src/calibre/__init__.py

@@ -310,9 +310,9 @@ def get_parsed_proxy(typ='http', debug=True):
     proxy = proxies.get(typ, None)
     if proxy:
         pattern = re.compile((
-            '(?:ptype://)?' \
+            '(?:ptype://)?'
-            '(?:(?P<user>\w+):(?P<pass>.*)@)?' \
+            '(?:(?P<user>\w+):(?P<pass>.*)@)?'
-            '(?P<host>[\w\-\.]+)' \
+            '(?P<host>[\w\-\.]+)'
             '(?::(?P<port>\d+))?').replace('ptype', typ)
         )
 
@@ -670,8 +670,8 @@ def human_readable(size, sep=' '):
     """ Convert a size in bytes into a human readable form """
     divisor, suffix = 1, "B"
     for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
-        if size < 1024**(i+1):
+        if size < (1 << ((i + 1) * 10)):
-            divisor, suffix = 1024**(i), candidate
+            divisor, suffix = (1 << (i * 10)), candidate
             break
     size = str(float(size)/divisor)
     if size.find(".") > -1:

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 9, 36)
+numeric_version = (0, 9, 37)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/db/cache.py

@@ -843,9 +843,25 @@ class Cache(object):
     @write_api
     def set_metadata(self, book_id, mi, ignore_errors=False, force_changes=False,
                      set_title=True, set_authors=True):
-        if callable(getattr(mi, 'to_book_metadata', None)):
+        '''
+        Set metadata for the book `id` from the `Metadata` object `mi`
+
+        Setting force_changes=True will force set_metadata to update fields even
+        if mi contains empty values. In this case, 'None' is distinguished from
+        'empty'. If mi.XXX is None, the XXX is not replaced, otherwise it is.
+        The tags, identifiers, and cover attributes are special cases. Tags and
+        identifiers cannot be set to None so then will always be replaced if
+        force_changes is true. You must ensure that mi contains the values you
+        want the book to have. Covers are always changed if a new cover is
+        provided, but are never deleted. Also note that force_changes has no
+        effect on setting title or authors.
+        '''
+
+        try:
             # Handle code passing in an OPF object instead of a Metadata object
             mi = mi.to_book_metadata()
+        except (AttributeError, TypeError):
+            pass
 
         def set_field(name, val, **kwargs):
             self._set_field(name, {book_id:val}, **kwargs)
@@ -864,7 +880,7 @@ class Cache(object):
             set_field('authors', authors, do_path_update=False)
 
         if path_changed:
-            self._update_path((book_id,))
+            self._update_path({book_id})
 
         def protected_set_field(name, val, **kwargs):
             try:
@@ -890,12 +906,16 @@ class Cache(object):
         if cdata is not None:
             self._set_cover({book_id: cdata})
 
-        for field in ('title_sort', 'author_sort', 'publisher', 'series',
+        for field in ('author_sort', 'publisher', 'series', 'tags', 'comments',
-            'tags', 'comments', 'languages', 'pubdate'):
+            'languages', 'pubdate'):
             val = mi.get(field, None)
             if (force_changes and val is not None) or not mi.is_null(field):
                 protected_set_field(field, val)
 
+        val = mi.get('title_sort', None)
+        if (force_changes and val is not None) or not mi.is_null('title_sort'):
+            protected_set_field('sort', val)
+
         # identifiers will always be replaced if force_changes is True
         mi_idents = mi.get_identifiers()
         if force_changes:
@@ -917,9 +937,11 @@ class Cache(object):
                 val = mi.get(key, None)
                 if force_changes or val is not None:
                     protected_set_field(key, val)
+                    idx = key + '_index'
+                    if idx in self.fields:
                         extra = mi.get_extra(key)
-                    if extra is not None:
+                        if extra is not None or force_changes:
-                        protected_set_field(key+'_index', extra)
+                            protected_set_field(idx, extra)
 
     # }}}
 

src/calibre/db/tests/base.py

@@ -78,7 +78,7 @@ class BaseTest(unittest.TestCase):
     def cloned_library(self):
         return self.clone_library(self.library_path)
 
-    def compare_metadata(self, mi1, mi2):
+    def compare_metadata(self, mi1, mi2, exclude=()):
         allfk1 = mi1.all_field_keys()
         allfk2 = mi2.all_field_keys()
         self.assertEqual(allfk1, allfk2)
@@ -88,7 +88,7 @@ class BaseTest(unittest.TestCase):
                     'ondevice_col', 'last_modified', 'has_cover',
                     'cover_data'}.union(allfk1)
         for attr in all_keys:
-            if attr == 'user_metadata':
+            if attr == 'user_metadata' or attr in exclude:
                 continue
             attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr)
             if attr == 'formats':
@@ -97,7 +97,7 @@ class BaseTest(unittest.TestCase):
                 attr1, attr2 = set(attr1), set(attr2)
             self.assertEqual(attr1, attr2,
                     '%s not the same: %r != %r'%(attr, attr1, attr2))
-            if attr.startswith('#'):
+            if attr.startswith('#') and attr + '_index' not in exclude:
                 attr1, attr2 = mi1.get_extra(attr), mi2.get_extra(attr)
                 self.assertEqual(attr1, attr2,
                     '%s {#extra} not the same: %r != %r'%(attr, attr1, attr2))

src/calibre/db/tests/legacy.py

@@ -120,7 +120,8 @@ class LegacyTest(BaseTest):
         for attr in dir(db):
             if attr in SKIP_ATTRS:
                 continue
-            self.assertTrue(hasattr(ndb, attr), 'The attribute %s is missing' % attr)
+            if not hasattr(ndb, attr):
+                raise AssertionError('The attribute %s is missing' % attr)
             obj, nobj  = getattr(db, attr), getattr(ndb, attr)
             if attr not in SKIP_ARGSPEC:
                 try:

src/calibre/db/tests/writing.py

@@ -376,7 +376,43 @@ class WritingTest(BaseTest):
             self.assertTrue(old.has_cover(book_id))
     # }}}
 
-    def test_set_metadata(self):
+    def test_set_metadata(self):  # {{{
         ' Test setting of metadata '
-        self.assertTrue(False, 'TODO: test set_metadata()')
+        ae = self.assertEqual
+        cache = self.init_cache(self.cloned_library)
+
+        # Check that changing title/author updates the path
+        mi = cache.get_metadata(1)
+        old_path = cache.field_for('path', 1)
+        old_title, old_author = mi.title, mi.authors[0]
+        ae(old_path, '%s/%s (1)' % (old_author, old_title))
+        mi.title, mi.authors = 'New Title', ['New Author']
+        cache.set_metadata(1, mi)
+        ae(cache.field_for('path', 1), '%s/%s (1)' % (mi.authors[0], mi.title))
+        p = cache.format_abspath(1, 'FMT1')
+        self.assertTrue(mi.authors[0] in p and mi.title in p)
+
+        # Compare old and new set_metadata()
+        db = self.init_old(self.cloned_library)
+        mi = db.get_metadata(1, index_is_id=True, get_cover=True, cover_as_data=True)
+        mi2 = db.get_metadata(3, index_is_id=True, get_cover=True, cover_as_data=True)
+        db.set_metadata(2, mi)
+        db.set_metadata(1, mi2, force_changes=True)
+        oldmi = db.get_metadata(2, index_is_id=True, get_cover=True, cover_as_data=True)
+        oldmi2 = db.get_metadata(1, index_is_id=True, get_cover=True, cover_as_data=True)
+        db.close()
+        del db
+        cache = self.init_cache(self.cloned_library)
+        cache.set_metadata(2, mi)
+        nmi = cache.get_metadata(2, get_cover=True, cover_as_data=True)
+        ae(oldmi.cover_data, nmi.cover_data)
+        self.compare_metadata(nmi, oldmi, exclude={'last_modified', 'format_metadata'})
+        cache.set_metadata(1, mi2, force_changes=True)
+        nmi2 = cache.get_metadata(1, get_cover=True, cover_as_data=True)
+        # The new code does not allow setting of #series_index to None, instead
+        # it is reset to 1.0
+        ae(nmi2.get_extra('#series'), 1.0)
+        self.compare_metadata(nmi2, oldmi2, exclude={'last_modified', 'format_metadata', '#series_index'})
+
+    # }}}
 

src/calibre/devices/misc.py

@@ -211,6 +211,7 @@ class ALURATEK_COLOR(USBMS):
     VENDOR_NAME = ['USB_2.0', 'EZREADER', 'C4+']
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['USB_FLASH_DRIVER', '.', 'TOUCH']
     SCAN_FROM_ROOT = True
+    SUPPORTS_SUB_DIRS_FOR_SCAN = True
 
 class TREKSTOR(USBMS):
 

src/calibre/ebooks/conversion/cli.py

@@ -94,6 +94,8 @@ def option_recommendation_to_cli_option(add_option, rec):
     if opt.long_switch == 'verbose':
         attrs['action'] = 'count'
         attrs.pop('type', '')
+    if opt.name == 'read_metadata_from_opf':
+        switches.append('--from-opf')
     if opt.name in DEFAULT_TRUE_OPTIONS and rec.recommended_value is True:
         switches = ['--disable-'+opt.long_switch]
     add_option(Option(*switches, **attrs))
@@ -136,7 +138,7 @@ def add_pipeline_options(parser, plumber):
                   [
                       'base_font_size', 'disable_font_rescaling',
                       'font_size_mapping', 'embed_font_family',
-                      'subset_embedded_fonts',
+                      'subset_embedded_fonts', 'embed_all_fonts',
                       'line_height', 'minimum_line_height',
                       'linearize_tables',
                       'extra_css', 'filter_css',
@@ -190,7 +192,7 @@ def add_pipeline_options(parser, plumber):
                   ),
 
               'METADATA' : (_('Options to set metadata in the output'),
-                            plumber.metadata_option_names,
+                            plumber.metadata_option_names + ['read_metadata_from_opf'],
                             ),
               'DEBUG': (_('Options to help with debugging the conversion'),
                         [
@@ -320,7 +322,7 @@ def main(args=sys.argv):
         opts.search_replace = read_sr_patterns(opts.search_replace, log)
 
     recommendations = [(n.dest, getattr(opts, n.dest),
-                        OptionRecommendation.HIGH) \
+                        OptionRecommendation.HIGH)
                                         for n in parser.options_iter()
                                         if n.dest]
     plumber.merge_ui_recommendations(recommendations)
@@ -342,3 +344,4 @@ def main(args=sys.argv):
 
 if __name__ == '__main__':
     sys.exit(main())
+

src/calibre/ebooks/conversion/plumber.py

@@ -205,6 +205,16 @@ OptionRecommendation(name='embed_font_family',
             'with some output formats, principally EPUB and AZW3.')
         ),
 
+OptionRecommendation(name='embed_all_fonts',
+        recommended_value=False, level=OptionRecommendation.LOW,
+        help=_(
+            'Embed every font that is referenced in the input document '
+            'but not already embedded. This will search your system for the '
+            'fonts, and if found, they will be embedded. Embedding will only work '
+            'if the format you are converting to supports embedded fonts, such as '
+            'EPUB, AZW3 or PDF.'
+        )),
+
 OptionRecommendation(name='subset_embedded_fonts',
         recommended_value=False, level=OptionRecommendation.LOW,
         help=_(
@@ -965,6 +975,9 @@ OptionRecommendation(name='search_replace',
         if self.for_regex_wizard and hasattr(self.opts, 'no_process'):
             self.opts.no_process = True
         self.flush()
+        if self.opts.embed_all_fonts or self.opts.embed_font_family:
+            # Start the threaded font scanner now, for performance
+            from calibre.utils.fonts.scanner import font_scanner  # noqa
         import cssutils, logging
         cssutils.log.setLevel(logging.WARN)
         get_types_map()  # Ensure the mimetypes module is intialized
@@ -1129,6 +1142,10 @@ OptionRecommendation(name='search_replace',
         RemoveFakeMargins()(self.oeb, self.log, self.opts)
         RemoveAdobeMargins()(self.oeb, self.log, self.opts)
 
+        if self.opts.embed_all_fonts:
+            from calibre.ebooks.oeb.transforms.embed_fonts import EmbedFonts
+            EmbedFonts()(self.oeb, self.log, self.opts)
+
         if self.opts.subset_embedded_fonts and self.output_plugin.file_type != 'pdf':
             from calibre.ebooks.oeb.transforms.subset import SubsetFonts
             SubsetFonts()(self.oeb, self.log, self.opts)

src/calibre/ebooks/lrf/html/convert_from.py

@@ -104,7 +104,7 @@ class HTMLConverter(object):
 
                         # Replace entities
                         (re.compile(ur'&(\S+?);'), partial(entity_to_unicode,
-                                                           exceptions=['lt', 'gt', 'amp'])),
+                                                           exceptions=['lt', 'gt', 'amp', 'quot'])),
                         # Remove comments from within style tags as they can mess up BeatifulSoup
                         (re.compile(r'(<style.*?</style>)', re.IGNORECASE|re.DOTALL),
                          strip_style_comments),

src/calibre/ebooks/metadata/opf2.py

@@ -1047,6 +1047,14 @@ class OPF(object):  # {{{
                     if raw:
                         return raw.rpartition(':')[-1]
 
+    @property
+    def page_progression_direction(self):
+        spine = self.XPath('descendant::*[re:match(name(), "spine", "i")][1]')(self.root)
+        if spine:
+            for k, v in spine[0].attrib.iteritems():
+                if k == 'page-progression-direction' or k.endswith('}page-progression-direction'):
+                    return v
+
     def guess_cover(self):
         '''
         Try to guess a cover. Needed for some old/badly formed OPF files.
@@ -1185,6 +1193,7 @@ class OPFCreator(Metadata):
         '''
         Metadata.__init__(self, title='', other=other)
         self.base_path = os.path.abspath(base_path)
+        self.page_progression_direction = None
         if self.application_id is None:
             self.application_id = str(uuid.uuid4())
         if not isinstance(self.toc, TOC):
@@ -1356,6 +1365,8 @@ class OPFCreator(Metadata):
         spine = E.spine()
         if self.toc is not None:
             spine.set('toc', 'ncx')
+        if self.page_progression_direction is not None:
+            spine.set('page-progression-direction', self.page_progression_direction)
         if self.spine is not None:
             for ref in self.spine:
                 if ref.id is not None:

src/calibre/ebooks/metadata/sources/cli.py

@@ -34,9 +34,9 @@ def option_parser():
     parser.add_option('-i', '--isbn', help='Book ISBN')
     parser.add_option('-v', '--verbose', default=False, action='store_true',
                       help='Print the log to the console (stderr)')
-    parser.add_option('-o', '--opf', help='Output the metadata in OPF format')
+    parser.add_option('-o', '--opf', help='Output the metadata in OPF format instead of human readable text.', action='store_true', default=False)
     parser.add_option('-c', '--cover',
-            help='Specify a filename. The cover, if available, will be saved to it')
+            help='Specify a filename. The cover, if available, will be saved to it. Without this option, no cover will be downloaded.')
     parser.add_option('-d', '--timeout', default='30',
             help='Timeout in seconds. Default is 30')
 
@@ -71,16 +71,14 @@ def main(args=sys.argv):
     if opts.cover and results:
         cover = download_cover(log, title=opts.title, authors=authors,
                 identifiers=result.identifiers, timeout=int(opts.timeout))
-        if cover is None:
+        if cover is None and not opts.opf:
             prints('No cover found', file=sys.stderr)
         else:
             save_cover_data_to(cover[-1], opts.cover)
             result.cover = cf = opts.cover
 
-
     log = buf.getvalue()
 
-
     result = (metadata_to_opf(result) if opts.opf else
                     unicode(result).encode('utf-8'))
 
@@ -95,3 +93,4 @@ def main(args=sys.argv):
 
 if __name__ == '__main__':
     sys.exit(main())
+

src/calibre/ebooks/mobi/reader/mobi8.py

@@ -20,7 +20,7 @@ from calibre.ebooks.mobi.reader.ncx import read_ncx, build_toc
 from calibre.ebooks.mobi.reader.markup import expand_mobi8_markup
 from calibre.ebooks.metadata.opf2 import Guide, OPFCreator
 from calibre.ebooks.metadata.toc import TOC
-from calibre.ebooks.mobi.utils import read_font_record
+from calibre.ebooks.mobi.utils import read_font_record, read_resc_record
 from calibre.ebooks.oeb.parse_utils import parse_html
 from calibre.ebooks.oeb.base import XPath, XHTML, xml2text
 from calibre.utils.imghdr import what
@@ -65,6 +65,7 @@ class Mobi8Reader(object):
         self.mobi6_reader, self.log = mobi6_reader, log
         self.header = mobi6_reader.book_header
         self.encrypted_fonts = []
+        self.resc_data = {}
 
     def __call__(self):
         self.mobi6_reader.check_for_drm()
@@ -389,9 +390,11 @@ class Mobi8Reader(object):
             data = sec[0]
             typ = data[:4]
             href = None
-            if typ in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n',
+            if typ in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n', b'BOUN',
-                    b'RESC', b'BOUN', b'FDST', b'DATP', b'AUDI', b'VIDE'}:
+                       b'FDST', b'DATP', b'AUDI', b'VIDE'}:
                 pass  # Ignore these records
+            elif typ == b'RESC':
+                self.resc_data = read_resc_record(data)
             elif typ == b'FONT':
                 font = read_font_record(data)
                 href = "fonts/%05d.%s" % (fname_idx, font['ext'])
@@ -452,6 +455,9 @@ class Mobi8Reader(object):
         opf.create_manifest_from_files_in([os.getcwdu()], exclude=exclude)
         opf.create_spine(spine)
         opf.set_toc(toc)
+        ppd = self.resc_data.get('page-progression-direction', None)
+        if ppd:
+            opf.page_progression_direction = ppd
 
         with open('metadata.opf', 'wb') as of, open('toc.ncx', 'wb') as ncx:
             opf.render(of, ncx, 'toc.ncx')

src/calibre/ebooks/mobi/utils.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import struct, string, zlib, os
+import struct, string, zlib, os, re
 from collections import OrderedDict
 from io import BytesIO
 
@@ -393,6 +393,15 @@ def mobify_image(data):
         data = im.export('gif')
     return data
 
+def read_resc_record(data):
+    ans = {}
+    match = re.search(br'''<spine [^>]*page-progression-direction=['"](.+?)['"]''', data)
+    if match is not None:
+        ppd = match.group(1).lower()
+        if ppd in {b'ltr', b'rtl'}:
+            ans['page-progression-direction'] = ppd.decode('ascii')
+    return ans
+
 # Font records {{{
 def read_font_record(data, extent=1040):
     '''

src/calibre/ebooks/oeb/base.py

@@ -1210,6 +1210,7 @@ class Spine(object):
     def __init__(self, oeb):
         self.oeb = oeb
         self.items = []
+        self.page_progression_direction = None
 
     def _linear(self, linear):
         if isinstance(linear, basestring):
@@ -1896,4 +1897,6 @@ class OEBBook(object):
                              attrib={'media-type': PAGE_MAP_MIME})
             spine.attrib['page-map'] = id
             results[PAGE_MAP_MIME] = (href, self.pages.to_page_map())
+        if self.spine.page_progression_direction in {'ltr', 'rtl'}:
+            spine.attrib['page-progression-direction'] = self.spine.page_progression_direction
         return results

src/calibre/ebooks/oeb/polish/toc.py

@@ -281,14 +281,17 @@ def find_text(node):
 
 def from_files(container):
     toc = TOC()
-    for spinepath in container.spine_items:
+    for i, spinepath in enumerate(container.spine_items):
         name = container.abspath_to_name(spinepath)
         root = container.parsed(name)
         body = XPath('//h:body')(root)
         if not body:
             continue
         text = find_text(body[0])
-        if text:
+        if not text:
+            text = name.rpartition('/')[-1]
+            if i == 0 and text.rpartition('.')[0].lower() in {'titlepage', 'cover'}:
+                text = _('Cover')
         toc.add(text, name)
     return toc
 

src/calibre/ebooks/oeb/reader.py

@@ -330,6 +330,9 @@ class OEBReader(object):
         if len(spine) == 0:
             raise OEBError("Spine is empty")
         self._spine_add_extra()
+        for val in xpath(opf, '/o2:package/o2:spine/@page-progression-direction'):
+            if val in {'ltr', 'rtl'}:
+                spine.page_progression_direction = val
 
     def _guide_from_opf(self, opf):
         guide = self.oeb.guide

src/calibre/ebooks/oeb/transforms/embed_fonts.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import logging
+from collections import defaultdict
+
+import cssutils
+from lxml import etree
+
+from calibre import guess_type
+from calibre.ebooks.oeb.base import XPath, CSS_MIME, XHTML
+from calibre.ebooks.oeb.transforms.subset import get_font_properties, find_font_face_rules, elem_style
+from calibre.utils.filenames import ascii_filename
+from calibre.utils.fonts.scanner import font_scanner, NoFonts
+
+def used_font(style, embedded_fonts):
+    ff = [unicode(f) for f in style.get('font-family', []) if unicode(f).lower() not in {
+        'serif', 'sansserif', 'sans-serif', 'fantasy', 'cursive', 'monospace'}]
+    if not ff:
+        return False, None
+    lnames = {unicode(x).lower() for x in ff}
+
+    matching_set = []
+
+    # Filter on font-family
+    for ef in embedded_fonts:
+        flnames = {x.lower() for x in ef.get('font-family', [])}
+        if not lnames.intersection(flnames):
+            continue
+        matching_set.append(ef)
+    if not matching_set:
+        return True, None
+
+    # Filter on font-stretch
+    widths = {x:i for i, x in enumerate(('ultra-condensed',
+            'extra-condensed', 'condensed', 'semi-condensed', 'normal',
+            'semi-expanded', 'expanded', 'extra-expanded', 'ultra-expanded'
+            ))}
+
+    width = widths[style.get('font-stretch', 'normal')]
+    for f in matching_set:
+        f['width'] = widths[style.get('font-stretch', 'normal')]
+
+    min_dist = min(abs(width-f['width']) for f in matching_set)
+    if min_dist > 0:
+        return True, None
+    nearest = [f for f in matching_set if abs(width-f['width']) ==
+        min_dist]
+    if width <= 4:
+        lmatches = [f for f in nearest if f['width'] <= width]
+    else:
+        lmatches = [f for f in nearest if f['width'] >= width]
+    matching_set = (lmatches or nearest)
+
+    # Filter on font-style
+    fs = style.get('font-style', 'normal')
+    matching_set = [f for f in matching_set if f.get('font-style', 'normal') == fs]
+
+    # Filter on font weight
+    fw = int(style.get('font-weight', '400'))
+    matching_set = [f for f in matching_set if f.get('weight', 400) == fw]
+
+    if not matching_set:
+        return True, None
+    return True, matching_set[0]
+
+
+class EmbedFonts(object):
+
+    '''
+    Embed all referenced fonts, if found on system. Must be called after CSS flattening.
+    '''
+
+    def __call__(self, oeb, log, opts):
+        self.oeb, self.log, self.opts = oeb, log, opts
+        self.sheet_cache = {}
+        self.find_style_rules()
+        self.find_embedded_fonts()
+        self.parser = cssutils.CSSParser(loglevel=logging.CRITICAL, log=logging.getLogger('calibre.css'))
+        self.warned = set()
+        self.warned2 = set()
+
+        for item in oeb.spine:
+            if not hasattr(item.data, 'xpath'):
+                continue
+            sheets = []
+            for href in XPath('//h:link[@href and @type="text/css"]/@href')(item.data):
+                sheet = self.oeb.manifest.hrefs.get(item.abshref(href), None)
+                if sheet is not None:
+                    sheets.append(sheet)
+            if sheets:
+                self.process_item(item, sheets)
+
+    def find_embedded_fonts(self):
+        '''
+        Find all @font-face rules and extract the relevant info from them.
+        '''
+        self.embedded_fonts = []
+        for item in self.oeb.manifest:
+            if not hasattr(item.data, 'cssRules'):
+                continue
+            self.embedded_fonts.extend(find_font_face_rules(item, self.oeb))
+
+    def find_style_rules(self):
+        '''
+        Extract all font related style information from all stylesheets into a
+        dict mapping classes to font properties specified by that class. All
+        the heavy lifting has already been done by the CSS flattening code.
+        '''
+        rules = defaultdict(dict)
+        for item in self.oeb.manifest:
+            if not hasattr(item.data, 'cssRules'):
+                continue
+            for i, rule in enumerate(item.data.cssRules):
+                if rule.type != rule.STYLE_RULE:
+                    continue
+                props = {k:v for k,v in
+                        get_font_properties(rule).iteritems() if v}
+                if not props:
+                    continue
+                for sel in rule.selectorList:
+                    sel = sel.selectorText
+                    if sel and sel.startswith('.'):
+                        # We dont care about pseudo-selectors as the worst that
+                        # can happen is some extra characters will remain in
+                        # the font
+                        sel = sel.partition(':')[0]
+                        rules[sel[1:]].update(props)
+
+        self.style_rules = dict(rules)
+
+    def get_page_sheet(self):
+        if self.page_sheet is None:
+            manifest = self.oeb.manifest
+            id_, href = manifest.generate('page_css', 'page_styles.css')
+            self.page_sheet = manifest.add(id_, href, CSS_MIME, data=self.parser.parseString('', validate=False))
+            head = self.current_item.xpath('//*[local-name()="head"][1]')
+            if head:
+                href = self.current_item.relhref(href)
+                l = etree.SubElement(head[0], XHTML('link'),
+                    rel='stylesheet', type=CSS_MIME, href=href)
+                l.tail = '\n'
+            else:
+                self.log.warn('No <head> cannot embed font rules')
+        return self.page_sheet
+
+    def process_item(self, item, sheets):
+        ff_rules = []
+        self.current_item = item
+        self.page_sheet = None
+        for sheet in sheets:
+            if 'page_css' in sheet.id:
+                ff_rules.extend(find_font_face_rules(sheet, self.oeb))
+                self.page_sheet = sheet
+
+        base = {'font-family':['serif'], 'font-weight': '400',
+                'font-style':'normal', 'font-stretch':'normal'}
+
+        for body in item.data.xpath('//*[local-name()="body"]'):
+            self.find_usage_in(body, base, ff_rules)
+
+    def find_usage_in(self, elem, inherited_style, ff_rules):
+        style = elem_style(self.style_rules, elem.get('class', '') or '', inherited_style)
+        for child in elem:
+            self.find_usage_in(child, style, ff_rules)
+        has_font, existing = used_font(style, ff_rules)
+        if not has_font:
+            return
+        if existing is None:
+            in_book = used_font(style, self.embedded_fonts)[1]
+            if in_book is None:
+                # Try to find the font in the system
+                added = self.embed_font(style)
+                if added is not None:
+                    ff_rules.append(added)
+                    self.embedded_fonts.append(added)
+            else:
+                # TODO: Create a page rule from the book rule (cannot use it
+                # directly as paths might be different)
+                item = in_book['item']
+                sheet = self.parser.parseString(in_book['rule'].cssText, validate=False)
+                rule = sheet.cssRules[0]
+                page_sheet = self.get_page_sheet()
+                href = page_sheet.abshref(item.href)
+                rule.style.setProperty('src', 'url(%s)' % href)
+                ff_rules.append(find_font_face_rules(sheet, self.oeb)[0])
+                page_sheet.data.insertRule(rule, len(page_sheet.data.cssRules))
+
+    def embed_font(self, style):
+        ff = [unicode(f) for f in style.get('font-family', []) if unicode(f).lower() not in {
+            'serif', 'sansserif', 'sans-serif', 'fantasy', 'cursive', 'monospace'}]
+        if not ff:
+            return
+        ff = ff[0]
+        if ff in self.warned:
+            return
+        try:
+            fonts = font_scanner.fonts_for_family(ff)
+        except NoFonts:
+            self.log.warn('Failed to find fonts for family:', ff, 'not embedding')
+            self.warned.add(ff)
+            return
+        try:
+            weight = int(style.get('font-weight', '400'))
+        except (ValueError, TypeError, AttributeError):
+            w = style['font-weight']
+            if w not in self.warned2:
+                self.log.warn('Invalid weight in font style: %r' % w)
+                self.warned2.add(w)
+            return
+        for f in fonts:
+            if f['weight'] == weight and f['font-style'] == style.get('font-style', 'normal') and f['font-stretch'] == style.get('font-stretch', 'normal'):
+                self.log('Embedding font %s from %s' % (f['full_name'], f['path']))
+                data = font_scanner.get_font_data(f)
+                name = f['full_name']
+                ext = 'otf' if f['is_otf'] else 'ttf'
+                name = ascii_filename(name).replace(' ', '-').replace('(', '').replace(')', '')
+                fid, href = self.oeb.manifest.generate(id=u'font', href=u'fonts/%s.%s'%(name, ext))
+                item = self.oeb.manifest.add(fid, href, guess_type('dummy.'+ext)[0], data=data)
+                item.unload_data_from_memory()
+                page_sheet = self.get_page_sheet()
+                href = page_sheet.relhref(item.href)
+                css = '''@font-face { font-family: "%s"; font-weight: %s; font-style: %s; font-stretch: %s; src: url(%s) }''' % (
+                    f['font-family'], f['font-weight'], f['font-style'], f['font-stretch'], href)
+                sheet = self.parser.parseString(css, validate=False)
+                page_sheet.data.insertRule(sheet.cssRules[0], len(page_sheet.data.cssRules))
+                return find_font_face_rules(sheet, self.oeb)[0]
+

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -194,7 +194,7 @@ class CSSFlattener(object):
         for i, font in enumerate(faces):
             ext = 'otf' if font['is_otf'] else 'ttf'
             fid, href = self.oeb.manifest.generate(id=u'font',
-                href=u'%s.%s'%(ascii_filename(font['full_name']).replace(u' ', u'-'), ext))
+                href=u'fonts/%s.%s'%(ascii_filename(font['full_name']).replace(u' ', u'-'), ext))
             item = self.oeb.manifest.add(fid, href,
                     guess_type('dummy.'+ext)[0],
                     data=font_scanner.get_font_data(font))

src/calibre/ebooks/oeb/transforms/subset.py

@@ -12,6 +12,111 @@ from collections import defaultdict
 from calibre.ebooks.oeb.base import urlnormalize
 from calibre.utils.fonts.sfnt.subset import subset, NoGlyphs, UnsupportedFont
 
+def get_font_properties(rule, default=None):
+    '''
+    Given a CSS rule, extract normalized font properties from
+    it. Note that shorthand font property should already have been expanded
+    by the CSS flattening code.
+    '''
+    props = {}
+    s = rule.style
+    for q in ('font-family', 'src', 'font-weight', 'font-stretch',
+            'font-style'):
+        g = 'uri' if q == 'src' else 'value'
+        try:
+            val = s.getProperty(q).propertyValue[0]
+            val = getattr(val, g)
+            if q == 'font-family':
+                val = [x.value for x in s.getProperty(q).propertyValue]
+                if val and val[0] == 'inherit':
+                    val = None
+        except (IndexError, KeyError, AttributeError, TypeError, ValueError):
+            val = None if q in {'src', 'font-family'} else default
+        if q in {'font-weight', 'font-stretch', 'font-style'}:
+            val = unicode(val).lower() if (val or val == 0) else val
+            if val == 'inherit':
+                val = default
+        if q == 'font-weight':
+            val = {'normal':'400', 'bold':'700'}.get(val, val)
+            if val not in {'100', '200', '300', '400', '500', '600', '700',
+                    '800', '900', 'bolder', 'lighter'}:
+                val = default
+            if val == 'normal':
+                val = '400'
+        elif q == 'font-style':
+            if val not in {'normal', 'italic', 'oblique'}:
+                val = default
+        elif q == 'font-stretch':
+            if val not in {'normal', 'ultra-condensed', 'extra-condensed',
+                    'condensed', 'semi-condensed', 'semi-expanded',
+                    'expanded', 'extra-expanded', 'ultra-expanded'}:
+                val = default
+        props[q] = val
+    return props
+
+
+def find_font_face_rules(sheet, oeb):
+    '''
+    Find all @font-face rules in the given sheet and extract the relevant info from them.
+    sheet can be either a ManifestItem or a CSSStyleSheet.
+    '''
+    ans = []
+    try:
+        rules = sheet.data.cssRules
+    except AttributeError:
+        rules = sheet.cssRules
+
+    for i, rule in enumerate(rules):
+        if rule.type != rule.FONT_FACE_RULE:
+            continue
+        props = get_font_properties(rule, default='normal')
+        if not props['font-family'] or not props['src']:
+            continue
+
+        try:
+            path = sheet.abshref(props['src'])
+        except AttributeError:
+            path = props['src']
+        ff = oeb.manifest.hrefs.get(urlnormalize(path), None)
+        if not ff:
+            continue
+        props['item'] = ff
+        if props['font-weight'] in {'bolder', 'lighter'}:
+            props['font-weight'] = '400'
+        props['weight'] = int(props['font-weight'])
+        props['rule'] = rule
+        props['chars'] = set()
+        ans.append(props)
+
+    return ans
+
+
+def elem_style(style_rules, cls, inherited_style):
+    '''
+    Find the effective style for the given element.
+    '''
+    classes = cls.split()
+    style = inherited_style.copy()
+    for cls in classes:
+        style.update(style_rules.get(cls, {}))
+    wt = style.get('font-weight', None)
+    pwt = inherited_style.get('font-weight', '400')
+    if wt == 'bolder':
+        style['font-weight'] = {
+                '100':'400',
+                '200':'400',
+                '300':'400',
+                '400':'700',
+                '500':'700',
+                }.get(pwt, '900')
+    elif wt == 'lighter':
+        style['font-weight'] = {
+                '600':'400', '700':'400',
+                '800':'700', '900':'700'}.get(pwt, '100')
+
+    return style
+
+
 class SubsetFonts(object):
 
     '''
@@ -76,72 +181,15 @@ class SubsetFonts(object):
             self.log('Reduced total font size to %.1f%% of original'%
                     (totals[0]/totals[1] * 100))
 
-    def get_font_properties(self, rule, default=None):
-        '''
-        Given a CSS rule, extract normalized font properties from
-        it. Note that shorthand font property should already have been expanded
-        by the CSS flattening code.
-        '''
-        props = {}
-        s = rule.style
-        for q in ('font-family', 'src', 'font-weight', 'font-stretch',
-                'font-style'):
-            g = 'uri' if q == 'src' else 'value'
-            try:
-                val = s.getProperty(q).propertyValue[0]
-                val = getattr(val, g)
-                if q == 'font-family':
-                    val = [x.value for x in s.getProperty(q).propertyValue]
-                    if val and val[0] == 'inherit':
-                        val = None
-            except (IndexError, KeyError, AttributeError, TypeError, ValueError):
-                val = None if q in {'src', 'font-family'} else default
-            if q in {'font-weight', 'font-stretch', 'font-style'}:
-                val = unicode(val).lower() if (val or val == 0) else val
-                if val == 'inherit':
-                    val = default
-            if q == 'font-weight':
-                val = {'normal':'400', 'bold':'700'}.get(val, val)
-                if val not in {'100', '200', '300', '400', '500', '600', '700',
-                        '800', '900', 'bolder', 'lighter'}:
-                    val = default
-                if val == 'normal': val = '400'
-            elif q == 'font-style':
-                if val not in {'normal', 'italic', 'oblique'}:
-                    val = default
-            elif q == 'font-stretch':
-                if val not in { 'normal', 'ultra-condensed', 'extra-condensed',
-                        'condensed', 'semi-condensed', 'semi-expanded',
-                        'expanded', 'extra-expanded', 'ultra-expanded'}:
-                    val = default
-            props[q] = val
-        return props
-
     def find_embedded_fonts(self):
         '''
         Find all @font-face rules and extract the relevant info from them.
         '''
         self.embedded_fonts = []
         for item in self.oeb.manifest:
-            if not hasattr(item.data, 'cssRules'): continue
+            if not hasattr(item.data, 'cssRules'):
-            for i, rule in enumerate(item.data.cssRules):
-                if rule.type != rule.FONT_FACE_RULE:
                 continue
-                props = self.get_font_properties(rule, default='normal')
+            self.embedded_fonts.extend(find_font_face_rules(item, self.oeb))
-                if not props['font-family'] or not props['src']:
-                    continue
-
-                path = item.abshref(props['src'])
-                ff = self.oeb.manifest.hrefs.get(urlnormalize(path), None)
-                if not ff:
-                    continue
-                props['item'] = ff
-                if props['font-weight'] in {'bolder', 'lighter'}:
-                    props['font-weight'] = '400'
-                props['weight'] = int(props['font-weight'])
-                props['chars'] = set()
-                props['rule'] = rule
-                self.embedded_fonts.append(props)
 
     def find_style_rules(self):
         '''
@@ -151,12 +199,13 @@ class SubsetFonts(object):
         '''
         rules = defaultdict(dict)
         for item in self.oeb.manifest:
-            if not hasattr(item.data, 'cssRules'): continue
+            if not hasattr(item.data, 'cssRules'):
+                continue
             for i, rule in enumerate(item.data.cssRules):
                 if rule.type != rule.STYLE_RULE:
                     continue
                 props = {k:v for k,v in
-                        self.get_font_properties(rule).iteritems() if v}
+                        get_font_properties(rule).iteritems() if v}
                 if not props:
                     continue
                 for sel in rule.selectorList:
@@ -172,41 +221,17 @@ class SubsetFonts(object):
 
     def find_font_usage(self):
         for item in self.oeb.manifest:
-            if not hasattr(item.data, 'xpath'): continue
+            if not hasattr(item.data, 'xpath'):
+                continue
             for body in item.data.xpath('//*[local-name()="body"]'):
                 base = {'font-family':['serif'], 'font-weight': '400',
                         'font-style':'normal', 'font-stretch':'normal'}
                 self.find_usage_in(body, base)
 
-    def elem_style(self, cls, inherited_style):
-        '''
-        Find the effective style for the given element.
-        '''
-        classes = cls.split()
-        style = inherited_style.copy()
-        for cls in classes:
-            style.update(self.style_rules.get(cls, {}))
-        wt = style.get('font-weight', None)
-        pwt = inherited_style.get('font-weight', '400')
-        if wt == 'bolder':
-            style['font-weight'] = {
-                    '100':'400',
-                    '200':'400',
-                    '300':'400',
-                    '400':'700',
-                    '500':'700',
-                    }.get(pwt, '900')
-        elif wt == 'lighter':
-            style['font-weight'] = {
-                    '600':'400', '700':'400',
-                    '800':'700', '900':'700'}.get(pwt, '100')
-
-        return style
-
     def used_font(self, style):
         '''
         Given a style find the embedded font that matches it. Returns None if
-        no match is found ( can happen if not family matches).
+        no match is found (can happen if no family matches).
         '''
         ff = style.get('font-family', [])
         lnames = {unicode(x).lower() for x in ff}
@@ -280,7 +305,7 @@ class SubsetFonts(object):
         return ans
 
     def find_usage_in(self, elem, inherited_style):
-        style = self.elem_style(elem.get('class', '') or '', inherited_style)
+        style = elem_style(self.style_rules, elem.get('class', '') or '', inherited_style)
         for child in elem:
             self.find_usage_in(child, style)
         font = self.used_font(style)
@@ -290,3 +315,4 @@ class SubsetFonts(object):
                 font['chars'] |= chars
 
 
+

src/calibre/ebooks/pdf/render/from_html.py

@@ -253,7 +253,7 @@ class PDFWriter(QObject):
             return self.loop.exit(1)
         try:
             if not self.render_queue:
-                if self.toc is not None and len(self.toc) > 0 and not hasattr(self, 'rendered_inline_toc'):
+                if self.opts.pdf_add_toc and self.toc is not None and len(self.toc) > 0 and not hasattr(self, 'rendered_inline_toc'):
                     return self.render_inline_toc()
                 self.loop.exit()
             else:

src/calibre/gui2/actions/edit_metadata.py

@@ -399,8 +399,7 @@ class EditMetadataAction(InterfaceAction):
         if safe_merge:
             if not confirm('<p>'+_(
                 'Book formats and metadata from the selected books '
-                'will be added to the <b>first selected book</b> (%s). '
+                'will be added to the <b>first selected book</b> (%s).<br> '
-                'ISBN will <i>not</i> be merged.<br><br> '
                 'The second and subsequently selected books will not '
                 'be deleted or changed.<br><br>'
                 'Please confirm you want to proceed.')%title
@@ -413,7 +412,7 @@ class EditMetadataAction(InterfaceAction):
                 'Book formats from the selected books will be merged '
                 'into the <b>first selected book</b> (%s). '
                 'Metadata in the first selected book will not be changed. '
-                'Author, Title, ISBN and all other metadata will <i>not</i> be merged.<br><br>'
+                'Author, Title and all other metadata will <i>not</i> be merged.<br><br>'
                 'After merger the second and subsequently '
                 'selected books, with any metadata they have will be <b>deleted</b>. <br><br>'
                 'All book formats of the first selected book will be kept '
@@ -427,8 +426,7 @@ class EditMetadataAction(InterfaceAction):
         else:
             if not confirm('<p>'+_(
                 'Book formats and metadata from the selected books will be merged '
-                'into the <b>first selected book</b> (%s). '
+                'into the <b>first selected book</b> (%s).<br><br>'
-                'ISBN will <i>not</i> be merged.<br><br>'
                 'After merger the second and '
                 'subsequently selected books will be <b>deleted</b>. <br><br>'
                 'All book formats of the first selected book will be kept '
@@ -490,11 +488,13 @@ class EditMetadataAction(InterfaceAction):
     def merge_metadata(self, dest_id, src_ids):
         db = self.gui.library_view.model().db
         dest_mi = db.get_metadata(dest_id, index_is_id=True)
+        merged_identifiers = db.get_identifiers(dest_id, index_is_id=True)
         orig_dest_comments = dest_mi.comments
         dest_cover = db.cover(dest_id, index_is_id=True)
         had_orig_cover = bool(dest_cover)
         for src_id in src_ids:
             src_mi = db.get_metadata(src_id, index_is_id=True)
+
             if src_mi.comments and orig_dest_comments != src_mi.comments:
                 if not dest_mi.comments:
                     dest_mi.comments = src_mi.comments
@@ -523,7 +523,15 @@ class EditMetadataAction(InterfaceAction):
             if not dest_mi.series:
                 dest_mi.series = src_mi.series
                 dest_mi.series_index = src_mi.series_index
+
+            src_identifiers = db.get_identifiers(src_id, index_is_id=True)
+            src_identifiers.update(merged_identifiers)
+            merged_identifiers = src_identifiers.copy()
+
+        if merged_identifiers:
+            dest_mi.set_identifiers(merged_identifiers)
         db.set_metadata(dest_id, dest_mi, ignore_errors=False)
+
         if not had_orig_cover and dest_cover:
             db.set_cover(dest_id, dest_cover)
 

src/calibre/gui2/convert/look_and_feel.py

@@ -32,7 +32,7 @@ class LookAndFeelWidget(Widget, Ui_Form):
         Widget.__init__(self, parent,
                 ['change_justification', 'extra_css', 'base_font_size',
                     'font_size_mapping', 'line_height', 'minimum_line_height',
-                    'embed_font_family', 'subset_embedded_fonts',
+                    'embed_font_family', 'embed_all_fonts', 'subset_embedded_fonts',
                     'smarten_punctuation', 'unsmarten_punctuation',
                     'disable_font_rescaling', 'insert_blank_line',
                     'remove_paragraph_spacing',

src/calibre/gui2/convert/look_and_feel.ui

@@ -14,6 +14,70 @@
    <string>Form</string>
   </property>
   <layout class="QGridLayout" name="gridLayout">
+   <item row="12" column="1" colspan="2">
+    <widget class="QCheckBox" name="opt_keep_ligatures">
+     <property name="text">
+      <string>Keep &amp;ligatures</string>
+     </property>
+    </widget>
+   </item>
+   <item row="12" column="3">
+    <widget class="QCheckBox" name="opt_linearize_tables">
+     <property name="text">
+      <string>&amp;Linearize tables</string>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="0">
+    <widget class="QLabel" name="label_18">
+     <property name="text">
+      <string>Base &amp;font size:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_base_font_size</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="9" column="3">
+    <widget class="QLabel" name="label_7">
+     <property name="text">
+      <string>&amp;Line size:</string>
+     </property>
+     <property name="alignment">
+      <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+     </property>
+     <property name="buddy">
+      <cstring>opt_insert_blank_line_size</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="5" column="1" colspan="2">
+    <widget class="EncodingComboBox" name="opt_input_encoding">
+     <property name="editable">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="8" column="0" colspan="2">
+    <widget class="QCheckBox" name="opt_remove_paragraph_spacing">
+     <property name="text">
+      <string>Remove &amp;spacing between paragraphs</string>
+     </property>
+    </widget>
+   </item>
+   <item row="8" column="3">
+    <widget class="QLabel" name="label_4">
+     <property name="text">
+      <string>&amp;Indent size:</string>
+     </property>
+     <property name="alignment">
+      <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+     </property>
+     <property name="buddy">
+      <cstring>opt_remove_paragraph_spacing_indent_size</cstring>
+     </property>
+    </widget>
+   </item>
    <item row="3" column="4">
     <widget class="QDoubleSpinBox" name="opt_line_height">
      <property name="suffix">
@@ -24,6 +88,57 @@
      </property>
     </widget>
    </item>
+   <item row="9" column="0" colspan="2">
+    <widget class="QCheckBox" name="opt_insert_blank_line">
+     <property name="text">
+      <string>Insert &amp;blank line between paragraphs</string>
+     </property>
+    </widget>
+   </item>
+   <item row="9" column="4">
+    <widget class="QDoubleSpinBox" name="opt_insert_blank_line_size">
+     <property name="suffix">
+      <string> em</string>
+     </property>
+     <property name="decimals">
+      <number>1</number>
+     </property>
+    </widget>
+   </item>
+   <item row="10" column="0">
+    <widget class="QLabel" name="label_5">
+     <property name="text">
+      <string>Text &amp;justification:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_change_justification</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="10" column="2" colspan="3">
+    <widget class="QComboBox" name="opt_change_justification"/>
+   </item>
+   <item row="11" column="0">
+    <widget class="QCheckBox" name="opt_smarten_punctuation">
+     <property name="text">
+      <string>Smarten &amp;punctuation</string>
+     </property>
+    </widget>
+   </item>
+   <item row="11" column="1" colspan="4">
+    <widget class="QCheckBox" name="opt_asciiize">
+     <property name="text">
+      <string>&amp;Transliterate unicode characters to ASCII</string>
+     </property>
+    </widget>
+   </item>
+   <item row="12" column="0">
+    <widget class="QCheckBox" name="opt_unsmarten_punctuation">
+     <property name="text">
+      <string>&amp;UnSmarten punctuation</string>
+     </property>
+    </widget>
+   </item>
    <item row="3" column="3">
     <widget class="QLabel" name="label">
      <property name="text">
@@ -44,51 +159,6 @@
      </property>
     </widget>
    </item>
-   <item row="3" column="1">
-    <widget class="QDoubleSpinBox" name="opt_minimum_line_height">
-     <property name="suffix">
-      <string> %</string>
-     </property>
-     <property name="decimals">
-      <number>1</number>
-     </property>
-     <property name="maximum">
-      <double>900.000000000000000</double>
-     </property>
-    </widget>
-   </item>
-   <item row="1" column="1">
-    <widget class="QDoubleSpinBox" name="opt_base_font_size">
-     <property name="suffix">
-      <string> pt</string>
-     </property>
-     <property name="decimals">
-      <number>1</number>
-     </property>
-     <property name="minimum">
-      <double>0.000000000000000</double>
-     </property>
-     <property name="maximum">
-      <double>50.000000000000000</double>
-     </property>
-     <property name="singleStep">
-      <double>1.000000000000000</double>
-     </property>
-     <property name="value">
-      <double>15.000000000000000</double>
-     </property>
-    </widget>
-   </item>
-   <item row="2" column="0">
-    <widget class="QLabel" name="label_2">
-     <property name="text">
-      <string>Font size &amp;key:</string>
-     </property>
-     <property name="buddy">
-      <cstring>opt_font_size_mapping</cstring>
-     </property>
-    </widget>
-   </item>
    <item row="2" column="1" colspan="3">
     <layout class="QHBoxLayout" name="horizontalLayout">
      <item>
@@ -133,56 +203,72 @@
      </property>
     </widget>
    </item>
-   <item row="5" column="1" colspan="2">
+   <item row="3" column="1">
-    <widget class="EncodingComboBox" name="opt_input_encoding">
+    <widget class="QDoubleSpinBox" name="opt_minimum_line_height">
-     <property name="editable">
-      <bool>true</bool>
-     </property>
-    </widget>
-   </item>
-   <item row="7" column="0" colspan="2">
-    <widget class="QCheckBox" name="opt_remove_paragraph_spacing">
-     <property name="text">
-      <string>Remove &amp;spacing between paragraphs</string>
-     </property>
-    </widget>
-   </item>
-   <item row="7" column="3">
-    <widget class="QLabel" name="label_4">
-     <property name="text">
-      <string>&amp;Indent size:</string>
-     </property>
-     <property name="alignment">
-      <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-     </property>
-     <property name="buddy">
-      <cstring>opt_remove_paragraph_spacing_indent_size</cstring>
-     </property>
-    </widget>
-   </item>
-   <item row="7" column="4">
-    <widget class="QDoubleSpinBox" name="opt_remove_paragraph_spacing_indent_size">
-     <property name="toolTip">
-      <string>&lt;p&gt;When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent.</string>
-     </property>
-     <property name="specialValueText">
-      <string>No change</string>
-     </property>
      <property name="suffix">
-      <string> em</string>
+      <string> %</string>
+     </property>
+     <property name="decimals">
+      <number>1</number>
+     </property>
+     <property name="maximum">
+      <double>900.000000000000000</double>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="1">
+    <widget class="QDoubleSpinBox" name="opt_base_font_size">
+     <property name="suffix">
+      <string> pt</string>
      </property>
      <property name="decimals">
       <number>1</number>
      </property>
      <property name="minimum">
-      <double>-0.100000000000000</double>
+      <double>0.000000000000000</double>
+     </property>
+     <property name="maximum">
+      <double>50.000000000000000</double>
      </property>
      <property name="singleStep">
-      <double>0.100000000000000</double>
+      <double>1.000000000000000</double>
+     </property>
+     <property name="value">
+      <double>15.000000000000000</double>
      </property>
     </widget>
    </item>
-   <item row="12" column="0" colspan="5">
+   <item row="0" column="0" colspan="5">
+    <widget class="QCheckBox" name="opt_disable_font_rescaling">
+     <property name="text">
+      <string>&amp;Disable font size rescaling</string>
+     </property>
+    </widget>
+   </item>
+   <item row="6" column="1" colspan="2">
+    <widget class="FontFamilyChooser" name="opt_embed_font_family" native="true"/>
+   </item>
+   <item row="2" column="0">
+    <widget class="QLabel" name="label_2">
+     <property name="text">
+      <string>Font size &amp;key:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_font_size_mapping</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="6" column="0">
+    <widget class="QLabel" name="label_10">
+     <property name="text">
+      <string>&amp;Embed font family:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_embed_font_family</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="13" column="0" colspan="5">
     <widget class="QTabWidget" name="tabWidget">
      <property name="currentIndex">
       <number>0</number>
@@ -300,121 +386,42 @@
      </widget>
     </widget>
    </item>
-   <item row="8" column="0" colspan="2">
-    <widget class="QCheckBox" name="opt_insert_blank_line">
-     <property name="text">
-      <string>Insert &amp;blank line between paragraphs</string>
-     </property>
-    </widget>
-   </item>
    <item row="8" column="4">
-    <widget class="QDoubleSpinBox" name="opt_insert_blank_line_size">
+    <widget class="QDoubleSpinBox" name="opt_remove_paragraph_spacing_indent_size">
+     <property name="toolTip">
+      <string>&lt;p&gt;When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent.</string>
+     </property>
+     <property name="specialValueText">
+      <string>No change</string>
+     </property>
      <property name="suffix">
       <string> em</string>
      </property>
      <property name="decimals">
       <number>1</number>
      </property>
-    </widget>
+     <property name="minimum">
-   </item>
+      <double>-0.100000000000000</double>
-   <item row="9" column="0">
-    <widget class="QLabel" name="label_5">
-     <property name="text">
-      <string>Text &amp;justification:</string>
      </property>
-     <property name="buddy">
+     <property name="singleStep">
-      <cstring>opt_change_justification</cstring>
+      <double>0.100000000000000</double>
      </property>
     </widget>
    </item>
-   <item row="9" column="2" colspan="3">
+   <item row="7" column="3">
-    <widget class="QComboBox" name="opt_change_justification"/>
-   </item>
-   <item row="10" column="0">
-    <widget class="QCheckBox" name="opt_smarten_punctuation">
-     <property name="text">
-      <string>Smarten &amp;punctuation</string>
-     </property>
-    </widget>
-   </item>
-   <item row="10" column="1" colspan="4">
-    <widget class="QCheckBox" name="opt_asciiize">
-     <property name="text">
-      <string>&amp;Transliterate unicode characters to ASCII</string>
-     </property>
-    </widget>
-   </item>
-   <item row="11" column="0">
-    <widget class="QCheckBox" name="opt_unsmarten_punctuation">
-     <property name="text">
-      <string>&amp;UnSmarten punctuation</string>
-     </property>
-    </widget>
-   </item>
-   <item row="11" column="1" colspan="2">
-    <widget class="QCheckBox" name="opt_keep_ligatures">
-     <property name="text">
-      <string>Keep &amp;ligatures</string>
-     </property>
-    </widget>
-   </item>
-   <item row="11" column="3">
-    <widget class="QCheckBox" name="opt_linearize_tables">
-     <property name="text">
-      <string>&amp;Linearize tables</string>
-     </property>
-    </widget>
-   </item>
-   <item row="1" column="0">
-    <widget class="QLabel" name="label_18">
-     <property name="text">
-      <string>Base &amp;font size:</string>
-     </property>
-     <property name="buddy">
-      <cstring>opt_base_font_size</cstring>
-     </property>
-    </widget>
-   </item>
-   <item row="8" column="3">
-    <widget class="QLabel" name="label_7">
-     <property name="text">
-      <string>&amp;Line size:</string>
-     </property>
-     <property name="alignment">
-      <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-     </property>
-     <property name="buddy">
-      <cstring>opt_insert_blank_line_size</cstring>
-     </property>
-    </widget>
-   </item>
-   <item row="6" column="0">
-    <widget class="QLabel" name="label_10">
-     <property name="text">
-      <string>&amp;Embed font family:</string>
-     </property>
-     <property name="buddy">
-      <cstring>opt_embed_font_family</cstring>
-     </property>
-    </widget>
-   </item>
-   <item row="0" column="0" colspan="5">
-    <widget class="QCheckBox" name="opt_disable_font_rescaling">
-     <property name="text">
-      <string>&amp;Disable font size rescaling</string>
-     </property>
-    </widget>
-   </item>
-   <item row="6" column="1" colspan="2">
-    <widget class="FontFamilyChooser" name="opt_embed_font_family" native="true"/>
-   </item>
-   <item row="6" column="3" colspan="2">
     <widget class="QCheckBox" name="opt_subset_embedded_fonts">
      <property name="text">
       <string>&amp;Subset all embedded fonts</string>
      </property>
     </widget>
    </item>
+   <item row="7" column="0" colspan="3">
+    <widget class="QCheckBox" name="opt_embed_all_fonts">
+     <property name="text">
+      <string>&amp;Embed referenced fonts</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <customwidgets>

src/calibre/gui2/dialogs/template_dialog.py

@@ -262,6 +262,8 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
             self.mi.rating = 4.0
             self.mi.tags = [_('Tag 1'), _('Tag 2')]
             self.mi.languages = ['eng']
+            if fm is not None:
+                self.mi.set_all_user_metadata(fm.custom_field_metadata())
 
         # Remove help icon on title bar
         icon = self.windowIcon()

src/calibre/gui2/email.py

@@ -92,7 +92,11 @@ class Sendmail(object):
                 raise worker.exception
 
     def sendmail(self, attachment, aname, to, subject, text, log):
+        logged = False
         while time.time() - self.last_send_time <= self.rate_limit:
+            if not logged and self.rate_limit > 0:
+                log('Waiting %s seconds before sending, to avoid being marked as spam.\nYou can control this delay via Preferences->Tweaks' % self.rate_limit)
+                logged = True
             time.sleep(1)
         try:
             opts = email_config().parse()
@@ -204,10 +208,10 @@ class EmailMixin(object): # {{{
                     if not components:
                         components = [mi.title]
                     subjects.append(os.path.join(*components))
-                a = authors_to_string(mi.authors if mi.authors else \
+                a = authors_to_string(mi.authors if mi.authors else
                         [_('Unknown')])
-                texts.append(_('Attached, you will find the e-book') + \
+                texts.append(_('Attached, you will find the e-book') +
-                        '\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' + \
+                        '\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' +
                         _('in the %s format.') %
                         os.path.splitext(f)[1][1:].upper())
                 prefix = ascii_filename(t+' - '+a)
@@ -227,7 +231,7 @@ class EmailMixin(object): # {{{
         auto = []
         if _auto_ids != []:
             for id in _auto_ids:
-                if specific_format == None:
+                if specific_format is None:
                     dbfmts = self.library_view.model().db.formats(id, index_is_id=True)
                     formats = [f.lower() for f in (dbfmts.split(',') if dbfmts else
                         [])]
@@ -298,8 +302,9 @@ class EmailMixin(object): # {{{
         sent_mails = email_news(mi, remove,
                 get_fmts, self.email_sent, self.job_manager)
         if sent_mails:
-            self.status_bar.show_message(_('Sent news to')+' '+\
+            self.status_bar.show_message(_('Sent news to')+' '+
                     ', '.join(sent_mails),  3000)
 
 # }}}
 
+

src/calibre/gui2/preferences/save_template.py

@@ -24,7 +24,7 @@ class SaveTemplate(QWidget, Ui_Form):
         Ui_Form.__init__(self)
         self.setupUi(self)
 
-    def initialize(self, name, default, help):
+    def initialize(self, name, default, help, field_metadata):
         variables = sorted(FORMAT_ARG_DESCS.keys())
         rows = []
         for var in variables:
@@ -36,6 +36,7 @@ class SaveTemplate(QWidget, Ui_Form):
         table = u'<table>%s</table>'%(u'\n'.join(rows))
         self.template_variables.setText(table)
 
+        self.field_metadata = field_metadata
         self.opt_template.initialize(name+'_template_history',
                 default, help)
         self.opt_template.editTextChanged.connect(self.changed)
@@ -44,7 +45,7 @@ class SaveTemplate(QWidget, Ui_Form):
         self.open_editor.clicked.connect(self.do_open_editor)
 
     def do_open_editor(self):
-        t = TemplateDialog(self, self.opt_template.text())
+        t = TemplateDialog(self, self.opt_template.text(), fm=self.field_metadata)
         t.setWindowTitle(_('Edit template'))
         if t.exec_():
             self.opt_template.set_value(t.rule[1])

src/calibre/gui2/preferences/saving.py

@@ -34,7 +34,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         ConfigWidgetBase.initialize(self)
         self.save_template.blockSignals(True)
         self.save_template.initialize('save_to_disk', self.proxy['template'],
-                self.proxy.help('template'))
+                self.proxy.help('template'),
+                self.gui.library_view.model().db.field_metadata)
         self.save_template.blockSignals(False)
 
     def restore_defaults(self):

src/calibre/gui2/preferences/sending.py

@@ -44,7 +44,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         ConfigWidgetBase.initialize(self)
         self.send_template.blockSignals(True)
         self.send_template.initialize('send_to_device', self.proxy['send_template'],
-                self.proxy.help('send_template'))
+                self.proxy.help('send_template'),
+                self.gui.library_view.model().db.field_metadata)
         self.send_template.blockSignals(False)
 
     def restore_defaults(self):

src/calibre/gui2/tools.py

@@ -116,7 +116,6 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
                     msg = _('This book has no actual ebook files')
                 res.append('%s - %s'%(title, msg))
 
-
             msg = '%s' % '\n'.join(res)
             warning_dialog(parent, _('Could not convert some books'),
                 _('Could not convert %(num)d of %(tot)d books, because no supported source'
@@ -266,6 +265,10 @@ def fetch_scheduled_recipe(arg): # {{{
     if 'output_profile' in ps:
         recs.append(('output_profile', ps['output_profile'],
             OptionRecommendation.HIGH))
+    for edge in ('left', 'top', 'bottom', 'right'):
+        edge = 'margin_' + edge
+        if edge in ps:
+            recs.append((edge, ps[edge], OptionRecommendation.HIGH))
 
     lf = load_defaults('look_and_feel')
     if lf.get('base_font_size', 0.0) != 0.0:
@@ -283,13 +286,19 @@ def fetch_scheduled_recipe(arg): # {{{
     if epub.get('epub_flatten', False):
         recs.append(('epub_flatten', True, OptionRecommendation.HIGH))
 
+    if fmt == 'pdf':
+        pdf = load_defaults('pdf_output')
+        from calibre.customize.ui import plugin_for_output_format
+        p = plugin_for_output_format('pdf')
+        for opt in p.options:
+            recs.append(opt.name, pdf.get(opt.name, opt.recommended_value), OptionRecommendation.HIGH)
+
     args = [arg['recipe'], pt.name, recs]
     if arg['username'] is not None:
         recs.append(('username', arg['username'], OptionRecommendation.HIGH))
     if arg['password'] is not None:
         recs.append(('password', arg['password'], OptionRecommendation.HIGH))
 
-
     return 'gui_convert', args, _('Fetch news from ')+arg['title'], fmt.upper(), [pt]
 
 # }}}
@@ -372,3 +381,4 @@ def convert_existing(parent, db, book_ids, output_format): # {{{
     return book_ids
 # }}}
 
+

src/calibre/web/fetch/javascript.py

@@ -128,6 +128,8 @@ def download_resources(browser, resource_cache, output_dir):
             else:
                 img_counter += 1
                 ext = what(None, raw) or 'jpg'
+                if ext == 'jpeg':
+                    ext = 'jpg'  # Apparently Moon+ cannot handle .jpeg
                 href = 'img_%d.%s' % (img_counter, ext)
             dest = os.path.join(output_dir, href)
             resource_cache[h] = dest