Sync to trunk.

installer/linux/freeze.py

@@ -49,6 +49,7 @@ def freeze():
                        '/usr/lib/libMagickCore.so',
                        '/usr/lib/libgcrypt.so.11',
                        '/usr/lib/libgpg-error.so.0',
+                       '/usr/lib/libphonon.so.4',
                        ]
 
     binary_includes += [os.path.join(QTDIR, 'lib%s.so.4'%x) for x in QTDLLS]

src/calibre/devices/cybookg3/driver.py

@@ -8,7 +8,7 @@ import os
 import shutil
 from itertools import cycle
 
-from calibre import sanitize_file_name as sanitize
+from calibre.utils.filenames import ascii_filename as sanitize
 from calibre.devices.usbms.driver import USBMS
 import calibre.devices.cybookg3.t2b as t2b
 

src/calibre/devices/jetbook/driver.py

@@ -8,7 +8,7 @@ import os, re, sys, shutil
 from itertools import cycle
 
 from calibre.devices.usbms.driver import USBMS
-from calibre import sanitize_file_name as sanitize
+from calibre.utils.filenames import ascii_filename as sanitize
 from calibre.ebooks.metadata import string_to_authors
 
 class JETBOOK(USBMS):

src/calibre/devices/usbms/device.py

@@ -15,7 +15,7 @@ from calibre.devices.interface import DevicePlugin
 from calibre.devices.errors import DeviceError
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre import iswindows, islinux, isosx, __appname__
-from calibre import sanitize_file_name as sanitize
+from calibre.utils.filenames import ascii_filename as sanitize
 
 class Device(DeviceConfig, DevicePlugin):
     '''

src/calibre/ebooks/comic/input.py

@@ -253,7 +253,6 @@ def process_pages(pages, opts, update, tdir):
 
     for job in jobs:
         if job.failed:
-            raw_input()
             raise Exception(_('Failed to process comic: \n\n%s')%
                     job.log_file.read())
         pages, failures_ = job.result

src/calibre/ebooks/lrf/lrs/convert_from.py

@@ -167,7 +167,8 @@ class LrsParser(object):
                 settings = self.attrs_to_dict(tag, map[tag.name][1]+['objid', 'objlabel'])
                 for a in ('pagestyle', 'blockstyle', 'textstyle'):
                     label = tag.get(a, False)
-                    if label:
+                    if label and \
+                        (label in self._style_labels or label in self.parsed_objects):
                         _obj = self.parsed_objects[label] if \
                             self.parsed_objects.has_key(label) else \
                             self._style_labels[label]

src/calibre/ebooks/mobi/reader.py

@@ -21,7 +21,8 @@ except ImportError:
 
 from lxml import html, etree
 
-from calibre import entity_to_unicode, sanitize_file_name
+from calibre import entity_to_unicode
+from calibre.utils.filenames import ascii_filename
 from calibre.ptempfile import TemporaryDirectory
 from calibre.ebooks import DRMError
 from calibre.ebooks.chardet import ENCODING_PATS
@@ -374,7 +375,7 @@ class MobiReader(object):
         fname = self.name.encode('ascii', 'replace')
         fname = re.sub(r'[\x08\x15\0]+', '', fname)
         htmlfile = os.path.join(output_dir,
-            sanitize_file_name(fname) + '.html')
+            ascii_filename(fname) + '.html')
         try:
             for ref in guide.xpath('descendant::reference'):
                 if ref.attrib.has_key('href'):

src/calibre/ebooks/oeb/transforms/jacket.py

@@ -65,9 +65,9 @@ class Jacket(object):
         if not comments.strip():
             comments = ''
         comments = comments.replace('\r\n', '\n').replace('\n\n', '<br/><br/>')
-        series = '<b>Series: </b>' + mi.series if mi.series else ''
+        series = '<b>Series: </b>' + escape(mi.series if mi.series else '')
         if series and mi.series_index is not None:
-            series += ' [%s]'%mi.format_series_index()
+            series += escape(' [%s]'%mi.format_series_index())
         tags = mi.tags
         if not tags:
             try:
@@ -75,7 +75,7 @@ class Jacket(object):
             except:
                 tags = []
         if tags:
-            tags = '<b>Tags: </b>' + self.opts.dest.tags_to_string(tags)
+            tags = '<b>Tags: </b>' + escape(self.opts.dest.tags_to_string(tags))
         else:
             tags = ''
         try:
@@ -84,8 +84,8 @@ class Jacket(object):
             title = _('Unknown')
         html = self.JACKET_TEMPLATE%dict(xmlns=XPNSMAP['h'],
                 title=escape(title), comments=escape(comments),
-                jacket=escape(_('Book Jacket')), series=escape(series),
+                jacket=escape(_('Book Jacket')), series=series,
-                tags=escape(tags))
+                tags=tags)
         id, href = self.oeb.manifest.generate('jacket', 'jacket.xhtml')
         root = etree.fromstring(html)
         item = self.oeb.manifest.add(id, href, guess_type(href)[0], data=root)

src/calibre/ebooks/unidecode/unidecoder.py

@@ -57,6 +57,7 @@ it under the same terms as Perl itself.
 import re
 
 from calibre.ebooks.unidecode.unicodepoints import CODEPOINTS
+from calibre.constants import preferred_encoding
 
 class Unidecoder(object):
 
@@ -70,7 +71,10 @@ class Unidecoder(object):
             try:
                 text = unicode(text)
             except:
-                text = text.decode('utf-8', 'ignore')
+                try:
+                    text = text.decode(preferred_encoding)
+                except:
+                    text = text.decode('utf-8', 'replace')
         # Replace characters larger than 127 with their ASCII equivelent.
         return re.sub('[^\x00-\x7f]', lambda x: self.replace_point(x.group()),
             text)
@@ -80,7 +84,7 @@ class Unidecoder(object):
         Returns the replacement character or ? if none can be found.
         '''
         try:
-            # Splite the unicode character xABCD into parts 0xAB and 0xCD.
+            # Split the unicode character xABCD into parts 0xAB and 0xCD.
             # 0xAB represents the group within CODEPOINTS to query and 0xCD
             # represents the position in the list of characters for the group.
             return CODEPOINTS[self.code_group(codepoint)][self.grouped_point(

src/calibre/gui2/__init__.py

@@ -342,6 +342,8 @@ class FileDialog(QObject):
                 ftext += '%s (%s);;'%(text, ' '.join(extensions))
         if add_all_files_filter or not ftext:
             ftext += 'All files (*)'
+        if ftext.endswith(';;'):
+            ftext = ftext[:-2]
 
         self.dialog_name = name if name else 'dialog_' + title
         self.selected_files = None

src/calibre/gui2/device.py

@@ -21,7 +21,7 @@ from calibre.gui2 import config, error_dialog, Dispatcher, dynamic, \
                                    pixmap_to_data, warning_dialog, \
                                    question_dialog
 from calibre.ebooks.metadata import authors_to_string
-from calibre import sanitize_file_name, preferred_encoding
+from calibre import preferred_encoding
 from calibre.utils.filenames import ascii_filename
 from calibre.devices.errors import FreeSpaceError
 from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \
@@ -542,7 +542,7 @@ class DeviceGUI(object):
                         '\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' + \
                         _('in the %s format.') %
                         os.path.splitext(f)[1][1:].upper())
-                prefix = sanitize_file_name(t+' - '+a)
+                prefix = ascii_filename(t+' - '+a)
                 if not isinstance(prefix, unicode):
                     prefix = prefix.decode(preferred_encoding, 'replace')
                 attachment_names.append(prefix + os.path.splitext(f)[1])
@@ -693,7 +693,7 @@ class DeviceGUI(object):
                     rows_are_ids=True)
             names = []
             for mi in metadata:
-                prefix = sanitize_file_name(mi['title'])
+                prefix = ascii_filename(mi['title'])
                 if not isinstance(prefix, unicode):
                     prefix = prefix.decode(preferred_encoding, 'replace')
                 prefix = ascii_filename(prefix)
@@ -758,7 +758,7 @@ class DeviceGUI(object):
                 a = mi['authors']
                 if not a:
                     a = _('Unknown')
-                prefix = sanitize_file_name(t+' - '+a)
+                prefix = ascii_filename(t+' - '+a)
                 if not isinstance(prefix, unicode):
                     prefix = prefix.decode(preferred_encoding, 'replace')
                 prefix = ascii_filename(prefix)

src/calibre/gui2/dialogs/metadata_single.py

@@ -526,3 +526,11 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         QDialog.accept(self)
         if callable(self.accepted_callback):
             self.accepted_callback(self.id)
+
+    def reject(self, *args):
+        cf = getattr(self, 'cover_fetcher', None)
+        if cf is not None and hasattr(cf, 'terminate'):
+            cf.terminate()
+            cf.wait()
+
+        QDialog.reject(self, *args)

src/calibre/gui2/main.py

@@ -14,8 +14,9 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
                      QMessageBox, QStackedLayout
 from PyQt4.QtSvg import QSvgRenderer
 
-from calibre import __version__, __appname__, sanitize_file_name, \
+from calibre import __version__, __appname__, \
                     iswindows, isosx, prints, patheq
+from calibre.utils.filenames import ascii_filename
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import prefs, dynamic
 from calibre.utils.ipc.server import Server
@@ -852,7 +853,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
     def _files_added(self, paths=[], names=[], infos=[], on_card=None):
         if paths:
             self.upload_books(paths,
-                                list(map(sanitize_file_name, names)),
+                                list(map(ascii_filename, names)),
                                 infos, on_card=on_card)
             self.status_bar.showMessage(
                     _('Uploading books to device.'), 2000)
@@ -888,7 +889,17 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                                    'removed from your computer. Are you sure?')
                                 +'</p>', 'library_delete_books', self):
                 return
+            ci = view.currentIndex()
+            row = None
+            if ci.isValid():
+                row = ci.row()
             view.model().delete_books(rows)
+            if row is not None:
+                ci = view.model().index(row, 0)
+                if ci.isValid():
+                    view.setCurrentIndex(ci)
+                    sm = view.selectionModel()
+                    sm.select(ci, sm.Select)
         else:
             if self.stack.currentIndex() == 1:
                 view = self.memory_view

src/calibre/library/database2.py

@@ -34,7 +34,7 @@ from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_e
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.customize.ui import run_plugins_on_import
 
-from calibre import sanitize_file_name
+from calibre.utils.filenames import ascii_filename
 from calibre.ebooks import BOOK_EXTENSIONS
 
 if iswindows:
@@ -652,8 +652,8 @@ class LibraryDatabase2(LibraryDatabase):
         authors = self.authors(id, index_is_id=True)
         if not authors:
             authors = _('Unknown')
-        author = sanitize_file_name(authors.split(',')[0][:self.PATH_LIMIT]).decode(filesystem_encoding, 'ignore')
+        author = ascii_filename(authors.split(',')[0][:self.PATH_LIMIT]).decode(filesystem_encoding, 'ignore')
-        title  = sanitize_file_name(self.title(id, index_is_id=True)[:self.PATH_LIMIT]).decode(filesystem_encoding, 'ignore')
+        title  = ascii_filename(self.title(id, index_is_id=True)[:self.PATH_LIMIT]).decode(filesystem_encoding, 'ignore')
         path   = author + '/' + title + ' (%d)'%id
         return path
 
@@ -664,8 +664,8 @@ class LibraryDatabase2(LibraryDatabase):
         authors = self.authors(id, index_is_id=True)
         if not authors:
             authors = _('Unknown')
-        author = sanitize_file_name(authors.split(',')[0][:self.PATH_LIMIT]).decode(filesystem_encoding, 'replace')
+        author = ascii_filename(authors.split(',')[0][:self.PATH_LIMIT]).decode(filesystem_encoding, 'replace')
-        title  = sanitize_file_name(self.title(id, index_is_id=True)[:self.PATH_LIMIT]).decode(filesystem_encoding, 'replace')
+        title  = ascii_filename(self.title(id, index_is_id=True)[:self.PATH_LIMIT]).decode(filesystem_encoding, 'replace')
         name   = title + ' - ' + author
         while name.endswith('.'):
             name = name[:-1]
@@ -1520,12 +1520,12 @@ class LibraryDatabase2(LibraryDatabase):
             x['cover'] = os.path.join(path, 'cover.jpg')
             if not self.has_cover(x['id'], index_is_id=True):
                 x['cover'] = None
-            path += os.sep +  self.construct_file_name(record[FIELD_MAP['id']]) + '.%s'
             formats = self.formats(record[FIELD_MAP['id']], index_is_id=True)
             if formats:
                 for fmt in formats.split(','):
-                    x['formats'].append(path%fmt.lower())
+                    path = self.format_abspath(x['id'], fmt, index_is_id=True)
-                    x['fmt_'+fmt.lower()] = path%fmt.lower()
+                    x['formats'].append(path)
+                    x['fmt_'+fmt.lower()] = path
                 x['available_formats'] = [i.upper() for i in formats.split(',')]
 
         return data
@@ -1602,12 +1602,12 @@ books_series_link      feeds
                 by_author[au] = []
             by_author[au].append(index)
         for au in by_author.keys():
-            apath = os.path.join(dir, sanitize_file_name(au))
+            apath = os.path.join(dir, ascii_filename(au))
             if not single_dir and not os.path.exists(apath):
                 os.mkdir(apath)
             for idx in by_author[au]:
                 title = re.sub(r'\s', ' ', self.title(idx, index_is_id=index_is_id))
-                tpath = os.path.join(apath, sanitize_file_name(title))
+                tpath = os.path.join(apath, ascii_filename(title))
                 id = idx if index_is_id else self.id(idx)
                 id = str(id)
                 if not single_dir and not os.path.exists(tpath):
@@ -1621,10 +1621,10 @@ books_series_link      feeds
                     mi.authors = [_('Unknown')]
                 cdata = self.cover(int(id), index_is_id=True)
                 if cdata is not None:
-                    cname = sanitize_file_name(name)+'.jpg'
+                    cname = ascii_filename(name)+'.jpg'
                     open(os.path.join(base, cname), 'wb').write(cdata)
                     mi.cover = cname
-                with open(os.path.join(base, sanitize_file_name(name)+'.opf'),
+                with open(os.path.join(base, ascii_filename(name)+'.opf'),
                         'wb') as f:
                     f.write(metadata_to_opf(mi))
 
@@ -1636,7 +1636,7 @@ books_series_link      feeds
                     if not data:
                         continue
                     fname = name +'.'+fmt.lower()
-                    fname = sanitize_file_name(fname)
+                    fname = ascii_filename(fname)
                     f = open(os.path.join(base, fname), 'w+b')
                     f.write(data)
                     f.flush()
@@ -1671,7 +1671,7 @@ books_series_link      feeds
             if not au:
                 au = _('Unknown')
             fname = '%s - %s.%s'%(title, au, format.lower())
-            fname = sanitize_file_name(fname)
+            fname = ascii_filename(fname)
             if not os.path.exists(dir):
                 os.makedirs(dir)
             f = open(os.path.join(dir, fname), 'w+b')

src/calibre/trac/plugins/templates/linux.html

@@ -82,10 +82,6 @@ sudo python -c "import urllib2; exec urllib2.urlopen('http://calibre.kovidgoyal.
                             </pre>
                             <h4>Note</h4>
                             <ul>
-                                <li>On some linux distributions, you have to install the
-                                    libphonon (may be called libphonon4) package for calibre
-                                    to work.
-                                </li>
                                 <li>
                                     When running the command line utilities,
                                     they will segfault after completion. This can

src/calibre/utils/filenames.py

@@ -1,97 +1,23 @@
-# -*- coding: utf-8 -*-
 '''
 Make strings safe for use as ASCII filenames, while trying to preserve as much
 meaning as possible.
 '''
 
-import re, string
+from calibre.ebooks.unidecode.unidecoder import Unidecoder
+from calibre import sanitize_file_name
+from calibre.constants import preferred_encoding
+udc = Unidecoder()
 
-MAP = {
+def ascii_text(orig):
-        u"‘" : "'",
+    try:
-        u"’" : "'",
+        ascii = udc.decode(orig)
-        u"«" : '"',
+    except:
-        u"»" : '"',
+        if isinstance(orig, unicode):
-        u"…" : "...",
+            ascii = orig.encode('ascii', 'replace')
-        u"№" : "#",
+        ascii = orig.decode(preferred_encoding,
-        u"Щ" : "Shh",
+                'replace').encode('ascii', 'replace')
-        u"Ё" : "Jo",
+    return ascii
-        u"Ж" : "Zh",
-        u"Ц" : "C",
-        u"Ч" : "Ch",
-        u"Ш" : "Sh",
-        u"Ы" : "Y",
-        u"Ю" : "Ju",
-        u"Я" : "Ja",
-        u"Б" : "B",
-        u"Г" : "G",
-        u"Д" : "D",
-        u"И" : "I",
-        u"Й" : "J",
-        u"К" : "K",
-        u"Л" : "L",
-        u"П" : "P",
-        u"Ф" : "F",
-        u"Э" : "E",
-        u"Ъ" : "`",
-        u"Ь" : "'",
-        u"щ" : "shh",
-        u"ё" : "jo",
-        u"ж" : "zh",
-        u"ц" : "c",
-        u"ч" : "ch",
-        u"ш" : "sh",
-        u"ы" : "y",
-        u"ю" : "ju",
-        u"я" : "ja",
-        u"б" : "b",
-        u"в" : "v",
-        u"г" : "g",
-        u"д" : "d",
-        u"з" : "z",
-        u"и" : "i",
-        u"й" : "j",
-        u"к" : "k",
-        u"л" : "l",
-        u"м" : "m",
-        u"н" : "n",
-        u"о" : "o",
-        u"п" : "p",
-        u"т" : "t",
-        u"ф" : "f",
-        u"э" : "e",
-        u"ъ" : "`",
-        u"ь" : "'",
-        u"А" : "A",
-        u"В" : "V",
-        u"Е" : "Je",
-        u"З" : "Z",
-        u"М" : "M",
-        u"Н" : "N",
-        u"О" : "O",
-        u"Р" : "R",
-        u"С" : "S",
-        u"Т" : "T",
-        u"У" : "U",
-        u"Х" : "Kh",
-        u"Є" : "Je",
-        u"Ї" : "Ji",
-        u"а" : "a",
-        u"е" : "je",
-        u"р" : "r",
-        u"с" : "s",
-        u"у" : "u",
-        u"х" : "kh",
-        u"є" : "je",
-}  #: Translation table
 
-for c in string.whitespace:
-    MAP[c] = ' '
-PAT = re.compile('['+u''.join(MAP.keys())+']')
 
 def ascii_filename(orig):
-    orig =  PAT.sub(lambda m:MAP[m.group()], orig)
+    return sanitize_file_name(ascii_text(orig).replace('?', '_'))
-    buf = []
-    for i in range(len(orig)):
-        val = ord(orig[i])
-        buf.append('_' if val < 33 or val > 126 else orig[i])
-    return (''.join(buf)).encode('ascii')

src/calibre/web/fetch/simple.py

@@ -14,8 +14,8 @@ from httplib import responses
 from PIL import Image
 from cStringIO import StringIO
 
-from calibre import browser, sanitize_file_name, \
+from calibre import browser, relpath, unicode_path
-                    relpath, unicode_path
+from calibre.utils.filenames import ascii_filename
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.utils.config import OptionParser
@@ -313,7 +313,7 @@ class RecursiveFetcher(object):
                 self.log.exception('Could not fetch image %s'% iurl)
                 continue
             c += 1
-            fname = sanitize_file_name('img'+str(c)+ext)
+            fname = ascii_filename('img'+str(c)+ext)
             if isinstance(fname, unicode):
                 fname = fname.encode('ascii', 'replace')
             imgpath = os.path.join(diskpath, fname+'.jpg')
@@ -416,7 +416,7 @@ class RecursiveFetcher(object):
                     if not isinstance(_fname, unicode):
                         _fname.decode('latin1', 'replace')
                     _fname = _fname.encode('ascii', 'replace').replace('%', '').replace(os.sep, '')
-                    _fname = sanitize_file_name(_fname)
+                    _fname = ascii_filename(_fname)
                     _fname = os.path.splitext(_fname)[0]+'.xhtml'
                     res = os.path.join(linkdiskpath, _fname)
                     self.downloaded_paths.append(res)