Merge from trunk

src/calibre/devices/usbms/cli.py

@@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en'
 import os, shutil, time
 
 from calibre.devices.errors import PathError
+from calibre.utils.filenames import case_preserving_open_file
 
 class File(object):
 
@@ -46,10 +47,8 @@ class CLI(object):
             path = os.path.join(path, infile.name)
         if not replace_file and os.path.exists(path):
             raise PathError('File already exists: ' + path)
-        d = os.path.dirname(path)
+        dest, actual_path = case_preserving_open_file(path)
-        if not os.path.exists(d):
+        with dest:
-            os.makedirs(d)
-        with open(path, 'w+b') as dest:
             try:
                 shutil.copyfileobj(infile, dest)
             except IOError:
@@ -62,6 +61,7 @@ class CLI(object):
             #if not check_transfer(infile, dest): raise Exception('Transfer failed')
         if close:
             infile.close()
+        return actual_path
 
     def munge_path(self, path):
         if path.startswith('/') and not (path.startswith(self._main_prefix) or \

src/calibre/devices/usbms/driver.py

@@ -13,7 +13,7 @@ for a particular device.
 import os, re, time, json, uuid, functools
 from itertools import cycle
 
-from calibre.constants import numeric_version, iswindows
+from calibre.constants import numeric_version
 from calibre import prints, isbytestring
 from calibre.constants import filesystem_encoding, DEBUG
 from calibre.devices.usbms.cli import CLI
@@ -260,8 +260,8 @@ class USBMS(CLI, Device):
             filepath = self.normalize_path(self.create_upload_path(path, mdata, fname))
             if not hasattr(infile, 'read'):
                 infile = self.normalize_path(infile)
-            self.put_file(infile, filepath, replace_file=True)
+            filepath = self.put_file(infile, filepath, replace_file=True)
-            paths.append(self.correct_case_of_filename(filepath))
+            paths.append(filepath)
             try:
                 self.upload_cover(os.path.dirname(filepath),
                                   os.path.splitext(os.path.basename(filepath))[0],
@@ -276,41 +276,6 @@ class USBMS(CLI, Device):
         debug_print('USBMS: finished uploading %d books'%(len(files)))
         return zip(paths, cycle([on_card]))
 
-    # Get the real case of the underlying filename. Can differ from what we
-    # have on case-insensitive file systems.
-    def correct_case_of_filename(self, filepath):
-        path = os.path.abspath(filepath);
-        comps = path.split(os.sep)
-        if not comps:
-            return filepath
-        res = comps[0]
-        if iswindows:
-            # must put a \ after the prefix or it will read the current directory
-            res += os.sep
-        else:
-            # for *nix file systems, must put a sep on the front.
-            res = os.sep + res
-
-        # read down the path directory by directory, doing a case-insensitive
-        # compare. Build a new path of the components with the case as on disk.
-        for comp in comps[1:]:
-            sc = os.listdir(res)
-            for c in sc:
-                if c.lower() == comp.lower():
-                    res = os.path.join(res, c);
-                    continue
-        # now see if the old and new path point at the same book. If we have
-        # a case-sensitive file system on the device, then we might have
-        # generated the wrong path. Books are considered the same if their
-        # mtime and size are the same.
-        before = os.stat(filepath)
-        after = os.stat(res)
-        if before.st_mtime == after.st_mtime and before.st_size == after.st_size:
-            # the same. the new path is valid. Might == the old one, but that is OK
-            return res
-        # not the same. The old path must be used.
-        return filepath
-
     def upload_cover(self, path, filename, metadata, filepath):
         '''
         Upload book cover to the device. Default implementation does nothing.

src/calibre/ebooks/__init__.py

@@ -28,8 +28,9 @@ class ParserError(ValueError):
 
 BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'text', 'htm', 'xhtm',
                    'html', 'htmlz', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc',
-                   'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
+                   'epub', 'fb2', 'djv', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
-                   'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'pmlz', 'mbp', 'tan', 'snb']
+                   'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'pmlz', 'mbp', 'tan', 'snb',
+                   'xps', 'oxps']
 
 class HTMLRenderer(object):
 

src/calibre/gui2/convert/heuristics.py

@@ -72,10 +72,11 @@ class HeuristicsWidget(Widget, Ui_Form):
             return True
 
     def load_histories(self):
+        val = unicode(self.opt_replace_scene_breaks.currentText())
+        
         self.opt_replace_scene_breaks.clear()
         self.opt_replace_scene_breaks.lineEdit().setText('')
 
-        val = unicode(self.opt_replace_scene_breaks.currentText())
         rssb_hist = gprefs.get('replace_scene_breaks_history', self.rssb_defaults)
         if val in rssb_hist:
             del rssb_hist[rssb_hist.index(val)]

src/calibre/gui2/preferences/adding.ui

@@ -130,7 +130,7 @@ Author matching is exact.</string>
    <item row="2" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_preserve_date_on_ctl">
      <property name="text">
-      <string>When &amp;copying books from one library to another, preserve the date</string>
+      <string>When using the  &quot;&amp;Copy to library&quot; action to copy books between libraries, preserve the date</string>
      </property>
     </widget>
    </item>

src/calibre/utils/filenames.py

@@ -3,11 +3,12 @@ Make strings safe for use as ASCII filenames, while trying to preserve as much
 meaning as possible.
 '''
 
-import os
+import os, errno
 from math import ceil
 
-from calibre import sanitize_file_name
+from calibre import sanitize_file_name, isbytestring, force_unicode
-from calibre.constants import preferred_encoding, iswindows
+from calibre.constants import (preferred_encoding, iswindows,
+        filesystem_encoding)
 from calibre.utils.localization import get_udc
 
 def ascii_text(orig):
@@ -114,3 +115,81 @@ def is_case_sensitive(path):
         os.remove(f1)
     return is_case_sensitive
 
+def case_preserving_open_file(path, mode='wb', mkdir_mode=0777):
+    '''
+    Open the file pointed to by path with the specified mode. If any
+    directories in path do not exist, they are created. Returns the
+    opened file object and the path to the opened file object. This path is
+    guaranteed to have the same case as the on disk path. For case insensitive
+    filesystems, the returned path may be different from the passed in path.
+    The returned path is always unicode and always an absolute path.
+
+    If mode is None, then this function assumes that path points to a directory
+    and return the path to the directory as the file object.
+
+    mkdir_mode specifies the mode with which any missing directories in path
+    are created.
+    '''
+    if isbytestring(path):
+        path = path.decode(filesystem_encoding)
+
+    path = os.path.abspath(path)
+
+    sep = force_unicode(os.sep, 'ascii')
+
+    if path.endswith(sep):
+        path = path[:-1]
+    if not path:
+        raise ValueError('Path must not point to root')
+
+    components = path.split(sep)
+    if not components:
+        raise ValueError('Invalid path: %r'%path)
+
+    cpath = sep
+    if iswindows:
+        # Always upper case the drive letter and add a trailing slash so that
+        # the first os.listdir works correctly
+        cpath = components[0].upper() + sep
+
+    # Create all the directories in path, putting the on disk case version of
+    # the directory into cpath
+    dirs = components[1:] if mode is None else components[1:-1]
+    for comp in dirs:
+        cdir = os.path.join(cpath, comp)
+        try:
+            os.mkdir(cdir, mkdir_mode)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                if not os.path.exists(cdir):
+                    # Check for exists again, as we could have got a permission
+                    # denied error
+                    raise
+            # This component already exists, ensure the case is correct
+            cl = comp.lower()
+            candidates = [c for c in os.listdir(cpath) if c.lower() == cl]
+            if len(candidates) == 1:
+                cdir = os.path.join(cpath, candidates[0])
+            # else: We are on a case sensitive file system so cdir must already
+            # be correct
+        cpath = cdir
+
+    if mode is None:
+        ans = fpath = cpath
+    else:
+        fname = components[-1]
+        ans = open(os.path.join(cpath, fname), mode)
+        # Ensure file and all its metadata is written to disk so that subsequent
+        # listdir() has file name in it. I don't know if this is actually
+        # necessary, but given the diversity of platforms, best to be safe.
+        ans.flush()
+        os.fsync(ans.fileno())
+
+        cl = fname.lower()
+        candidates = [c for c in os.listdir(cpath) if c.lower() == cl]
+        if len(candidates) == 1:
+            fpath = os.path.join(cpath, candidates[0])
+        else:
+            fpath = ans.name
+    return ans, fpath
+