diff --git a/src/calibre/utils/shared_file.py b/src/calibre/utils/shared_file.py new file mode 100644 index 0000000000..1a2a3d1523 --- /dev/null +++ b/src/calibre/utils/shared_file.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2015, Kovid Goyal ' + +import os +from calibre.constants import iswindows, plugins + +''' +This modeule defines a share_open() function which is a replacement for +python's builtin open() function. + +This replacement, opens 'shareable' files on all platforms. That is files that +can be read from and written to and deleted at the same time by multiple +processes. All file handles are non-inheritable, as in Python 3, but unlike, +Python 2. Non-inheritance is atomic. + +Caveats on windows: On windows sharing is co-operative, i.e. it only works if +all processes involved open the file with share_open(). Also while you can +delete a file that is open, you cannot open a new file with the same filename +until all open file handles are closed. You also cannot delete the containing +directory until all file handles are closed. To get around this, rename the +file before deleting it. +''' + +speedup, err = plugins['speedup'] + +if not speedup: + raise RuntimeError('Failed to load the speedup plugin with error: %s' % err) + +valid_modes = {'a', 'a+', 'a+b', 'ab', 'r', 'rb', 'r+', 'r+b', 'w', 'wb', 'w+', 'w+b'} + +def validate_mode(mode): + return mode in valid_modes + +class FlagConstants(object): + + def __init__(self): + for x in 'APPEND CREAT TRUNC EXCL RDWR RDONLY WRONLY'.split(): + x = 'O_' + x + setattr(self, x, getattr(os, x)) + for x in 'RANDOM SEQUENTIAL TEXT BINARY'.split(): + x = 'O_' + x + setattr(self, x, getattr(os, x, 0)) +fc = FlagConstants() + +def flags_from_mode(mode): + if not validate_mode(mode): + raise ValueError('The mode is invalid') + m = mode[0] + random = '+' in mode + binary = 'b' in mode + if m == 'a': + flags = fc.O_APPEND | fc.O_CREAT + if random: + flags |= fc.O_RDWR | fc.O_RANDOM + else: + flags |= fc.O_WRONLY | fc.O_SEQUENTIAL + elif m == 'r': + if random: + flags = fc.O_RDWR | fc.O_RANDOM + else: + flags = fc.O_RDONLY | fc.O_SEQUENTIAL + elif m == 'w': + if random: + flags = fc.O_RDWR | fc.O_RANDOM + else: + flags = fc.O_WRONLY | fc.O_SEQUENTIAL + flags |= fc.O_TRUNC | fc.O_CREAT + flags |= (fc.O_BINARY if binary else fc.O_TEXT) + return flags + +if iswindows: + from numbers import Integral + import msvcrt + import win32file + CREATE_NEW = win32file.CREATE_NEW + CREATE_ALWAYS = win32file.CREATE_ALWAYS + OPEN_EXISTING = win32file.OPEN_EXISTING + OPEN_ALWAYS = win32file.OPEN_ALWAYS + TRUNCATE_EXISTING = win32file.TRUNCATE_EXISTING + FILE_SHARE_READ = win32file.FILE_SHARE_READ + FILE_SHARE_WRITE = win32file.FILE_SHARE_WRITE + FILE_SHARE_DELETE = win32file.FILE_SHARE_DELETE + FILE_SHARE_VALID_FLAGS = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE + FILE_ATTRIBUTE_READONLY = win32file.FILE_ATTRIBUTE_READONLY + FILE_ATTRIBUTE_NORMAL = win32file.FILE_ATTRIBUTE_NORMAL + FILE_ATTRIBUTE_TEMPORARY = win32file.FILE_ATTRIBUTE_TEMPORARY + FILE_FLAG_DELETE_ON_CLOSE = win32file.FILE_FLAG_DELETE_ON_CLOSE + FILE_FLAG_SEQUENTIAL_SCAN = win32file.FILE_FLAG_SEQUENTIAL_SCAN + FILE_FLAG_RANDOM_ACCESS = win32file.FILE_FLAG_RANDOM_ACCESS + GENERIC_READ = win32file.GENERIC_READ & 0xffffffff + GENERIC_WRITE = win32file.GENERIC_WRITE & 0xffffffff + DELETE = 0x00010000 + + _ACCESS_MASK = os.O_RDONLY | os.O_WRONLY | os.O_RDWR + _ACCESS_MAP = { + os.O_RDONLY : GENERIC_READ, + os.O_WRONLY : GENERIC_WRITE, + os.O_RDWR : GENERIC_READ | GENERIC_WRITE + } + + _CREATE_MASK = os.O_CREAT | os.O_EXCL | os.O_TRUNC + _CREATE_MAP = { + 0 : OPEN_EXISTING, + os.O_EXCL : OPEN_EXISTING, + os.O_CREAT : OPEN_ALWAYS, + os.O_CREAT | os.O_EXCL : CREATE_NEW, + os.O_CREAT | os.O_TRUNC | os.O_EXCL : CREATE_NEW, + os.O_TRUNC : TRUNCATE_EXISTING, + os.O_TRUNC | os.O_EXCL : TRUNCATE_EXISTING, + os.O_CREAT | os.O_TRUNC : CREATE_ALWAYS + } + + def os_open(path, flags, mode=0o777, share_flags=FILE_SHARE_VALID_FLAGS): + ''' + Replacement for os.open() allowing moving or unlinking before closing + ''' + if not isinstance(flags, Integral): + raise TypeError('flags must be an integer') + if not isinstance(mode, Integral): + raise TypeError('mode must be an integer') + + if share_flags & ~FILE_SHARE_VALID_FLAGS: + raise ValueError('bad share_flags: %r' % share_flags) + + access_flags = _ACCESS_MAP[flags & _ACCESS_MASK] + create_flags = _CREATE_MAP[flags & _CREATE_MASK] + attrib_flags = FILE_ATTRIBUTE_NORMAL + + if flags & os.O_CREAT and mode & ~0o444 == 0: + attrib_flags = FILE_ATTRIBUTE_READONLY + + if flags & os.O_TEMPORARY: + share_flags |= FILE_SHARE_DELETE + attrib_flags |= FILE_FLAG_DELETE_ON_CLOSE + access_flags |= DELETE + + if flags & os.O_SHORT_LIVED: + attrib_flags |= FILE_ATTRIBUTE_TEMPORARY + + if flags & os.O_SEQUENTIAL: + attrib_flags |= FILE_FLAG_SEQUENTIAL_SCAN + + if flags & os.O_RANDOM: + attrib_flags |= FILE_FLAG_RANDOM_ACCESS + + h = win32file.CreateFileW( + path, access_flags, share_flags, None, create_flags, attrib_flags, None) + ans = msvcrt.open_osfhandle(h, flags | os.O_NOINHERIT) + h.Detach() # We dont want the handle to be automatically closed when h is deleted + return ans + + def share_open(path, mode='r', buffering=-1): + flags = flags_from_mode(mode) + return speedup.fdopen(os_open(path, flags), path, mode, buffering) + +else: + def share_open(path, mode='r', buffering=-1): + flags = flags_from_mode(mode) | speedup.O_CLOEXEC + return speedup.fdopen(os.open(path, flags), path, mode, buffering) + +def test(): + import repr as reprlib + + def eq(x, y): + if x != y: + raise AssertionError('%s != %s' % (reprlib.repr(x), reprlib.repr(y))) + + from calibre.ptempfile import TemporaryDirectory + with TemporaryDirectory() as tdir: + fname = os.path.join(tdir, 'test.txt') + with share_open(fname, 'wb') as f: + f.write(b'a' * 20 * 1024) + eq(fname, f.name) + f = share_open(fname, 'rb') + eq(f.read(1), b'a') + if iswindows: + os.rename(fname, fname+'.moved') + os.remove(fname+'.moved') + else: + os.remove(fname) + eq(f.read(1), b'a') + f2 = share_open(fname, 'w+b') + f2.write(b'b' * 10 * 1024) + f2.seek(0) + eq(f.read(10000), b'a'*10000) + eq(f2.read(100), b'b' * 100) + f3 = share_open(fname, 'rb') + eq(f3.read(100), b'b' * 100) + diff --git a/src/calibre/utils/speedup.c b/src/calibre/utils/speedup.c index ac0b5ece87..e11b9cd0b8 100644 --- a/src/calibre/utils/speedup.c +++ b/src/calibre/utils/speedup.c @@ -113,6 +113,28 @@ speedup_detach(PyObject *self, PyObject *args) { Py_RETURN_NONE; } +static PyObject* +speedup_fdopen(PyObject *self, PyObject *args) { + PyObject *ans = NULL, *name = NULL; + PyFileObject *t = NULL; + FILE *fp = NULL; + int fd = -1, bufsize = -1; + char *mode = NULL; + + if (!PyArg_ParseTuple(args, "iOs|i", &fd, &name, &mode, &bufsize)) return NULL; + fp = fdopen(fd, mode); + if (fp == NULL) return PyErr_SetFromErrno(PyExc_OSError); + ans = PyFile_FromFile(fp, "", mode, fclose); + if (ans != NULL) { + t = (PyFileObject*)ans; + Py_XDECREF(t->f_name); + t->f_name = name; + Py_INCREF(name); + PyFile_SetBufSize(ans, bufsize); + } + return ans; +} + static void calculate_gaussian_kernel(Py_ssize_t size, double *kernel, double radius) { const double sqr = radius * radius; const double factor = 1.0 / (2 * M_PI * sqr); @@ -218,6 +240,10 @@ static PyMethodDef speedup_methods[] = { " This function returns an image (bytestring) in the PPM format as the texture." }, + {"fdopen", speedup_fdopen, METH_VARARGS, + "fdopen(fd, name, mode [, bufsize=-1)\n\nCreate a python file object from an OS file descriptor with a name. Note that this does not do any validation of mode, so you must ensure fd already has the correct flags set." + }, + {NULL, NULL, 0, NULL} };