Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Code to open shared files on windows

Browse Source
This commit is contained in:
Kovid Goyal 2015-06-16 13:58:07 +05:30
parent 0732802ffc
commit ea033b0157
2 changed files with 220 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

194
src/calibre/utils/shared_file.py Normal file
View File

@ -0,0 +1,194 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os
from calibre.constants import iswindows, plugins
'''
This modeule defines a share_open() function which is a replacement for
python's builtin open() function.
This replacement, opens 'shareable' files on all platforms. That is files that
can be read from and written to and deleted at the same time by multiple
processes. All file handles are non-inheritable, as in Python 3, but unlike,
Python 2. Non-inheritance is atomic.
Caveats on windows: On windows sharing is co-operative, i.e. it only works if
all processes involved open the file with share_open(). Also while you can
delete a file that is open, you cannot open a new file with the same filename
until all open file handles are closed. You also cannot delete the containing
directory until all file handles are closed. To get around this, rename the
file before deleting it.
'''
speedup, err = plugins['speedup']
if not speedup:
raise RuntimeError('Failed to load the speedup plugin with error: %s' % err)
valid_modes = {'a', 'a+', 'a+b', 'ab', 'r', 'rb', 'r+', 'r+b', 'w', 'wb', 'w+', 'w+b'}
def validate_mode(mode):
return mode in valid_modes
class FlagConstants(object):
def __init__(self):
for x in 'APPEND CREAT TRUNC EXCL RDWR RDONLY WRONLY'.split():
x = 'O_' + x
setattr(self, x, getattr(os, x))
for x in 'RANDOM SEQUENTIAL TEXT BINARY'.split():
x = 'O_' + x
setattr(self, x, getattr(os, x, 0))
fc = FlagConstants()
def flags_from_mode(mode):
if not validate_mode(mode):
raise ValueError('The mode is invalid')
m = mode[0]
random = '+' in mode
binary = 'b' in mode
if m == 'a':
flags = fc.O_APPEND | fc.O_CREAT
if random:
flags |= fc.O_RDWR | fc.O_RANDOM
else:
flags |= fc.O_WRONLY | fc.O_SEQUENTIAL
elif m == 'r':
if random:
flags = fc.O_RDWR | fc.O_RANDOM
else:
flags = fc.O_RDONLY | fc.O_SEQUENTIAL
elif m == 'w':
if random:
flags = fc.O_RDWR | fc.O_RANDOM
else:
flags = fc.O_WRONLY | fc.O_SEQUENTIAL
flags |= fc.O_TRUNC | fc.O_CREAT
flags |= (fc.O_BINARY if binary else fc.O_TEXT)
return flags
if iswindows:
from numbers import Integral
import msvcrt
import win32file
CREATE_NEW = win32file.CREATE_NEW
CREATE_ALWAYS = win32file.CREATE_ALWAYS
OPEN_EXISTING = win32file.OPEN_EXISTING
OPEN_ALWAYS = win32file.OPEN_ALWAYS
TRUNCATE_EXISTING = win32file.TRUNCATE_EXISTING
FILE_SHARE_READ = win32file.FILE_SHARE_READ
FILE_SHARE_WRITE = win32file.FILE_SHARE_WRITE
FILE_SHARE_DELETE = win32file.FILE_SHARE_DELETE
FILE_SHARE_VALID_FLAGS = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE
FILE_ATTRIBUTE_READONLY = win32file.FILE_ATTRIBUTE_READONLY
FILE_ATTRIBUTE_NORMAL = win32file.FILE_ATTRIBUTE_NORMAL
FILE_ATTRIBUTE_TEMPORARY = win32file.FILE_ATTRIBUTE_TEMPORARY
FILE_FLAG_DELETE_ON_CLOSE = win32file.FILE_FLAG_DELETE_ON_CLOSE
FILE_FLAG_SEQUENTIAL_SCAN = win32file.FILE_FLAG_SEQUENTIAL_SCAN
FILE_FLAG_RANDOM_ACCESS = win32file.FILE_FLAG_RANDOM_ACCESS
GENERIC_READ = win32file.GENERIC_READ & 0xffffffff
GENERIC_WRITE = win32file.GENERIC_WRITE & 0xffffffff
DELETE = 0x00010000
_ACCESS_MASK = os.O_RDONLY | os.O_WRONLY | os.O_RDWR
_ACCESS_MAP = {
os.O_RDONLY : GENERIC_READ,
os.O_WRONLY : GENERIC_WRITE,
os.O_RDWR : GENERIC_READ | GENERIC_WRITE
}
_CREATE_MASK = os.O_CREAT | os.O_EXCL | os.O_TRUNC
_CREATE_MAP = {
0 : OPEN_EXISTING,
os.O_EXCL : OPEN_EXISTING,
os.O_CREAT : OPEN_ALWAYS,
os.O_CREAT | os.O_EXCL : CREATE_NEW,
os.O_CREAT | os.O_TRUNC | os.O_EXCL : CREATE_NEW,
os.O_TRUNC : TRUNCATE_EXISTING,
os.O_TRUNC | os.O_EXCL : TRUNCATE_EXISTING,
os.O_CREAT | os.O_TRUNC : CREATE_ALWAYS
}
def os_open(path, flags, mode=0o777, share_flags=FILE_SHARE_VALID_FLAGS):
'''
Replacement for os.open() allowing moving or unlinking before closing
'''
if not isinstance(flags, Integral):
raise TypeError('flags must be an integer')
if not isinstance(mode, Integral):
raise TypeError('mode must be an integer')
if share_flags & ~FILE_SHARE_VALID_FLAGS:
raise ValueError('bad share_flags: %r' % share_flags)
access_flags = _ACCESS_MAP[flags & _ACCESS_MASK]
create_flags = _CREATE_MAP[flags & _CREATE_MASK]
attrib_flags = FILE_ATTRIBUTE_NORMAL
if flags & os.O_CREAT and mode & ~0o444 == 0:
attrib_flags = FILE_ATTRIBUTE_READONLY
if flags & os.O_TEMPORARY:
share_flags |= FILE_SHARE_DELETE
attrib_flags |= FILE_FLAG_DELETE_ON_CLOSE
access_flags |= DELETE
if flags & os.O_SHORT_LIVED:
attrib_flags |= FILE_ATTRIBUTE_TEMPORARY
if flags & os.O_SEQUENTIAL:
attrib_flags |= FILE_FLAG_SEQUENTIAL_SCAN
if flags & os.O_RANDOM:
attrib_flags |= FILE_FLAG_RANDOM_ACCESS
h = win32file.CreateFileW(
path, access_flags, share_flags, None, create_flags, attrib_flags, None)
ans = msvcrt.open_osfhandle(h, flags | os.O_NOINHERIT)
h.Detach() # We dont want the handle to be automatically closed when h is deleted
return ans
def share_open(path, mode='r', buffering=-1):
flags = flags_from_mode(mode)
return speedup.fdopen(os_open(path, flags), path, mode, buffering)
else:
def share_open(path, mode='r', buffering=-1):
flags = flags_from_mode(mode) | speedup.O_CLOEXEC
return speedup.fdopen(os.open(path, flags), path, mode, buffering)
def test():
import repr as reprlib
def eq(x, y):
if x != y:
raise AssertionError('%s != %s' % (reprlib.repr(x), reprlib.repr(y)))
from calibre.ptempfile import TemporaryDirectory
with TemporaryDirectory() as tdir:
fname = os.path.join(tdir, 'test.txt')
with share_open(fname, 'wb') as f:
f.write(b'a' * 20 * 1024)
eq(fname, f.name)
f = share_open(fname, 'rb')
eq(f.read(1), b'a')
if iswindows:
os.rename(fname, fname+'.moved')
os.remove(fname+'.moved')
else:
os.remove(fname)
eq(f.read(1), b'a')
f2 = share_open(fname, 'w+b')
f2.write(b'b' * 10 * 1024)
f2.seek(0)
eq(f.read(10000), b'a'*10000)
eq(f2.read(100), b'b' * 100)
f3 = share_open(fname, 'rb')
eq(f3.read(100), b'b' * 100)

26
src/calibre/utils/speedup.c
View File

@ -113,6 +113,28 @@ speedup_detach(PyObject *self, PyObject *args) {
Py_RETURN_NONE;
}
static PyObject*
speedup_fdopen(PyObject *self, PyObject *args) {
PyObject *ans = NULL, *name = NULL;
PyFileObject *t = NULL;
FILE *fp = NULL;
int fd = -1, bufsize = -1;
char *mode = NULL;
if (!PyArg_ParseTuple(args, "iOs|i", &fd, &name, &mode, &bufsize)) return NULL;
fp = fdopen(fd, mode);
if (fp == NULL) return PyErr_SetFromErrno(PyExc_OSError);
ans = PyFile_FromFile(fp, "<fdopen>", mode, fclose);
if (ans != NULL) {
t = (PyFileObject*)ans;
Py_XDECREF(t->f_name);
t->f_name = name;
Py_INCREF(name);
PyFile_SetBufSize(ans, bufsize);
}
return ans;
}
static void calculate_gaussian_kernel(Py_ssize_t size, double *kernel, double radius) {
const double sqr = radius * radius;
const double factor = 1.0 / (2 * M_PI * sqr);
@ -218,6 +240,10 @@ static PyMethodDef speedup_methods[] = {
" This function returns an image (bytestring) in the PPM format as the texture."
},
{"fdopen", speedup_fdopen, METH_VARARGS,
"fdopen(fd, name, mode [, bufsize=-1)\n\nCreate a python file object from an OS file descriptor with a name. Note that this does not do any validation of mode, so you must ensure fd already has the correct flags set."
},
{NULL, NULL, 0, NULL}
};