diff --git a/session.vim b/session.vim index 6b965cff2f..54fb6305c2 100644 --- a/session.vim +++ b/session.vim @@ -1,5 +1,5 @@ " Project wide builtins -let g:pyflakes_builtins += ["dynamic_property", "__", "P", "I"] +let g:pyflakes_builtins += ["dynamic_property", "__", "P", "I", "lopen"] python << EOFPY import os diff --git a/setup/check.py b/setup/check.py index 9106ca6871..57c72e08c1 100644 --- a/setup/check.py +++ b/setup/check.py @@ -63,7 +63,7 @@ class Check(Command): description = 'Check for errors in the calibre source code' - BUILTINS = ['_', '__', 'dynamic_property', 'I', 'P'] + BUILTINS = ['_', '__', 'dynamic_property', 'I', 'P', 'lopen'] CACHE = '.check-cache.pickle' def get_files(self, cache): diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 179262dedc..c22f9e00b0 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -112,7 +112,7 @@ class MetadataBackup(Thread): # {{{ traceback.print_exc() def write(self, path, raw): - with open(path, 'wb') as f: + with lopen(path, 'wb') as f: f.write(raw) diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index 28a0de2153..c4f6908002 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -1333,7 +1333,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE; id = obj.lastrowid self.conn.commit() self.set_metadata(id, mi) - stream = path if hasattr(path, 'read') else open(path, 'rb') + stream = path if hasattr(path, 'read') else lopen(path, 'rb') stream.seek(0, 2) usize = stream.tell() stream.seek(0) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 9d9ebf64c5..4de8c3d552 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -453,7 +453,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if current_path and os.path.exists(spath): # Migrate existing files cdata = self.cover(id, index_is_id=True) if cdata is not None: - with open(os.path.join(tpath, 'cover.jpg'), 'wb') as f: + with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f: f.write(cdata) for format in formats: # Get data as string (can't use file as source and target files may be the same) @@ -526,10 +526,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if as_path: return path try: - f = open(path, 'rb') + f = lopen(path, 'rb') except (IOError, OSError): time.sleep(0.2) - f = open(path, 'rb') + f = lopen(path, 'rb') if as_image: img = QImage() img.loadFromData(f.read()) @@ -607,7 +607,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): continue try: raw = metadata_to_opf(mi) - with open(path, 'wb') as f: + with lopen(path, 'wb') as f: f.write(raw) except: # Something went wrong. Put the book back on the dirty list @@ -906,7 +906,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ''' path = self.format_abspath(index, format, index_is_id=index_is_id) if path is not None: - f = open(path, mode) + f = lopen(path, mode) try: ret = f if as_file else f.read() except IOError: @@ -922,7 +922,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): path=None, notify=True): npath = self.run_import_plugins(fpath, format) format = os.path.splitext(npath)[-1].lower().replace('.', '').upper() - stream = open(npath, 'rb') + stream = lopen(npath, 'rb') format = check_ebook_format(stream, format) return self.add_format(index, format, stream, index_is_id=index_is_id, path=path, notify=notify) @@ -943,7 +943,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): pdir = os.path.dirname(dest) if not os.path.exists(pdir): os.makedirs(pdir) - with open(dest, 'wb') as f: + with lopen(dest, 'wb') as f: shutil.copyfileobj(stream, f) stream.seek(0, 2) size=stream.tell() @@ -1271,7 +1271,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if mi.cover_data[1] is not None: doit(self.set_cover, id, mi.cover_data[1]) # doesn't use commit elif mi.cover is not None and os.access(mi.cover, os.R_OK): - doit(self.set_cover, id, open(mi.cover, 'rb')) + doit(self.set_cover, id, lopen(mi.cover, 'rb')) if mi.tags: doit(self.set_tags, id, mi.tags, notify=False, commit=False) if mi.comments: @@ -1923,7 +1923,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def add_catalog(self, path, title): format = os.path.splitext(path)[1][1:].lower() - with open(path, 'rb') as stream: + with lopen(path, 'rb') as stream: matches = self.data.get_matches('title', '='+title) if matches: tag_matches = self.data.get_matches('tags', '='+_('Catalog')) @@ -1958,7 +1958,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def add_news(self, path, arg): format = os.path.splitext(path)[1][1:].lower() - stream = path if hasattr(path, 'read') else open(path, 'rb') + stream = path if hasattr(path, 'read') else lopen(path, 'rb') stream.seek(0) mi = get_metadata(stream, format, use_libprs_metadata=False) stream.seek(0) @@ -2084,7 +2084,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.set_metadata(id, mi) npath = self.run_import_plugins(path, format) format = os.path.splitext(npath)[-1].lower().replace('.', '').upper() - stream = open(npath, 'rb') + stream = lopen(npath, 'rb') format = check_ebook_format(stream, format) self.add_format(id, format, stream, index_is_id=True) stream.close() @@ -2128,7 +2128,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if import_hooks: self.add_format_with_hooks(id, ext, path, index_is_id=True) else: - with open(path, 'rb') as f: + with lopen(path, 'rb') as f: self.add_format(id, ext, f, index_is_id=True) self.conn.commit() self.data.refresh_ids(self, [id]) # Needed to update format list and size diff --git a/src/calibre/startup.py b/src/calibre/startup.py index 75aac7c277..9f745f60ff 100644 --- a/src/calibre/startup.py +++ b/src/calibre/startup.py @@ -21,6 +21,12 @@ from calibre.constants import iswindows, preferred_encoding, plugins _run_once = False winutil = winutilerror = None +try: + import fcntl +except: + fcntl + fcntl = None + if not _run_once: _run_once = True @@ -106,5 +112,83 @@ if not _run_once: os.path.join = my_join + def local_open(name, mode='r', bufsize=-1): + ''' + Open a file that wont be inherited by child processes + + Only supports the following modes: + r, w, a, rb, wb, ab, r+, w+, a+, r+b, w+b, a+b + ''' + if iswindows: + m = mode[0] + random = len(mode) > 1 and mode[1] == '+' + binary = mode[-1] == 'b' + + if m == 'a': + flags = os._O_APPEND| os._O_RDWR + flags |= os._O_RANDOM if random else os._O_SEQUENTIAL + elif m == 'r': + if random: + flags = os._O_RDWR | os._O_RANDOM + else: + flags = os._O_RDONLY | os._O_SEQUENTIAL + elif m == 'w': + if random: + flags = os._O_RDWR | os._O_RANDOM + else: + flags = os._WRONLY | os._O_SEQUENTIAL + flags |= os._O_TRUNC | os._O_CREAT + if binary: + flags |= os._O_BINARY + else: + flags |= os._O_TEXT + flags |= os._O_NOINHERIT + fd = os.open(name, flags) + ans = os.fdopen(fd, mode, bufsize) + else: + try: + cloexec_flag = fcntl.FD_CLOEXEC + except AttributeError: + cloexec_flag = 1 + ans = open(name, mode, bufsize) + old = fcntl.fcntl(ans, fcntl.F_GETFD) + fcntl.fcntl(ans, fcntl.F_SETFD, old | cloexec_flag) + return ans + + __builtin__.__dict__['lopen'] = local_open + +def test_lopen(): + from calibre.ptempfile import TemporaryDirectory + from calibre import CurrentDir + n = u'f\xe4llen' + + with TemporaryDirectory() as tdir: + with CurrentDir(tdir): + with lopen(n, 'w') as f: + f.write('one') + print 'O_CREAT tested' + with lopen(n, 'w+b') as f: + f.write('two') + with lopen(n, 'r') as f: + if f.read() == 'two': + print 'O_TRUNC tested' + else: + raise Exception('O_TRUNC failed') + with lopen(n, 'ab') as f: + f.write('three') + with lopen(n, 'r+') as f: + if f.read() == 'twothree': + print 'O_APPEND tested' + else: + raise Exception('O_APPEND failed') + with lopen(n, 'r+') as f: + f.seek(3) + f.write('xxxxx') + f.seek(0) + if f.read() == 'twoxxxxx': + print 'O_RANDOM tested' + else: + raise Exception('O_RANDOM failed') +