Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Use process local file descriptors in db2

Browse Source
This commit is contained in:
Kovid Goyal 2010-09-30 16:47:17 -06:00
parent cb24b0312e
commit 625e478ce0
6 changed files with 100 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
session.vim
View File

@ -1,5 +1,5 @@
" Project wide builtins
let g:pyflakes_builtins += ["dynamic_property", "__", "P", "I"]
let g:pyflakes_builtins += ["dynamic_property", "__", "P", "I", "lopen"]
python << EOFPY
import os

2
setup/check.py
View File

@ -63,7 +63,7 @@ class Check(Command):
description = 'Check for errors in the calibre source code'
BUILTINS = ['_', '__', 'dynamic_property', 'I', 'P']
BUILTINS = ['_', '__', 'dynamic_property', 'I', 'P', 'lopen']
CACHE = '.check-cache.pickle'
def get_files(self, cache):

2
src/calibre/library/caches.py
View File

@ -112,7 +112,7 @@ class MetadataBackup(Thread): # {{{
traceback.print_exc()
def write(self, path, raw):
with open(path, 'wb') as f:
with lopen(path, 'wb') as f:
f.write(raw)

2
src/calibre/library/database.py
View File

@ -1333,7 +1333,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
id = obj.lastrowid
self.conn.commit()
self.set_metadata(id, mi)
stream = path if hasattr(path, 'read') else open(path, 'rb')
stream = path if hasattr(path, 'read') else lopen(path, 'rb')
stream.seek(0, 2)
usize = stream.tell()
stream.seek(0)

24
src/calibre/library/database2.py
View File

@ -453,7 +453,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if current_path and os.path.exists(spath): # Migrate existing files
cdata = self.cover(id, index_is_id=True)
if cdata is not None:
with open(os.path.join(tpath, 'cover.jpg'), 'wb') as f:
with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f:
f.write(cdata)
for format in formats:
# Get data as string (can't use file as source and target files may be the same)
@ -526,10 +526,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if as_path:
return path
try:
f = open(path, 'rb')
f = lopen(path, 'rb')
except (IOError, OSError):
time.sleep(0.2)
f = open(path, 'rb')
f = lopen(path, 'rb')
if as_image:
img = QImage()
img.loadFromData(f.read())
@ -607,7 +607,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
continue
try:
raw = metadata_to_opf(mi)
with open(path, 'wb') as f:
with lopen(path, 'wb') as f:
f.write(raw)
except:
# Something went wrong. Put the book back on the dirty list
@ -906,7 +906,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
'''
path = self.format_abspath(index, format, index_is_id=index_is_id)
if path is not None:
f = open(path, mode)
f = lopen(path, mode)
try:
ret = f if as_file else f.read()
except IOError:
@ -922,7 +922,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
path=None, notify=True):
npath = self.run_import_plugins(fpath, format)
format = os.path.splitext(npath)[-1].lower().replace('.', '').upper()
stream = open(npath, 'rb')
stream = lopen(npath, 'rb')
format = check_ebook_format(stream, format)
return self.add_format(index, format, stream,
index_is_id=index_is_id, path=path, notify=notify)
@ -943,7 +943,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
pdir = os.path.dirname(dest)
if not os.path.exists(pdir):
os.makedirs(pdir)
with open(dest, 'wb') as f:
with lopen(dest, 'wb') as f:
shutil.copyfileobj(stream, f)
stream.seek(0, 2)
size=stream.tell()
@ -1271,7 +1271,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if mi.cover_data[1] is not None:
doit(self.set_cover, id, mi.cover_data[1]) # doesn't use commit
elif mi.cover is not None and os.access(mi.cover, os.R_OK):
doit(self.set_cover, id, open(mi.cover, 'rb'))
doit(self.set_cover, id, lopen(mi.cover, 'rb'))
if mi.tags:
doit(self.set_tags, id, mi.tags, notify=False, commit=False)
if mi.comments:
@ -1923,7 +1923,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def add_catalog(self, path, title):
format = os.path.splitext(path)[1][1:].lower()
with open(path, 'rb') as stream:
with lopen(path, 'rb') as stream:
matches = self.data.get_matches('title', '='+title)
if matches:
tag_matches = self.data.get_matches('tags', '='+_('Catalog'))
@ -1958,7 +1958,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def add_news(self, path, arg):
format = os.path.splitext(path)[1][1:].lower()
stream = path if hasattr(path, 'read') else open(path, 'rb')
stream = path if hasattr(path, 'read') else lopen(path, 'rb')
stream.seek(0)
mi = get_metadata(stream, format, use_libprs_metadata=False)
stream.seek(0)
@ -2084,7 +2084,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.set_metadata(id, mi)
npath = self.run_import_plugins(path, format)
format = os.path.splitext(npath)[-1].lower().replace('.', '').upper()
stream = open(npath, 'rb')
stream = lopen(npath, 'rb')
format = check_ebook_format(stream, format)
self.add_format(id, format, stream, index_is_id=True)
stream.close()
@ -2128,7 +2128,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if import_hooks:
self.add_format_with_hooks(id, ext, path, index_is_id=True)
else:
with open(path, 'rb') as f:
with lopen(path, 'rb') as f:
self.add_format(id, ext, f, index_is_id=True)
self.conn.commit()
self.data.refresh_ids(self, [id]) # Needed to update format list and size

84
src/calibre/startup.py
View File

@ -21,6 +21,12 @@ from calibre.constants import iswindows, preferred_encoding, plugins
_run_once = False
winutil = winutilerror = None
try:
import fcntl
except:
fcntl
fcntl = None
if not _run_once:
_run_once = True
@ -106,5 +112,83 @@ if not _run_once:
os.path.join = my_join
def local_open(name, mode='r', bufsize=-1):
'''
Open a file that wont be inherited by child processes
Only supports the following modes:
r, w, a, rb, wb, ab, r+, w+, a+, r+b, w+b, a+b
'''
if iswindows:
m = mode[0]
random = len(mode) > 1 and mode[1] == '+'
binary = mode[-1] == 'b'
if m == 'a':
flags = os._O_APPEND| os._O_RDWR
flags |= os._O_RANDOM if random else os._O_SEQUENTIAL
elif m == 'r':
if random:
flags = os._O_RDWR | os._O_RANDOM
else:
flags = os._O_RDONLY | os._O_SEQUENTIAL
elif m == 'w':
if random:
flags = os._O_RDWR | os._O_RANDOM
else:
flags = os._WRONLY | os._O_SEQUENTIAL
flags |= os._O_TRUNC | os._O_CREAT
if binary:
flags |= os._O_BINARY
else:
flags |= os._O_TEXT
flags |= os._O_NOINHERIT
fd = os.open(name, flags)
ans = os.fdopen(fd, mode, bufsize)
else:
try:
cloexec_flag = fcntl.FD_CLOEXEC
except AttributeError:
cloexec_flag = 1
ans = open(name, mode, bufsize)
old = fcntl.fcntl(ans, fcntl.F_GETFD)
fcntl.fcntl(ans, fcntl.F_SETFD, old | cloexec_flag)
return ans
__builtin__.__dict__['lopen'] = local_open
def test_lopen():
from calibre.ptempfile import TemporaryDirectory
from calibre import CurrentDir
n = u'f\xe4llen'
with TemporaryDirectory() as tdir:
with CurrentDir(tdir):
with lopen(n, 'w') as f:
f.write('one')
print 'O_CREAT tested'
with lopen(n, 'w+b') as f:
f.write('two')
with lopen(n, 'r') as f:
if f.read() == 'two':
print 'O_TRUNC tested'
else:
raise Exception('O_TRUNC failed')
with lopen(n, 'ab') as f:
f.write('three')
with lopen(n, 'r+') as f:
if f.read() == 'twothree':
print 'O_APPEND tested'
else:
raise Exception('O_APPEND failed')
with lopen(n, 'r+') as f:
f.seek(3)
f.write('xxxxx')
f.seek(0)
if f.read() == 'twoxxxxx':
print 'O_RANDOM tested'
else:
raise Exception('O_RANDOM failed')