Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-06-23 15:30:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

Use one less worker process

Browse Source
This commit is contained in:
Kovid Goyal 2019-10-26 14:22:05 +05:30
parent 9a44ef24ff
commit 2d2ee3f85c
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/calibre/srv/render_book.py
View File

@ -450,16 +450,17 @@ class RenderManager(object):
def __call__(self, names, args, in_process_container): def __call__(self, names, args, in_process_container):
num_workers = min(detect_ncpus(), len(names)) num_workers = min(detect_ncpus(), len(names))
if num_workers > 1: if num_workers > 1:
total_sz = sum(os.path.getsize(in_process_container.name_path_map[n]) for n in names) if len(names) < 3 or sum(os.path.getsize(in_process_container.name_path_map[n]) for n in names) < 128 * 1024:
if total_sz < 128 * 1024:
num_workers = 1 num_workers = 1
if num_workers == 1: if num_workers == 1:
return [process_book_files(names, *args, container=in_process_container)] return [process_book_files(names, *args, container=in_process_container)]
while len(self.workers) < num_workers: num_other_workers = num_workers - 1
while len(self.workers) < num_other_workers:
self.launch_worker() self.launch_worker()
group_sz = int(ceil(len(names) / num_workers)) group_sz = int(ceil(len(names) / num_workers))
for group, worker in zip(grouper(group_sz, names), self.workers): groups = tuple(grouper(group_sz, names))
for group, worker in zip(groups[:-1], self.workers):
worker.stdin.write(as_bytes(msgpack_dumps((worker.output_path, group,) + args))) worker.stdin.write(as_bytes(msgpack_dumps((worker.output_path, group,) + args)))
worker.stdin.flush(), worker.stdin.close() worker.stdin.flush(), worker.stdin.close()
worker.job_sent = True worker.job_sent = True
@ -469,7 +470,7 @@ class RenderManager(object):
worker.stdin.write(b'_'), worker.stdin.flush(), worker.stdin.close() worker.stdin.write(b'_'), worker.stdin.flush(), worker.stdin.close()
error = None error = None
results = [] results = [process_book_files(groups[-1], *args, container=in_process_container)]
for worker in self.workers: for worker in self.workers:
if not hasattr(worker, 'job_sent'): if not hasattr(worker, 'job_sent'):
worker.wait() worker.wait()