Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix #951 ([comic2lrf] on windows multi-threading isn't working)

Browse Source
This commit is contained in:
Kovid Goyal 2008-08-27 11:57:48 -07:00
parent c3cced43ec
commit 606f3a6dd1
8 changed files with 91 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
installer/windows/freeze.py
View File

@ -161,12 +161,12 @@ def main(args=sys.argv):
'win32process', 'win32api', 'msvcrt',
'win32event', 'calibre.ebooks.lrf.any.*',
'calibre.ebooks.lrf.feeds.*',
'lxml', 'lxml._elementpath', 'genshi',
'genshi',
'path', 'pydoc', 'IPython.Extensions.*',
'calibre.web.feeds.recipes.*',
'PyQt4.QtWebKit', 'PyQt4.QtNetwork',
],
'packages' : ['PIL'],
'packages' : ['PIL', 'lxml'],
'excludes' : ["Tkconstants", "Tkinter", "tcl",
"_imagingtk", "ImageTk", "FixTk"
],

4
src/calibre/debug.py
View File

@ -25,9 +25,7 @@ Run an embedded python interpreter.
def update_zipfile(zipfile, mod, path):
if 'win32' in sys.platform:
print 'WARNING: On Windows Vista you must run this from a console that has been started in Administrator mode.'
print 'Press Enter to continue if this is an Administrator console or Ctrl-C to Cancel'
raw_input()
print 'WARNING: On Windows Vista using this option may cause windows to put library.zip into the Virtual Store (typically located in c:\Users\username\AppData\Local\VirtualStore). If it does this you must delete it from there after you\'re done debugging).'
pat = re.compile(mod.replace('.', '/')+r'\.py[co]*')
name = mod.replace('.', '/') + os.path.splitext(path)[-1]
update(zipfile, [pat], [path], [name])

20
src/calibre/ebooks/epub/__init__.py
View File

@ -8,8 +8,28 @@ Conversion to EPUB.
'''
import sys
from calibre.utils.config import Config, StringConfig
from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED
from calibre.ebooks.html import config as common_config
def initialize_container(path_to_container, opf_name='metadata.opf'):
'''
Create an empty EPUB document, with a default skeleton.
'''
CONTAINER='''\
<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="%s" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>
'''%opf_name
zf = ZipFile(path_to_container, 'w')
zf.writestr('mimetype', 'application/epub+zip', compression=ZIP_DEFLATED)
zf.writestr('META-INF/', '', 0700)
zf.writestr('META-INF/container.xml', CONTAINER)
return zf
def config(defaults=None):
desc = _('Options to control the conversion to EPUB')
if defaults is None:

6
src/calibre/ebooks/epub/from_html.py
View File

@ -36,7 +36,7 @@ class HTMLProcessor(Parser):
for text in get_text(self.body if self.body is not None else self.root):
length, parent = len(re.sub(r'\s+', '', text)), text.getparent()
#TODO: Use cssutils on self.raw_css to figure out the font size
# of this piece text and update statistics accordingly
# of this piece of text and update statistics accordingly
def split(self):
''' Split into individual flows to accommodate Adobe's incompetence '''
@ -74,13 +74,17 @@ def convert(htmlfile, opts, notification=None):
mi = merge_metadata(htmlfile, opf, opts)
opts.chapter = XPath(opts.chapter,
namespaces={'re':'http://exslt.org/regular-expressions'})
resource_map = parse_content(filelist, opts)
resources = [os.path.join(opts.output, 'content', f) for f in resource_map.values()]
if opf.cover and os.access(opf.cover, os.R_OK):
shutil.copyfile(opf.cover, os.path.join(opts.output, 'content', 'resources', '_cover_'+os.path.splitext(opf.cover)))
cpath = os.path.join(opts.output, 'content', 'resources', '_cover_'+os.path.splitext(opf.cover))
shutil.copyfile(opf.cover, cpath)
resources.append(cpath)
mi.cover = cpath
def main(args=sys.argv):
parser = option_parser()

73
src/calibre/ebooks/lrf/comic/convert_from.py
View File

@ -7,14 +7,13 @@ __docformat__ = 'restructuredtext en'
Based on ideas from comiclrf created by FangornUK.
'''
import os, sys, traceback, shutil
import os, sys, traceback, shutil, threading
from uuid import uuid4
from calibre import extract, detect_ncpus, terminal_controller, \
__appname__, __version__
from calibre import extract, terminal_controller, __appname__, __version__
from calibre.utils.config import Config, StringConfig
from calibre.ptempfile import PersistentTemporaryDirectory
from calibre.utils.threadpool import ThreadPool, WorkRequest
from calibre.parallel import Server, ParallelJob
from calibre.utils.terminfo import ProgressBar
from calibre.ebooks.lrf.pylrs.pylrs import Book, BookSetting, ImageStream, ImageBlock
try:
@ -84,12 +83,13 @@ class PageProcessor(list):
'''
def __init__(self, path_to_page, dest, opts, num):
self.path_to_page = path_to_page
self.opts = opts
self.num = num
self.dest = dest
self.rotate = False
list.__init__(self)
self.path_to_page = path_to_page
self.opts = opts
self.num = num
self.dest = dest
self.rotate = False
def __call__(self):
try:
@ -100,7 +100,6 @@ class PageProcessor(list):
raise IOError('Failed to read image from: %'%self.path_to_page)
width = MagickGetImageWidth(img)
height = MagickGetImageHeight(img)
if self.num == 0: # First image so create a thumbnail from it
thumb = CloneMagickWand(img)
if thumb < 0:
@ -122,7 +121,6 @@ class PageProcessor(list):
MagickCropImage(split1, (width/2)-1, height, 0, 0)
MagickCropImage(split2, (width/2)-1, height, width/2, 0 )
self.pages = [split2, split1] if self.opts.right2left else [split1, split2]
self.process_pages()
except Exception, err:
print 'Failed to process page: %s'%os.path.basename(self.path_to_page)
@ -138,23 +136,20 @@ class PageProcessor(list):
PixelSetColor(pw, 'white')
MagickSetImageBorderColor(wand, pw)
if self.rotate:
MagickRotateImage(wand, pw, -90)
# 25 percent fuzzy trim?
MagickTrimImage(wand, 25*65535/100)
MagickSetImagePage(wand, 0,0,0,0) #Clear page after trim, like a "+repage"
# Do the Photoshop "Auto Levels" equivalent
if not self.opts.dont_normalize:
MagickNormalizeImage(wand)
sizex = MagickGetImageWidth(wand)
sizey = MagickGetImageHeight(wand)
SCRWIDTH, SCRHEIGHT = PROFILES[self.opts.profile]
print 77777, threading.currentThread()
if self.opts.keep_aspect_ratio:
# Preserve the aspect ratio by adding border
aspect = float(sizex) / float(sizey)
@ -168,7 +163,6 @@ class PageProcessor(list):
newsizey = int(newsizex / aspect)
deltax = 0
deltay = (SCRHEIGHT - newsizey) / 2
MagickResizeImage(wand, newsizex, newsizey, CatromFilter, 1.0)
MagickSetImageBorderColor(wand, pw)
MagickBorderImage(wand, pw, deltax, deltay)
@ -193,6 +187,12 @@ class PageProcessor(list):
DestroyPixelWand(pw)
wand = DestroyMagickWand(wand)
def process_page(path, dest, opts, num):
pp = PageProcessor(path, dest, opts, num)
with ImageMagick():
pp()
return list(pp)
class Progress(object):
def __init__(self, total, update):
@ -200,10 +200,11 @@ class Progress(object):
self.update = update
self.done = 0
def __call__(self, req, res):
def __call__(self, job):
self.done += 1
self.update(float(self.done)/self.total,
_('Rendered %s')%os.path.basename(req.callable.path_to_page))
msg = _('Rendered %s') if job.result else _('Failed %s')
msg = msg%os.path.basename(job.args[0])
self.update(float(self.done)/self.total, msg)
def process_pages(pages, opts, update):
'''
@ -211,23 +212,25 @@ def process_pages(pages, opts, update):
'''
if not _imagemagick_loaded:
raise RuntimeError('Failed to load ImageMagick')
with ImageMagick():
tdir = PersistentTemporaryDirectory('_comic2lrf_pp')
processed_pages = [PageProcessor(path, tdir, opts, i) for i, path in enumerate(pages)]
tp = ThreadPool(detect_ncpus())
update(0, '')
notify = Progress(len(pages), update)
for pp in processed_pages:
tp.putRequest(WorkRequest(pp, callback=notify))
tp.wait()
ans, failures = [], []
tdir = PersistentTemporaryDirectory('_comic2lrf_pp')
notify = Progress(len(pages), update)
server = Server()
jobs = []
for i, path in enumerate(pages):
jobs.append(ParallelJob('render_page', notify, args=[path, tdir, opts, i]))
server.add_job(jobs[-1])
server.wait()
server.killall()
server.close()
ans, failures = [], []
for pp in processed_pages:
if len(pp) == 0:
failures.append(os.path.basename(pp.path_to_page))
else:
ans += pp
return ans, failures, tdir
for job in jobs:
if not job.result:
failures.append(os.path.basename(job.args[0]))
else:
ans += job.result
return ans, failures, tdir
def config(defaults=None):
desc = _('Options to control the conversion of comics (CBR, CBZ) files into ebooks')

28
src/calibre/parallel.py
View File

@ -36,19 +36,22 @@ DEBUG = False
#: A mapping from job names to functions that perform the jobs
PARALLEL_FUNCS = {
'any2lrf' :
'any2lrf' :
('calibre.ebooks.lrf.any.convert_from', 'main', dict(gui_mode=True), None),
'lrfviewer' :
'lrfviewer' :
('calibre.gui2.lrf_renderer.main', 'main', {}, None),
'feeds2lrf' :
'feeds2lrf' :
('calibre.ebooks.lrf.feeds.convert_from', 'main', {}, 'notification'),
'render_table' :
'render_table' :
('calibre.ebooks.lrf.html.table_as_image', 'do_render', {}, None),
'render_page' :
('calibre.ebooks.lrf.comic.convert_from', 'process_page', {}, None),
'comic2lrf' :
'comic2lrf' :
('calibre.ebooks.lrf.comic.convert_from', 'do_convert', {}, 'notification'),
}
@ -657,6 +660,21 @@ class Server(Thread):
if job.job_manager is not None:
job.job_manager.add_job(job)
def poll(self):
'''
Return True if the server has either working or queued jobs
'''
with self.job_lock:
with self.working_lock:
return len(self.jobs) + len(self.working) > 0
def wait(self, sleep=1):
'''
Wait until job queue is empty
'''
while self.poll():
time.sleep(sleep)
def run(self):
while True:
job = None

1
src/calibre/utils/threadpool.py
View File

@ -46,7 +46,6 @@ __date__ = "$Date: 2006/06/23 12:32:25 $"
__license__ = 'Python license'
# standard library modules
import sys
import threading
import Queue

4
src/calibre/utils/zipfile.py
View File

@ -1136,7 +1136,7 @@ class ZipFile:
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def writestr(self, zinfo_or_arcname, bytes, permissions=0600):
def writestr(self, zinfo_or_arcname, bytes, permissions=0600, compression=ZIP_DEFLATED):
"""Write a file into the archive. The contents is the string
'bytes'. 'zinfo_or_arcname' is either a ZipInfo instance or
the name of the file in the archive."""
@ -1145,7 +1145,7 @@ class ZipFile:
zinfo_or_arcname = zinfo_or_arcname.encode('utf-8')
zinfo = ZipInfo(filename=zinfo_or_arcname,
date_time=time.localtime(time.time())[:6])
zinfo.compress_type = self.compression
zinfo.compress_type = compression
zinfo.external_attr = permissions << 16
else:
zinfo = zinfo_or_arcname