Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-11-03 19:17:02 -05:00
Code Issues Packages Projects Releases Wiki Activity

IGN:More string processing optimizations

Browse Source
This commit is contained in:
Kovid Goyal 2009-09-02 17:00:33 -06:00
parent f6acabca50
commit 3e7df1bf84
3 changed files with 71 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
src/calibre/ebooks/fb2/fb2ml.py
View File

@ -8,7 +8,6 @@ __docformat__ = 'restructuredtext en'
Transform OEB content into FB2 markup Transform OEB content into FB2 markup
''' '''
import os
import cStringIO import cStringIO
from base64 import b64encode from base64 import b64encode
@ -67,14 +66,14 @@ class FB2MLizer(object):
def fb2mlize_spine(self): def fb2mlize_spine(self):
self.image_hrefs = {} self.image_hrefs = {}
self.link_hrefs = {} self.link_hrefs = {}
output = self.fb2_header() output = [self.fb2_header()]
output += self.get_cover_page() output.append(self.get_cover_page())
output += u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk' output.append(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk')
output += self.get_text() output.append(self.get_text())
output += self.fb2_body_footer() output.append(self.fb2_body_footer())
output += self.fb2mlize_images() output.append(self.fb2mlize_images())
output += self.fb2_footer() output.append(self.fb2_footer())
output = output.replace(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk', self.get_toc()) output = ''.join(output).replace(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk', self.get_toc())
return u'<?xml version="1.0" encoding="UTF-8"?>\n%s' % etree.tostring(etree.fromstring(output), encoding=unicode, pretty_print=True) return u'<?xml version="1.0" encoding="UTF-8"?>\n%s' % etree.tostring(etree.fromstring(output), encoding=unicode, pretty_print=True)
def fb2_header(self): def fb2_header(self):
@ -117,25 +116,25 @@ class FB2MLizer(object):
return output return output
def get_toc(self): def get_toc(self):
toc = u'' toc = [u'']
if self.opts.inline_toc: if self.opts.inline_toc:
self.log.debug('Generating table of contents...') self.log.debug('Generating table of contents...')
toc += u'<p>%s</p>' % _('Table of Contents:') toc.append(u'<p>%s</p>' % _('Table of Contents:'))
for item in self.oeb_book.toc: for item in self.oeb_book.toc:
if item.href in self.link_hrefs.keys(): if item.href in self.link_hrefs.keys():
toc += '<p><a xlink:href="#%s">%s</a></p>\n' % (self.link_hrefs[item.href], item.title) toc.append('<p><a xlink:href="#%s">%s</a></p>\n' % (self.link_hrefs[item.href], item.title))
else: else:
self.oeb.warn('Ignoring toc item: %s not found in document.' % item) self.oeb.warn('Ignoring toc item: %s not found in document.' % item)
return toc return ''.join(toc)
def get_text(self): def get_text(self):
text = u'' text = [u'']
for item in self.oeb_book.spine: for item in self.oeb_book.spine:
self.log.debug('Converting %s to FictionBook2 XML' % item.href) self.log.debug('Converting %s to FictionBook2 XML' % item.href)
stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts.output_profile) stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts.output_profile)
text += self.add_page_anchor(item) text.append(self.add_page_anchor(item))
text += self.dump_text(item.data.find(XHTML('body')), stylizer, item) text += self.dump_text(item.data.find(XHTML('body')), stylizer, item)
return text return ''.join(text)
def fb2_body_footer(self): def fb2_body_footer(self):
return u'\n</section>\n</body>' return u'\n</section>\n</body>'
@ -155,7 +154,7 @@ class FB2MLizer(object):
return '<v id="%s"></v>' % aid return '<v id="%s"></v>' % aid
def fb2mlize_images(self): def fb2mlize_images(self):
images = u'' images = [u'']
for item in self.oeb_book.manifest: for item in self.oeb_book.manifest:
if item.media_type in OEB_RASTER_IMAGES: if item.media_type in OEB_RASTER_IMAGES:
try: try:
@ -174,23 +173,23 @@ class FB2MLizer(object):
col = 1 col = 1
col += 1 col += 1
data += char data += char
images += '<binary id="%s" content-type="%s">%s\n</binary>' % (self.image_hrefs.get(item.href, '0000.JPEG'), item.media_type, data) images.append('<binary id="%s" content-type="%s">%s\n</binary>' % (self.image_hrefs.get(item.href, '0000.JPEG'), item.media_type, data))
except Exception as e: except Exception as e:
self.log.error('Error: Could not include file %s becuase ' \ self.log.error('Error: Could not include file %s becuase ' \
'%s.' % (item.href, e)) '%s.' % (item.href, e))
return images return ''.join(images)
def dump_text(self, elem, stylizer, page, tag_stack=[]): def dump_text(self, elem, stylizer, page, tag_stack=[]):
if not isinstance(elem.tag, basestring) \ if not isinstance(elem.tag, basestring) \
or namespace(elem.tag) != XHTML_NS: or namespace(elem.tag) != XHTML_NS:
return u'' return [u'']
fb2_text = u'' fb2_text = [u'']
style = stylizer.style(elem) style = stylizer.style(elem)
if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \ if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \
or style['visibility'] == 'hidden': or style['visibility'] == 'hidden':
return u'' return [u'']
tag = barename(elem.tag) tag = barename(elem.tag)
tag_count = 0 tag_count = 0
@ -199,33 +198,33 @@ class FB2MLizer(object):
if elem.attrib.get('src', None): if elem.attrib.get('src', None):
if page.abshref(elem.attrib['src']) not in self.image_hrefs.keys(): if page.abshref(elem.attrib['src']) not in self.image_hrefs.keys():
self.image_hrefs[page.abshref(elem.attrib['src'])] = '%s.jpg' % len(self.image_hrefs.keys()) self.image_hrefs[page.abshref(elem.attrib['src'])] = '%s.jpg' % len(self.image_hrefs.keys())
fb2_text += '<image xlink:href="#%s" />' % self.image_hrefs[page.abshref(elem.attrib['src'])] fb2_text.append('<image xlink:href="#%s" />' % self.image_hrefs[page.abshref(elem.attrib['src'])])
if tag in TAG_LINKS: if tag in TAG_LINKS:
href = elem.get('href') href = elem.get('href')
if href: if href:
href = prepare_string_for_xml(page.abshref(href)) href = prepare_string_for_xml(page.abshref(href))
if '://' in href: if '://' in href:
fb2_text += '<a xlink:href="%s">' % href fb2_text.append('<a xlink:href="%s">' % href)
else: else:
if '#' not in href: if '#' not in href:
href += '#' href += '#'
if href not in self.link_hrefs.keys(): if href not in self.link_hrefs.keys():
self.link_hrefs[href] = 'calibre_link-%s' % len(self.link_hrefs.keys()) self.link_hrefs[href] = 'calibre_link-%s' % len(self.link_hrefs.keys())
href = self.link_hrefs[href] href = self.link_hrefs[href]
fb2_text += '<a xlink:href="#%s">' % href fb2_text.append('<a xlink:href="#%s">' % href)
tag_count += 1 tag_count += 1
tag_stack.append('a') tag_stack.append('a')
# Anchor ids # Anchor ids
id_name = elem.get('id') id_name = elem.get('id')
if id_name: if id_name:
fb2_text += self.get_anchor(page, id_name) fb2_text.append(self.get_anchor(page, id_name))
fb2_tag = TAG_MAP.get(tag, None) fb2_tag = TAG_MAP.get(tag, None)
if fb2_tag and fb2_tag not in tag_stack: if fb2_tag and fb2_tag not in tag_stack:
tag_count += 1 tag_count += 1
fb2_text += '<%s>' % fb2_tag fb2_text.append('<%s>' % fb2_tag)
tag_stack.append(fb2_tag) tag_stack.append(fb2_tag)
# Processes style information # Processes style information
@ -233,15 +232,15 @@ class FB2MLizer(object):
style_tag = s[1].get(style[s[0]], None) style_tag = s[1].get(style[s[0]], None)
if style_tag: if style_tag:
tag_count += 1 tag_count += 1
fb2_text += '<%s>' % style_tag fb2_text.append('<%s>' % style_tag)
tag_stack.append(style_tag) tag_stack.append(style_tag)
if tag in TAG_SPACE: if tag in TAG_SPACE:
if not fb2_text or fb2_text[-1] != ' ': if not fb2_text or fb2_text[-1] != ' ':
fb2_text += ' ' fb2_text.append(' ')
if hasattr(elem, 'text') and elem.text != None: if hasattr(elem, 'text') and elem.text != None:
fb2_text += prepare_string_for_xml(elem.text) fb2_text.append(prepare_string_for_xml(elem.text))
for item in elem: for item in elem:
fb2_text += self.dump_text(item, stylizer, page, tag_stack) fb2_text += self.dump_text(item, stylizer, page, tag_stack)
@ -253,16 +252,16 @@ class FB2MLizer(object):
if hasattr(elem, 'tail') and elem.tail != None: if hasattr(elem, 'tail') and elem.tail != None:
if 'p' not in tag_stack: if 'p' not in tag_stack:
fb2_text += '<p>%s</p>' % prepare_string_for_xml(elem.tail) fb2_text.append('<p>%s</p>' % prepare_string_for_xml(elem.tail))
else: else:
fb2_text += prepare_string_for_xml(elem.tail) fb2_text.append(prepare_string_for_xml(elem.tail))
return fb2_text return fb2_text
def close_tags(self, tags): def close_tags(self, tags):
fb2_text = u'' fb2_text = [u'']
for i in range(0, len(tags)): for i in range(0, len(tags)):
fb2_tag = tags.pop() fb2_tag = tags.pop()
fb2_text += '</%s>' % fb2_tag fb2_text.append('</%s>' % fb2_tag)
return fb2_text return fb2_text

54
src/calibre/ebooks/rb/rbml.py
View File

@ -8,12 +8,12 @@ __docformat__ = 'restructuredtext en'
Transform OEB content into RB compatible markup. Transform OEB content into RB compatible markup.
''' '''
import os
import re import re
from calibre import prepare_string_for_xml from calibre import prepare_string_for_xml
from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace
from calibre.ebooks.oeb.stylizer import Stylizer from calibre.ebooks.oeb.stylizer import Stylizer
from calibre.ebooks.rb import unique_name
TAGS = [ TAGS = [
'b', 'b',
@ -71,12 +71,12 @@ class RBMLizer(object):
def mlize_spine(self): def mlize_spine(self):
self.link_hrefs = {} self.link_hrefs = {}
output = u'<HTML><HEAD><TITLE></TITLE></HEAD><BODY>' output = [u'<HTML><HEAD><TITLE></TITLE></HEAD><BODY>']
output += self.get_cover_page() output.append(self.get_cover_page())
output += u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk' output.append(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk')
output += self.get_text() output.append(self.get_text())
output += u'</BODY></HTML>' output.append(u'</BODY></HTML>')
output = output.replace(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk', self.get_toc()) output = ''.join(output).replace(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk', self.get_toc())
output = self.clean_text(output) output = self.clean_text(output)
return output return output
@ -92,26 +92,26 @@ class RBMLizer(object):
return output return output
def get_toc(self): def get_toc(self):
toc = u'' toc = [u'']
if self.opts.inline_toc: if self.opts.inline_toc:
self.log.debug('Generating table of contents...') self.log.debug('Generating table of contents...')
toc += u'<H1>%s</H1><UL>\n' % _('Table of Contents:') toc.append(u'<H1>%s</H1><UL>\n' % _('Table of Contents:'))
for item in self.oeb_book.toc: for item in self.oeb_book.toc:
if item.href in self.link_hrefs.keys(): if item.href in self.link_hrefs.keys():
toc += '<LI><A HREF="#%s">%s</A></LI>\n' % (self.link_hrefs[item.href], item.title) toc.append('<LI><A HREF="#%s">%s</A></LI>\n' % (self.link_hrefs[item.href], item.title))
else: else:
self.oeb.warn('Ignoring toc item: %s not found in document.' % item) self.oeb.warn('Ignoring toc item: %s not found in document.' % item)
toc += '</UL>' toc.append('</UL>')
return toc return ''.join(toc)
def get_text(self): def get_text(self):
output = u'' output = [u'']
for item in self.oeb_book.spine: for item in self.oeb_book.spine:
self.log.debug('Converting %s to RocketBook HTML...' % item.href) self.log.debug('Converting %s to RocketBook HTML...' % item.href)
stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts.output_profile) stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts.output_profile)
output += self.add_page_anchor(item) output.append(self.add_page_anchor(item))
output += self.dump_text(item.data.find(XHTML('body')), stylizer, item) output += self.dump_text(item.data.find(XHTML('body')), stylizer, item)
return output return ''.join(output)
def add_page_anchor(self, page): def add_page_anchor(self, page):
return self.get_anchor(page, '') return self.get_anchor(page, '')
@ -135,14 +135,14 @@ class RBMLizer(object):
def dump_text(self, elem, stylizer, page, tag_stack=[]): def dump_text(self, elem, stylizer, page, tag_stack=[]):
if not isinstance(elem.tag, basestring) \ if not isinstance(elem.tag, basestring) \
or namespace(elem.tag) != XHTML_NS: or namespace(elem.tag) != XHTML_NS:
return u'' return [u'']
text = u'' text = [u'']
style = stylizer.style(elem) style = stylizer.style(elem)
if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \ if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \
or style['visibility'] == 'hidden': or style['visibility'] == 'hidden':
return u'' return [u'']
tag = barename(elem.tag) tag = barename(elem.tag)
tag_count = 0 tag_count = 0
@ -153,12 +153,12 @@ class RBMLizer(object):
if elem.attrib.get('src', None): if elem.attrib.get('src', None):
if page.abshref(elem.attrib['src']) not in self.name_map.keys(): if page.abshref(elem.attrib['src']) not in self.name_map.keys():
self.name_map[page.abshref(elem.attrib['src'])] = unique_name('%s' % len(self.image_hrefs.keys()), self.image_hrefs.keys(), self.name_map.keys()) self.name_map[page.abshref(elem.attrib['src'])] = unique_name('%s' % len(self.image_hrefs.keys()), self.image_hrefs.keys(), self.name_map.keys())
text += '<IMG SRC="%s">' % self.name_map[page.abshref(elem.attrib['src'])] text.append('<IMG SRC="%s">' % self.name_map[page.abshref(elem.attrib['src'])])
rb_tag = tag.upper() if tag in TAGS else None rb_tag = tag.upper() if tag in TAGS else None
if rb_tag: if rb_tag:
tag_count += 1 tag_count += 1
text += '<%s>' % rb_tag text.append('<%s>' % rb_tag)
tag_stack.append(rb_tag) tag_stack.append(rb_tag)
# Anchors links # Anchors links
@ -172,14 +172,14 @@ class RBMLizer(object):
if href not in self.link_hrefs.keys(): if href not in self.link_hrefs.keys():
self.link_hrefs[href] = 'calibre_link-%s' % len(self.link_hrefs.keys()) self.link_hrefs[href] = 'calibre_link-%s' % len(self.link_hrefs.keys())
href = self.link_hrefs[href] href = self.link_hrefs[href]
text += '<A HREF="#%s">' % href text.append('<A HREF="#%s">' % href)
tag_count += 1 tag_count += 1
tag_stack.append('A') tag_stack.append('A')
# Anchor ids # Anchor ids
id_name = elem.get('id') id_name = elem.get('id')
if id_name: if id_name:
text += self.get_anchor(page, id_name) text.append(self.get_anchor(page, id_name))
# Processes style information # Processes style information
for s in STYLES: for s in STYLES:
@ -187,12 +187,12 @@ class RBMLizer(object):
if style_tag: if style_tag:
style_tag = style_tag.upper() style_tag = style_tag.upper()
tag_count += 1 tag_count += 1
text += '<%s>' % style_tag text.append('<%s>' % style_tag)
tag_stack.append(style_tag) tag_stack.append(style_tag)
# Proccess tags that contain text. # Proccess tags that contain text.
if hasattr(elem, 'text') and elem.text != None and elem.text.strip() != '': if hasattr(elem, 'text') and elem.text != None and elem.text.strip() != '':
text += prepare_string_for_xml(elem.text) text.append(prepare_string_for_xml(elem.text))
for item in elem: for item in elem:
text += self.dump_text(item, stylizer, page, tag_stack) text += self.dump_text(item, stylizer, page, tag_stack)
@ -204,14 +204,14 @@ class RBMLizer(object):
text += self.close_tags(close_tag_list) text += self.close_tags(close_tag_list)
if hasattr(elem, 'tail') and elem.tail != None and elem.tail.strip() != '': if hasattr(elem, 'tail') and elem.tail != None and elem.tail.strip() != '':
text += prepare_string_for_xml(elem.tail) text.append(prepare_string_for_xml(elem.tail))
return text return text
def close_tags(self, tags): def close_tags(self, tags):
text = u'' text = [u'']
for i in range(0, len(tags)): for i in range(0, len(tags)):
tag = tags.pop() tag = tags.pop()
text += '</%s>' % tag text.append('</%s>' % tag)
return text return text

13
src/calibre/ebooks/txt/txtml.py
View File

@ -52,7 +52,7 @@ class TXTMLizer(object):
stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts.output_profile) stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts.output_profile)
content = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode)) content = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
content = self.remove_newlines(content) content = self.remove_newlines(content)
output.append(self.dump_text(etree.fromstring(content), stylizer)) output += self.dump_text(etree.fromstring(content), stylizer)
output = self.cleanup_text(u''.join(output)) output = self.cleanup_text(u''.join(output))
return output return output
@ -107,17 +107,6 @@ class TXTMLizer(object):
return text return text
def get_text(self, elem, stylizer):
'''
@elem: The element in the etree that we are working on.
@stylizer: The style information attached to the element.
@end: The last two characters of the text from the previous element.
This is used to determine if a blank line is needed when starting
a new block element.
'''
return u''.join(self.dump_text(elem, stylizer))
def dump_text(self, elem, stylizer, end=''): def dump_text(self, elem, stylizer, end=''):
''' '''
@elem: The element in the etree that we are working on. @elem: The element in the etree that we are working on.