From d4f76eba5b58692fd4be1285cbc5e032f649484d Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 3 Dec 2010 19:08:19 -0500 Subject: [PATCH 01/11] FB2 Output: Make FB2 output more compliant. This removes some features such as inline TOC and links. --- src/calibre/ebooks/fb2/fb2ml.py | 218 +++++++++++++------------------- 1 file changed, 87 insertions(+), 131 deletions(-) diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 2a9a92612e..1c24213b46 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -32,7 +32,7 @@ TAG_MAP = { 'p' : 'p', 'li' : 'p', 'div': 'p', - 'br' : 'p', + 'br' : 'empty-line', } TAG_SPACE = [] @@ -42,7 +42,6 @@ TAG_IMAGES = [ ] TAG_LINKS = [ - 'a', ] BLOCK = [ @@ -56,9 +55,8 @@ STYLES = [ class FB2MLizer(object): ''' - Todo: * Ensure all style tags are inside of the p tags. - * Include more FB2 specific tags in the conversion. - * Handle reopening of a tag properly. + Todo: * Include more FB2 specific tags in the conversion. + * Handle a tags. * Figure out some way to turn oeb_book.toc items into
<p> to allow for readers to generate toc from the document. ''' @@ -66,7 +64,8 @@ class FB2MLizer(object): def __init__(self, log): self.log = log self.image_hrefs = {} - self.link_hrefs = {} + # Used to ensure text and tags are always within <p> and </p> + self.in_p = False def extract_content(self, oeb_book, opts): self.log.info('Converting XHTML to FB2 markup...') @@ -78,17 +77,13 @@ class FB2MLizer(object): self.image_hrefs = {} self.link_hrefs = {} output = [self.fb2_header()] - output.append(self.get_cover_page()) - output.append(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk') output.append(self.get_text()) output.append(self.fb2_body_footer()) output.append(self.fb2mlize_images()) output.append(self.fb2_footer()) - output = ''.join(output).replace(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk', self.get_toc()) - output = self.clean_text(output) - if self.opts.sectionize_chapters: - output = self.sectionize_chapters(output) - return u'<?xml version="1.0" encoding="UTF-8"?>\n%s' % etree.tostring(etree.fromstring(output), encoding=unicode, pretty_print=True) + output = self.clean_text(u''.join(output)) + #return u'<?xml version="1.0" encoding="UTF-8"?>\n%s' % etree.tostring(etree.fromstring(output), encoding=unicode, pretty_print=True) + return u'<?xml version="1.0" encoding="UTF-8"?>' + output def clean_text(self, text): text = re.sub(r'(?miu)<section>\s*</section>', '', text) @@ -116,88 +111,40 @@ class FB2MLizer(object): author_middle = ' '.join(author_parts[1:-2]) author_last = author_parts[-1] - return u'<FictionBook xmlns:xlink="http://www.w3.org/1999/xlink" ' \ - 'xmlns="http://www.gribuser.ru/xml/fictionbook/2.0">\n' \ - '<description>\n<title-info>\n ' \ - '<author>\n<first-name>%s</first-name>\n<middle-name>%s' \ - '</middle-name>\n<last-name>%s</last-name>\n</author>\n' \ - '<book-title>%s</book-title> ' \ - '</title-info><document-info> ' \ - '<program-used>%s - %s</program-used></document-info>\n' \ - '</description>\n<body>\n<section>' % tuple(map(prepare_string_for_xml, - (author_first, author_middle, - author_last, self.oeb_book.metadata.title[0].value, - __appname__, __version__))) - - def get_cover_page(self): - output = u'' - if 'cover' in self.oeb_book.guide: - output += '<image xlink:href="#cover.jpg" />' - self.image_hrefs[self.oeb_book.guide['cover'].href] = 'cover.jpg' - if 'titlepage' in self.oeb_book.guide: - self.log.debug('Generating cover page...') - href = self.oeb_book.guide['titlepage'].href - item = self.oeb_book.manifest.hrefs[href] - if item.spine_position is None: - stylizer = Stylizer(item.data, item.href, self.oeb_book, - self.opts, self.opts.output_profile) - output += ''.join(self.dump_text(item.data.find(XHTML('body')), stylizer, item)) - return output - - def get_toc(self): - toc = [] - if self.opts.inline_toc: - self.log.debug('Generating table of contents...') - toc.append(u'<p>%s</p>' % _('Table of Contents:')) - for item in self.oeb_book.toc: - if item.href in self.link_hrefs.keys(): - toc.append('<p><a xlink:href="#%s">%s</a></p>\n' % (self.link_hrefs[item.href], item.title)) - else: - self.oeb.warn('Ignoring toc item: %s not found in document.' % item) - return ''.join(toc) - - def sectionize_chapters(self, text): - def remove_p(t): - t = t.replace('<p>', '') - t = t.replace('</p>', '') - return t - text = re.sub(r'(?imsu)(<p>)\s*(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(</p>)\s*(<p>)\s*(?P<strong><strong>.+?</strong>)\s*(</p>)', lambda mo: '</section><section>%s<title><p>%s</p>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text) - text = re.sub(r'(?imsu)(

)\s*(?P)\s*(

)\s*(?P.+?)', lambda mo: '
%s<p>%s</p>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text) - text = re.sub(r'(?imsu)(?P)\s*(

)\s*(?P.+?)\s*(

)', lambda mo: '
%s<p>%s</p>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text) - text = re.sub(r'(?imsu)(

)\s*(?P)\s*(?P.+?)\s*(

)', lambda mo: '
%s<p>%s</p>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text) - text = re.sub(r'(?imsu)(?P)\s*(?P.+?)', lambda mo: '
%s<p>%s</p>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text) - return text + return u'' \ + '' \ + '' \ + '' \ + '' \ + '%s' \ + '%s' \ + '%s' \ + '' \ + '%s' \ + '

' \ + '' \ + '' \ + '%s - %s' \ + '' \ + '' % tuple(map(prepare_string_for_xml, (author_first, author_middle, author_last, + self.oeb_book.metadata.title[0].value, __appname__, __version__))) def get_text(self): text = [] - for i, item in enumerate(self.oeb_book.spine): - if self.opts.sectionize_chapters_using_file_structure and i is not 0: - text.append('

') + for item in self.oeb_book.spine: self.log.debug('Converting %s to FictionBook2 XML' % item.href) stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts, self.opts.output_profile) - text.append(self.add_page_anchor(item)) + text.append('
') text += self.dump_text(item.data.find(XHTML('body')), stylizer, item) - if self.opts.sectionize_chapters_using_file_structure and i is not len(self.oeb_book.spine) - 1: - text.append('
') + text.append('
') return ''.join(text) def fb2_body_footer(self): - return u'\n
\n' + return u'' def fb2_footer(self): return u'' - def add_page_anchor(self, page): - return self.get_anchor(page, '') - - def get_anchor(self, page, aid): - aid = prepare_string_for_xml(aid) - aid = '%s#%s' % (page.href, aid) - if aid not in self.link_hrefs.keys(): - self.link_hrefs[aid] = 'calibre_link-%s' % len(self.link_hrefs.keys()) - aid = self.link_hrefs[aid] - return '' % aid - def fb2mlize_images(self): images = [] for item in self.oeb_book.manifest: @@ -218,12 +165,42 @@ class FB2MLizer(object): col = 1 col += 1 data += char - images.append('%s\n' % (self.image_hrefs.get(item.href, '0000.JPEG'), item.media_type, data)) + images.append('%s\n' % (self.image_hrefs.get(item.href, '_0000.JPEG'), item.media_type, data)) except Exception as e: - self.log.error('Error: Could not include file %s becuase ' \ + self.log.error('Error: Could not include file %s because ' \ '%s.' % (item.href, e)) return ''.join(images) + def ensure_p(self): + if self.in_p: + return [], [] + else: + self.in_p = True + return ['

'], ['p'] + + def close_open_p(self, tags): + text = [''] + added_p = False + + if self.in_p: + # Close all up to p. Close p. Reopen all closed tags including p. + closed_tags = [] + tags.reverse() + for t in tags: + text.append('' % t) + closed_tags.append(t) + if t == 'p': + break + closed_tags.reverse() + for t in closed_tags: + text.append('<%s>' % t) + else: + text.append('

') + added_p = True + self.in_p = True + + return text, added_p + def dump_text(self, elem, stylizer, page, tag_stack=[]): if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: @@ -242,53 +219,26 @@ class FB2MLizer(object): if tag in TAG_IMAGES: if elem.attrib.get('src', None): if page.abshref(elem.attrib['src']) not in self.image_hrefs.keys(): - self.image_hrefs[page.abshref(elem.attrib['src'])] = '%s.jpg' % len(self.image_hrefs.keys()) + self.image_hrefs[page.abshref(elem.attrib['src'])] = '_%s.jpg' % len(self.image_hrefs.keys()) + p_txt, p_tag = self.ensure_p() + fb2_text += p_txt + tags += p_tag fb2_text.append('' % self.image_hrefs[page.abshref(elem.attrib['src'])]) - if tag in TAG_LINKS: - href = elem.get('href') - if href: - href = prepare_string_for_xml(page.abshref(href)) - href = href.replace('"', '"') - if '://' in href: - fb2_text.append('' % href) - else: - if href.startswith('#'): - href = href[1:] - if href not in self.link_hrefs.keys(): - self.link_hrefs[href] = 'calibre_link-%s' % len(self.link_hrefs.keys()) - href = self.link_hrefs[href] - fb2_text.append('' % href) - tags.append('a') - - # Anchor ids - id_name = elem.get('id') - if id_name: - fb2_text.append(self.get_anchor(page, id_name)) - if tag == 'h1' and self.opts.h1_to_title or tag == 'h2' and self.opts.h2_to_title or tag == 'h3' and self.opts.h3_to_title: fb2_text.append('') tags.append('title') fb2_tag = TAG_MAP.get(tag, None) if fb2_tag == 'p': - if 'p' in tag_stack+tags: - # Close all up to p. Close p. Reopen all closed tags including p. - all_tags = tag_stack+tags - closed_tags = [] - all_tags.reverse() - for t in all_tags: - fb2_text.append('</%s>' % t) - closed_tags.append(t) - if t == 'p': - break - closed_tags.reverse() - for t in closed_tags: - fb2_text.append('<%s>' % t) - else: - fb2_text.append('<p>') + p_text, added_p = self.close_open_p(tag_stack+tags) + fb2_text += p_text + if added_p: tags.append('p') elif fb2_tag and fb2_tag not in tag_stack+tags: + p_text, p_tags = self.ensure_p() + fb2_text += p_text + tags += p_tags fb2_text.append('<%s>' % fb2_tag) tags.append(fb2_tag) @@ -296,18 +246,21 @@ class FB2MLizer(object): for s in STYLES: style_tag = s[1].get(style[s[0]], None) if style_tag and style_tag not in tag_stack+tags: + p_text, p_tags = self.ensure_p() + fb2_text += p_text + tags += p_tags fb2_text.append('<%s>' % style_tag) tags.append(style_tag) if tag in TAG_SPACE: - if not fb2_text or fb2_text[-1] != ' ' or not fb2_text[-1].endswith(' '): - fb2_text.append(' ') + fb2_text.append(' ') if hasattr(elem, 'text') and elem.text: - if 'p' not in tag_stack+tags: - fb2_text.append('<p>%s</p>' % prepare_string_for_xml(elem.text)) - else: - fb2_text.append(prepare_string_for_xml(elem.text)) + if not self.in_p: + fb2_text.append('<p>') + fb2_text.append(prepare_string_for_xml(elem.text)) + if not self.in_p: + fb2_text.append('</p>') for item in elem: fb2_text += self.dump_text(item, stylizer, page, tag_stack+tags) @@ -316,10 +269,11 @@ class FB2MLizer(object): fb2_text += self.close_tags(tags) if hasattr(elem, 'tail') and elem.tail: - if 'p' not in tag_stack: - fb2_text.append('<p>%s</p>' % prepare_string_for_xml(elem.tail)) - else: - fb2_text.append(prepare_string_for_xml(elem.tail)) + if not self.in_p: + fb2_text.append('<p>') + fb2_text.append(prepare_string_for_xml(elem.tail)) + if not self.in_p: + fb2_text.append('</p>') return fb2_text @@ -327,5 +281,7 @@ class FB2MLizer(object): text = [] for tag in tags: text.append('</%s>' % tag) + if tag == 'p': + self.in_p = False return text From 692b6bcf02e6dbc75f5f5088a122626ece3e4d42 Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Fri, 3 Dec 2010 19:11:33 -0500 Subject: [PATCH 02/11] FB2 Output: Remove FB2 options that are no longer implemented. --- src/calibre/ebooks/fb2/output.py | 14 ------------- src/calibre/gui2/convert/fb2_output.py | 4 +--- src/calibre/gui2/convert/fb2_output.ui | 27 +++----------------------- 3 files changed, 4 insertions(+), 41 deletions(-) diff --git a/src/calibre/ebooks/fb2/output.py b/src/calibre/ebooks/fb2/output.py index bacaf0da91..88508b83e0 100644 --- a/src/calibre/ebooks/fb2/output.py +++ b/src/calibre/ebooks/fb2/output.py @@ -16,20 +16,6 @@ class FB2Output(OutputFormatPlugin): file_type = 'fb2' options = set([ - OptionRecommendation(name='inline_toc', - recommended_value=False, level=OptionRecommendation.LOW, - help=_('Add Table of Contents to beginning of the book.')), - OptionRecommendation(name='sectionize_chapters', - recommended_value=False, level=OptionRecommendation.LOW, - help=_('Try to turn chapters into individual sections. ' \ - 'WARNING: ' \ - 'This option is experimental. It can cause conversion ' \ - 'to fail. It can also produce unexpected output.')), - OptionRecommendation(name='sectionize_chapters_using_file_structure', - recommended_value=False, level=OptionRecommendation.LOW, - help=_('Try to turn chapters into individual sections using the ' \ - 'internal structure of the ebook. This works well for EPUB ' \ - 'books that have been internally split by chapter.')), OptionRecommendation(name='h1_to_title', recommended_value=False, level=OptionRecommendation.LOW, help=_('Wrap all h1 tags with fb2 title elements.')), diff --git a/src/calibre/gui2/convert/fb2_output.py b/src/calibre/gui2/convert/fb2_output.py index 5d927146a5..6b1497a9db 100644 --- a/src/calibre/gui2/convert/fb2_output.py +++ b/src/calibre/gui2/convert/fb2_output.py @@ -17,8 +17,6 @@ class PluginWidget(Widget, Ui_Form): ICON = I('mimetypes/fb2.png') def __init__(self, parent, get_option, get_help, db=None, book_id=None): - Widget.__init__(self, parent, ['inline_toc', 'sectionize_chapters', - 'sectionize_chapters_using_file_structure', 'h1_to_title', - 'h2_to_title', 'h3_to_title']) + Widget.__init__(self, parent, ['h1_to_title', 'h2_to_title', 'h3_to_title']) self.db, self.book_id = db, book_id self.initialize_options(get_option, get_help, db, book_id) diff --git a/src/calibre/gui2/convert/fb2_output.ui b/src/calibre/gui2/convert/fb2_output.ui index a90ecd615e..436719aed4 100644 --- a/src/calibre/gui2/convert/fb2_output.ui +++ b/src/calibre/gui2/convert/fb2_output.ui @@ -14,7 +14,7 @@ <string>Form</string> </property> <layout class="QGridLayout" name="gridLayout"> - <item row="6" column="0"> + <item row="3" column="0"> <spacer name="verticalSpacer"> <property name="orientation"> <enum>Qt::Vertical</enum> @@ -28,41 +28,20 @@ </spacer> </item> <item row="0" column="0"> - <widget class="QCheckBox" name="opt_inline_toc"> - <property name="text"> - <string>&Inline TOC</string> - </property> - </widget> - </item> - <item row="1" column="0"> - <widget class="QCheckBox" name="opt_sectionize_chapters"> - <property name="text"> - <string>Sectionize Chapters (Use with care!)</string> - </property> - </widget> - </item> - <item row="2" column="0"> - <widget class="QCheckBox" name="opt_sectionize_chapters_using_file_structure"> - <property name="text"> - <string>Sectionize Chapters using file structure</string> - </property> - </widget> - </item> - <item row="3" column="0"> <widget class="QCheckBox" name="opt_h1_to_title"> <property name="text"> <string>Wrap h1 tags with <title> elements</string> </property> </widget> </item> - <item row="4" column="0"> + <item row="1" column="0"> <widget class="QCheckBox" name="opt_h2_to_title"> <property name="text"> <string>Wrap h2 tags with <title> elements</string> </property> </widget> </item> - <item row="5" column="0"> + <item row="2" column="0"> <widget class="QCheckBox" name="opt_h3_to_title"> <property name="text"> <string>Wrap h3 tags with <title> elements</string> From 1fd503a12ea53196d12a1969220e42a23c027a98 Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Fri, 3 Dec 2010 19:14:09 -0500 Subject: [PATCH 03/11] FB2 Output: Use pretty print option. --- src/calibre/ebooks/fb2/fb2ml.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 1c24213b46..e658dce25a 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -82,8 +82,10 @@ class FB2MLizer(object): output.append(self.fb2mlize_images()) output.append(self.fb2_footer()) output = self.clean_text(u''.join(output)) - #return u'<?xml version="1.0" encoding="UTF-8"?>\n%s' % etree.tostring(etree.fromstring(output), encoding=unicode, pretty_print=True) - return u'<?xml version="1.0" encoding="UTF-8"?>' + output + if self.opts.pretty_print: + return u'<?xml version="1.0" encoding="UTF-8"?>\n%s' % etree.tostring(etree.fromstring(output), encoding=unicode, pretty_print=True) + else: + return u'<?xml version="1.0" encoding="UTF-8"?>' + output def clean_text(self, text): text = re.sub(r'(?miu)<section>\s*</section>', '', text) From 9409781f4a4446078e2a1637d7b4303abdc81bff Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Fri, 3 Dec 2010 19:43:50 -0500 Subject: [PATCH 04/11] FB2 Output: Insert empty lines properly. --- src/calibre/ebooks/fb2/fb2ml.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index e658dce25a..252453d25e 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -32,7 +32,6 @@ TAG_MAP = { 'p' : 'p', 'li' : 'p', 'div': 'p', - 'br' : 'empty-line', } TAG_SPACE = [] @@ -126,7 +125,7 @@ class FB2MLizer(object): '<annotation><p/></annotation>' \ '</title-info>' \ '<document-info>' \ - '<program-used>%s - %s</program-used>' \ + '<program-used>%s %s</program-used>' \ '</document-info>' \ '</description><body>' % tuple(map(prepare_string_for_xml, (author_first, author_middle, author_last, self.oeb_book.metadata.title[0].value, __appname__, __version__))) @@ -180,6 +179,24 @@ class FB2MLizer(object): self.in_p = True return ['<p>'], ['p'] + def insert_empty_line(self, tags): + if self.in_p: + text = [''] + closed_tags = [] + tags.reverse() + for t in tags: + text.append('</%s>' % t) + closed_tags.append(t) + if t == 'p': + break + text.append('<empty-line />') + closed_tags.reverse() + for t in closed_tags: + text.append('<%s>' % t) + return text + else: + return ['<empty-line />'] + def close_open_p(self, tags): text = [''] added_p = False @@ -230,6 +247,8 @@ class FB2MLizer(object): if tag == 'h1' and self.opts.h1_to_title or tag == 'h2' and self.opts.h2_to_title or tag == 'h3' and self.opts.h3_to_title: fb2_text.append('<title>') tags.append('title') + if tag == 'br': + fb2_text += self.insert_empty_line(tag_stack+tags) fb2_tag = TAG_MAP.get(tag, None) if fb2_tag == 'p': From 6ec27b1234d40ac866ab36d4f4d941fbecfc01d2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Fri, 3 Dec 2010 19:36:31 -0700 Subject: [PATCH 05/11] ... --- setup/installer/linux/freeze2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index 684b33b80d..693bf28121 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -14,7 +14,7 @@ from setup import Command, modules, basenames, functions, __version__, \ SITE_PACKAGES = ['IPython', 'PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize', 'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml', - 'sipconfig.py', 'xdg'] + 'sipconfig.py', 'xdg', 'dbus'] QTDIR = '/usr/lib/qt4' QTDLLS = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus') From 38be2c7fc60dc9f6285494388df3073e81833fc4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Fri, 3 Dec 2010 20:06:13 -0700 Subject: [PATCH 06/11] ... --- setup/installer/linux/freeze2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index 693bf28121..cefc193f18 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -14,7 +14,8 @@ from setup import Command, modules, basenames, functions, __version__, \ SITE_PACKAGES = ['IPython', 'PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize', 'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml', - 'sipconfig.py', 'xdg', 'dbus'] + 'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so', 'dbus_bindings.py', + '_dbus_glib_bindings.so'] QTDIR = '/usr/lib/qt4' QTDLLS = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus') From 67367f521d7c640ce046ed614b37f382e55653ec Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Fri, 3 Dec 2010 20:18:56 -0700 Subject: [PATCH 07/11] Use ICU for sorting --- setup/build_environment.py | 4 + setup/extensions.py | 14 +- setup/installer/windows/freeze.py | 2 +- setup/installer/windows/notes.rst | 9 ++ src/calibre/constants.py | 3 +- src/calibre/library/caches.py | 9 +- src/calibre/utils/icu.c | 220 ++++++++++++++++++++++++++++++ src/calibre/utils/icu.py | 53 +++++++ 8 files changed, 306 insertions(+), 8 deletions(-) create mode 100644 src/calibre/utils/icu.c create mode 100644 src/calibre/utils/icu.py diff --git a/setup/build_environment.py b/setup/build_environment.py index c021ebc6a6..d6581a907d 100644 --- a/setup/build_environment.py +++ b/setup/build_environment.py @@ -91,11 +91,15 @@ podofo_inc = '/usr/include/podofo' podofo_lib = '/usr/lib' chmlib_inc_dirs = chmlib_lib_dirs = [] sqlite_inc_dirs = [] +icu_inc_dirs = [] +icu_lib_dirs = [] if iswindows: prefix = r'C:\cygwin\home\kovid\sw' sw_inc_dir = os.path.join(prefix, 'include') sw_lib_dir = os.path.join(prefix, 'lib') + icu_inc_dirs = [sw_inc_dir] + icu_lib_dirs = [sw_lib_dir] sqlite_inc_dirs = [sw_inc_dir] fc_inc = os.path.join(sw_inc_dir, 'fontconfig') fc_lib = sw_lib_dir diff --git a/setup/extensions.py b/setup/extensions.py index d4ac8e188c..3862cce62a 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -18,7 +18,8 @@ from setup.build_environment import fc_inc, fc_lib, chmlib_inc_dirs, \ QMAKE, msvc, MT, win_inc, win_lib, png_inc_dirs, win_ddk, \ magick_inc_dirs, magick_lib_dirs, png_lib_dirs, png_libs, \ magick_error, magick_libs, ft_lib_dirs, ft_libs, jpg_libs, \ - jpg_lib_dirs, chmlib_lib_dirs, sqlite_inc_dirs + jpg_lib_dirs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs, \ + icu_lib_dirs MT isunix = islinux or isosx or isfreebsd @@ -56,8 +57,19 @@ pdfreflow_libs = [] if iswindows: pdfreflow_libs = ['advapi32', 'User32', 'Gdi32', 'zlib'] +icu_libs = ['icudata', 'icui18n', 'icuuc', 'icuio'] +if iswindows: + icu_libs = ['icudt', 'icuin', 'icuuc', 'icuio'] + extensions = [ + Extension('icu', + ['calibre/utils/icu.c'], + libraries=icu_libs, + lib_dirs=icu_lib_dirs, + inc_dirs=icu_inc_dirs, + ), + Extension('sqlite_custom', ['calibre/library/sqlite_custom.c'], inc_dirs=sqlite_inc_dirs diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index 30cc2a97af..7d8ea4d80a 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -199,7 +199,7 @@ class Win32Freeze(Command, WixMixIn): for pat in ('*.dll',): for f in glob.glob(os.path.join(bindir, pat)): ok = True - for ex in ('expatw',): + for ex in ('expatw', 'testplug'): if ex in f.lower(): ok = False if not ok: continue diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 9c553c42e8..af4c871dac 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -77,6 +77,15 @@ Test it on the target system with calibre-debug -c "import _imaging, _imagingmath, _imagingft, _imagingcms" +ICU +------- + +Download the win32 msvc9 binary from http://www.icu-project.org/download/4.4.html + +Note that 4.4 is the last version of ICU that can be compiled (is precompiled) with msvc9 + +Put the dlls into sw/bin and the unicode dir into sw/include and the contents of lib int sw/lib + Libunrar ---------- diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 197fe5888a..f9c177e7a8 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -67,7 +67,8 @@ if plugins is None: 'pdfreflow', 'progress_indicator', 'chmlib', - 'chm_extra' + 'chm_extra', + 'icu', ] + \ (['winutil'] if iswindows else []) + \ (['usbobserver'] if isosx else []): diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 7b4c66c8b8..7c1dea792c 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -796,11 +796,13 @@ class SortKey(object): class SortKeyGenerator(object): def __init__(self, fields, field_metadata, data): + from calibre.utils.icu import sort_key self.field_metadata = field_metadata self.orders = [-1 if x[1] else 1 for x in fields] self.entries = [(x[0], field_metadata[x[0]]) for x in fields] self.library_order = tweaks['title_series_sorting'] == 'library_order' self.data = data + self.string_sort_key = sort_key def __call__(self, record): values = tuple(self.itervals(self.data[record])) @@ -821,17 +823,14 @@ class SortKeyGenerator(object): if val is None: val = ('', 1) else: - val = val.lower() if self.library_order: val = title_sort(val) sidx_fm = self.field_metadata[name + '_index'] sidx = record[sidx_fm['rec_index']] - val = (val, sidx) + val = (self.string_sort_key(val), sidx) elif dt in ('text', 'comments', 'composite', 'enumeration'): - if val is None: - val = '' - val = val.lower() + val = self.string_sort_key(val) elif dt == 'bool': val = {True: 1, False: 2, None: 3}.get(val, 3) diff --git a/src/calibre/utils/icu.c b/src/calibre/utils/icu.c new file mode 100644 index 0000000000..6e06d54dff --- /dev/null +++ b/src/calibre/utils/icu.c @@ -0,0 +1,220 @@ +#define UNICODE +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <unicode/utypes.h> +#include <unicode/uclean.h> +#include <unicode/ucol.h> +#include <unicode/ustring.h> + + +// Collator object definition {{{ +typedef struct { + PyObject_HEAD + // Type-specific fields go here. + UCollator *collator; + +} icu_Collator; + +static void +icu_Collator_dealloc(icu_Collator* self) +{ + if (self->collator != NULL) ucol_close(self->collator); + self->collator = NULL; + self->ob_type->tp_free((PyObject*)self); +} + +static PyObject * +icu_Collator_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + icu_Collator *self; + const char *loc; + UErrorCode status = U_ZERO_ERROR; + + if (!PyArg_ParseTuple(args, "s", &loc)) return NULL; + + self = (icu_Collator *)type->tp_alloc(type, 0); + if (self != NULL) { + self->collator = ucol_open(loc, &status); + if (self->collator == NULL || U_FAILURE(status)) { + PyErr_SetString(PyExc_Exception, "Failed to create collator."); + self->collator = NULL; + Py_DECREF(self); + return NULL; + } + } + + return (PyObject *)self; +} + +// Collator.display_name {{{ +static PyObject * +icu_Collator_display_name(icu_Collator *self, void *closure) { + const char *loc = NULL; + UErrorCode status = U_ZERO_ERROR; + UChar dname[400]; + char buf[100]; + + loc = ucol_getLocaleByType(self->collator, ULOC_ACTUAL_LOCALE, &status); + if (loc == NULL || U_FAILURE(status)) { + PyErr_SetString(PyExc_Exception, "Failed to get actual locale"); return NULL; + } + ucol_getDisplayName(loc, "en", dname, 100, &status); + if (U_FAILURE(status)) return PyErr_NoMemory(); + + u_strToUTF8(buf, 100, NULL, dname, -1, &status); + if (U_FAILURE(status)) { + PyErr_SetString(PyExc_Exception, "Failed ot convert dname to UTF-8"); return NULL; + } + return Py_BuildValue("s", buf); +} + +// }}} + +// Collator.actual_locale {{{ +static PyObject * +icu_Collator_actual_locale(icu_Collator *self, void *closure) { + const char *loc = NULL; + UErrorCode status = U_ZERO_ERROR; + + loc = ucol_getLocaleByType(self->collator, ULOC_ACTUAL_LOCALE, &status); + if (loc == NULL || U_FAILURE(status)) { + PyErr_SetString(PyExc_Exception, "Failed to get actual locale"); return NULL; + } + return Py_BuildValue("s", loc); +} + +// }}} + +// Collator.sort_key {{{ +static PyObject * +icu_Collator_sort_key(icu_Collator *self, PyObject *args, PyObject *kwargs) { + PyObject *o; + Py_ssize_t sz; + wchar_t *buf; + UChar *buf2; + uint8_t *buf3; + PyObject *ans; + UErrorCode status = U_ZERO_ERROR; + + if (!PyArg_ParseTuple(args, "U", &o)) return NULL; + + sz = PyUnicode_GetSize(o); + + buf = (wchar_t*)calloc(sz*2 + 1, sizeof(wchar_t)); + buf2 = (UChar*)calloc(sz*2 + 1, sizeof(UChar)); + buf3 = (uint8_t*)calloc(sz*4 + 1, sizeof(uint8_t)); + + if (buf == NULL || buf2 == NULL || buf3 == NULL) return PyErr_NoMemory(); + + PyUnicode_AsWideChar((PyUnicodeObject *)o, buf, sz); + + u_strFromWCS(buf2, 2*sz+1, NULL, buf, -1, &status); + if (U_SUCCESS(status)) + ucol_getSortKey(self->collator, buf2, -1, buf3, sz*4+1); + + ans = PyBytes_FromString((char *)buf3); + free(buf3); free(buf); free(buf2); + if (ans == NULL) return PyErr_NoMemory(); + + return ans; +} + +static PyMethodDef icu_Collator_methods[] = { + {"sort_key", (PyCFunction)icu_Collator_sort_key, METH_VARARGS, + "sort_key(unicode object) -> Return a sort key for the given object as a bytestring. The idea is that these bytestring will sort using the builtin cmp function, just like the original unicode strings would sort in the current locale with ICU." + }, + + {NULL} /* Sentinel */ +}; + +static PyGetSetDef icu_Collator_getsetters[] = { + {(char *)"actual_locale", + (getter)icu_Collator_actual_locale, NULL, + (char *)"Actual locale used by this collator.", + NULL}, + + {(char *)"display_name", + (getter)icu_Collator_display_name, NULL, + (char *)"Display name of this collator in English. The name reflects the actual data source used.", + NULL}, + + {NULL} /* Sentinel */ +}; + +static PyTypeObject icu_CollatorType = { // {{{ + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "icu.Collator", /*tp_name*/ + sizeof(icu_Collator), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)icu_Collator_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "Collator", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + icu_Collator_methods, /* tp_methods */ + 0, /* tp_members */ + icu_Collator_getsetters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + icu_Collator_new, /* tp_new */ +}; // }}} + +// }} + + +// }}} + + +// Module initialization {{{ + +static PyMethodDef icu_methods[] = { + {NULL} /* Sentinel */ +}; + + +PyMODINIT_FUNC +initicu(void) +{ + PyObject* m; + UErrorCode status = U_ZERO_ERROR; + + u_init(&status); + + + if (PyType_Ready(&icu_CollatorType) < 0) + return; + + m = Py_InitModule3("icu", icu_methods, + "Wrapper for the ICU internationalization library"); + + Py_INCREF(&icu_CollatorType); + PyModule_AddObject(m, "Collator", (PyObject *)&icu_CollatorType); + // uint8_t must be the same size as char + PyModule_AddIntConstant(m, "ok", (U_SUCCESS(status) && sizeof(uint8_t) == sizeof(char)) ? 1 : 0); + +} +// }}} diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py new file mode 100644 index 0000000000..5b432747f0 --- /dev/null +++ b/src/calibre/utils/icu.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' +__docformat__ = 'restructuredtext en' + +from functools import partial + +from calibre.constants import plugins + +_icu = _collator = None + +_none = u'' +_none2 = b'' + +def load_icu(): + global _icu + if _icu is None: + _icu = plugins['icu'][0] + if _icu is None: + print plugins['icu'][1] + else: + if not _icu.ok: + print 'icu not ok' + _icu = None + return _icu + +def load_collator(): + global _collator + from calibre.utils.localization import get_lang + if _collator is None: + icu = load_icu() + if icu is not None: + _collator = icu.Collator(get_lang()) + return _collator + + +def py_sort_key(obj): + if not obj: + return _none + return obj.lower() + +def icu_sort_key(collator, obj): + if not obj: + return _none2 + return collator.sort_key(obj.lower()) + +load_icu() +load_collator() +sort_key = py_sort_key if _icu is None or _collator is None else \ + partial(icu_sort_key, _collator) + From 74a801f4cce869b25cd70277881b006fe3c145d8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Fri, 3 Dec 2010 20:36:56 -0700 Subject: [PATCH 08/11] Add ICU to linux binary build --- setup/installer/linux/freeze2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index cefc193f18..df2c1d6480 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -50,6 +50,10 @@ binary_includes = [ '/lib/libreadline.so.6', '/usr/lib/libchm.so.0', '/usr/lib/liblcms2.so.2', + '/usr/lib/libicudata.so.46', + '/usr/lib/libicui18n.so.46', + '/usr/lib/libicuuc.so.46', + '/usr/lib/libicuio.so.46', ] binary_includes += [os.path.join(QTDIR, 'lib%s.so.4'%x) for x in QTDLLS] From 0b70f40709fca1197a8fb634b0dfa4958eedb8be Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 4 Dec 2010 00:06:49 -0700 Subject: [PATCH 09/11] Fix ICU py->UChar string conversion and add support for OS X --- setup/extensions.py | 6 ++++++ src/calibre/utils/icu.c | 37 ++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/setup/extensions.py b/setup/extensions.py index 3862cce62a..6a9cce7625 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -58,8 +58,13 @@ if iswindows: pdfreflow_libs = ['advapi32', 'User32', 'Gdi32', 'zlib'] icu_libs = ['icudata', 'icui18n', 'icuuc', 'icuio'] +icu_cflags = [] if iswindows: icu_libs = ['icudt', 'icuin', 'icuuc', 'icuio'] +if isosx: + icu_libs = ['icucore'] + icu_cflags = ['-DU_DISABLE_RENAMING'] # Needed to use system libicucore.dylib + extensions = [ @@ -68,6 +73,7 @@ extensions = [ libraries=icu_libs, lib_dirs=icu_lib_dirs, inc_dirs=icu_inc_dirs, + cflags=icu_cflags ), Extension('sqlite_custom', diff --git a/src/calibre/utils/icu.c b/src/calibre/utils/icu.c index 6e06d54dff..1da14a0252 100644 --- a/src/calibre/utils/icu.c +++ b/src/calibre/utils/icu.c @@ -88,32 +88,34 @@ icu_Collator_actual_locale(icu_Collator *self, void *closure) { // Collator.sort_key {{{ static PyObject * icu_Collator_sort_key(icu_Collator *self, PyObject *args, PyObject *kwargs) { - PyObject *o; + char *input; Py_ssize_t sz; - wchar_t *buf; - UChar *buf2; - uint8_t *buf3; + UChar *buf; + uint8_t *buf2; PyObject *ans; + int32_t key_size; UErrorCode status = U_ZERO_ERROR; - - if (!PyArg_ParseTuple(args, "U", &o)) return NULL; + + if (!PyArg_ParseTuple(args, "es", "UTF-8", &input)) return NULL; - sz = PyUnicode_GetSize(o); + sz = strlen(input); - buf = (wchar_t*)calloc(sz*2 + 1, sizeof(wchar_t)); - buf2 = (UChar*)calloc(sz*2 + 1, sizeof(UChar)); - buf3 = (uint8_t*)calloc(sz*4 + 1, sizeof(uint8_t)); + buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar)); - if (buf == NULL || buf2 == NULL || buf3 == NULL) return PyErr_NoMemory(); + if (buf == NULL) return PyErr_NoMemory(); - PyUnicode_AsWideChar((PyUnicodeObject *)o, buf, sz); + u_strFromUTF8(buf, sz*4 + 1, &key_size, input, sz, &status); - u_strFromWCS(buf2, 2*sz+1, NULL, buf, -1, &status); - if (U_SUCCESS(status)) - ucol_getSortKey(self->collator, buf2, -1, buf3, sz*4+1); + if (U_SUCCESS(status)) { + key_size = ucol_getSortKey(self->collator, buf, -1, NULL, 0); + buf2 = (uint8_t*)calloc(key_size + 1, sizeof(uint8_t)); + if (buf2 == NULL) return PyErr_NoMemory(); + ucol_getSortKey(self->collator, buf, -1, buf2, key_size+1); + ans = PyBytes_FromString((char *)buf2); + free(buf2); + } else ans = PyBytes_FromString(""); - ans = PyBytes_FromString((char *)buf3); - free(buf3); free(buf); free(buf2); + free(buf); if (ans == NULL) return PyErr_NoMemory(); return ans; @@ -188,6 +190,7 @@ static PyTypeObject icu_CollatorType = { // {{{ // }}} +// }}} // Module initialization {{{ From fc9ac1d715a26d0a567349d976c0ba2105032c5d Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 4 Dec 2010 00:30:42 -0700 Subject: [PATCH 10/11] Add ICU dorting test --- src/calibre/utils/icu.py | 107 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index 5b432747f0..74fbe182f5 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -51,3 +51,110 @@ load_collator() sort_key = py_sort_key if _icu is None or _collator is None else \ partial(icu_sort_key, _collator) + +def test(): # {{{ + # Data {{{ + german = ''' + Sonntag +Montag +Dienstag +Januar +Februar +März +Fuße +Fluße +Flusse +flusse +fluße +flüße +flüsse +''' + german_good = ''' + Dienstag +Februar +flusse +Flusse +fluße +Fluße +flüsse +flüße +Fuße +Januar +März +Montag +Sonntag''' + french = ''' +dimanche +lundi +mardi +janvier +février +mars +déjà +Meme +deja +même +dejà +bpef +bœg +Boef +Mémé +bœf +boef +bnef +pêche +pèché +pêché +pêche +pêché''' + french_good = ''' + bnef + boef + Boef + bœf + bœg + bpef + deja + dejà + déjà + dimanche + février + janvier + lundi + mardi + mars + Meme + Mémé + même + pèché + pêche + pêche + pêché + pêché''' + # }}} + + def create(l): + l = l.decode('utf-8').splitlines() + return [x.strip() for x in l if x.strip()] + + german = create(german) + c = _icu.Collator('de') + print 'Sorted german:: (%s)'%c.actual_locale + gs = list(sorted(german, key=c.sort_key)) + for x in gs: + print '\t', x.encode('utf-8') + if gs != create(german_good): + print 'German failed' + return + print + french = create(french) + c = _icu.Collator('fr') + print 'Sorted french:: (%s)'%c.actual_locale + fs = list(sorted(french, key=c.sort_key)) + for x in fs: + print '\t', x.encode('utf-8') + if fs != create(french_good): + print 'French failed' + return +# }}} + From a9983208d7c992a393ce2c2bc6af965b3d5bc268 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 4 Dec 2010 00:40:38 -0700 Subject: [PATCH 11/11] ... --- src/calibre/utils/icu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index 74fbe182f5..7c2fd31f78 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -154,7 +154,7 @@ pêché''' for x in fs: print '\t', x.encode('utf-8') if fs != create(french_good): - print 'French failed' + print 'French failed (note that French fails with icu < 4.6 i.e. on windows and OS X)' return # }}}