Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge from trunk

Browse Source
This commit is contained in:
Charles Haley 2010-12-04 09:33:48 +00:00
commit 39447744f8
13 changed files with 540 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
setup/build_environment.py
View File

@ -91,11 +91,15 @@ podofo_inc = '/usr/include/podofo'
podofo_lib = '/usr/lib'
chmlib_inc_dirs = chmlib_lib_dirs = []
sqlite_inc_dirs = []
icu_inc_dirs = []
icu_lib_dirs = []
if iswindows:
prefix = r'C:\cygwin\home\kovid\sw'
sw_inc_dir = os.path.join(prefix, 'include')
sw_lib_dir = os.path.join(prefix, 'lib')
icu_inc_dirs = [sw_inc_dir]
icu_lib_dirs = [sw_lib_dir]
sqlite_inc_dirs = [sw_inc_dir]
fc_inc = os.path.join(sw_inc_dir, 'fontconfig')
fc_lib = sw_lib_dir

20
setup/extensions.py
View File

@ -18,7 +18,8 @@ from setup.build_environment import fc_inc, fc_lib, chmlib_inc_dirs, \
QMAKE, msvc, MT, win_inc, win_lib, png_inc_dirs, win_ddk, \
magick_inc_dirs, magick_lib_dirs, png_lib_dirs, png_libs, \
magick_error, magick_libs, ft_lib_dirs, ft_libs, jpg_libs, \
jpg_lib_dirs, chmlib_lib_dirs, sqlite_inc_dirs
jpg_lib_dirs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs, \
icu_lib_dirs
MT
isunix = islinux or isosx or isfreebsd
@ -56,8 +57,25 @@ pdfreflow_libs = []
if iswindows:
pdfreflow_libs = ['advapi32', 'User32', 'Gdi32', 'zlib']
icu_libs = ['icudata', 'icui18n', 'icuuc', 'icuio']
icu_cflags = []
if iswindows:
icu_libs = ['icudt', 'icuin', 'icuuc', 'icuio']
if isosx:
icu_libs = ['icucore']
icu_cflags = ['-DU_DISABLE_RENAMING'] # Needed to use system libicucore.dylib
extensions = [
Extension('icu',
['calibre/utils/icu.c'],
libraries=icu_libs,
lib_dirs=icu_lib_dirs,
inc_dirs=icu_inc_dirs,
cflags=icu_cflags
),
Extension('sqlite_custom',
['calibre/library/sqlite_custom.c'],
inc_dirs=sqlite_inc_dirs

7
setup/installer/linux/freeze2.py
View File

@ -14,7 +14,8 @@ from setup import Command, modules, basenames, functions, __version__, \
SITE_PACKAGES = ['IPython', 'PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize',
'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml',
'sipconfig.py', 'xdg']
'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so', 'dbus_bindings.py',
'_dbus_glib_bindings.so']
QTDIR = '/usr/lib/qt4'
QTDLLS = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus')
@ -49,6 +50,10 @@ binary_includes = [
'/lib/libreadline.so.6',
'/usr/lib/libchm.so.0',
'/usr/lib/liblcms2.so.2',
'/usr/lib/libicudata.so.46',
'/usr/lib/libicui18n.so.46',
'/usr/lib/libicuuc.so.46',
'/usr/lib/libicuio.so.46',
]
binary_includes += [os.path.join(QTDIR, 'lib%s.so.4'%x) for x in QTDLLS]

2
setup/installer/windows/freeze.py
View File

@ -199,7 +199,7 @@ class Win32Freeze(Command, WixMixIn):
for pat in ('*.dll',):
for f in glob.glob(os.path.join(bindir, pat)):
ok = True
for ex in ('expatw',):
for ex in ('expatw', 'testplug'):
if ex in f.lower():
ok = False
if not ok: continue

9
setup/installer/windows/notes.rst
View File

@ -77,6 +77,15 @@ Test it on the target system with
calibre-debug -c "import _imaging, _imagingmath, _imagingft, _imagingcms"
ICU
-------
Download the win32 msvc9 binary from http://www.icu-project.org/download/4.4.html
Note that 4.4 is the last version of ICU that can be compiled (is precompiled) with msvc9
Put the dlls into sw/bin and the unicode dir into sw/include and the contents of lib int sw/lib
Libunrar
----------

3
src/calibre/constants.py
View File

@ -67,7 +67,8 @@ if plugins is None:
'pdfreflow',
'progress_indicator',
'chmlib',
'chm_extra'
'chm_extra',
'icu',
] + \
(['winutil'] if iswindows else []) + \
(['usbobserver'] if isosx else []):

239
src/calibre/ebooks/fb2/fb2ml.py
View File

@ -32,7 +32,6 @@ TAG_MAP = {
'p' : 'p',
'li' : 'p',
'div': 'p',
'br' : 'p',
}
TAG_SPACE = []
@ -42,7 +41,6 @@ TAG_IMAGES = [
]
TAG_LINKS = [
'a',
]
BLOCK = [
@ -56,9 +54,8 @@ STYLES = [
class FB2MLizer(object):
'''
Todo: * Ensure all style tags are inside of the p tags.
* Include more FB2 specific tags in the conversion.
* Handle reopening of a tag properly.
Todo: * Include more FB2 specific tags in the conversion.
* Handle a tags.
* Figure out some way to turn oeb_book.toc items into <section><title>
<p> to allow for readers to generate toc from the document.
'''
@ -66,7 +63,8 @@ class FB2MLizer(object):
def __init__(self, log):
self.log = log
self.image_hrefs = {}
self.link_hrefs = {}
# Used to ensure text and tags are always within <p> and </p>
self.in_p = False
def extract_content(self, oeb_book, opts):
self.log.info('Converting XHTML to FB2 markup...')
@ -78,17 +76,15 @@ class FB2MLizer(object):
self.image_hrefs = {}
self.link_hrefs = {}
output = [self.fb2_header()]
output.append(self.get_cover_page())
output.append(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk')
output.append(self.get_text())
output.append(self.fb2_body_footer())
output.append(self.fb2mlize_images())
output.append(self.fb2_footer())
output = ''.join(output).replace(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk', self.get_toc())
output = self.clean_text(output)
if self.opts.sectionize_chapters:
output = self.sectionize_chapters(output)
return u'<?xml version="1.0" encoding="UTF-8"?>\n%s' % etree.tostring(etree.fromstring(output), encoding=unicode, pretty_print=True)
output = self.clean_text(u''.join(output))
if self.opts.pretty_print:
return u'<?xml version="1.0" encoding="UTF-8"?>\n%s' % etree.tostring(etree.fromstring(output), encoding=unicode, pretty_print=True)
else:
return u'<?xml version="1.0" encoding="UTF-8"?>' + output
def clean_text(self, text):
text = re.sub(r'(?miu)<section>\s*</section>', '', text)
@ -116,88 +112,40 @@ class FB2MLizer(object):
author_middle = ' '.join(author_parts[1:-2])
author_last = author_parts[-1]
return u'<FictionBook xmlns:xlink="http://www.w3.org/1999/xlink" ' \
'xmlns="http://www.gribuser.ru/xml/fictionbook/2.0">\n' \
'<description>\n<title-info>\n ' \
'<author>\n<first-name>%s</first-name>\n<middle-name>%s' \
'</middle-name>\n<last-name>%s</last-name>\n</author>\n' \
'<book-title>%s</book-title> ' \
'</title-info><document-info> ' \
'<program-used>%s - %s</program-used></document-info>\n' \
'</description>\n<body>\n<section>' % tuple(map(prepare_string_for_xml,
(author_first, author_middle,
author_last, self.oeb_book.metadata.title[0].value,
__appname__, __version__)))
def get_cover_page(self):
output = u''
if 'cover' in self.oeb_book.guide:
output += '<image xlink:href="#cover.jpg" />'
self.image_hrefs[self.oeb_book.guide['cover'].href] = 'cover.jpg'
if 'titlepage' in self.oeb_book.guide:
self.log.debug('Generating cover page...')
href = self.oeb_book.guide['titlepage'].href
item = self.oeb_book.manifest.hrefs[href]
if item.spine_position is None:
stylizer = Stylizer(item.data, item.href, self.oeb_book,
self.opts, self.opts.output_profile)
output += ''.join(self.dump_text(item.data.find(XHTML('body')), stylizer, item))
return output
def get_toc(self):
toc = []
if self.opts.inline_toc:
self.log.debug('Generating table of contents...')
toc.append(u'<p>%s</p>' % _('Table of Contents:'))
for item in self.oeb_book.toc:
if item.href in self.link_hrefs.keys():
toc.append('<p><a xlink:href="#%s">%s</a></p>\n' % (self.link_hrefs[item.href], item.title))
else:
self.oeb.warn('Ignoring toc item: %s not found in document.' % item)
return ''.join(toc)
def sectionize_chapters(self, text):
def remove_p(t):
t = t.replace('<p>', '')
t = t.replace('</p>', '')
return t
text = re.sub(r'(?imsu)(<p>)\s*(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(</p>)\s*(<p>)\s*(?P<strong><strong>.+?</strong>)\s*(</p>)', lambda mo: '</section><section>%s<title><p>%s</p></title>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text)
text = re.sub(r'(?imsu)(<p>)\s*(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(</p>)\s*(?P<strong><strong>.+?</strong>)', lambda mo: '</section><section>%s<title><p>%s</p></title>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text)
text = re.sub(r'(?imsu)(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(<p>)\s*(?P<strong><strong>.+?</strong>)\s*(</p>)', lambda mo: '</section><section>%s<title><p>%s</p></title>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text)
text = re.sub(r'(?imsu)(<p>)\s*(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(?P<strong><strong>.+?</strong>)\s*(</p>)', lambda mo: '</section><section>%s<title><p>%s</p></title>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text)
text = re.sub(r'(?imsu)(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(?P<strong><strong>.+?</strong>)', lambda mo: '</section><section>%s<title><p>%s</p></title>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text)
return text
return u'<FictionBook xmlns="http://www.gribuser.ru/xml/fictionbook/2.0" xmlns:xlink="http://www.w3.org/1999/xlink">' \
'<description>' \
'<title-info>' \
'<genre></genre>' \
'<author>' \
'<first-name>%s</first-name>' \
'<middle-name>%s</middle-name>' \
'<last-name>%s</last-name>' \
'</author>' \
'<book-title>%s</book-title>' \
'<annotation><p/></annotation>' \
'</title-info>' \
'<document-info>' \
'<program-used>%s %s</program-used>' \
'</document-info>' \
'</description><body>' % tuple(map(prepare_string_for_xml, (author_first, author_middle, author_last,
self.oeb_book.metadata.title[0].value, __appname__, __version__)))
def get_text(self):
text = []
for i, item in enumerate(self.oeb_book.spine):
if self.opts.sectionize_chapters_using_file_structure and i is not 0:
text.append('<section>')
for item in self.oeb_book.spine:
self.log.debug('Converting %s to FictionBook2 XML' % item.href)
stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts, self.opts.output_profile)
text.append(self.add_page_anchor(item))
text.append('<section>')
text += self.dump_text(item.data.find(XHTML('body')), stylizer, item)
if self.opts.sectionize_chapters_using_file_structure and i is not len(self.oeb_book.spine) - 1:
text.append('</section>')
text.append('</section>')
return ''.join(text)
def fb2_body_footer(self):
return u'\n</section>\n</body>'
return u'</body>'
def fb2_footer(self):
return u'</FictionBook>'
def add_page_anchor(self, page):
return self.get_anchor(page, '')
def get_anchor(self, page, aid):
aid = prepare_string_for_xml(aid)
aid = '%s#%s' % (page.href, aid)
if aid not in self.link_hrefs.keys():
self.link_hrefs[aid] = 'calibre_link-%s' % len(self.link_hrefs.keys())
aid = self.link_hrefs[aid]
return '<a id="%s" />' % aid
def fb2mlize_images(self):
images = []
for item in self.oeb_book.manifest:
@ -218,12 +166,60 @@ class FB2MLizer(object):
col = 1
col += 1
data += char
images.append('<binary id="%s" content-type="%s">%s\n</binary>' % (self.image_hrefs.get(item.href, '0000.JPEG'), item.media_type, data))
images.append('<binary id="%s" content-type="%s">%s\n</binary>' % (self.image_hrefs.get(item.href, '_0000.JPEG'), item.media_type, data))
except Exception as e:
self.log.error('Error: Could not include file %s becuase ' \
self.log.error('Error: Could not include file %s because ' \
'%s.' % (item.href, e))
return ''.join(images)
def ensure_p(self):
if self.in_p:
return [], []
else:
self.in_p = True
return ['<p>'], ['p']
def insert_empty_line(self, tags):
if self.in_p:
text = ['']
closed_tags = []
tags.reverse()
for t in tags:
text.append('</%s>' % t)
closed_tags.append(t)
if t == 'p':
break
text.append('<empty-line />')
closed_tags.reverse()
for t in closed_tags:
text.append('<%s>' % t)
return text
else:
return ['<empty-line />']
def close_open_p(self, tags):
text = ['']
added_p = False
if self.in_p:
# Close all up to p. Close p. Reopen all closed tags including p.
closed_tags = []
tags.reverse()
for t in tags:
text.append('</%s>' % t)
closed_tags.append(t)
if t == 'p':
break
closed_tags.reverse()
for t in closed_tags:
text.append('<%s>' % t)
else:
text.append('<p>')
added_p = True
self.in_p = True
return text, added_p
def dump_text(self, elem, stylizer, page, tag_stack=[]):
if not isinstance(elem.tag, basestring) \
or namespace(elem.tag) != XHTML_NS:
@ -242,53 +238,28 @@ class FB2MLizer(object):
if tag in TAG_IMAGES:
if elem.attrib.get('src', None):
if page.abshref(elem.attrib['src']) not in self.image_hrefs.keys():
self.image_hrefs[page.abshref(elem.attrib['src'])] = '%s.jpg' % len(self.image_hrefs.keys())
self.image_hrefs[page.abshref(elem.attrib['src'])] = '_%s.jpg' % len(self.image_hrefs.keys())
p_txt, p_tag = self.ensure_p()
fb2_text += p_txt
tags += p_tag
fb2_text.append('<image xlink:href="#%s" />' % self.image_hrefs[page.abshref(elem.attrib['src'])])
if tag in TAG_LINKS:
href = elem.get('href')
if href:
href = prepare_string_for_xml(page.abshref(href))
href = href.replace('"', '&quot;')
if '://' in href:
fb2_text.append('<a xlink:href="%s">' % href)
else:
if href.startswith('#'):
href = href[1:]
if href not in self.link_hrefs.keys():
self.link_hrefs[href] = 'calibre_link-%s' % len(self.link_hrefs.keys())
href = self.link_hrefs[href]
fb2_text.append('<a xlink:href="#%s">' % href)
tags.append('a')
# Anchor ids
id_name = elem.get('id')
if id_name:
fb2_text.append(self.get_anchor(page, id_name))
if tag == 'h1' and self.opts.h1_to_title or tag == 'h2' and self.opts.h2_to_title or tag == 'h3' and self.opts.h3_to_title:
fb2_text.append('<title>')
tags.append('title')
if tag == 'br':
fb2_text += self.insert_empty_line(tag_stack+tags)
fb2_tag = TAG_MAP.get(tag, None)
if fb2_tag == 'p':
if 'p' in tag_stack+tags:
# Close all up to p. Close p. Reopen all closed tags including p.
all_tags = tag_stack+tags
closed_tags = []
all_tags.reverse()
for t in all_tags:
fb2_text.append('</%s>' % t)
closed_tags.append(t)
if t == 'p':
break
closed_tags.reverse()
for t in closed_tags:
fb2_text.append('<%s>' % t)
else:
fb2_text.append('<p>')
p_text, added_p = self.close_open_p(tag_stack+tags)
fb2_text += p_text
if added_p:
tags.append('p')
elif fb2_tag and fb2_tag not in tag_stack+tags:
p_text, p_tags = self.ensure_p()
fb2_text += p_text
tags += p_tags
fb2_text.append('<%s>' % fb2_tag)
tags.append(fb2_tag)
@ -296,18 +267,21 @@ class FB2MLizer(object):
for s in STYLES:
style_tag = s[1].get(style[s[0]], None)
if style_tag and style_tag not in tag_stack+tags:
p_text, p_tags = self.ensure_p()
fb2_text += p_text
tags += p_tags
fb2_text.append('<%s>' % style_tag)
tags.append(style_tag)
if tag in TAG_SPACE:
if not fb2_text or fb2_text[-1] != ' ' or not fb2_text[-1].endswith(' '):
fb2_text.append(' ')
fb2_text.append(' ')
if hasattr(elem, 'text') and elem.text:
if 'p' not in tag_stack+tags:
fb2_text.append('<p>%s</p>' % prepare_string_for_xml(elem.text))
else:
fb2_text.append(prepare_string_for_xml(elem.text))
if not self.in_p:
fb2_text.append('<p>')
fb2_text.append(prepare_string_for_xml(elem.text))
if not self.in_p:
fb2_text.append('</p>')
for item in elem:
fb2_text += self.dump_text(item, stylizer, page, tag_stack+tags)
@ -316,10 +290,11 @@ class FB2MLizer(object):
fb2_text += self.close_tags(tags)
if hasattr(elem, 'tail') and elem.tail:
if 'p' not in tag_stack:
fb2_text.append('<p>%s</p>' % prepare_string_for_xml(elem.tail))
else:
fb2_text.append(prepare_string_for_xml(elem.tail))
if not self.in_p:
fb2_text.append('<p>')
fb2_text.append(prepare_string_for_xml(elem.tail))
if not self.in_p:
fb2_text.append('</p>')
return fb2_text
@ -327,5 +302,7 @@ class FB2MLizer(object):
text = []
for tag in tags:
text.append('</%s>' % tag)
if tag == 'p':
self.in_p = False
return text

14
src/calibre/ebooks/fb2/output.py
View File

@ -16,20 +16,6 @@ class FB2Output(OutputFormatPlugin):
file_type = 'fb2'
options = set([
OptionRecommendation(name='inline_toc',
recommended_value=False, level=OptionRecommendation.LOW,
help=_('Add Table of Contents to beginning of the book.')),
OptionRecommendation(name='sectionize_chapters',
recommended_value=False, level=OptionRecommendation.LOW,
help=_('Try to turn chapters into individual sections. ' \
'WARNING: ' \
'This option is experimental. It can cause conversion ' \
'to fail. It can also produce unexpected output.')),
OptionRecommendation(name='sectionize_chapters_using_file_structure',
recommended_value=False, level=OptionRecommendation.LOW,
help=_('Try to turn chapters into individual sections using the ' \
'internal structure of the ebook. This works well for EPUB ' \
'books that have been internally split by chapter.')),
OptionRecommendation(name='h1_to_title',
recommended_value=False, level=OptionRecommendation.LOW,
help=_('Wrap all h1 tags with fb2 title elements.')),

4
src/calibre/gui2/convert/fb2_output.py
View File

@ -17,8 +17,6 @@ class PluginWidget(Widget, Ui_Form):
ICON = I('mimetypes/fb2.png')
def __init__(self, parent, get_option, get_help, db=None, book_id=None):
Widget.__init__(self, parent, ['inline_toc', 'sectionize_chapters',
'sectionize_chapters_using_file_structure', 'h1_to_title',
'h2_to_title', 'h3_to_title'])
Widget.__init__(self, parent, ['h1_to_title', 'h2_to_title', 'h3_to_title'])
self.db, self.book_id = db, book_id
self.initialize_options(get_option, get_help, db, book_id)

27
src/calibre/gui2/convert/fb2_output.ui
View File

@ -14,7 +14,7 @@
<string>Form</string>
</property>
<layout class="QGridLayout" name="gridLayout">
<item row="6" column="0">
<item row="3" column="0">
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
@ -28,41 +28,20 @@
</spacer>
</item>
<item row="0" column="0">
<widget class="QCheckBox" name="opt_inline_toc">
<property name="text">
<string>&amp;Inline TOC</string>
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QCheckBox" name="opt_sectionize_chapters">
<property name="text">
<string>Sectionize Chapters (Use with care!)</string>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QCheckBox" name="opt_sectionize_chapters_using_file_structure">
<property name="text">
<string>Sectionize Chapters using file structure</string>
</property>
</widget>
</item>
<item row="3" column="0">
<widget class="QCheckBox" name="opt_h1_to_title">
<property name="text">
<string>Wrap h1 tags with &lt;title&gt; elements</string>
</property>
</widget>
</item>
<item row="4" column="0">
<item row="1" column="0">
<widget class="QCheckBox" name="opt_h2_to_title">
<property name="text">
<string>Wrap h2 tags with &lt;title&gt; elements</string>
</property>
</widget>
</item>
<item row="5" column="0">
<item row="2" column="0">
<widget class="QCheckBox" name="opt_h3_to_title">
<property name="text">
<string>Wrap h3 tags with &lt;title&gt; elements</string>

9
src/calibre/library/caches.py
View File

@ -796,11 +796,13 @@ class SortKey(object):
class SortKeyGenerator(object):
def __init__(self, fields, field_metadata, data):
from calibre.utils.icu import sort_key
self.field_metadata = field_metadata
self.orders = [-1 if x[1] else 1 for x in fields]
self.entries = [(x[0], field_metadata[x[0]]) for x in fields]
self.library_order = tweaks['title_series_sorting'] == 'library_order'
self.data = data
self.string_sort_key = sort_key
def __call__(self, record):
values = tuple(self.itervals(self.data[record]))
@ -821,17 +823,14 @@ class SortKeyGenerator(object):
if val is None:
val = ('', 1)
else:
val = val.lower()
if self.library_order:
val = title_sort(val)
sidx_fm = self.field_metadata[name + '_index']
sidx = record[sidx_fm['rec_index']]
val = (val, sidx)
val = (self.string_sort_key(val), sidx)
elif dt in ('text', 'comments', 'composite', 'enumeration'):
if val is None:
val = ''
val = val.lower()
val = self.string_sort_key(val)
elif dt == 'bool':
val = {True: 1, False: 2, None: 3}.get(val, 3)

223
src/calibre/utils/icu.c Normal file
View File

@ -0,0 +1,223 @@
#define UNICODE
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <unicode/utypes.h>
#include <unicode/uclean.h>
#include <unicode/ucol.h>
#include <unicode/ustring.h>
// Collator object definition {{{
typedef struct {
PyObject_HEAD
// Type-specific fields go here.
UCollator *collator;
} icu_Collator;
static void
icu_Collator_dealloc(icu_Collator* self)
{
if (self->collator != NULL) ucol_close(self->collator);
self->collator = NULL;
self->ob_type->tp_free((PyObject*)self);
}
static PyObject *
icu_Collator_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
icu_Collator *self;
const char *loc;
UErrorCode status = U_ZERO_ERROR;
if (!PyArg_ParseTuple(args, "s", &loc)) return NULL;
self = (icu_Collator *)type->tp_alloc(type, 0);
if (self != NULL) {
self->collator = ucol_open(loc, &status);
if (self->collator == NULL || U_FAILURE(status)) {
PyErr_SetString(PyExc_Exception, "Failed to create collator.");
self->collator = NULL;
Py_DECREF(self);
return NULL;
}
}
return (PyObject *)self;
}
// Collator.display_name {{{
static PyObject *
icu_Collator_display_name(icu_Collator *self, void *closure) {
const char *loc = NULL;
UErrorCode status = U_ZERO_ERROR;
UChar dname[400];
char buf[100];
loc = ucol_getLocaleByType(self->collator, ULOC_ACTUAL_LOCALE, &status);
if (loc == NULL || U_FAILURE(status)) {
PyErr_SetString(PyExc_Exception, "Failed to get actual locale"); return NULL;
}
ucol_getDisplayName(loc, "en", dname, 100, &status);
if (U_FAILURE(status)) return PyErr_NoMemory();
u_strToUTF8(buf, 100, NULL, dname, -1, &status);
if (U_FAILURE(status)) {
PyErr_SetString(PyExc_Exception, "Failed ot convert dname to UTF-8"); return NULL;
}
return Py_BuildValue("s", buf);
}
// }}}
// Collator.actual_locale {{{
static PyObject *
icu_Collator_actual_locale(icu_Collator *self, void *closure) {
const char *loc = NULL;
UErrorCode status = U_ZERO_ERROR;
loc = ucol_getLocaleByType(self->collator, ULOC_ACTUAL_LOCALE, &status);
if (loc == NULL || U_FAILURE(status)) {
PyErr_SetString(PyExc_Exception, "Failed to get actual locale"); return NULL;
}
return Py_BuildValue("s", loc);
}
// }}}
// Collator.sort_key {{{
static PyObject *
icu_Collator_sort_key(icu_Collator *self, PyObject *args, PyObject *kwargs) {
char *input;
Py_ssize_t sz;
UChar *buf;
uint8_t *buf2;
PyObject *ans;
int32_t key_size;
UErrorCode status = U_ZERO_ERROR;
if (!PyArg_ParseTuple(args, "es", "UTF-8", &input)) return NULL;
sz = strlen(input);
buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar));
if (buf == NULL) return PyErr_NoMemory();
u_strFromUTF8(buf, sz*4 + 1, &key_size, input, sz, &status);
if (U_SUCCESS(status)) {
key_size = ucol_getSortKey(self->collator, buf, -1, NULL, 0);
buf2 = (uint8_t*)calloc(key_size + 1, sizeof(uint8_t));
if (buf2 == NULL) return PyErr_NoMemory();
ucol_getSortKey(self->collator, buf, -1, buf2, key_size+1);
ans = PyBytes_FromString((char *)buf2);
free(buf2);
} else ans = PyBytes_FromString("");
free(buf);
if (ans == NULL) return PyErr_NoMemory();
return ans;
}
static PyMethodDef icu_Collator_methods[] = {
{"sort_key", (PyCFunction)icu_Collator_sort_key, METH_VARARGS,
"sort_key(unicode object) -> Return a sort key for the given object as a bytestring. The idea is that these bytestring will sort using the builtin cmp function, just like the original unicode strings would sort in the current locale with ICU."
},
{NULL} /* Sentinel */
};
static PyGetSetDef icu_Collator_getsetters[] = {
{(char *)"actual_locale",
(getter)icu_Collator_actual_locale, NULL,
(char *)"Actual locale used by this collator.",
NULL},
{(char *)"display_name",
(getter)icu_Collator_display_name, NULL,
(char *)"Display name of this collator in English. The name reflects the actual data source used.",
NULL},
{NULL} /* Sentinel */
};
static PyTypeObject icu_CollatorType = { // {{{
PyObject_HEAD_INIT(NULL)
0, /*ob_size*/
"icu.Collator", /*tp_name*/
sizeof(icu_Collator), /*tp_basicsize*/
0, /*tp_itemsize*/
(destructor)icu_Collator_dealloc, /*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_compare*/
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash */
0, /*tp_call*/
0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/
"Collator", /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
icu_Collator_methods, /* tp_methods */
0, /* tp_members */
icu_Collator_getsetters, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
icu_Collator_new, /* tp_new */
}; // }}}
// }}
// }}}
// }}}
// Module initialization {{{
static PyMethodDef icu_methods[] = {
{NULL} /* Sentinel */
};
PyMODINIT_FUNC
initicu(void)
{
PyObject* m;
UErrorCode status = U_ZERO_ERROR;
u_init(&status);
if (PyType_Ready(&icu_CollatorType) < 0)
return;
m = Py_InitModule3("icu", icu_methods,
"Wrapper for the ICU internationalization library");
Py_INCREF(&icu_CollatorType);
PyModule_AddObject(m, "Collator", (PyObject *)&icu_CollatorType);
// uint8_t must be the same size as char
PyModule_AddIntConstant(m, "ok", (U_SUCCESS(status) && sizeof(uint8_t) == sizeof(char)) ? 1 : 0);
}
// }}}

160
src/calibre/utils/icu.py Normal file
View File

@ -0,0 +1,160 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
__license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
from functools import partial
from calibre.constants import plugins
_icu = _collator = None
_none = u''
_none2 = b''
def load_icu():
global _icu
if _icu is None:
_icu = plugins['icu'][0]
if _icu is None:
print plugins['icu'][1]
else:
if not _icu.ok:
print 'icu not ok'
_icu = None
return _icu
def load_collator():
global _collator
from calibre.utils.localization import get_lang
if _collator is None:
icu = load_icu()
if icu is not None:
_collator = icu.Collator(get_lang())
return _collator
def py_sort_key(obj):
if not obj:
return _none
return obj.lower()
def icu_sort_key(collator, obj):
if not obj:
return _none2
return collator.sort_key(obj.lower())
load_icu()
load_collator()
sort_key = py_sort_key if _icu is None or _collator is None else \
partial(icu_sort_key, _collator)
def test(): # {{{
# Data {{{
german = '''
Sonntag
Montag
Dienstag
Januar
Februar
März
Fuße
Fluße
Flusse
flusse
fluße
flüße
flüsse
'''
german_good = '''
Dienstag
Februar
flusse
Flusse
fluße
Fluße
flüsse
flüße
Fuße
Januar
März
Montag
Sonntag'''
french = '''
dimanche
lundi
mardi
janvier
février
mars
déjà
Meme
deja
même
dejà
bpef
bœg
Boef
Mémé
bœf
boef
bnef
pêche
pèché
pêché
pêche
pêché'''
french_good = '''
bnef
boef
Boef
bœf
bœg
bpef
deja
dejà
déjà
dimanche
février
janvier
lundi
mardi
mars
Meme
Mémé
même
pèché
pêche
pêche
pêché
pêché'''
# }}}
def create(l):
l = l.decode('utf-8').splitlines()
return [x.strip() for x in l if x.strip()]
german = create(german)
c = _icu.Collator('de')
print 'Sorted german:: (%s)'%c.actual_locale
gs = list(sorted(german, key=c.sort_key))
for x in gs:
print '\t', x.encode('utf-8')
if gs != create(german_good):
print 'German failed'
return
print
french = create(french)
c = _icu.Collator('fr')
print 'Sorted french:: (%s)'%c.actual_locale
fs = list(sorted(french, key=c.sort_key))
for x in fs:
print '\t', x.encode('utf-8')
if fs != create(french_good):
print 'French failed (note that French fails with icu < 4.6 i.e. on windows and OS X)'
return
# }}}