From 0f3228e6585dadcf6f4aa6110ed3619966bbfff2 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 11 Apr 2011 19:04:56 -0400 Subject: [PATCH] Basic plucker working (text, non-composite images). --- src/calibre/ebooks/pdb/plucker/reader.py | 455 +++++++++++++++++++++-- 1 file changed, 425 insertions(+), 30 deletions(-) diff --git a/src/calibre/ebooks/pdb/plucker/reader.py b/src/calibre/ebooks/pdb/plucker/reader.py index d1e5931580..502682baba 100644 --- a/src/calibre/ebooks/pdb/plucker/reader.py +++ b/src/calibre/ebooks/pdb/plucker/reader.py @@ -10,9 +10,13 @@ import os import struct import zlib +from collections import OrderedDict + from calibre import CurrentDir from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.ebooks.pdb.formatreader import FormatReader +from calibre.ptempfile import TemporaryFile +from calibre.utils.magick import Image DATATYPE_PHTML = 0 DATATYPE_PHTML_COMPRESSED = 1 @@ -38,6 +42,100 @@ DATATYPE_EXT_ANCHOR_INDEX = 20 DATATYPE_EXT_ANCHOR = 21 DATATYPE_EXT_ANCHOR_COMPRESSED = 22 +# IETF IANA MIBenum value for the character set. +# See the http://www.iana.org/assignments/character-sets for valid values. +# Not all character sets are handled by Python. This is a small subset that +# the MIBenum maps to Python standard encodings +# from http://docs.python.org/library/codecs.html#standard-encodings +MIBNUM_TO_NAME = { + 3: 'ascii', + 4: 'latin_1', + 5: 'iso8859_2', + 6: 'iso8859_3', + 7: 'iso8859_4', + 8: 'iso8859_5', + 9: 'iso8859_6', + 10: 'iso8859_7', + 11: 'iso8859_8', + 12: 'iso8859_9', + 13: 'iso8859_10', + 17: 'shift_jis', + 18: 'euc_jp', + 27: 'utf_7', + 36: 'euc_kr', + 37: 'iso2022_kr', + 38: 'euc_kr', + 39: 'iso2022_jp', + 40: 'iso2022_jp_2', + 106: 'utf-8', + 109: 'iso8859_13', + 110: 'iso8859_14', + 111: 'iso8859_15', + 112: 'iso8859_16', + 1013: 'utf_16_be', + 1014: 'utf_16_le', + 1015: 'utf_16', + 2009: 'cp850', + 2010: 'cp852', + 2011: 'cp437', + 2013: 'cp862', + 2025: 'gb2312', + 2026: 'big5', + 2028: 'cp037', + 2043: 'cp424', + 2044: 'cp500', + 2046: 'cp855', + 2047: 'cp857', + 2048: 'cp860', + 2049: 'cp861', + 2050: 'cp863', + 2051: 'cp864', + 2052: 'cp865', + 2054: 'cp869', + 2063: 'cp1026', + 2085: 'hz', + 2086: 'cp866', + 2087: 'cp775', + 2089: 'cp858', + 2091: 'cp1140', + 2102: 'big5hkscs', + 2250: 'cp1250', + 2251: 'cp1251', + 2252: 'cp1252', + 2253: 'cp1253', + 2254: 'cp1254', + 2255: 'cp1255', + 2256: 'cp1256', + 2257: 'cp1257', + 2258: 'cp1258', +} + +def decompress_doc(data): + buffer = [ord(i) for i in data] + res = [] + i = 0 + while i < len(buffer): + c = buffer[i] + i += 1 + if c >= 1 and c <= 8: + res.extend(buffer[i:i+c]) + i += c + elif c <= 0x7f: + res.append(c) + elif c >= 0xc0: + res.extend( (ord(' '), c^0x80) ) + else: + c = (c << 8) + buffer[i] + i += 1 + di = (c & 0x3fff) >> 3 + j = len(res) + num = (c & ((1 << 3) - 1)) + 3 + + for k in range( num ): + res.append(res[j - di+k]) + + return ''.join([chr(i) for i in res]) + class HeaderRecord(object): def __init__(self, raw): @@ -68,14 +166,62 @@ class SectionHeader(object): class SectionHeaderText(object): - def __init__(self, data_header, raw): + def __init__(self, section_header, raw): self.sizes = [] self.attributes = [] - for i in xrange(data_header.paragraphs): + for i in xrange(section_header.paragraphs): adv = 4*i - self.sizes.append(struct.unpack('>H', raw[8+adv:10+adv])[0]) - self.attributes.append(struct.unpack('>H', raw[10+adv:12+adv])[0]) + self.sizes.append(struct.unpack('>H', raw[adv:2+adv])[0]) + self.attributes.append(struct.unpack('>H', raw[2+adv:4+adv])[0]) + +class SectionMetadata(object): + + def __init__(self, raw): + self.default_encoding = 'utf-8' + self.exceptional_uid_encodings = {} + self.owner_id = None + + record_count, = struct.unpack('>H', raw[0:2]) + + adv = 0 + for i in xrange(record_count): + type, = struct.unpack('>H', raw[2+adv:4+adv]) + length, = struct.unpack('>H', raw[4+adv:6+adv]) + + # CharSet + if type == 1: + val, = struct.unpack('>H', raw[6+adv:8+adv]) + self.default_encoding = MIBNUM_TO_NAME.get(val, 'utf-8') + # ExceptionalCharSets + elif type == 2: + ii_adv = 0 + for ii in xrange(length / 2): + uid, = struct.unpack('>H', raw[6+adv+ii_adv:8+adv+ii_adv]) + mib, = struct.unpack('>H', raw[8+adv+ii_adv:10+adv+ii_adv]) + self.exceptional_uid_encodings[uid] = MIBNUM_TO_NAME.get(mib, 'utf-8') + ii_adv += 4 + # OwnerID + elif type == 3: + self.owner_id = struct.unpack('>I', raw[6+adv:10+adv]) + # Author, Title, PubDate + # Ignored here. The metadata reader plugin + # will get this info because if it's missing + # the metadata reader plugin will use fall + # back data from elsewhere in the file. + elif type in (4, 5, 6): + pass + # Linked Documents + elif type == 7: + pass + + adv += 2*length + +class SectionText(object): + + def __init__(self, section_header, raw): + self.header = SectionHeaderText(section_header, raw) + self.data = raw[section_header.paragraphs * 4:] class Reader(FormatReader): @@ -84,53 +230,302 @@ class Reader(FormatReader): self.stream = stream self.log = log self.options = options - - self.sections = [] - for i in range(1, header.num_sections): - start = 8 - raw_data = header.section_data(i) - data_header = SectionHeader(raw_data) - sub_header = None - if data_header.type in (DATATYPE_PHTML, DATATYPE_PHTML_COMPRESSED): - sub_header = SectionHeaderText(data_header, raw_data) - start += data_header.paragraphs * 4 - self.sections.append((data_header, sub_header, raw_data[start:])) + # Mapping of section uid to our internal + # list of sections. + self.uid_section_number = OrderedDict() + self.uid_text_secion_number = OrderedDict() + self.uid_text_secion_encoding = {} + self.uid_image_section_number = {} + self.metadata_section_number = None + self.default_encoding = 'utf-8' + self.owner_id = None + self.sections = [] + self.header_record = HeaderRecord(header.section_data(0)) + + for i in range(1, header.num_sections): + section_number = i - 1 + start = 8 + section = None + + raw_data = header.section_data(i) + section_header = SectionHeader(raw_data) + + self.uid_section_number[section_header.uid] = section_number + + if section_header.type in (DATATYPE_PHTML, DATATYPE_PHTML_COMPRESSED): + self.uid_text_secion_number[section_header.uid] = section_number + section = SectionText(section_header, raw_data[start:]) + elif section_header.type in (DATATYPE_TBMP, DATATYPE_TBMP_COMPRESSED): + self.uid_image_section_number[section_header.uid] = section_number + section = raw_data[start:] + elif section_header.type == DATATYPE_METADATA: + self.metadata_section_number = section_number + section = SectionMetadata(raw_data[start:]) + elif section_header.type == DATATYPE_COMPOSITE_IMAGE: + + + self.sections.append((section_header, section)) + + if self.metadata_section_number: + mdata_section = self.sections[self.metadata_section_number][1] + for k, v in mdata_section.exceptional_uid_encodings.items(): + self.uid_text_secion_encoding[k] = v + self.default_encoding = mdata_section.default_encoding + self.owner_id = mdata_section.owner_id from calibre.ebooks.metadata.pdb import get_metadata self.mi = get_metadata(stream, False) def extract_content(self, output_dir): - html = u'' + html = u'' images = [] - - for header, sub_header, data in self.sections: - if header.type == DATATYPE_PHTML: - html += data - elif header.type == DATATYPE_PHTML_COMPRESSED: - d = self.decompress_phtml(data).decode('latin-1', 'replace') - print len(d) == header.size - html += d - - print html + + for uid, num in self.uid_text_secion_number.items(): + section_header, section_data = self.sections[num] + if section_header.type == DATATYPE_PHTML: + html += self.process_phtml(section_data.header, section_data.data.decode(self.get_text_uid_encoding(section_header.uid), 'replace')) + elif section_header.type == DATATYPE_PHTML_COMPRESSED: + d = self.decompress_phtml(section_data.data).decode(self.get_text_uid_encoding(section_header.uid), 'replace') + html += self.process_phtml(section_data.header, d) + + html += '' + with CurrentDir(output_dir): with open('index.html', 'wb') as index: self.log.debug('Writing text to index.html') index.write(html.encode('utf-8')) - + + if not os.path.exists(os.path.join(output_dir, 'images/')): + os.makedirs(os.path.join(output_dir, 'images/')) + with CurrentDir(os.path.join(output_dir, 'images/')): + #im.read('/Users/john/Tmp/plkr/apnx.palm') + for uid, num in self.uid_image_section_number.items(): + section_header, section_data = self.sections[num] + if section_data: + idata = None + if section_header.type == DATATYPE_TBMP: + idata = section_data + elif section_header.type == DATATYPE_TBMP_COMPRESSED: + if self.header_record.compression == 1: + idata = decompress_doc(section_data) + elif self.header_record.compression == 2: + idata = zlib.decompress(section_data) + try: + with TemporaryFile(suffix='.palm') as itn: + with open(itn, 'wb') as itf: + itf.write(idata) + im = Image() + im.read(itn) + im.set_compression_quality(70) + im.save('%s.jpg' % uid) + self.log.debug('Wrote image with uid %s to images/%s.jpg' % (uid, uid)) + except Exception as e: + self.log.error('Failed to write image with uid %s: %s' % (uid, e)) + images.append('%s.jpg' % uid) + else: + self.log.error('Failed to write image with uid %s: No data.' % uid) + opf_path = self.create_opf(output_dir, images) return opf_path def decompress_phtml(self, data): if self.header_record.compression == 2: - raise NotImplementedError - #return zlib.decompress(data) + if self.owner_id: + raise NotImplementedError + return zlib.decompress(data) elif self.header_record.compression == 1: - from calibre.ebooks.compression.palmdoc import decompress_doc + #from calibre.ebooks.compression.palmdoc import decompress_doc return decompress_doc(data) + def process_phtml(self, sub_header, d): + html = u'' + offset = 0 + paragraph_open = False + paragraph_offsets = [] + running_offset = 0 + for size in sub_header.sizes: + running_offset += size + paragraph_offsets.append(running_offset) + + while offset < len(d): + if not paragraph_open: + html += u'

' + paragraph_open = True + + c = ord(d[offset]) + if c == 0x0: + offset += 1 + c = ord(d[offset]) + # Page link begins + # 2 Bytes + # record ID + if c == 0x0a: + offset += 2 + # Targeted page link begins + # 3 Bytes + # record ID, target + elif c == 0x0b: + offset += 3 + # Paragraph link begins + # 4 Bytes + # record ID, paragraph number + elif c == 0x0c: + offset += 4 + # Targeted paragraph link begins + # 5 Bytes + # record ID, paragraph number, target + elif c == 0x0d: + offset += 5 + # Link ends + # 0 Bytes + elif c == 0x08: + pass + # Set font + # 1 Bytes + # font specifier + elif c == 0x11: + offset += 1 + # Embedded image + # 2 Bytes + # image record ID + elif c == 0x1a: + offset += 1 + uid = struct.unpack('>H', d[offset:offset+2])[0] + html += '' % uid + offset += 1 + # Set margin + # 2 Bytes + # left margin, right margin + elif c == 0x22: + offset += 2 + # Alignment of text + # 1 Bytes + # alignment + elif c == 0x29: + offset += 1 + # Horizontal rule + # 3 Bytes + # 8-bit height, 8-bit width (pixels), 8-bit width (%, 1-100) + elif c == 0x33: + offset += 3 + if paragraph_open: + html += u'

' + paragraph_open = False + html += u'
' + # New line + # 0 Bytes + elif c == 0x38: + if paragraph_open: + html += u'

\n' + paragraph_open = False + # Italic text begins + # 0 Bytes + elif c == 0x40: + html += u'' + # Italic text ends + # 0 Bytes + elif c == 0x48: + html += u'' + # Set text color + # 3 Bytes + # 8-bit red, 8-bit green, 8-bit blue + elif c == 0x53: + offset += 3 + # Multiple embedded image + # 4 Bytes + # alternate image record ID, image record ID + elif c == 0x5c: + offset += 4 + # Underline text begins + # 0 Bytes + elif c == 0x60: + html += u'' + # Underline text ends + # 0 Bytes + elif c == 0x68: + html += u'' + # Strike-through text begins + # 0 Bytes + elif c == 0x70: + html += u'' + # Strike-through text ends + # 0 Bytes + elif c == 0x78: + html += u'' + # 16-bit Unicode character + # 3 Bytes + # alternate text length, 16-bit unicode character + elif c == 0x83: + #offset += 2 + #c16 = d[offset:offset+2] + #html += c16.decode('utf-16') + #offset += 1 + offset += 3 + # 32-bit Unicode character + # 5 Bytes + # alternate text length, 32-bit unicode character + elif c == 0x85: + #offset += 2 + #c32 = d[offset:offset+4] + #html += c32.decode('utf-32') + #offset += 3 + offset += 5 + # Begin custom font span + # 6 Bytes + # font page record ID, X page position, Y page position + elif c == 0x8e: + offset += 6 + # Adjust custom font glyph position + # 4 Bytes + # X page position, Y page position + elif c == 0x8c: + offset += 4 + # Change font page + # 2 Bytes + # font record ID + elif c == 0x8a: + offset += 2 + # End custom font span + # 0 Bytes + elif c == 0x88: + pass + # Begin new table row + # 0 Bytes + elif c == 0x90: + pass + # Insert table (or table link) + # 2 Bytes + # table record ID + elif c == 0x92: + offset += 2 + # Table cell data + # 7 Bytes + # 8-bit alignment, 16-bit image record ID, 8-bit columns, 8-bit rows, 16-bit text length + elif c == 0x97: + offset += 7 + # Exact link modifier + # 2 Bytes + # Paragraph Offset (The Exact Link Modifier modifies a Paragraph Link or Targeted Paragraph Link function to specify an exact byte offset within the paragraph. This function must be followed immediately by the function it modifies). + elif c == 0x9a: + offset += 2 + else: + html += unichr(c) + offset += 1 + if offset in paragraph_offsets: + if paragraph_open: + html += u'

\n' + paragraph_open = False + + if paragraph_open: + html += u'

' + + return html + + def get_text_uid_encoding(self, uid): + return self.uid_text_secion_encoding.get(uid, self.default_encoding) def create_opf(self, output_dir, images): with CurrentDir(output_dir):