diff --git a/src/calibre/ebooks/rtf/input.py b/src/calibre/ebooks/rtf/input.py index 8c7561f68c..fdd501495b 100644 --- a/src/calibre/ebooks/rtf/input.py +++ b/src/calibre/ebooks/rtf/input.py @@ -198,21 +198,21 @@ class RTFInput(InputFormatPlugin): with open('styles.css', 'ab') as f: f.write(css) - def preprocess(self, fname): - self.log('\tPreprocessing to convert unicode characters') - try: - data = open(fname, 'rb').read() - from calibre.ebooks.rtf.preprocess import RtfTokenizer, RtfTokenParser - tokenizer = RtfTokenizer(data) - tokens = RtfTokenParser(tokenizer.tokens) - data = tokens.toRTF() - fname = 'preprocessed.rtf' - with open(fname, 'wb') as f: - f.write(data) - except: - self.log.exception( - 'Failed to preprocess RTF to convert unicode sequences, ignoring...') - return fname + # def preprocess(self, fname): + # self.log('\tPreprocessing to convert unicode characters') + # try: + # data = open(fname, 'rb').read() + # from calibre.ebooks.rtf.preprocess import RtfTokenizer, RtfTokenParser + # tokenizer = RtfTokenizer(data) + # tokens = RtfTokenParser(tokenizer.tokens) + # data = tokens.toRTF() + # fname = 'preprocessed.rtf' + # with open(fname, 'wb') as f: + # f.write(data) + # except: + # self.log.exception( + # 'Failed to preprocess RTF to convert unicode sequences, ignoring...') + # return fname def convert_borders(self, doc): border_styles = [] @@ -249,9 +249,9 @@ class RTFInput(InputFormatPlugin): self.log = log self.log('Converting RTF to XML...') #Name of the preprocesssed RTF file - fname = self.preprocess(stream.name) + # fname = self.preprocess(stream.name) try: - xml = self.generate_xml(fname) + xml = self.generate_xml(stream.name) except RtfInvalidCodeException, e: raise ValueError(_('This RTF file has a feature calibre does not ' 'support. Convert it to HTML first and then try it.\n%s')%e) diff --git a/src/calibre/ebooks/rtf2xml/ParseRtf.py b/src/calibre/ebooks/rtf2xml/ParseRtf.py index 7b89407f79..4b0bb41d42 100755 --- a/src/calibre/ebooks/rtf2xml/ParseRtf.py +++ b/src/calibre/ebooks/rtf2xml/ParseRtf.py @@ -17,7 +17,8 @@ ######################################################################### # $Revision: 1.41 $ # $Date: 2006/03/24 23:50:07 $ -import sys,os +import sys, os + from calibre.ebooks.rtf2xml import headings_to_sections, \ line_endings, footnote, fields_small, default_encoding, \ make_lists, preamble_div, header, colors, group_borders, \ @@ -90,7 +91,6 @@ class ParseRtf: out_file = '', out_dir = None, dtd = '', - #debug = 0, #why? calibre deb_dir = None, convert_symbol = None, convert_wingdings = None, @@ -107,6 +107,7 @@ class ParseRtf: no_dtd = 0, char_data = '', ): + """ Requires: 'file' --file to parse @@ -119,12 +120,11 @@ class ParseRtf: script tries to output to directory where is script is exectued.) 'deb_dir' --debug directory. If a debug_dir is provided, the script will copy each run through as a file to examine in the debug_dir - 'perl_script'--use perl to make tokens. This runs just a bit faster. - (I will probably phase this out.) 'check_brackets' -- make sure the brackets match up after each run through a file. Only for debugging. Returns: Nothing """ + self.__file = in_file self.__out_file = out_file self.__out_dir = out_dir @@ -132,7 +132,7 @@ class ParseRtf: self.__dtd_path = dtd self.__check_file(in_file,"file_to_parse") self.__char_data = char_data - self.__debug_dir = deb_dir #self.__debug_dir = debug calibre + self.__debug_dir = deb_dir self.__check_dir(self.__temp_dir) self.__copy = self.__check_dir(self.__debug_dir) self.__convert_caps = convert_caps @@ -155,25 +155,24 @@ class ParseRtf: if hasattr(the_file, 'read'): return if the_file == None: if type == "file_to_parse": - message = "You must provide a file for the script to work" - msg = message + msg = _("\nYou must provide a file for the script to work") raise RtfInvalidCodeException, msg elif os.path.exists(the_file): pass # do nothing else: - message = "The file '%s' cannot be found" % the_file - msg = message + msg = _("\nThe file '%s' cannot be found") % the_file raise RtfInvalidCodeException, msg + def __check_dir(self, the_dir): """Check to see if directory exists""" if not the_dir : return dir_exists = os.path.isdir(the_dir) if not dir_exists: - message = "%s is not a directory" % the_dir - msg = message + msg = _("\n%s is not a directory") % the_dir raise RtfInvalidCodeException, msg return 1 + def parse_rtf(self): """ Parse the file by calling on other classes. @@ -194,13 +193,14 @@ class ParseRtf: copy_obj.set_dir(self.__debug_dir) copy_obj.remove_files() copy_obj.copy_file(self.__temp_file, "original_file") - # new as of 2005-08-02. Do I want this? + # Function to check if bracket are well handled if self.__debug_dir or self.__run_level > 2: self.__check_brack_obj = check_brackets.CheckBrackets\ (file = self.__temp_file, bug_handler = RtfInvalidCodeException, ) - # convert Macintosh line endings to Unix line endings + # convert Macintosh and Windows line endings to Unix line endings + #why do this if you don't wb after? line_obj = line_endings.FixLineEndings( in_file = self.__temp_file, bug_handler = RtfInvalidCodeException, @@ -208,13 +208,13 @@ class ParseRtf: run_level = self.__run_level, replace_illegals = self.__replace_illegals, ) - return_value = line_obj.fix_endings() + return_value = line_obj.fix_endings() #calibre return what? self.__return_code(return_value) tokenize_obj = tokenize.Tokenize( bug_handler = RtfInvalidCodeException, in_file = self.__temp_file, copy = self.__copy, - run_level = self.__run_level,) + run_level = self.__run_level) tokenize_obj.tokenize() process_tokens_obj = process_tokens.ProcessTokens( in_file = self.__temp_file, @@ -230,11 +230,13 @@ class ParseRtf: os.remove(self.__temp_file) except OSError: pass + #Check to see if the file is correct ascii check_encoding_obj = check_encoding.CheckEncoding( - bug_handler = RtfInvalidCodeException, - ) - check_encoding_obj.check_encoding(self.__file) - sys.stderr.write('File "%s" does not appear to be RTF.\n' % self.__file if isinstance(self.__file, str) else self.__file.encode('utf-8')) + bug_handler = RtfInvalidCodeException, + ) + if check_encoding_obj.check_encoding(self.__file): + sys.stderr.write(_('File "%s" does not appear to be ascii.\n') \ + % self.__file if isinstance(self.__file, str) else self.__file.encode('utf-8')) raise InvalidRtfException, msg delete_info_obj = delete_info.DeleteInfo( in_file = self.__temp_file, @@ -370,10 +372,10 @@ class ParseRtf: sys.stderr.write('File could be older RTF...\n') if found_destination: if self.__run_level > 1: - sys.stderr.write( + sys.stderr.write(_( 'File also has newer RTF.\n' 'Will do the best to convert.\n' - ) + )) add_brackets_obj = add_brackets.AddBrackets( in_file = self.__temp_file, bug_handler = RtfInvalidCodeException, @@ -520,35 +522,28 @@ class ParseRtf: output_obj.output() os.remove(self.__temp_file) return self.__exit_level + def __bracket_match(self, file_name): if self.__run_level > 2: good_br, msg = self.__check_brack_obj.check_brackets() if good_br: pass - # sys.stderr.write( msg + ' in ' + file_name + "\n") + #sys.stderr.write( msg + ' in ' + file_name + "\n") else: - msg += msg + " in file '" + file_name + "'\n" + msg = _('%s in file %s\n') % (msg, file_name) raise RtfInvalidCodeException, msg + def __return_code(self, num): - if num == None: - return - if int(num) > self.__exit_level: - self.__exit_level = num + if num == None: + return + if int(num) > self.__exit_level: + self.__exit_level = num + def __make_temp_file(self,file): """Make a temporary file to parse""" write_file="rtf_write_file" read_obj = file if hasattr(file, 'read') else open(file,'r') - write_obj = open(write_file, 'w') - line = "dummy" - while line: - line = read_obj.read(1000) - write_obj.write(line ) - write_obj.close() - return write_file - """ -mi1\n -mi33\n -mi 3: - msg = 'number "%s" cannot be converted to integer\n' % num + msg = _('Number "%s" cannot be converted to integer\n') % num raise self.__bug_handler, msg type = self.__number_type_dict.get(num) if type == None: if self.__run_level > 3: - msg = 'No type for "%s" in self.__number_type_dict\n' + msg = _('No type for "%s" in self.__number_type_dict\n') raise self.__bug_handler type = 'Arabic' return 'cw<%s<%snum<%s\n' % (token, num) + def divide_by_2(self, pre, token, num): num = self.divide_num(num, 2) return 'cw<%s<%s%s<%s\n' % (token, num, token) + def divide_by_20(self, pre, token, num): num = self.divide_num(num, 20) return 'cw<%s<%s%s<%s\n' % (token, num, token) + def text_func(self, pre, token, num=None): return 'tx%s<%s\n' % (third_field, token, num, token) + def bool_st_func(self, pre, token, num): if num is None or num == '' or num == '1': return 'cw<%s<%s 3: msg = 'no number to process?\n' @@ -698,6 +715,7 @@ class ProcessTokens: if string_num[-2:] == ".0": string_num = string_num[:-2] return string_num + def split_let_num(self, token): match_obj = re.search(self.__num_exp,token) if match_obj != None: @@ -714,6 +732,7 @@ class ProcessTokens: raise self.__bug_handler return token, 0 return first, second + def convert_to_hex(self,number): """Convert a string to uppercase hexidecimal""" num = int(number) @@ -722,6 +741,7 @@ class ProcessTokens: return hex_num except: raise self.__bug_handler + def process_cw(self, token): """Change the value of the control word by determining what dictionary it belongs to""" @@ -737,92 +757,60 @@ class ProcessTokens: pre, token, action = self.dict_token.get(token, (None, None, None)) if action: return action(pre, token, num) - # unused function - def initiate_token_actions(self): - self.action_for_token={ - '{' : self.ob_func, - '}' : self.cb_func, - '\\' : self.process_cw, - } - # unused function - def evaluate_token(self,token): - """Evaluate tokens. Return a value if the token is not a - control word. Otherwise, pass token onto another method - for further evaluation.""" - token, action = self.dict_token.get(token[0:1]) - if action: - line = action(token) - return line - else : - return 'tx -1: + msg ='Invalid RTF: token "\\ " not valid.\n' raise self.__exception_handler, msg - first_token = 1 - elif first_token and not second_token: - if token[0:4] != '\\rtf': - msg ='Invalid RTF: document doesn\'t start with \\rtf \n' - raise self.__exception_handler, msg - second_token = 1 - ##token = self.evaluate_token(token) - the_index = token.find('\\ ') - if token != None and the_index > -1: - msg ='Invalid RTF: token "\\ " not valid. \n' - raise self.__exception_handler, msg - elif token[0:1] == "\\": - line = self.process_cw(token) - if line != None: - write_obj.write(line) - else: - fields = re.split(self.__utf_exp, token) - for field in fields: - if not field: - continue - if field[0:1] == '&': - write_obj.write('tx", ">") - line = line.replace("\\~", "\\~ ") - line = line.replace("\\_", "\\_ ") - line = line.replace("\\:", "\\: ") - line = line.replace("\\-", "\\- ") - # turn into a generic token to eliminate special - # cases and make processing easier - line = line.replace("\\{", "\\ob ") - # turn into a generic token to eliminate special - # cases and make processing easier - line = line.replace("\\}", "\\cb ") - # put a backslash in front of to eliminate special cases and - # make processing easier - line = line.replace("{", "\\{") - # put a backslash in front of to eliminate special cases and - # make processing easier - line = line.replace("}", "\\}") - line = re.sub(self.__utf_exp, self.__from_ms_to_utf8, line) - # line = re.sub( self.__neg_utf_exp, self.__neg_unicode_func, line) - line = re.sub(self.__ms_hex_exp, "\\mshex0\g<1> ", line) - ##line = line.replace("\\backslash", "\\\\") - # this is for older RTF - line = re.sub(self.__par_exp, '\\par ', line) - return line - def __compile_expressions(self): - self.__ms_hex_exp = re.compile(r"\\\'(..)") - self.__utf_exp = re.compile(r"\\u(-?\d{3,6}) {0,1}") - self.__splitexp = re.compile(r"(\\[\\{}]|{|}|\\[^\s\\{}&]+(?:\s)?)") - self.__par_exp = re.compile(r'\\$') - self.__mixed_exp = re.compile(r"(\\[a-zA-Z]+\d+)(\D+)") - ##self.num_exp = re.compile(r"(\*|:|[a-zA-Z]+)(.*)") - def __create_tokens(self): self.__compile_expressions() - read_obj = open(self.__file, 'r') - write_obj = open(self.__write_to, 'w') - line_to_read = "dummy" - while line_to_read: - line_to_read = read_obj.readline() - line = line_to_read - line = line.replace("\n", "") - line = self.__sub_line_reg(line) - tokens = re.split(self.__splitexp, line) - ##print tokens - for token in tokens: - if token != "": - write_obj.write(token + "\n") - """ - match_obj = re.search(self.__mixed_exp, token) - if match_obj != None: - first = match_obj.group(1) - second = match_obj.group(2) - write_obj.write(first + "\n") - write_obj.write(second + "\n") - else: - write_obj.write(token + "\n") - """ - read_obj.close() - write_obj.close() + #variables + self.__uc_char = 0 + self.__uc_bin = False + self.__uc_value = [1] + + def __reini_utf8_counters(self): + self.__uc_char = 0 + self.__uc_bin = False + + def __remove_uc_chars(self, startchar, token): + for i in xrange(startchar, len(token)): + if token[i] == " ": + continue + elif self.__uc_char: + self.__uc_char -= 1 + else: + return token[i:] + #if only " " and char to skip + return '' + + def __unicode_process(self, token): + #change scope in + if token == '\{': + self.__uc_value.append(self.__uc_value[-1]) + #basic error handling + self.__reini_utf8_counters() + return token + #change scope out + elif token == '\}': + self.__uc_value.pop() + self.__reini_utf8_counters() + return token + #add a uc control + elif token[:3] == '\uc': + self.__uc_value[-1] = int(token[3:]) + self.__reini_utf8_counters() + return token + #bin data to slip + elif self.__uc_bin: + self.__uc_bin = False + return '' + #uc char to remove + elif self.__uc_char: + #handle \bin tag in case of uc char to skip + if token[:4] == '\bin': + self.__uc_char -=1 + self.__uc_bin = True + return '' + elif token[:1] == "\\" : + self.__uc_char -=1 + return '' + else: + return self.__remove_uc_chars(0, token) + #go for real \u token + match_obj = self.__utf_exp.match(token) + if match_obj is not None: + self.__reini_utf8_counters() + #get value and handle negative case + uni_char = int(match_obj.group(1)) + uni_len = len(match_obj.group(1)) + 2 + if uni_char < 0: + uni_char += 65536 + uni_char = unichr(uni_char).encode('ascii', 'xmlcharrefreplace') + self.__uc_char = self.__uc_value[-1] + #there is only an unicode char + if len(token)<= uni_len: + return uni_char + #an unicode char and something else + #must be after as it is splited on \ + #necessary? maybe for \bin? + elif not self.__uc_char: + return uni_char + token[uni_len:] + #if not uc0 and chars + else: + return uni_char + self.__remove_uc_chars(uni_len, token) + #default + return token + + def __sub_reg_split(self,input_file): + input_file = self.__replace_spchar.mreplace(input_file) + input_file = self.__ms_hex_exp.sub("\\mshex0\g<1> ", input_file) + input_file = self.__utf_ud.sub("\\{\\uc0 \g<1>\\}", input_file) + #remove \n in bin data + input_file = self.__bin_exp.sub(lambda x: \ + x.group().replace('\n', '') +'\n', input_file) + #split + tokens = re.split(self.__splitexp, input_file) + #remove empty tokens and \n + return filter(lambda x: len(x) > 0 and x != '\n', tokens) + #input_file = re.sub(self.__utf_exp, self.__from_ms_to_utf8, input_file) + # line = re.sub( self.__neg_utf_exp, self.__neg_unicode_func, line) + # this is for older RTF + #line = re.sub(self.__par_exp, '\\par ', line) + #return filter(lambda x: len(x) > 0, \ + #(self.__remove_line.sub('', x) for x in tokens)) + + def __compile_expressions(self): + SIMPLE_RPL = { + "\\\\": "\\backslash ", + "\\~": "\\~ ", + "\\;": "\\; ", + "&": "&", + "<": "<", + ">": ">", + "\\~": "\\~ ", + "\\_": "\\_ ", + "\\:": "\\: ", + "\\-": "\\- ", + # turn into a generic token to eliminate special + # cases and make processing easier + "\\{": "\\ob ", + # turn into a generic token to eliminate special + # cases and make processing easier + "\\}": "\\cb ", + # put a backslash in front of to eliminate special cases and + # make processing easier + "{": "\\{", + # put a backslash in front of to eliminate special cases and + # make processing easier + "}": "\\}", + # this is for older RTF + r'\\$': '\\par ', + } + self.__replace_spchar = MReplace(SIMPLE_RPL) + #add ;? in case of char following \u + self.__ms_hex_exp = re.compile(r"\\\'([0-9a-fA-F]{2})") #r"\\\'(..)" + self.__utf_exp = re.compile(r"\\u(-?\d{3,6}) ?") + self.__bin_exp = re.compile(r"(?:\\bin(-?\d{0,10})[\n ]+)[01\n]+") + #manage upr/ud situations + self.__utf_ud = re.compile(r"\\{[\n ]?\\upr[\n ]?(?:\\{.*?\\})[\n ]?" + \ + r"\\{[\n ]?\\*[\n ]?\\ud[\n ]?(\\{.*?\\})[\n ]?\\}[\n ]?\\}") + #add \n in split for whole file reading + #why keep backslash whereas \is replaced before? + #remove \n from endline char + self.__splitexp = re.compile(r"(\\[{}]|\n|\\[^\s\\{}&]+(?:[ \t\r\f\v])?)") + #self.__bin_exp = re.compile(r"\\bin(-?\d{1,8}) {0,1}") + #self.__utf_exp = re.compile(r"^\\u(-?\d{3,6})") + #self.__splitexp = re.compile(r"(\\[\\{}]|{|}|\n|\\[^\s\\{}&]+(?:\s)?)") + #self.__par_exp = re.compile(r'\\$') + #self.__remove_line = re.compile(r'\n+') + #self.__mixed_exp = re.compile(r"(\\[a-zA-Z]+\d+)(\D+)") + ##self.num_exp = re.compile(r"(\*|:|[a-zA-Z]+)(.*)") + def tokenize(self): - """Main class for handling other methods. Reads in one line \ - at a time, usues method self.sub_line to make basic substitutions,\ - uses ? to process tokens""" - self.__create_tokens() + """Main class for handling other methods. Reads the file \ + , uses method self.sub_reg to make basic substitutions,\ + and process tokens by itself""" + #read + with open(self.__file, 'r') as read_obj: + input_file = read_obj.read() + + #process simple replacements and split giving us a correct list + #remove '' and \n in the process + tokens = self.__sub_reg_split(input_file) + #correct unicode + tokens = map(self.__unicode_process, tokens) + #remove empty items created by removing \uc + tokens = filter(lambda x: len(x) > 0, tokens) + + #write + with open(self.__write_to, 'wb') as write_obj: + write_obj.write('\n'.join(tokens)) + #Move and copy copy_obj = copy.Copy(bug_handler = self.__bug_handler) if self.__copy: copy_obj.copy_file(self.__write_to, "tokenize.data") copy_obj.rename(self.__write_to, self.__file) os.remove(self.__write_to) + + #self.__special_tokens = [ '_', '~', "'", '{', '}' ] \ No newline at end of file