py3: centralize creation of rtf file objects so they can be controlled easily

src/calibre/ebooks/rtf2xml/ParseRtf.py

@@ -25,6 +25,7 @@ from calibre.ebooks.rtf2xml import headings_to_sections, \
     body_styles, preamble_rest, group_styles, \
     inline
 from calibre.ebooks.rtf2xml.old_rtf import OldRtf
+from . import open_for_read, open_for_write
 
 """
 Here is an example script using the ParseRTF module directly
@@ -562,8 +563,8 @@ class ParseRtf:
     def __make_temp_file(self,file):
         """Make a temporary file to parse"""
         write_file="rtf_write_file"
-        read_obj = file if hasattr(file, 'read') else open(file,'rb')
+        read_obj = file if hasattr(file, 'read') else open_for_read(file)
-        with open(write_file, 'wb') as write_obj:
+        with open_for_write(write_file) as write_obj:
             for line in read_obj:
                 write_obj.write(line)
         return write_file

src/calibre/ebooks/rtf2xml/__init__.py

@@ -1,3 +1,12 @@
-'''
+from __future__ import unicode_literals
-modules for rtf2xml
+
-'''
+import io
+
+
+def open_for_read(path):
+    return io.open(path, encoding='utf-8', errors='replace')
+
+
+def open_for_write(path, append=False):
+    mode = 'a' if append else 'w'
+    return io.open(path, mode, encoding='utf-8', errors='replace', newline='')

src/calibre/ebooks/rtf2xml/add_brackets.py

@@ -16,6 +16,7 @@ import sys, os
 from calibre.ebooks.rtf2xml import copy, check_brackets
 from calibre.ptempfile import better_mktemp
 from polyglot.builtins import iteritems
+from . import open_for_read, open_for_write
 
 
 class AddBrackets:
@@ -202,8 +203,8 @@ class AddBrackets:
         """
         """
         self.__initiate_values()
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     if self.__token_info == 'ob<nu<open-brack':

src/calibre/ebooks/rtf2xml/body_styles.py

@@ -13,6 +13,7 @@
 import os
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 """
 Simply write the list of strings after style table
@@ -53,8 +54,8 @@ class BodyStyles:
     def insert_info(self):
         """
         """
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/check_brackets.py

@@ -12,6 +12,9 @@
 #########################################################################
 
 
+from . import open_for_read
+
+
 class CheckBrackets:
     """Check that brackets match up"""
 
@@ -41,7 +44,7 @@ class CheckBrackets:
 
     def check_brackets(self):
         line_count = 0
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
             for line in read_obj:
                 line_count += 1
                 self.__token_info = line[:16]

src/calibre/ebooks/rtf2xml/check_encoding.py

@@ -13,17 +13,17 @@ class CheckEncoding:
             char_position +=1
             try:
                 char.decode(encoding)
-            except UnicodeError as msg:
+            except ValueError as msg:
                 sys.stderr.write('line: %s char: %s\n%s\n' %  (line_num, char_position, str(msg)))
 
     def check_encoding(self, path, encoding='us-ascii', verbose=True):
         line_num = 0
-        with open(path, 'r') as read_obj:
+        with open(path, 'rb') as read_obj:
             for line in read_obj:
                 line_num += 1
                 try:
                     line.decode(encoding)
-                except UnicodeError:
+                except ValueError:
                     if verbose:
                         if len(line) < 1000:
                             self.__get_position_error(line, encoding, line_num)

src/calibre/ebooks/rtf2xml/colors.py

@@ -14,6 +14,7 @@ import sys, os, re
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class Colors:
@@ -236,8 +237,8 @@ class Colors:
             info, and substitute the number with the hex number.
         """
         self.__initiate_values()
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
                 for line in read_obj:
                     self.__line+=1
                     self.__token_info = line[:16]

src/calibre/ebooks/rtf2xml/combine_borders.py

@@ -14,6 +14,7 @@ import os
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class CombineBorders:
@@ -76,8 +77,8 @@ class CombineBorders:
             self.add_to_border_desc(line)
 
     def combine_borders(self):
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as write_obj:
+            with open_for_write(self.__write_to) as write_obj:
                 for line in read_obj:
                     self.__first_five = line[0:5]
                     if self.__state == 'border':

src/calibre/ebooks/rtf2xml/configure_txt.py

@@ -1,4 +1,5 @@
 import os, sys
+from . import open_for_read
 
 
 class Configure:
@@ -31,7 +32,7 @@ class Configure:
         if self.__show_config_file and not self.__configuration_file:
             sys.stderr.write('No configuraiton file found; using default values\n')
         if self.__configuration_file:
-            read_obj = open(self.__configuration_file, 'r')
+            read_obj = open_for_read(self.__configuration_file)
             line_to_read = 1
             line_num = 0
             while line_to_read:

src/calibre/ebooks/rtf2xml/convert_to_tags.py

@@ -3,6 +3,7 @@ from codecs import EncodedFile
 
 from calibre.ebooks.rtf2xml import copy, check_encoding
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 public_dtd = 'rtf2xml1.0.dtd'
 
@@ -260,9 +261,9 @@ class ConvertToTags:
             an empty tag function.
             """
         self.__initiate_values()
-        with open(self.__write_to, 'w') as self.__write_obj:
+        with open_for_write(self.__write_to) as self.__write_obj:
             self.__write_dec()
-            with open(self.__file, 'r') as read_obj:
+            with open_for_read(self.__file) as read_obj:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     action = self.__state_dict.get(self.__token_info)
@@ -275,8 +276,8 @@ class ConvertToTags:
             file_encoding = "utf-8"
             if self.__bad_encoding:
                 file_encoding = "us-ascii"
-            with open(self.__file, 'r') as read_obj:
+            with open_for_read(self.__file) as read_obj:
-                with open(self.__write_to, 'w') as write_obj:
+                with open_for_write(self.__write_to) as write_obj:
                     write_objenc = EncodedFile(write_obj, self.__encoding,
                                     file_encoding, 'replace')
                     for line in read_obj:

src/calibre/ebooks/rtf2xml/default_encoding.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 #########################################################################
 #                                                                       #
 #   copyright 2002 Paul Henry Tremblay                                  #
@@ -55,8 +56,8 @@ Codepages as to RTF 1.9.1:
     57010	Gujarati
     57011	Punjabi
 '''
-from __future__ import print_function
 import re
+from . import open_for_read
 
 
 class DefaultEncoding:
@@ -125,7 +126,7 @@ class DefaultEncoding:
         return self.__platform
 
     def _encoding(self):
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
             cpfound = False
             if not self.__fetchraw:
                 for line in read_obj:

src/calibre/ebooks/rtf2xml/delete_info.py

@@ -14,6 +14,7 @@ import sys, os
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class DeleteInfo:
@@ -185,8 +186,8 @@ class DeleteInfo:
     def delete_info(self):
         """Main method for handling other methods. Read one line at
         a time, and determine whether to print the line based on the state."""
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
                 for line in read_obj:
                     # ob<nu<open-brack<0001
                     self.__token_info = line[:16]

src/calibre/ebooks/rtf2xml/fields_large.py

@@ -13,6 +13,7 @@
 import sys, os
 from calibre.ebooks.rtf2xml import field_strings, copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class FieldsLarge:
@@ -351,8 +352,8 @@ Examples
             If the state is body, send the line to the body method.
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/fields_small.py

@@ -14,6 +14,7 @@ import sys, os, re
 
 from calibre.ebooks.rtf2xml import field_strings, copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class FieldsSmall:
@@ -438,8 +439,8 @@ file.
            bookmark.
         """
         self.__initiate_values()
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     if self.__token_info == 'ob<nu<open-brack':

src/calibre/ebooks/rtf2xml/fonts.py

@@ -14,6 +14,7 @@ import sys, os
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class Fonts:
@@ -204,8 +205,8 @@ class Fonts:
             info. Substitute a font name for a font number.
             """
         self.__initiate_values()
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     action = self.__state_dict.get(self.__state)

src/calibre/ebooks/rtf2xml/footnote.py

@@ -14,6 +14,7 @@ import os
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class Footnote:
@@ -118,9 +119,9 @@ class Footnote:
         """
         self.__initiate_sep_values()
         self.__footnote_holder = better_mktemp()
-        with open(self.__file) as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
-                with open(self.__footnote_holder, 'w') as self.__write_to_foot_obj:
+                with open_for_write(self.__footnote_holder) as self.__write_to_foot_obj:
                     for line in read_obj:
                         self.__token_info = line[:16]
                         # keep track of opening and closing brackets
@@ -134,8 +135,8 @@ class Footnote:
                         # not in the middle of footnote text
                         else:
                             self.__default_sep(line)
-        with open(self.__footnote_holder, 'r') as read_obj:
+        with open_for_read(self.__footnote_holder) as read_obj:
-            with open(self.__write_to, 'a') as write_obj:
+            with open_for_write(self.__write_to, append=True) as write_obj:
                 write_obj.write(
                     'mi<mk<sect-close\n'
                     'mi<mk<body-close\n'
@@ -188,9 +189,9 @@ class Footnote:
         These two functions do the work of separating the footnotes form the
         body.
         """
-        with open(self.__file) as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
-                with open(self.__footnote_holder, 'w') as self.__write_to_foot_obj:
+                with open_for_write(self.__footnote_holder) as self.__write_to_foot_obj:
                     for line in read_obj:
                         self.__token_info = line[:16]
                         if self.__state == 'body':
@@ -226,9 +227,9 @@ class Footnote:
         print out to the third file.
         If no footnote marker is found, simply print out the token (line).
         """
-        with open(self.__footnote_holder, 'r') as self.__read_from_foot_obj:
+        with open_for_read(self.__footnote_holder) as self.__read_from_foot_obj:
-            with open(self.__write_to, 'r') as read_obj:
+            with open_for_read(self.__write_to) as read_obj:
-                with open(self.__write_to2, 'w') as self.__write_obj:
+                with open_for_write(self.__write_to2) as self.__write_obj:
                     for line in read_obj:
                         if line[:16] == 'mi<mk<footnt-ind':
                             line = self.__get_foot_from_temp(line[17:-1])

src/calibre/ebooks/rtf2xml/group_borders.py

@@ -13,6 +13,7 @@
 import sys, os, re
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class GroupBorders:
@@ -285,8 +286,8 @@ class GroupBorders:
         Logic:
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/group_styles.py

@@ -13,6 +13,7 @@
 import sys, os,  re
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class GroupStyles:
@@ -231,8 +232,8 @@ class GroupStyles:
         Logic:
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/header.py

@@ -14,6 +14,7 @@ import sys, os
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class Header:
@@ -120,9 +121,9 @@ class Header:
         """
         self.__initiate_sep_values()
         self.__header_holder = better_mktemp()
-        with open(self.__file) as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
-                with open(self.__header_holder, 'w') as self.__write_to_head_obj:
+                with open_for_write(self.__header_holder) as self.__write_to_head_obj:
                     for line in read_obj:
                         self.__token_info = line[:16]
                         # keep track of opening and closing brackets
@@ -137,8 +138,8 @@ class Header:
                         else:
                             self.__default_sep(line)
 
-        with open(self.__header_holder, 'r') as read_obj:
+        with open_for_read(self.__header_holder) as read_obj:
-            with open(self.__write_to, 'a') as write_obj:
+            with open_for_write(self.__write_to, append=True) as write_obj:
                 write_obj.write(
                 'mi<mk<header-beg\n')
                 for line in read_obj:
@@ -187,9 +188,9 @@ class Header:
         These two functions do the work of separating the footnotes form the
         body.
         """
-        with open(self.__file) as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
-                with open(self.__header_holder, 'w') as self.__write_to_head_obj:
+                with open_for_write(self.__header_holder) as self.__write_to_head_obj:
                     for line in read_obj:
                         self.__token_info = line[:16]
                         if self.__state == 'body':
@@ -225,9 +226,9 @@ class Header:
         print out to the third file.
         If no footnote marker is found, simply print out the token (line).
         """
-        self.__read_from_head_obj = open(self.__header_holder, 'r')
+        self.__read_from_head_obj = open_for_read(self.__header_holder)
-        self.__write_obj = open(self.__write_to2, 'w')
+        self.__write_obj = open_for_write(self.__write_to2)
-        with open(self.__write_to, 'r') as read_obj:
+        with open_for_read(self.__write_to) as read_obj:
             for line in read_obj:
                 if line[:16] == 'mi<mk<header-ind':
                     line = self.__get_head_from_temp(line[17:-1])

src/calibre/ebooks/rtf2xml/headings_to_sections.py

@@ -13,6 +13,7 @@
 import os, re
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class HeadingsToSections:
@@ -207,8 +208,8 @@ class HeadingsToSections:
         Logic:
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/hex_2_utf8.py

@@ -15,6 +15,7 @@ import sys, os, io
 from calibre.ebooks.rtf2xml import get_char_map, copy
 from calibre.ebooks.rtf2xml.char_set import char_set
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class Hex2Utf8:
@@ -282,8 +283,8 @@ class Hex2Utf8:
 
     def __convert_preamble(self):
         self.__state = 'preamble'
-        with open(self.__write_to, 'w') as self.__write_obj:
+        with open_for_write(self.__write_to) as self.__write_obj:
-            with open(self.__file, 'r') as read_obj:
+            with open_for_read(self.__file) as read_obj:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     action = self.__preamble_state_dict.get(self.__state)
@@ -540,8 +541,8 @@ class Hex2Utf8:
 
     def __convert_body(self):
         self.__state = 'body'
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     action = self.__body_state_dict.get(self.__state)

src/calibre/ebooks/rtf2xml/info.py

@@ -14,6 +14,7 @@ import sys, os, re
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class Info:
@@ -267,8 +268,8 @@ class Info:
             information table, simply write the line to the output file.
         """
         self.__initiate_values()
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'wb') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     action = self.__state_dict.get(self.__state)

src/calibre/ebooks/rtf2xml/inline.py

@@ -2,6 +2,7 @@ import sys, os
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 """
 States.
@@ -397,8 +398,8 @@ class Inline:
             the state.
         """
         self.__initiate_values()
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
                 for line in read_obj:
                     token = line[0:-1]
                     self.__token_info = ''

src/calibre/ebooks/rtf2xml/list_numbers.py

@@ -13,6 +13,7 @@
 import os
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class ListNumbers:
@@ -177,8 +178,8 @@ class ListNumbers:
             print out self.__list_chunk and the line.
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/make_lists.py

@@ -13,6 +13,7 @@
 import sys, os, re
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class MakeLists:
@@ -439,8 +440,8 @@ class MakeLists:
         Logic:
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/old_rtf.py

@@ -11,6 +11,7 @@
 #                                                                       #
 #########################################################################
 import sys
+from . import open_for_read
 
 
 class OldRtf:
@@ -106,7 +107,7 @@ class OldRtf:
         """
         self.__initiate_values()
         line_num = 0
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
             for line in read_obj:
                 line_num += 1
                 self.__token_info = line[:16]

src/calibre/ebooks/rtf2xml/output.py

@@ -12,6 +12,7 @@
 #########################################################################
 import sys, os
 from polyglot.builtins import raw_input
+from . import open_for_read, open_for_write
 # , codecs
 
 
@@ -84,8 +85,8 @@ class Output:
             sys.stderr.write(msg)
             user_response = raw_input()
         if user_response == 'o':
-            with open(self.__file, 'r') as read_obj:
+            with open_for_read(self.__file) as read_obj:
-                with open(self.output_file, 'w') as write_obj:
+                with open_for_write(self.output_file) as write_obj:
                     for line in read_obj:
                         write_obj.write(line)
         else:
@@ -100,8 +101,8 @@ class Output:
         Logic:
             read one line at a time. Output to standard
         """
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__out_file, 'w') as write_obj:
+            with open_for_write(self.__out_file) as write_obj:
                 for line in read_obj:
                     write_obj.write(line)
 
@@ -114,24 +115,6 @@ class Output:
         Logic:
             read one line at a time. Output to standard
         """
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
             for line in read_obj:
                 sys.stdout.write(line)
-
-    # def __output_xml(self, in_file, out_file):
-        # """
-        # output the ill-formed xml file
-        # """
-        # (utf8_encode, utf8_decode, utf8_reader, utf8_writer) = codecs.lookup("utf-8")
-        # write_obj = utf8_writer(open(out_file, 'w'))
-        # write_obj = open(out_file, 'w')
-        # read_obj = utf8_writer(open(in_file, 'r'))
-        # read_obj = open(in_file, 'r')
-        # line = 1
-        # while line:
-            # line = read_obj.readline()
-            # if isinstance(line, type(u"")):
-                # line = line.encode("utf-8")
-            # write_obj.write(line)
-        # read_obj.close()
-        # write_obj.close()

src/calibre/ebooks/rtf2xml/paragraph_def.py

@@ -13,6 +13,7 @@
 import sys, os
 from calibre.ebooks.rtf2xml import copy, border_parse
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class ParagraphDef:
@@ -736,8 +737,8 @@ if another paragraph_def is found, the state changes to collect_tokens.
             the state.
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/paragraphs.py

@@ -14,6 +14,7 @@ import sys, os
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class Paragraphs:
@@ -242,8 +243,8 @@ class Paragraphs:
             only other state is 'paragraph'.
         """
         self.__initiate_values()
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     action = self.__state_dict.get(self.__state)

src/calibre/ebooks/rtf2xml/pict.py

@@ -14,6 +14,7 @@ import sys, os
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class Pict:
@@ -93,7 +94,7 @@ class Pict:
         """Create a file for all the pict data to be written to.
         """
         self.__pict_file = os.path.join(self.__dir_name, 'picts.rtf')
-        self.__write_pic_obj = open(self.__pict_file, 'a')
+        self.__write_pic_obj = open_for_write(self.__pict_file, append=True)
 
     def __in_pict_func(self, line):
         if self.__cb_count == self.__pict_br_count:
@@ -143,8 +144,8 @@ class Pict:
 
     def process_pict(self):
         self.__make_dir()
-        with open(self.__file) as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as write_obj:
+            with open_for_write(self.__write_to) as write_obj:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     if self.__token_info == 'ob<nu<open-brack':

src/calibre/ebooks/rtf2xml/preamble_div.py

@@ -14,6 +14,7 @@ from __future__ import print_function
 import sys, os
 from calibre.ebooks.rtf2xml import copy, override_table, list_table
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class PreambleDiv:
@@ -562,8 +563,8 @@ cw<ci<font-style<nu<0
 
     def make_preamble_divisions(self):
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/preamble_rest.py

@@ -13,6 +13,7 @@
 import sys,os
 
 from calibre.ebooks.rtf2xml import copy
+from . import open_for_read, open_for_write
 
 
 class Preamble:
@@ -139,8 +140,8 @@ class Preamble:
             the list table.
         """
         self.__initiate_values()
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as self.__write_obj:
+            with open_for_write(self.__write_to) as self.__write_obj:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     action = self.__state_dict.get(self.__state)

src/calibre/ebooks/rtf2xml/process_tokens.py

@@ -14,6 +14,7 @@ import os, re
 
 from calibre.ebooks.rtf2xml import copy, check_brackets
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class ProcessTokens:
@@ -43,8 +44,8 @@ class ProcessTokens:
         self.__bug_handler = bug_handler
 
     def compile_expressions(self):
-        self.__num_exp = re.compile(br"([a-zA-Z]+)(.*)")
+        self.__num_exp = re.compile(r"([a-zA-Z]+)(.*)")
-        self.__utf_exp = re.compile(br'(&.*?;)')
+        self.__utf_exp = re.compile(r'(&.*?;)')
 
     def initiate_token_dict(self):
         self.__return_code = 0
@@ -762,10 +763,10 @@ class ProcessTokens:
     def process_cw(self, token):
         """Change the value of the control word by determining what dictionary
         it belongs to"""
-        special = [b'*', b':', b'}', b'{', b'~', b'_', b'-', b';']
+        special = ['*', ':', '}', '{', '~', '_', '-', ';']
         # if token != "{" or token != "}":
         token = token[1:]  # strip off leading \
-        token = token.replace(b" ", b"")
+        token = token.replace(" ", "")
         # if not token: return
         only_alpha = token.isalpha()
         num = None
@@ -784,30 +785,24 @@ class ProcessTokens:
     def process_tokens(self):
         """Main method for handling other methods. """
         line_count = 0
-        with open(self.__file, 'rb') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'wb') as write_obj:
+            with open_for_write(self.__write_to) as write_obj:
                 for line in read_obj:
-                    token = line.replace(b"\n",b"")
+                    token = line.replace("\n", "")
                     line_count += 1
-                    if line_count == 1 and token != b'\\{':
+                    if line_count == 1 and token != '\\{':
                         msg = '\nInvalid RTF: document doesn\'t start with {\n'
                         raise self.__exception_handler(msg)
-                    elif line_count == 2 and token[0:4] != b'\\rtf':
+                    elif line_count == 2 and token[0:4] != '\\rtf':
                         msg = '\nInvalid RTF: document doesn\'t start with \\rtf \n'
                         raise self.__exception_handler(msg)
 
-                    the_index = token.find(b'\\ ')
+                    the_index = token.find('\\ ')
                     if token is not None and the_index > -1:
                         msg = '\nInvalid RTF: token "\\ " not valid.\nError at line %d'\
                             % line_count
                         raise self.__exception_handler(msg)
-                    elif token[:1] == b"\\":
+                    elif token[:1] == "\\":
-                        try:
-                            token.decode('us-ascii')
-                        except UnicodeError as msg:
-                            msg = '\nInvalid RTF: Tokens not ascii encoded.\n%s\nError at line %d'\
-                                % (str(msg), line_count)
-                            raise self.__exception_handler(msg)
                         line = self.process_cw(token)
                         if line is not None:
                             write_obj.write(line)
@@ -816,10 +811,10 @@ class ProcessTokens:
                         for field in fields:
                             if not field:
                                 continue
-                            if field[0:1] == b'&':
+                            if field[0:1] == '&':
-                                write_obj.write(b'tx<ut<__________<%s\n' % field)
+                                write_obj.write('tx<ut<__________<%s\n' % field)
                             else:
-                                write_obj.write(b'tx<nu<__________<%s\n' % field)
+                                write_obj.write('tx<nu<__________<%s\n' % field)
 
         if not line_count:
             msg = '\nInvalid RTF: file appears to be empty.\n'

src/calibre/ebooks/rtf2xml/replace_illegals.py

@@ -15,6 +15,7 @@ import os
 from calibre.ebooks.rtf2xml import copy
 from calibre.utils.cleantext import clean_ascii_chars
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class ReplaceIllegals:
@@ -35,8 +36,8 @@ class ReplaceIllegals:
     def replace_illegals(self):
         """
         """
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
-            with open(self.__write_to, 'w') as write_obj:
+            with open_for_write(self.__write_to) as write_obj:
                 for line in read_obj:
                     write_obj.write(clean_ascii_chars(line))
         copy_obj = copy.Copy()

src/calibre/ebooks/rtf2xml/sections.py

@@ -14,6 +14,7 @@ import sys, os
 
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class Sections:
@@ -513,8 +514,8 @@ class Sections:
             If the state is body, send the line to the body method.
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/styles.py

@@ -13,6 +13,7 @@
 import sys, os
 from calibre.ebooks.rtf2xml import copy, border_parse
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 
 class Styles:
@@ -700,8 +701,8 @@ class Styles:
             info, and substitute the number with the name of the style.
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/table.py

@@ -13,6 +13,7 @@
 import sys, os
 from calibre.ebooks.rtf2xml import copy, border_parse
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 """
 States.
@@ -540,8 +541,8 @@ class Table:
             the state.
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/table_info.py

@@ -13,6 +13,7 @@
 import os
 from calibre.ebooks.rtf2xml import copy
 from calibre.ptempfile import better_mktemp
+from . import open_for_read, open_for_write
 
 # note to self. This is the first module in which I use tempfile. A good idea?
 """
@@ -53,8 +54,8 @@ class TableInfo:
     def insert_info(self):
         """
         """
-        read_obj = open(self.__file, 'r')
+        read_obj = open_for_read(self.__file)
-        self.__write_obj = open(self.__write_to, 'w')
+        self.__write_obj = open_for_write(self.__write_to)
         line_to_read = 1
         while line_to_read:
             line_to_read = read_obj.readline()

src/calibre/ebooks/rtf2xml/tokenize.py

@@ -16,6 +16,7 @@ from calibre.ebooks.rtf2xml import copy
 from calibre.utils.mreplace import MReplace
 from calibre.ptempfile import better_mktemp
 from polyglot.builtins import codepoint_to_chr, range, filter
+from . import open_for_read, open_for_write
 
 
 class Tokenize:
@@ -175,7 +176,7 @@ class Tokenize:
         , uses method self.sub_reg to make basic substitutions,\
         and process tokens by itself"""
         # read
-        with open(self.__file, 'r') as read_obj:
+        with open_for_read(self.__file) as read_obj:
             input_file = read_obj.read()
 
         # process simple replacements and split giving us a correct list
@@ -187,8 +188,8 @@ class Tokenize:
         tokens = list(filter(lambda x: len(x) > 0, tokens))
 
         # write
-        with open(self.__write_to, 'wb') as write_obj:
+        with open_for_write(self.__write_to) as write_obj:
-            write_obj.write('\n'.join(tokens).encode('utf-8'))
+            write_obj.write('\n'.join(tokens))
         # Move and copy
         copy_obj = copy.Copy(bug_handler=self.__bug_handler)
         if self.__copy: