Use unicode paths everywhere

src/calibre/__init__.py

@@ -419,7 +419,7 @@ class CurrentDir(object):
         self.cwd = None
 
     def __enter__(self, *args):
-        self.cwd = os.getcwd()
+        self.cwd = os.getcwdu()
         os.chdir(self.path)
         return self.cwd
 

src/calibre/customize/builtins.py

@@ -284,7 +284,7 @@ class OPFMetadataReader(MetadataReaderPlugin):
 
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.metadata.opf2 import OPF
-        return OPF(stream, os.getcwd()).to_book_metadata()
+        return OPF(stream, os.getcwdu()).to_book_metadata()
 
 class PDBMetadataReader(MetadataReaderPlugin):
 

src/calibre/debug.py

@@ -137,7 +137,7 @@ def add_simple_plugin(path_to_plugin):
     tdir = tempfile.mkdtemp()
     open(os.path.join(tdir, 'custom_plugin.py'),
             'wb').write(open(path_to_plugin, 'rb').read())
-    odir = os.getcwd()
+    odir = os.getcwdu()
     os.chdir(tdir)
     zf = zipfile.ZipFile('plugin.zip', 'w')
     zf.write('custom_plugin.py')

src/calibre/ebooks/conversion/plugins/azw4_input.py

@@ -22,6 +22,6 @@ class AZW4Input(InputFormatPlugin):
 
         header = PdbHeaderReader(stream)
         reader = Reader(header, stream, log, options)
-        opf = reader.extract_content(os.getcwd())
+        opf = reader.extract_content(os.getcwdu())
 
         return opf

src/calibre/ebooks/conversion/plugins/comic_input.py

@@ -173,7 +173,7 @@ class ComicInput(InputFormatPlugin):
         comics = []
         for i, x in enumerate(comics_):
             title, fname = x
-            cdir = 'comic_%d'%(i+1) if len(comics_) > 1 else '.'
+            cdir = u'comic_%d'%(i+1) if len(comics_) > 1 else u'.'
             cdir = os.path.abspath(cdir)
             if not os.path.exists(cdir):
                 os.makedirs(cdir)
@@ -187,7 +187,7 @@ class ComicInput(InputFormatPlugin):
 
         mi  = MetaInformation(os.path.basename(stream.name).rpartition('.')[0],
             [_('Unknown')])
-        opf = OPFCreator(os.path.abspath('.'), mi)
+        opf = OPFCreator(os.getcwdu(), mi)
         entries = []
 
         def href(x):
@@ -225,9 +225,9 @@ class ComicInput(InputFormatPlugin):
                                 _('Page')+' %d'%(i+1), play_order=po)
                         po += 1
         opf.set_toc(toc)
-        m, n = open('metadata.opf', 'wb'), open('toc.ncx', 'wb')
+        m, n = open(u'metadata.opf', 'wb'), open('toc.ncx', 'wb')
-        opf.render(m, n, 'toc.ncx')
+        opf.render(m, n, u'toc.ncx')
-        return os.path.abspath('metadata.opf')
+        return os.path.abspath(u'metadata.opf')
 
     def create_wrappers(self, pages):
         from calibre.ebooks.oeb.base import XHTML_NS
@@ -252,7 +252,7 @@ class ComicInput(InputFormatPlugin):
         dir = os.path.dirname(pages[0])
         for i, page in enumerate(pages):
             wrapper = WRAPPER%(XHTML_NS, i+1, os.path.basename(page), i+1)
-            page = os.path.join(dir, 'page_%d.xhtml'%(i+1))
+            page = os.path.join(dir, u'page_%d.xhtml'%(i+1))
             open(page, 'wb').write(wrapper)
             wrappers.append(page)
         return wrappers

src/calibre/ebooks/conversion/plugins/epub_input.py

@@ -138,7 +138,7 @@ class EPUBInput(InputFormatPlugin):
         from calibre.ebooks import DRMError
         from calibre.ebooks.metadata.opf2 import OPF
         zf = ZipFile(stream)
-        zf.extractall(os.getcwd())
+        zf.extractall(os.getcwdu())
         encfile = os.path.abspath(os.path.join('META-INF', 'encryption.xml'))
         opf = self.find_opf()
         if opf is None:
@@ -150,7 +150,7 @@ class EPUBInput(InputFormatPlugin):
         path = getattr(stream, 'name', 'stream')
 
         if opf is None:
-            raise ValueError('%s is not a valid EPUB file'%path)
+            raise ValueError('%s is not a valid EPUB file (could not find opf)'%path)
 
         opf = os.path.relpath(opf, os.getcwdu())
         parts = os.path.split(opf)
@@ -197,4 +197,4 @@ class EPUBInput(InputFormatPlugin):
         with open('content.opf', 'wb') as nopf:
             nopf.write(opf.render())
 
-        return os.path.abspath('content.opf')
+        return os.path.abspath(u'content.opf')

src/calibre/ebooks/conversion/plugins/epub_output.py

@@ -196,7 +196,7 @@ class EPUBOutput(OutputFormatPlugin):
             uuid = str(uuid4())
             oeb.metadata.add('identifier', uuid, scheme='uuid', id=uuid)
 
-        with TemporaryDirectory('_epub_output') as tdir:
+        with TemporaryDirectory(u'_epub_output') as tdir:
             from calibre.customize.ui import plugin_for_output_format
             metadata_xml = None
             extra_entries = []
@@ -204,7 +204,7 @@ class EPUBOutput(OutputFormatPlugin):
                 if self.opts.output_profile.epub_periodical_format == 'sony':
                     from calibre.ebooks.epub.periodical import sony_metadata
                     metadata_xml, atom_xml = sony_metadata(oeb)
-                    extra_entries = [('atom.xml', 'application/atom+xml', atom_xml)]
+                    extra_entries = [(u'atom.xml', 'application/atom+xml', atom_xml)]
             oeb_output = plugin_for_output_format('oeb')
             oeb_output.convert(oeb, tdir, input_plugin, opts, log)
             opf = [x for x in os.listdir(tdir) if x.endswith('.opf')][0]

src/calibre/ebooks/conversion/plugins/fb2_input.py

@@ -33,8 +33,6 @@ class FB2Input(InputFormatPlugin):
         ),
     ])
 
-
-
     def convert(self, stream, options, file_ext, log,
                 accelerators):
         from lxml import etree
@@ -92,8 +90,8 @@ class FB2Input(InputFormatPlugin):
             src = img.get('src')
             img.set('src', self.binary_map.get(src, src))
         index = transform.tostring(result)
-        open('index.xhtml', 'wb').write(index)
+        open(u'index.xhtml', 'wb').write(index)
-        open('inline-styles.css', 'wb').write(css)
+        open(u'inline-styles.css', 'wb').write(css)
         stream.seek(0)
         mi = get_metadata(stream, 'fb2')
         if not mi.title:
@@ -102,9 +100,9 @@ class FB2Input(InputFormatPlugin):
             mi.authors = [_('Unknown')]
         cpath = None
         if mi.cover_data and mi.cover_data[1]:
-            with open('fb2_cover_calibre_mi.jpg', 'wb') as f:
+            with open(u'fb2_cover_calibre_mi.jpg', 'wb') as f:
                 f.write(mi.cover_data[1])
-            cpath = os.path.abspath('fb2_cover_calibre_mi.jpg')
+            cpath = os.path.abspath(u'fb2_cover_calibre_mi.jpg')
         else:
             for img in doc.xpath('//f:coverpage/f:image', namespaces=NAMESPACES):
                 href = img.get('{%s}href'%XLINK_NS, img.get('href', None))
@@ -115,14 +113,14 @@ class FB2Input(InputFormatPlugin):
                     break
 
         opf = OPFCreator(os.getcwdu(), mi)
-        entries = [(f, guess_type(f)[0]) for f in os.listdir('.')]
+        entries = [(f, guess_type(f)[0]) for f in os.listdir(u'.')]
         opf.create_manifest(entries)
-        opf.create_spine(['index.xhtml'])
+        opf.create_spine([u'index.xhtml'])
         if cpath:
             opf.guide.set_cover(cpath)
-        with open('metadata.opf', 'wb') as f:
+        with open(u'metadata.opf', 'wb') as f:
             opf.render(f)
-        return os.path.join(os.getcwd(), 'metadata.opf')
+        return os.path.join(os.getcwdu(), u'metadata.opf')
 
     def extract_embedded_content(self, doc):
         self.binary_map = {}

src/calibre/ebooks/conversion/plugins/html_input.py

@@ -57,7 +57,7 @@ class HTMLInput(InputFormatPlugin):
     def convert(self, stream, opts, file_ext, log,
                 accelerators):
         self._is_case_sensitive = None
-        basedir = os.getcwd()
+        basedir = os.getcwdu()
         self.opts = opts
 
         fname = None

src/calibre/ebooks/conversion/plugins/htmlz_input.py

@@ -37,18 +37,18 @@ class HTMLZInput(InputFormatPlugin):
         index = u''
         multiple_html = False
         # Get a list of all top level files in the archive.
-        for x in os.listdir('.'):
+        for x in os.listdir(u'.'):
             if os.path.isfile(x):
                 top_levels.append(x)
         # Try to find an index. file.
         for x in top_levels:
-            if x.lower() in ('index.html', 'index.xhtml', 'index.htm'):
+            if x.lower() in (u'index.html', u'index.xhtml', u'index.htm'):
                 index = x
                 break
         # Look for multiple HTML files in the archive. We look at the
         # top level files only as only they matter in HTMLZ.
         for x in top_levels:
-            if os.path.splitext(x)[1].lower() in ('.html', '.xhtml', '.htm'):
+            if os.path.splitext(x)[1].lower() in (u'.html', u'.xhtml', u'.htm'):
                 # Set index to the first HTML file found if it's not
                 # called index.
                 if not index:
@@ -85,11 +85,11 @@ class HTMLZInput(InputFormatPlugin):
             setattr(options, opt.option.name, opt.recommended_value)
         options.input_encoding = 'utf-8'
         base = os.getcwdu()
-        fname = os.path.join(base, 'index.html')
+        fname = os.path.join(base, u'index.html')
         c = 0
         while os.path.exists(fname):
             c += 1
-            fname = 'index%d.html'%c
+            fname = u'index%d.html'%c
         htmlfile = open(fname, 'wb')
         with htmlfile:
             htmlfile.write(html.encode('utf-8'))
@@ -111,16 +111,16 @@ class HTMLZInput(InputFormatPlugin):
         cover_path = None
         opf = None
         for x in top_levels:
-            if os.path.splitext(x)[1].lower() in ('.opf'):
+            if os.path.splitext(x)[1].lower() == u'.opf':
                 opf = x
                 break
         if opf:
-            opf = OPF(opf, basedir=os.getcwd())
+            opf = OPF(opf, basedir=os.getcwdu())
             cover_path = opf.raster_cover
         # Set the cover.
         if cover_path:
             cdata = None
-            with open(os.path.join(os.getcwd(), cover_path), 'rb') as cf:
+            with open(os.path.join(os.getcwdu(), cover_path), 'rb') as cf:
                 cdata = cf.read()
             cover_name = os.path.basename(cover_path)
             id, href = oeb.manifest.generate('cover', cover_name)

src/calibre/ebooks/conversion/plugins/htmlz_output.py

@@ -55,30 +55,30 @@ class HTMLZOutput(OutputFormatPlugin):
         else:
             from calibre.ebooks.htmlz.oeb2html import OEB2HTMLClassCSSizer as OEB2HTMLizer
 
-        with TemporaryDirectory('_htmlz_output') as tdir:
+        with TemporaryDirectory(u'_htmlz_output') as tdir:
             htmlizer = OEB2HTMLizer(log)
             html = htmlizer.oeb2html(oeb_book, opts)
 
-            with open(os.path.join(tdir, 'index.html'), 'wb') as tf:
+            with open(os.path.join(tdir, u'index.html'), 'wb') as tf:
                 tf.write(html)
 
             # CSS
             if opts.htmlz_css_type == 'class' and opts.htmlz_class_style == 'external':
-                with open(os.path.join(tdir, 'style.css'), 'wb') as tf:
+                with open(os.path.join(tdir, u'style.css'), 'wb') as tf:
                     tf.write(htmlizer.get_css(oeb_book))
 
             # Images
             images = htmlizer.images
             if images:
-                if not os.path.exists(os.path.join(tdir, 'images')):
+                if not os.path.exists(os.path.join(tdir, u'images')):
-                    os.makedirs(os.path.join(tdir, 'images'))
+                    os.makedirs(os.path.join(tdir, u'images'))
                 for item in oeb_book.manifest:
                     if item.media_type in OEB_IMAGES and item.href in images:
                         if item.media_type == SVG_MIME:
                             data = unicode(etree.tostring(item.data, encoding=unicode))
                         else:
                             data = item.data
-                        fname = os.path.join(tdir, 'images', images[item.href])
+                        fname = os.path.join(tdir, u'images', images[item.href])
                         with open(fname, 'wb') as img:
                             img.write(data)
 
@@ -91,7 +91,7 @@ class HTMLZOutput(OutputFormatPlugin):
                     cover_data = oeb_book.guide[term].item.data
                 if cover_data:
                     from calibre.utils.magick.draw import save_cover_data_to
-                    cover_path = os.path.join(tdir, 'cover.jpg')
+                    cover_path = os.path.join(tdir, u'cover.jpg')
                     with open(cover_path, 'w') as cf:
                         cf.write('')
                     save_cover_data_to(cover_data, cover_path)
@@ -100,11 +100,11 @@ class HTMLZOutput(OutputFormatPlugin):
                 traceback.print_exc()
 
             # Metadata
-            with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf:
+            with open(os.path.join(tdir, u'metadata.opf'), 'wb') as mdataf:
                 opf = OPF(StringIO(etree.tostring(oeb_book.metadata.to_opf1())))
                 mi = opf.to_book_metadata()
                 if cover_path:
-                    mi.cover = 'cover.jpg'
+                    mi.cover = u'cover.jpg'
                 mdataf.write(metadata_to_opf(mi))
 
             htmlz = ZipFile(output_path, 'w')

src/calibre/ebooks/conversion/plugins/lrf_input.py

@@ -28,7 +28,7 @@ class LRFInput(InputFormatPlugin):
         d.parse()
         xml = d.to_xml(write_files=True)
         if options.verbose > 2:
-            open('lrs.xml', 'wb').write(xml.encode('utf-8'))
+            open(u'lrs.xml', 'wb').write(xml.encode('utf-8'))
         parser = etree.XMLParser(no_network=True, huge_tree=True)
         try:
             doc = etree.fromstring(xml, parser=parser)
@@ -84,4 +84,4 @@ class LRFInput(InputFormatPlugin):
         with open('content.opf', 'wb') as f:
             f.write(result)
         styles.write()
-        return os.path.abspath('content.opf')
+        return os.path.abspath(u'content.opf')

src/calibre/ebooks/conversion/plugins/lrf_output.py

@@ -182,7 +182,7 @@ class LRFOutput(OutputFormatPlugin):
         self.flatten_toc()
 
         from calibre.ptempfile import TemporaryDirectory
-        with TemporaryDirectory('_lrf_output') as tdir:
+        with TemporaryDirectory(u'_lrf_output') as tdir:
             from calibre.customize.ui import plugin_for_output_format
             oeb_output = plugin_for_output_format('oeb')
             oeb_output.convert(oeb, tdir, input_plugin, opts, log)

src/calibre/ebooks/conversion/plugins/mobi_input.py

@@ -20,17 +20,17 @@ class MOBIInput(InputFormatPlugin):
         try:
             mr = MobiReader(stream, log, options.input_encoding,
                         options.debug_pipeline)
-            mr.extract_content('.', parse_cache)
+            mr.extract_content(u'.', parse_cache)
         except:
             mr = MobiReader(stream, log, options.input_encoding,
                         options.debug_pipeline, try_extra_data_fix=True)
-            mr.extract_content('.', parse_cache)
+            mr.extract_content(u'.', parse_cache)
 
         raw = parse_cache.pop('calibre_raw_mobi_markup', False)
         if raw:
             if isinstance(raw, unicode):
                 raw = raw.encode('utf-8')
-            open('debug-raw.html', 'wb').write(raw)
+            open(u'debug-raw.html', 'wb').write(raw)
         for f, root in parse_cache.items():
             with open(f, 'wb') as q:
                 q.write(html.tostring(root, encoding='utf-8', method='xml',

src/calibre/ebooks/conversion/plugins/pdb_input.py

@@ -29,6 +29,6 @@ class PDBInput(InputFormatPlugin):
         log.debug('Detected ebook format as: %s with identity: %s' % (IDENTITY_TO_NAME[header.ident], header.ident))
 
         reader = Reader(header, stream, log, options)
-        opf = reader.extract_content(os.getcwd())
+        opf = reader.extract_content(os.getcwdu())
 
         return opf

src/calibre/ebooks/conversion/plugins/pdf_input.py

@@ -35,9 +35,9 @@ class PDFInput(InputFormatPlugin):
         if pdfreflow_err:
             raise RuntimeError('Failed to load pdfreflow: ' + pdfreflow_err)
         pdfreflow.reflow(stream.read(), 1, -1)
-        xml = clean_ascii_chars(open('index.xml', 'rb').read())
+        xml = clean_ascii_chars(open(u'index.xml', 'rb').read())
         PDFDocument(xml, self.opts, self.log)
-        return os.path.join(os.getcwd(), 'metadata.opf')
+        return os.path.join(os.getcwdu(), u'metadata.opf')
 
 
     def convert(self, stream, options, file_ext, log,
@@ -50,25 +50,25 @@ class PDFInput(InputFormatPlugin):
         self.opts, self.log = options, log
         if options.new_pdf_engine:
             return self.convert_new(stream, accelerators)
-        pdftohtml(os.getcwd(), stream.name, options.no_images)
+        pdftohtml(os.getcwdu(), stream.name, options.no_images)
 
         from calibre.ebooks.metadata.meta import get_metadata
         log.debug('Retrieving document metadata...')
         mi = get_metadata(stream, 'pdf')
-        opf = OPFCreator(os.getcwd(), mi)
+        opf = OPFCreator(os.getcwdu(), mi)
 
-        manifest = [('index.html', None)]
+        manifest = [(u'index.html', None)]
 
-        images = os.listdir(os.getcwd())
+        images = os.listdir(os.getcwdu())
         images.remove('index.html')
         for i in images:
             manifest.append((i, None))
         log.debug('Generating manifest...')
         opf.create_manifest(manifest)
 
-        opf.create_spine(['index.html'])
+        opf.create_spine([u'index.html'])
         log.debug('Rendering manifest...')
-        with open('metadata.opf', 'wb') as opffile:
+        with open(u'metadata.opf', 'wb') as opffile:
             opf.render(opffile)
 
-        return os.path.join(os.getcwd(), 'metadata.opf')
+        return os.path.join(os.getcwdu(), u'metadata.opf')

src/calibre/ebooks/conversion/plugins/pml_input.py

@@ -69,12 +69,12 @@ class PMLInput(InputFormatPlugin):
                 imgs = glob.glob(os.path.join(tdir, os.path.splitext(os.path.basename(stream.name))[0] + '_img', '*.png'))
         # No images in Dropbook location try generic images directory
         if not imgs:
-            imgs = glob.glob(os.path.join(os.path.join(tdir, 'images'), '*.png'))
+            imgs = glob.glob(os.path.join(os.path.join(tdir, u'images'), u'*.png'))
         if imgs:
-            os.makedirs(os.path.join(os.getcwd(), 'images'))
+            os.makedirs(os.path.join(os.getcwdu(), u'images'))
         for img in imgs:
             pimg_name = os.path.basename(img)
-            pimg_path = os.path.join(os.getcwd(), 'images', pimg_name)
+            pimg_path = os.path.join(os.getcwdu(), 'images', pimg_name)
 
             images.append('images/' + pimg_name)
 
@@ -94,14 +94,14 @@ class PMLInput(InputFormatPlugin):
 
         if file_ext == 'pmlz':
             log.debug('De-compressing content to temporary directory...')
-            with TemporaryDirectory('_unpmlz') as tdir:
+            with TemporaryDirectory(u'_unpmlz') as tdir:
                 zf = ZipFile(stream)
                 zf.extractall(tdir)
 
-                pmls = glob.glob(os.path.join(tdir, '*.pml'))
+                pmls = glob.glob(os.path.join(tdir, u'*.pml'))
                 for pml in pmls:
                     html_name = os.path.splitext(os.path.basename(pml))[0]+'.html'
-                    html_path = os.path.join(os.getcwd(), html_name)
+                    html_path = os.path.join(os.getcwdu(), html_name)
 
                     pages.append(html_name)
                     log.debug('Processing PML item %s...' % pml)
@@ -109,8 +109,8 @@ class PMLInput(InputFormatPlugin):
                     toc += ttoc
                 images = self.get_images(stream, tdir, True)
         else:
-            toc = self.process_pml(stream, 'index.html')
+            toc = self.process_pml(stream, u'index.html')
-            pages.append('index.html')
+            pages.append(u'index.html')
 
             if hasattr(stream, 'name'):
                 images = self.get_images(stream, os.path.abspath(os.path.dirname(stream.name)))
@@ -126,14 +126,14 @@ class PMLInput(InputFormatPlugin):
         log.debug('Reading metadata from input file...')
         mi = get_metadata(stream, 'pml')
         if 'images/cover.png' in images:
-            mi.cover = 'images/cover.png'
+            mi.cover = u'images/cover.png'
-        opf = OPFCreator(os.getcwd(), mi)
+        opf = OPFCreator(os.getcwdu(), mi)
         log.debug('Generating manifest...')
         opf.create_manifest(manifest_items)
         opf.create_spine(pages)
         opf.set_toc(toc)
-        with open('metadata.opf', 'wb') as opffile:
+        with open(u'metadata.opf', 'wb') as opffile:
-            with open('toc.ncx', 'wb') as tocfile:
+            with open(u'toc.ncx', 'wb') as tocfile:
-                opf.render(opffile, tocfile, 'toc.ncx')
+                opf.render(opffile, tocfile, u'toc.ncx')
 
-        return os.path.join(os.getcwd(), 'metadata.opf')
+        return os.path.join(os.getcwdu(), u'metadata.opf')

src/calibre/ebooks/conversion/plugins/rb_input.py

@@ -20,6 +20,6 @@ class RBInput(InputFormatPlugin):
         from calibre.ebooks.rb.reader import Reader
 
         reader = Reader(stream, log, options.input_encoding)
-        opf = reader.extract_content(os.getcwd())
+        opf = reader.extract_content(os.getcwdu())
 
         return opf

src/calibre/ebooks/conversion/plugins/recipe_input.py

@@ -58,7 +58,7 @@ class RecipeInput(InputFormatPlugin):
             zf = ZipFile(recipe_or_file, 'r')
             zf.extractall()
             zf.close()
-            self.recipe_source = open('download.recipe', 'rb').read()
+            self.recipe_source = open(u'download.recipe', 'rb').read()
             recipe = compile_recipe(self.recipe_source)
             recipe.needs_subscription = False
             self.recipe_object = recipe(opts, log, self.report_progress)
@@ -108,11 +108,11 @@ class RecipeInput(InputFormatPlugin):
         for key, val in self.recipe_object.conversion_options.items():
             setattr(opts, key, val)
 
-        for f in os.listdir('.'):
+        for f in os.listdir(u'.'):
             if f.endswith('.opf'):
                 return os.path.abspath(f)
 
-        for f in walk('.'):
+        for f in walk(u'.'):
             if f.endswith('.opf'):
                 return os.path.abspath(f)
 

src/calibre/ebooks/conversion/plugins/rtf_input.py

@@ -47,12 +47,12 @@ class RTFInput(InputFormatPlugin):
 
     def generate_xml(self, stream):
         from calibre.ebooks.rtf2xml.ParseRtf import ParseRtf
-        ofile = 'dataxml.xml'
+        ofile = u'dataxml.xml'
         run_lev, debug_dir, indent_out = 1, None, 0
         if getattr(self.opts, 'debug_pipeline', None) is not None:
             try:
-                os.mkdir('rtfdebug')
+                os.mkdir(u'rtfdebug')
-                debug_dir = 'rtfdebug'
+                debug_dir = u'rtfdebug'
                 run_lev = 4
                 indent_out = 1
                 self.log('Running RTFParser in debug mode')
@@ -124,7 +124,7 @@ class RTFInput(InputFormatPlugin):
             if fmt is None:
                 fmt = 'wmf'
             count += 1
-            name = '%04d.%s' % (count, fmt)
+            name = u'%04d.%s' % (count, fmt)
             with open(name, 'wb') as f:
                 f.write(data)
             imap[count] = name
@@ -201,7 +201,7 @@ class RTFInput(InputFormatPlugin):
         for cls, val in border_styles.iteritems():
             css += '\n\n.%s {\n%s\n}'%(cls, val)
 
-        with open('styles.css', 'ab') as f:
+        with open(u'styles.css', 'ab') as f:
             f.write(css)
 
     def convert_borders(self, doc):
@@ -271,7 +271,7 @@ class RTFInput(InputFormatPlugin):
         extensions = { ('calibre', 'inline-class') : inline_class }
         transform = etree.XSLT(styledoc, extensions=extensions)
         result = transform(doc)
-        html = 'index.xhtml'
+        html = u'index.xhtml'
         with open(html, 'wb') as f:
             res = transform.tostring(result)
             # res = res[:100].replace('xmlns:html', 'xmlns') + res[100:]
@@ -289,10 +289,10 @@ class RTFInput(InputFormatPlugin):
             mi.title = _('Unknown')
         if not mi.authors:
             mi.authors = [_('Unknown')]
-        opf = OPFCreator(os.getcwd(), mi)
+        opf = OPFCreator(os.getcwdu(), mi)
-        opf.create_manifest([('index.xhtml', None)])
+        opf.create_manifest([(u'index.xhtml', None)])
-        opf.create_spine(['index.xhtml'])
+        opf.create_spine([u'index.xhtml'])
-        opf.render(open('metadata.opf', 'wb'))
+        opf.render(open(u'metadata.opf', 'wb'))
-        return os.path.abspath('metadata.opf')
+        return os.path.abspath(u'metadata.opf')
 
 

src/calibre/ebooks/lrf/html/convert_from.py

@@ -374,13 +374,12 @@ class HTMLConverter(object):
             else:
                 self.css[selector] = self.override_css[selector]
 
-        upath = path.encode(sys.getfilesystemencoding()) if isinstance(path, unicode) else path
+        self.file_name = os.path.basename(path)
-        self.file_name = os.path.basename(upath.decode(sys.getfilesystemencoding()))
+        self.log.info(_('Processing %s')%( path if self.verbose else self.file_name))
-        self.log.info(_('Processing %s')%( repr(upath) if self.verbose else repr(self.file_name)))
 
-        if not os.path.exists(upath):
+        if not os.path.exists(path):
-            upath = upath.replace('&', '%26') #convertlit replaces & with %26 in file names
+            path = path.replace('&', '%26') #convertlit replaces & with %26 in file names
-        f = open(upath, 'rb')
+        f = open(path, 'rb')
         raw = f.read()
         if self.pdftohtml: # Bug in pdftohtml that causes it to output invalid UTF-8 files
             raw = raw.decode('utf-8', 'ignore')
@@ -1938,7 +1937,7 @@ def process_file(path, options, logger):
     if not oname:
         suffix = '.lrs' if options.lrs else '.lrf'
         name = os.path.splitext(os.path.basename(path))[0] + suffix
-        oname = os.path.join(os.getcwd(), name)
+        oname = os.path.join(os.getcwdu(), name)
     oname = os.path.abspath(os.path.expanduser(oname))
     conv.writeto(oname, lrs=options.lrs)
     conv.cleanup()

src/calibre/ebooks/metadata/__init__.py

@@ -187,7 +187,7 @@ class Resource(object):
 
     '''
 
-    def __init__(self, href_or_path, basedir=os.getcwd(), is_path=True):
+    def __init__(self, href_or_path, basedir=os.getcwdu(), is_path=True):
         self._href = None
         self._basedir = basedir
         self.path = None
@@ -230,7 +230,7 @@ class Resource(object):
             if self._basedir:
                 basedir = self._basedir
             else:
-                basedir = os.getcwd()
+                basedir = os.getcwdu()
         if self.path is None:
             return self._href
         f = self.fragment.encode('utf-8') if isinstance(self.fragment, unicode) else self.fragment

src/calibre/ebooks/metadata/lit.py

@@ -14,7 +14,7 @@ def get_metadata(stream):
     litfile = LitContainer(stream, Log())
     src = litfile.get_metadata().encode('utf-8')
     litfile = litfile._litfile
-    opf = OPF(cStringIO.StringIO(src), os.getcwd())
+    opf = OPF(cStringIO.StringIO(src), os.getcwdu())
     mi = opf.to_book_metadata()
     covers = []
     for item in opf.iterguide():

src/calibre/ebooks/metadata/meta.py

@@ -199,7 +199,7 @@ def metadata_from_filename(name, pat=None):
 def opf_metadata(opfpath):
     if hasattr(opfpath, 'read'):
         f = opfpath
-        opfpath = getattr(f, 'name', os.getcwd())
+        opfpath = getattr(f, 'name', os.getcwdu())
     else:
         f = open(opfpath, 'rb')
     try:

src/calibre/ebooks/metadata/opf2.py

@@ -36,7 +36,7 @@ class Resource(object): # {{{
     :method:`href`
     '''
 
-    def __init__(self, href_or_path, basedir=os.getcwd(), is_path=True):
+    def __init__(self, href_or_path, basedir=os.getcwdu(), is_path=True):
         self.orig = href_or_path
         self._href = None
         self._basedir = basedir
@@ -81,7 +81,7 @@ class Resource(object): # {{{
             if self._basedir:
                 basedir = self._basedir
             else:
-                basedir = os.getcwd()
+                basedir = os.getcwdu()
         if self.path is None:
             return self._href
         f = self.fragment.encode('utf-8') if isinstance(self.fragment, unicode) else self.fragment
@@ -1487,7 +1487,7 @@ class OPFTest(unittest.TestCase):
 </package>
 '''
         )
-        self.opf = OPF(self.stream, os.getcwd())
+        self.opf = OPF(self.stream, os.getcwdu())
 
     def testReading(self, opf=None):
         if opf is None:
@@ -1518,11 +1518,11 @@ class OPFTest(unittest.TestCase):
         self.opf.render()
 
     def testCreator(self):
-        opf = OPFCreator(os.getcwd(), self.opf)
+        opf = OPFCreator(os.getcwdu(), self.opf)
         buf = cStringIO.StringIO()
         opf.render(buf)
         raw = buf.getvalue()
-        self.testReading(opf=OPF(cStringIO.StringIO(raw), os.getcwd()))
+        self.testReading(opf=OPF(cStringIO.StringIO(raw), os.getcwdu()))
 
     def testSmartUpdate(self):
         self.opf.smart_update(MetaInformation(self.opf))
@@ -1547,7 +1547,7 @@ def test_user_metadata():
         }
     mi.set_all_user_metadata(um)
     raw = metadata_to_opf(mi)
-    opfc = OPFCreator(os.getcwd(), other=mi)
+    opfc = OPFCreator(os.getcwdu(), other=mi)
     out = StringIO()
     opfc.render(out)
     raw2 = out.getvalue()

src/calibre/ebooks/metadata/toc.py

@@ -29,9 +29,9 @@ C = ElementMaker(namespace=CALIBRE_NS, nsmap=NSMAP)
 
 class TOC(list):
 
-    def __init__(self, href=None, fragment=None, text=None, parent=None, play_order=0,
+    def __init__(self, href=None, fragment=None, text=None, parent=None,
-                 base_path=os.getcwd(), type='unknown', author=None,
+            play_order=0, base_path=os.getcwdu(), type='unknown', author=None,
-                 description=None, toc_thumbnail=None):
+            description=None, toc_thumbnail=None):
         self.href = href
         self.fragment = fragment
         if not self.fragment:
@@ -272,7 +272,7 @@ class TOC(list):
                 elem.append(C.meta(desc, name='description'))
             idx = getattr(np, 'toc_thumbnail', None)
             if idx:
-                elem.append(C.meta(idx, name='toc_thumbnail'))   
+                elem.append(C.meta(idx, name='toc_thumbnail'))
             parent.append(elem)
             for np2 in np:
                 navpoint(elem, np2)

src/calibre/ebooks/metadata/zip.py

@@ -44,7 +44,7 @@ def zip_opf_metadata(opfpath, zf):
     from calibre.ebooks.metadata.opf2 import OPF
     if hasattr(opfpath, 'read'):
         f = opfpath
-        opfpath = getattr(f, 'name', os.getcwd())
+        opfpath = getattr(f, 'name', os.getcwdu())
     else:
         f = open(opfpath, 'rb')
     opf = OPF(f, os.path.dirname(opfpath))

src/calibre/ebooks/mobi/reader.py

@@ -785,11 +785,11 @@ class MobiReader(object):
             mi = MetaInformation(self.book_header.title, [_('Unknown')])
         opf = OPFCreator(os.path.dirname(htmlfile), mi)
         if hasattr(self.book_header.exth, 'cover_offset'):
-            opf.cover = 'images/%05d.jpg' % (self.book_header.exth.cover_offset + 1)
+            opf.cover = u'images/%05d.jpg' % (self.book_header.exth.cover_offset + 1)
         elif mi.cover is not None:
             opf.cover = mi.cover
         else:
-            opf.cover = 'images/%05d.jpg' % 1
+            opf.cover = u'images/%05d.jpg' % 1
             if not os.path.exists(os.path.join(os.path.dirname(htmlfile),
                 * opf.cover.split('/'))):
                 opf.cover = None
@@ -799,7 +799,7 @@ class MobiReader(object):
         if cover is not None:
             cover = cover.replace('/', os.sep)
             if os.path.exists(cover):
-                ncover = 'images'+os.sep+'calibre_cover.jpg'
+                ncover = u'images'+os.sep+u'calibre_cover.jpg'
                 if os.path.exists(ncover):
                     os.remove(ncover)
                 shutil.copyfile(cover, ncover)
@@ -807,7 +807,7 @@ class MobiReader(object):
                 opf.cover = ncover.replace(os.sep, '/')
 
         manifest = [(htmlfile, 'application/xhtml+xml'),
-            (os.path.abspath('styles.css'), 'text/css')]
+            (os.path.abspath(u'styles.css'), 'text/css')]
         bp = os.path.dirname(htmlfile)
         added = set([])
         for i in getattr(self, 'image_names', []):

src/calibre/ebooks/oeb/base.py

@@ -774,6 +774,8 @@ class Manifest(object):
 
         def __init__(self, oeb, id, href, media_type,
                      fallback=None, loader=str, data=None):
+            if href:
+                href = unicode(href)
             self.oeb = oeb
             self.id = id
             self.href = self.path = urlnormalize(href)
@@ -1106,7 +1108,7 @@ class Manifest(object):
             while href.lower() in lhrefs:
                 href = base + str(index) + ext
                 index += 1
-        return id, href
+        return id, unicode(href)
 
     def __iter__(self):
         for item in self.items:
@@ -1320,6 +1322,8 @@ class Guide(object):
 
     def add(self, type, title, href):
         """Add a new reference to the `Guide`."""
+        if href:
+            href = unicode(href)
         ref = self.Reference(self.oeb, type, title, href)
         self.refs[type] = ref
         return ref

src/calibre/ebooks/oeb/transforms/cover.py

@@ -104,7 +104,7 @@ class CoverManager(object):
             img_data = calibre_cover(title, authors_to_string(authors),
                     series_string=series_string)
             id, href = self.oeb.manifest.generate('cover',
-                    'cover_image.jpg')
+                    u'cover_image.jpg')
             item = self.oeb.manifest.add(id, href, guess_type('t.jpg')[0],
                         data=img_data)
             m.clear('cover')
@@ -154,7 +154,7 @@ class CoverManager(object):
                 templ = self.non_svg_template if self.no_svg_cover \
                         else self.svg_template
                 tp = templ%unquote(href)
-                id, href = m.generate('titlepage', 'titlepage.xhtml')
+                id, href = m.generate('titlepage', u'titlepage.xhtml')
                 item = m.add(id, href, guess_type('t.xhtml')[0],
                         data=etree.fromstring(tp))
         else:

src/calibre/ebooks/oeb/transforms/split.py

@@ -188,7 +188,7 @@ class FlowSplitter(object):
         self.csp_counter    = 0
 
         base, ext = os.path.splitext(self.base)
-        self.base = base.replace('%', '%%')+'_split_%.3d'+ext
+        self.base = base.replace('%', '%%')+u'_split_%.3d'+ext
 
         self.trees = [self.item.data.getroottree()]
         self.splitting_on_page_breaks = True

src/calibre/ebooks/pdf/pdftohtml.py

@@ -39,7 +39,7 @@ def pdftohtml(output_dir, pdf_path, no_images):
         raise ConversionError('Cannot read from ' + pdf_path)
 
     with CurrentDir(output_dir):
-        index = os.path.join(os.getcwd(), 'index.html')
+        index = os.path.join(os.getcwdu(), 'index.html')
         # This is neccessary as pdftohtml doesn't always (linux) respect absolute paths
         pdf_path = os.path.abspath(pdf_path)
         cmd = [PDFTOHTML, '-enc', 'UTF-8', '-noframes', '-p', '-nomerge', '-nodrm', '-q', pdf_path, os.path.basename(index)]

src/calibre/ebooks/pdf/writer.py

@@ -126,7 +126,7 @@ class PDFWriter(QObject): # {{{
                 type=Qt.QueuedConnection)
         self.render_queue = []
         self.combine_queue = []
-        self.tmp_path = PersistentTemporaryDirectory('_pdf_output_parts')
+        self.tmp_path = PersistentTemporaryDirectory(u'_pdf_output_parts')
 
         self.opts = opts
         self.size = get_printer_page_size(opts)
@@ -152,7 +152,7 @@ class PDFWriter(QObject): # {{{
             self._render_next()
 
     def _render_next(self):
-        item = str(self.render_queue.pop(0))
+        item = unicode(self.render_queue.pop(0))
         self.combine_queue.append(os.path.join(self.tmp_path, '%i.pdf' % (len(self.combine_queue) + 1)))
 
         self.logger.debug('Processing %s...' % item)

src/calibre/library/cli.py

@@ -443,7 +443,7 @@ def do_show_metadata(db, id, as_opf):
         raise ValueError('Id #%d is not present in database.'%id)
     mi = db.get_metadata(id, index_is_id=True)
     if as_opf:
-        mi = OPFCreator(os.getcwd(), mi)
+        mi = OPFCreator(os.getcwdu(), mi)
         mi.render(sys.stdout)
     else:
         prints(unicode(mi))

src/calibre/libunrar.py

@@ -170,7 +170,7 @@ def extract(path, dir):
     """
     open_archive_data = RAROpenArchiveDataEx(ArcName=path, OpenMode=RAR_OM_EXTRACT, CmtBuf=None)
     arc_data = _libunrar.RAROpenArchiveEx(byref(open_archive_data))
-    cwd = os.getcwd()
+    cwd = os.getcwdu()
     if not os.path.isdir( dir ):
         os.mkdir( dir )
     os.chdir( dir )

src/calibre/ptempfile.py

@@ -67,7 +67,7 @@ def base_dir():
 
         # Tell the tempfile module to in future always use our temp dir
         # This also means that it will return unicode paths, instead of
-        # bytestrings
+        # bytestrings. This is particularly important on windows.
         tempfile.tempdir = _base_dir
 
     return _base_dir

src/calibre/utils/ipc/server.py

@@ -117,8 +117,8 @@ class Server(Thread):
         with self._worker_launch_lock:
             self.launched_worker_count += 1
             id = self.launched_worker_count
-        fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_'%(self.id, id),
+        fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_'%(self.id, id),
-                dir=base_dir(), suffix='.pickle')
+                dir=base_dir(), suffix=u'.pickle')
         os.close(fd)
         if redirect_output is None:
             redirect_output = not gui
@@ -127,7 +127,7 @@ class Server(Thread):
                 'CALIBRE_WORKER_ADDRESS' :
                     hexlify(cPickle.dumps(self.listener.address, -1)),
                 'CALIBRE_WORKER_KEY' : hexlify(self.auth_key),
-                'CALIBRE_WORKER_RESULT' : hexlify(rfile),
+                'CALIBRE_WORKER_RESULT' : hexlify(rfile.encode('utf-8')),
               }
         for i in range(2):
             # Try launch twice as occasionally on OS X

src/calibre/utils/ipc/worker.py

@@ -176,7 +176,7 @@ def main():
         return
     address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
     key     = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
-    resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT'])
+    resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT']).decode('utf-8')
     with closing(Client(address, authkey=key)) as conn:
         name, args, kwargs, desc = conn.recv()
         if desc:

src/calibre/utils/zipfile.py

@@ -1066,7 +1066,7 @@ class ZipFile:
             member = self.getinfo(member)
 
         if path is None:
-            path = os.getcwd()
+            path = os.getcwdu()
 
         return self._extract_member(member, path, pwd)
 
@@ -1298,7 +1298,7 @@ class ZipFile:
         '''
         if prefix:
             self.writestr(prefix+'/', '', 0755)
-        cwd = os.path.abspath(os.getcwd())
+        cwd = os.path.abspath(os.getcwdu())
         try:
             os.chdir(path)
             fp = (prefix + ('/' if prefix else '')).replace('//', '/')

src/calibre/web/feeds/recipes/__init__.py

@@ -42,7 +42,7 @@ def compile_recipe(src):
             'AutomaticNewsRecipe':AutomaticNewsRecipe,
             'time':time, 're':re,
             'BeautifulSoup':BeautifulSoup
-        }
+    }
     exec src in namespace
 
     for x in namespace.itervalues():

src/calibre/web/fetch/simple.py

@@ -14,6 +14,7 @@ from PIL import Image
 from cStringIO import StringIO
 
 from calibre import browser, relpath, unicode_path
+from calibre.constants import filesystem_encoding
 from calibre.utils.filenames import ascii_filename
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
 from calibre.ebooks.chardet import xml_to_unicode
@@ -101,7 +102,11 @@ class RecursiveFetcher(object):
     default_timeout = socket.getdefaulttimeout() # Needed here as it is used in __del__
 
     def __init__(self, options, log, image_map={}, css_map={}, job_info=None):
-        self.base_dir = os.path.abspath(os.path.expanduser(options.dir))
+        bd = options.dir
+        if not isinstance(bd, unicode):
+            bd = bd.decode(filesystem_encoding)
+
+        self.base_dir = os.path.abspath(os.path.expanduser(bd))
         if not os.path.exists(self.base_dir):
             os.makedirs(self.base_dir)
         self.log = log