From cb29d11996adb33dcdaa3c1f0b64ac7cc92bd980 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 9 Sep 2019 17:42:46 -0400 Subject: [PATCH 1/8] sorted always returns a list --- src/calibre/library/database2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 843e6ca450..af0639ed4b 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -427,7 +427,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): custom_map = self.custom_columns_in_meta() # custom col labels are numbers (the id in the custom_columns table) - custom_cols = list(sorted(custom_map.keys())) + custom_cols = sorted(custom_map.keys()) lines.extend([custom_map[x] for x in custom_cols]) self.FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'timestamp':3, From 9320e2fe2207dc7dea7ac27c8d0829c5d8f57ed0 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 9 Sep 2019 17:43:28 -0400 Subject: [PATCH 2/8] where possible, open files using context managers --- src/calibre/library/catalogs/epub_mobi_builder.py | 3 ++- src/calibre/library/database2.py | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index e6e4ebfb22..b629718bcf 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -2961,7 +2961,8 @@ class CatalogBuilder(object): left = max(int((MI_WIDTH - width) / 2.), 0) top = max(int((MI_HEIGHT - height) / 2.), 0) draw.text((left, top), text, fill=(0, 0, 0), font=font) - img.save(open(out_path, 'wb'), 'GIF') + with open(out_path, 'wb') as f: + img.save(f, 'GIF') def generate_ncx_header(self): """ Generate the basic NCX file. diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index af0639ed4b..1b1fbfa1f0 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -3517,10 +3517,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.set_metadata(id, mi, commit=True, ignore_errors=True) npath = self.run_import_plugins(path, format) format = os.path.splitext(npath)[-1].lower().replace('.', '').upper() - stream = lopen(npath, 'rb') - format = check_ebook_format(stream, format) - self.add_format(id, format, stream, index_is_id=True) - stream.close() + with lopen(npath, 'rb') as stream: + format = check_ebook_format(stream, format) + self.add_format(id, format, stream, index_is_id=True) postimport.append((id, format)) self.conn.commit() self.data.refresh_ids(self, ids) # Needed to update format list and size From 1c5ea10a347c3f696cdc53bb4806e7327f181b9e Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 9 Sep 2019 17:44:04 -0400 Subject: [PATCH 3/8] list comprehension is faster than the function call overhead of list() --- src/calibre/library/database2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 1b1fbfa1f0..c31de99a00 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -3526,9 +3526,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for book_id, fmt in postimport: run_plugins_on_postimport(self, book_id, fmt) if duplicates: - paths = list(duplicate[0] for duplicate in duplicates) - formats = list(duplicate[1] for duplicate in duplicates) - metadata = list(duplicate[2] for duplicate in duplicates) + paths = [duplicate[0] for duplicate in duplicates] + formats = [duplicate[1] for duplicate in duplicates] + metadata = [duplicate[2] for duplicate in duplicates] return (paths, formats, metadata), (ids if return_ids else len(ids)) return None, (ids if return_ids else len(ids)) From 670c5ebe6ab7ca7d4e8e5c43aab849a54712aa47 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 9 Sep 2019 20:17:24 -0400 Subject: [PATCH 4/8] misc cleanup --- src/calibre/library/catalogs/bibtex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/catalogs/bibtex.py b/src/calibre/library/catalogs/bibtex.py index 93ea6db277..f098a2a642 100644 --- a/src/calibre/library/catalogs/bibtex.py +++ b/src/calibre/library/catalogs/bibtex.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' import re, codecs, os, numbers from collections import namedtuple -from calibre import (strftime) +from calibre import strftime from calibre.customize import CatalogPlugin from calibre.library.catalogs import FIELDS, TEMPLATE_ALLOWED_FIELDS from calibre.customize.conversion import DummyReporter From d78e4807c3b51fcef7508df2678b13b9c1437f81 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 9 Sep 2019 20:38:25 -0400 Subject: [PATCH 5/8] py3: write unicode bom without using bytes() type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since the file is opened in utf-8 mode, it needs to be written to using unicode text, not a b'' string. In python3, '\xef\xbb\xbf' becomes '' which is definitely not a BOM. Writing the unicode escaped codepoint allows the codecs.open() encoding to correctly choose the right bytes for the BOM and insert it as needed. --- src/calibre/library/catalogs/csv_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/catalogs/csv_xml.py b/src/calibre/library/catalogs/csv_xml.py index bed17ff771..8576e3849b 100644 --- a/src/calibre/library/catalogs/csv_xml.py +++ b/src/calibre/library/catalogs/csv_xml.py @@ -107,7 +107,7 @@ class CSV_XML(CatalogPlugin): outfile = codecs.open(path_to_output, 'w', 'utf8') # Write a UTF-8 BOM - outfile.write('\xef\xbb\xbf') + outfile.write(u'\ufeff') # Output the field headers outfile.write(u'%s\n' % u','.join(fields)) From 8078bf0931ddf9cf1e05f575c1b79ac710094113 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 9 Sep 2019 21:59:46 -0400 Subject: [PATCH 6/8] py3: etree.tostring emits bytes, and must be written using binary mode --- src/calibre/library/catalogs/csv_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/catalogs/csv_xml.py b/src/calibre/library/catalogs/csv_xml.py index 8576e3849b..1267a628ab 100644 --- a/src/calibre/library/catalogs/csv_xml.py +++ b/src/calibre/library/catalogs/csv_xml.py @@ -235,6 +235,6 @@ class CSV_XML(CatalogPlugin): if 'library_name' in fields: record.append(E.library_name(current_library)) - with open(path_to_output, 'w') as f: + with open(path_to_output, 'wb') as f: f.write(etree.tostring(root, encoding='utf-8', xml_declaration=True, pretty_print=True)) From 975b9ac168711d25a9b2f9007de28ee5ad041e5d Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 9 Sep 2019 20:20:34 -0400 Subject: [PATCH 7/8] py3: more work toward universal _future__s --- src/calibre/library/__init__.py | 6 +- src/calibre/library/add_to_library.py | 1 + src/calibre/library/caches.py | 12 ++-- src/calibre/library/catalogs/bibtex.py | 67 +++++++++--------- src/calibre/library/catalogs/csv_xml.py | 21 +++--- src/calibre/library/catalogs/epub_mobi.py | 25 +++---- .../library/catalogs/epub_mobi_builder.py | 69 +++++++++---------- src/calibre/library/catalogs/utils.py | 14 ++-- src/calibre/library/database.py | 5 +- src/calibre/library/database2.py | 36 +++++----- src/calibre/library/restore.py | 1 + src/calibre/library/save_to_disk.py | 8 +-- src/calibre/library/schema_upgrades.py | 4 +- src/calibre/library/sqlite.py | 22 +++--- 14 files changed, 152 insertions(+), 139 deletions(-) diff --git a/src/calibre/library/__init__.py b/src/calibre/library/__init__.py index 316961fb21..655ca84169 100644 --- a/src/calibre/library/__init__.py +++ b/src/calibre/library/__init__.py @@ -1,6 +1,8 @@ -from __future__ import print_function +from __future__ import absolute_import, division, print_function, unicode_literals + __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' + ''' Code to manage ebook library''' @@ -64,7 +66,7 @@ def generate_test_db(library_path, # {{{ t = time.time() - start print('\nGenerated', num_of_records, 'records in:', t, 'seconds') - print('Time per record:', t/float(num_of_records)) + print('Time per record:', t/num_of_records) # }}} diff --git a/src/calibre/library/add_to_library.py b/src/calibre/library/add_to_library.py index 78fa2be2e2..c284fdb141 100644 --- a/src/calibre/library/add_to_library.py +++ b/src/calibre/library/add_to_library.py @@ -1,5 +1,6 @@ #!/usr/bin/env python2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import absolute_import, division, print_function, unicode_literals __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 6cc8634b71..86d5a65efc 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -1,6 +1,6 @@ #!/usr/bin/env python2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement +from __future__ import absolute_import, division, print_function, unicode_literals __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' @@ -466,7 +466,7 @@ class ResultCache(SearchQueryParser): # {{{ cast = lambda x: int(x) elif dt == 'rating': cast = lambda x: 0 if x is None else int(x) - adjust = lambda x: x/2 + adjust = lambda x: x//2 elif dt in ('float', 'composite'): cast = lambda x : float(x) else: # count operation @@ -851,7 +851,7 @@ class ResultCache(SearchQueryParser): # {{{ def _build_restriction_string(self, restriction): if self.base_restriction: if restriction: - return u'(%s) and (%s)' % (self.base_restriction, restriction) + return '(%s) and (%s)' % (self.base_restriction, restriction) else: return self.base_restriction else: @@ -867,7 +867,7 @@ class ResultCache(SearchQueryParser): # {{{ else: q = query if search_restriction: - q = u'(%s) and (%s)' % (search_restriction, query) + q = '(%s) and (%s)' % (search_restriction, query) if not q: if set_restriction_count: self.search_restriction_book_count = len(self._map) @@ -924,7 +924,7 @@ class ResultCache(SearchQueryParser): # {{{ ''' if not hasattr(id_dict, 'items'): # Simple list. Make it a dict of string 'true' - self.marked_ids_dict = dict.fromkeys(id_dict, u'true') + self.marked_ids_dict = dict.fromkeys(id_dict, 'true') else: # Ensure that all the items in the dict are text self.marked_ids_dict = dict(zip(iter(id_dict), map(unicode_type, @@ -1214,7 +1214,7 @@ class SortKeyGenerator(object): else: if self.library_order: try: - lang = record[self.lang_idx].partition(u',')[0] + lang = record[self.lang_idx].partition(',')[0] except (AttributeError, ValueError, KeyError, IndexError, TypeError): lang = None diff --git a/src/calibre/library/catalogs/bibtex.py b/src/calibre/library/catalogs/bibtex.py index f098a2a642..e352900e65 100644 --- a/src/calibre/library/catalogs/bibtex.py +++ b/src/calibre/library/catalogs/bibtex.py @@ -1,5 +1,6 @@ #!/usr/bin/env python2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import absolute_import, division, print_function, unicode_literals __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' @@ -14,7 +15,7 @@ from calibre.library.catalogs import FIELDS, TEMPLATE_ALLOWED_FIELDS from calibre.customize.conversion import DummyReporter from calibre.constants import preferred_encoding from calibre.ebooks.metadata import format_isbn -from polyglot.builtins import string_or_bytes, filter +from polyglot.builtins import filter, string_or_bytes, unicode_type class BIBTEX(CatalogPlugin): @@ -126,9 +127,9 @@ class BIBTEX(CatalogPlugin): bibtex_entry = [] if mode != "misc" and check_entry_book_valid(entry) : - bibtex_entry.append(u'@book{') + bibtex_entry.append('@book{') elif mode != "book" : - bibtex_entry.append(u'@misc{') + bibtex_entry.append('@misc{') else : # case strict book return '' @@ -137,7 +138,7 @@ class BIBTEX(CatalogPlugin): # Citation tag bibtex_entry.append(make_bibtex_citation(entry, template_citation, bibtexdict)) - bibtex_entry = [u' '.join(bibtex_entry)] + bibtex_entry = [' '.join(bibtex_entry)] for field in fields: if field.startswith('#'): @@ -161,68 +162,68 @@ class BIBTEX(CatalogPlugin): pass if field == 'authors' : - bibtex_entry.append(u'author = "%s"' % bibtexdict.bibtex_author_format(item)) + bibtex_entry.append('author = "%s"' % bibtexdict.bibtex_author_format(item)) elif field == 'id' : - bibtex_entry.append(u'calibreid = "%s"' % int(item)) + bibtex_entry.append('calibreid = "%s"' % int(item)) elif field == 'rating' : - bibtex_entry.append(u'rating = "%s"' % int(item)) + bibtex_entry.append('rating = "%s"' % int(item)) elif field == 'size' : - bibtex_entry.append(u'%s = "%s octets"' % (field, int(item))) + bibtex_entry.append('%s = "%s octets"' % (field, int(item))) elif field == 'tags' : # A list to flatten - bibtex_entry.append(u'tags = "%s"' % bibtexdict.utf8ToBibtex(u', '.join(item))) + bibtex_entry.append('tags = "%s"' % bibtexdict.utf8ToBibtex(', '.join(item))) elif field == 'comments' : # \n removal - item = item.replace(u'\r\n',u' ') - item = item.replace(u'\n',u' ') + item = item.replace('\r\n', ' ') + item = item.replace('\n', ' ') # unmatched brace removal (users should use \leftbrace or \rightbrace for single braces) - item = bibtexdict.stripUnmatchedSyntax(item, u'{', u'}') + item = bibtexdict.stripUnmatchedSyntax(item, '{', '}') # html to text try: item = html2text(item) except: log.warn("Failed to convert comments to text") - bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item)) + bibtex_entry.append('note = "%s"' % bibtexdict.utf8ToBibtex(item)) elif field == 'isbn' : # Could be 9, 10 or 13 digits - bibtex_entry.append(u'isbn = "%s"' % format_isbn(item)) + bibtex_entry.append('isbn = "%s"' % format_isbn(item)) elif field == 'formats' : # Add file path if format is selected formats = [format.rpartition('.')[2].lower() for format in item] - bibtex_entry.append(u'formats = "%s"' % u', '.join(formats)) + bibtex_entry.append('formats = "%s"' % ', '.join(formats)) if calibre_files: - files = [u':%s:%s' % (format, format.rpartition('.')[2].upper()) + files = [':%s:%s' % (format, format.rpartition('.')[2].upper()) for format in item] - bibtex_entry.append(u'file = "%s"' % u', '.join(files)) + bibtex_entry.append('file = "%s"' % ', '.join(files)) elif field == 'series_index' : - bibtex_entry.append(u'volume = "%s"' % int(item)) + bibtex_entry.append('volume = "%s"' % int(item)) elif field == 'timestamp' : - bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0]) + bibtex_entry.append('timestamp = "%s"' % isoformat(item).partition('T')[0]) elif field == 'pubdate' : - bibtex_entry.append(u'year = "%s"' % item.year) - bibtex_entry.append(u'month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item))) + bibtex_entry.append('year = "%s"' % item.year) + bibtex_entry.append('month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item))) elif field.startswith('#') and isinstance(item, string_or_bytes): - bibtex_entry.append(u'custom_%s = "%s"' % (field[1:], + bibtex_entry.append('custom_%s = "%s"' % (field[1:], bibtexdict.utf8ToBibtex(item))) elif isinstance(item, string_or_bytes): # elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice', # 'author_sort', 'series', 'title_sort'] : - bibtex_entry.append(u'%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item))) + bibtex_entry.append('%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item))) - bibtex_entry = u',\n '.join(bibtex_entry) - bibtex_entry += u' }\n\n' + bibtex_entry = ',\n '.join(bibtex_entry) + bibtex_entry += ' }\n\n' return bibtex_entry @@ -241,7 +242,7 @@ class BIBTEX(CatalogPlugin): # define a function to replace the template entry by its value def tpl_replace(objtplname) : - tpl_field = re.sub(u'[\\{\\}]', u'', objtplname.group()) + tpl_field = re.sub('[\\{\\}]', '', objtplname.group()) if tpl_field in TEMPLATE_ALLOWED_FIELDS : if tpl_field in ['pubdate', 'timestamp'] : @@ -249,26 +250,26 @@ class BIBTEX(CatalogPlugin): elif tpl_field in ['tags', 'authors'] : tpl_field =entry[tpl_field][0] elif tpl_field in ['id', 'series_index'] : - tpl_field = str(entry[tpl_field]) + tpl_field = unicode_type(entry[tpl_field]) else : tpl_field = entry[tpl_field] return ascii_text(tpl_field) else: - return u'' + return '' if len(template_citation) >0 : tpl_citation = bibtexclass.utf8ToBibtex( - bibtexclass.ValidateCitationKey(re.sub(u'\\{[^{}]*\\}', + bibtexclass.ValidateCitationKey(re.sub('\\{[^{}]*\\}', tpl_replace, template_citation))) if len(tpl_citation) >0 : return tpl_citation if len(entry["isbn"]) > 0 : - template_citation = u'%s' % re.sub(u'[\\D]',u'', entry["isbn"]) + template_citation = '%s' % re.sub('[\\D]','', entry["isbn"]) else : - template_citation = u'%s' % str(entry["id"]) + template_citation = '%s' % unicode_type(entry["id"]) return bibtexclass.ValidateCitationKey(template_citation) @@ -394,8 +395,8 @@ class BIBTEX(CatalogPlugin): for entry in data: entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice'] - outfile.write(u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries)) - outfile.write(u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n' + outfile.write('%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries)) + outfile.write('@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n' % (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(preferred_encoding))) for entry in data: diff --git a/src/calibre/library/catalogs/csv_xml.py b/src/calibre/library/catalogs/csv_xml.py index 1267a628ab..382b55ac44 100644 --- a/src/calibre/library/catalogs/csv_xml.py +++ b/src/calibre/library/catalogs/csv_xml.py @@ -1,5 +1,6 @@ #!/usr/bin/env python2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import absolute_import, division, print_function, unicode_literals __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' @@ -107,10 +108,10 @@ class CSV_XML(CatalogPlugin): outfile = codecs.open(path_to_output, 'w', 'utf8') # Write a UTF-8 BOM - outfile.write(u'\ufeff') + outfile.write('\ufeff') # Output the field headers - outfile.write(u'%s\n' % u','.join(fields)) + outfile.write('%s\n' % ','.join(fields)) # Output the entry fields for entry in data: @@ -144,14 +145,14 @@ class CSV_XML(CatalogPlugin): item = ', '.join(item) elif field == 'isbn': # Could be 9, 10 or 13 digits, with hyphens, possibly ending in 'X' - item = u'%s' % re.sub(r'[^\dX-]', '', item) + item = '%s' % re.sub(r'[^\dX-]', '', item) elif fm.get(field, {}).get('datatype') == 'datetime': item = isoformat(item, as_utc=False) elif field == 'comments': - item = item.replace(u'\r\n', u' ') - item = item.replace(u'\n', u' ') + item = item.replace('\r\n', ' ') + item = item.replace('\n', ' ') elif fm.get(field, {}).get('datatype', None) == 'rating' and item: - item = u'%.2g' % (item / 2.0) + item = '%.2g' % (item / 2) # Convert HTML to markdown text if isinstance(item, unicode_type): @@ -161,9 +162,9 @@ class CSV_XML(CatalogPlugin): if closing_tag: item = html2text(item) - outstr.append(u'"%s"' % unicode_type(item).replace('"', '""')) + outstr.append('"%s"' % unicode_type(item).replace('"', '""')) - outfile.write(u','.join(outstr) + u'\n') + outfile.write(','.join(outstr) + '\n') outfile.close() elif self.fmt == 'xml': @@ -191,7 +192,7 @@ class CSV_XML(CatalogPlugin): if not isinstance(val, (bytes, unicode_type)): if (fm.get(field, {}).get('datatype', None) == 'rating' and val): - val = u'%.2g' % (val / 2.0) + val = '%.2g' % (val / 2) val = unicode_type(val) item = getattr(E, field)(val) record.append(item) @@ -221,7 +222,7 @@ class CSV_XML(CatalogPlugin): if 'series' in fields and r['series']: record.append(E.series(r['series'], - index=str(r['series_index']))) + index=unicode_type(r['series_index']))) if 'cover' in fields and r['cover']: record.append(E.cover(r['cover'].replace(os.sep, '/'))) diff --git a/src/calibre/library/catalogs/epub_mobi.py b/src/calibre/library/catalogs/epub_mobi.py index cbe65380fa..112e7c91b7 100644 --- a/src/calibre/library/catalogs/epub_mobi.py +++ b/src/calibre/library/catalogs/epub_mobi.py @@ -16,6 +16,7 @@ from calibre.library import current_library_name from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.localization import calibre_langcode_to_name, canonicalize_lang, get_lang +from polyglot.builtins import unicode_type Option = namedtuple('Option', 'option, default, dest, action, help') @@ -60,7 +61,7 @@ class EPUB_MOBI(CatalogPlugin): "Default: '%default'\n" "Applies to: AZW3, EPUB, MOBI output formats")), Option('--exclude-genre', - default='\\[.+\\]|^\\+$', + default=r'\[.+\]|^\+$', dest='exclude_genre', action=None, help=_("Regex describing tags to exclude as genres.\n" @@ -264,7 +265,7 @@ class EPUB_MOBI(CatalogPlugin): build_log = [] - build_log.append(u"%s('%s'): Generating %s %sin %s environment, locale: '%s'" % + build_log.append("%s('%s'): Generating %s %sin %s environment, locale: '%s'" % (self.name, current_library_name(), self.fmt, @@ -282,23 +283,23 @@ class EPUB_MOBI(CatalogPlugin): if opts.connected_device['is_device_connected'] and \ opts.connected_device['kind'] == 'device': if opts.connected_device['serial']: - build_log.append(u" connected_device: '%s' #%s%s " % + build_log.append(" connected_device: '%s' #%s%s " % (opts.connected_device['name'], opts.connected_device['serial'][0:4], 'x' * (len(opts.connected_device['serial']) - 4))) for storage in opts.connected_device['storage']: if storage: - build_log.append(u" mount point: %s" % storage) + build_log.append(" mount point: %s" % storage) else: - build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) + build_log.append(" connected_device: '%s'" % opts.connected_device['name']) try: for storage in opts.connected_device['storage']: if storage: - build_log.append(u" mount point: %s" % storage) + build_log.append(" mount point: %s" % storage) except: - build_log.append(u" (no mount points)") + build_log.append(" (no mount points)") else: - build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) + build_log.append(" connected_device: '%s'" % opts.connected_device['name']) opts_dict = vars(opts) if opts_dict['ids']: @@ -337,7 +338,7 @@ class EPUB_MOBI(CatalogPlugin): sections_list.insert(0, 'Authors') opts.generate_authors = True - opts.log(u" Sections: %s" % ', '.join(sections_list)) + opts.log(" Sections: %s" % ', '.join(sections_list)) opts.section_list = sections_list # Limit thumb_width to 1.0" - 2.0" @@ -397,7 +398,7 @@ class EPUB_MOBI(CatalogPlugin): if opts.verbose: log.info(" Begin catalog source generation (%s)" % - str(datetime.timedelta(seconds=int(time.time() - opts.start_time)))) + unicode_type(datetime.timedelta(seconds=int(time.time() - opts.start_time)))) # Launch the Catalog builder catalog = CatalogBuilder(db, opts, self, report_progress=notification) @@ -406,7 +407,7 @@ class EPUB_MOBI(CatalogPlugin): catalog.build_sources() if opts.verbose: log.info(" Completed catalog source generation (%s)\n" % - str(datetime.timedelta(seconds=int(time.time() - opts.start_time)))) + unicode_type(datetime.timedelta(seconds=int(time.time() - opts.start_time)))) except (AuthorSortMismatchException, EmptyCatalogException) as e: log.error(" *** Terminated catalog generation: %s ***" % e) except: @@ -499,7 +500,7 @@ class EPUB_MOBI(CatalogPlugin): if opts.verbose: log.info(" Catalog creation complete (%s)\n" % - str(datetime.timedelta(seconds=int(time.time() - opts.start_time)))) + unicode_type(datetime.timedelta(seconds=int(time.time() - opts.start_time)))) # returns to gui2.actions.catalog:catalog_generated() return catalog.error diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index b629718bcf..a12db3641f 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -1,8 +1,7 @@ #!/usr/bin/env python2 # vim:fileencoding=utf-8 # License: GPLv3 Copyright: 2010, Greg Riker - -from __future__ import print_function +from __future__ import absolute_import, division, print_function, unicode_literals import datetime import os @@ -44,7 +43,7 @@ from calibre.utils.localization import get_lang, lang_as_iso639_1 from calibre.utils.zipfile import ZipFile from polyglot.builtins import unicode_type, iteritems, map, zip -NBSP = u'\u00a0' +NBSP = '\u00a0' class Formatter(TemplateFormatter): @@ -241,7 +240,7 @@ class CatalogBuilder(object): index = book['series_index'] integer = int(index) fraction = index - integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + series_index = '%04d%s' % (integer, unicode_type('%0.4f' % fraction).lstrip('0')) key = '%s ~%s %s' % (self._kf_author_to_author_sort(book['author']), self.generate_sort_title(book['series']), series_index) @@ -260,15 +259,15 @@ class CatalogBuilder(object): (str): sort key """ if not book['series']: - fs = u'{:<%d}!{!s}' % longest_author_sort + fs = '{:<%d}!{!s}' % longest_author_sort key = fs.format(capitalize(book['author_sort']), capitalize(book['title_sort'])) else: index = book['series_index'] integer = int(index) fraction = index - integer - series_index = u'%04d%s' % (integer, str(u'%0.4f' % fraction).lstrip(u'0')) - fs = u'{:<%d}~{!s}{!s}' % longest_author_sort + series_index = '%04d%s' % (integer, unicode_type('%0.4f' % fraction).lstrip('0')) + fs = '{:<%d}~{!s}{!s}' % longest_author_sort key = fs.format(capitalize(book['author_sort']), self.generate_sort_title(book['series']), series_index) @@ -278,7 +277,7 @@ class CatalogBuilder(object): index = book['series_index'] integer = int(index) fraction = index - integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + series_index = '%04d%s' % (integer, unicode_type('%0.4f' % fraction).lstrip('0')) key = '%s %s' % (self.generate_sort_title(book['series']), series_index) return key @@ -677,9 +676,9 @@ class CatalogBuilder(object): # Hack to force the cataloged leading letter to be # an unadorned character if the accented version sorts before the unaccented exceptions = { - u'Ä': u'A', - u'Ö': u'O', - u'Ü': u'U' + 'Ä': 'A', + 'Ö': 'O', + 'Ü': 'U' } if key is not None: @@ -699,7 +698,7 @@ class CatalogBuilder(object): # Hackhackhackhackhack # icu returns bogus results with curly apostrophes, maybe others under OS X 10.6.x # When we see the magic combo of 0/-1 for ordnum/ordlen, special case the logic - last_c = u'' + last_c = '' if ordnum == 0 and ordlen == -1: if icu_upper(c[0]) != last_c: last_c = icu_upper(c[0]) @@ -955,7 +954,7 @@ class CatalogBuilder(object): if is_date_undefined(record['pubdate']): this_title['date'] = None else: - this_title['date'] = strftime(u'%B %Y', as_local_time(record['pubdate']).timetuple()) + this_title['date'] = strftime('%B %Y', as_local_time(record['pubdate']).timetuple()) this_title['timestamp'] = record['timestamp'] @@ -1091,11 +1090,11 @@ class CatalogBuilder(object): from calibre.devices.kindle.bookmark import Bookmark from calibre.ebooks.metadata import MetaInformation - MBP_FORMATS = [u'azw', u'mobi', u'prc', u'txt'] + MBP_FORMATS = ['azw', 'mobi', 'prc', 'txt'] mbp_formats = set(MBP_FORMATS) - PDR_FORMATS = [u'pdf'] + PDR_FORMATS = ['pdf'] pdr_formats = set(PDR_FORMATS) - TAN_FORMATS = [u'tpz', u'azw1'] + TAN_FORMATS = ['tpz', 'azw1'] tan_formats = set(TAN_FORMATS) class BookmarkDevice(Device): @@ -1174,7 +1173,7 @@ class CatalogBuilder(object): book['percent_read'] = min(float(100 * myBookmark.last_read / myBookmark.book_length), 100) except: book['percent_read'] = 0 - dots = int((book['percent_read'] + 5) / 10) + dots = int((book['percent_read'] + 5) // 10) dot_string = self.SYMBOL_PROGRESS_READ * dots empty_dots = self.SYMBOL_PROGRESS_UNREAD * (10 - dots) book['reading_progress'] = '%s%s' % (dot_string, empty_dots) @@ -1411,7 +1410,7 @@ class CatalogBuilder(object): Return: (dict): formatted args for templating """ - series_index = str(book['series_index']) + series_index = unicode_type(book['series_index']) if series_index.endswith('.0'): series_index = series_index[:-2] args = dict( @@ -1661,7 +1660,7 @@ class CatalogBuilder(object): key=lambda x: sort_key(self._kf_books_by_author_sorter_author_sort(x, len(las)))) # Create a new month anchor - date_string = strftime(u'%B %Y', current_date.timetuple()) + date_string = strftime('%B %Y', current_date.timetuple()) pIndexTag = soup.new_tag("p") pIndexTag['class'] = "date_index" aTag = soup.new_tag("a") @@ -1919,7 +1918,7 @@ class CatalogBuilder(object): def _add_books_to_html_by_day(todays_list, dtc): if len(todays_list): # Create a new day anchor - date_string = strftime(u'%A, %B %d', current_date.timetuple()) + date_string = strftime('%A, %B %d', current_date.timetuple()) pIndexTag = soup.new_tag("p") pIndexTag['class'] = "date_index" aTag = soup.new_tag("a") @@ -1981,7 +1980,7 @@ class CatalogBuilder(object): ptc = 0 # Percent read - dots = int((new_entry['percent_read'] + 5) / 10) + dots = int((new_entry['percent_read'] + 5) // 10) dot_string = self.SYMBOL_PROGRESS_READ * dots empty_dots = self.SYMBOL_PROGRESS_UNREAD * (10 - dots) pBookTag.insert(ptc, NavigableString('%s%s' % (dot_string, empty_dots))) @@ -2690,7 +2689,7 @@ class CatalogBuilder(object): series_index = '' if book['series']: series = book['series'] - series_index = str(book['series_index']) + series_index = unicode_type(book['series_index']) if series_index.endswith('.0'): series_index = series_index[:-2] @@ -2754,7 +2753,7 @@ class CatalogBuilder(object): publisher = book['publisher'] # Rating - stars = int(book['rating']) / 2 + stars = int(book['rating']) // 2 rating = '' if stars: star_string = self.SYMBOL_FULL_RATING * stars @@ -2958,8 +2957,8 @@ class CatalogBuilder(object): font = ImageFont.truetype(default_font, 48) text = self.opts.catalog_title.encode('utf-8') width, height = draw.textsize(text, font=font) - left = max(int((MI_WIDTH - width) / 2.), 0) - top = max(int((MI_HEIGHT - height) / 2.), 0) + left = max(int((MI_WIDTH - width) / 2), 0) + top = max(int((MI_HEIGHT - height) / 2), 0) draw.text((left, top), text, fill=(0, 0, 0), font=font) with open(out_path, 'wb') as f: img.save(f, 'GIF') @@ -3100,7 +3099,7 @@ class CatalogBuilder(object): navLabelTag = ncx_soup.new_tag("navLabel") textTag = ncx_soup.new_tag("text") if book['series']: - series_index = str(book['series_index']) + series_index = unicode_type(book['series_index']) if series_index.endswith('.0'): series_index = series_index[:-2] if self.generate_for_kindle_mobi: @@ -3694,7 +3693,7 @@ class CatalogBuilder(object): # Add *article* entries for each populated month # master_months_list{}: [0]:titles list [1]:date for books_by_month in master_month_list: - datestr = strftime(u'%B %Y', books_by_month[1].timetuple()) + datestr = strftime('%B %Y', books_by_month[1].timetuple()) navPointByMonthTag = ncx_soup.new_tag('navPoint') if self.generate_for_kindle_mobi: navPointByMonthTag['class'] = "article" @@ -3844,7 +3843,7 @@ class CatalogBuilder(object): # Add *article* entries for each populated day # master_day_list{}: [0]:titles list [1]:date for books_by_day in master_day_list: - datestr = strftime(u'%A, %B %d', books_by_day[1].timetuple()) + datestr = strftime('%A, %B %d', books_by_day[1].timetuple()) navPointByDayTag = ncx_soup.new_tag('navPoint') if self.generate_for_kindle_mobi: navPointByDayTag['class'] = "article" @@ -3995,7 +3994,7 @@ class CatalogBuilder(object): for title in genre['books']: titles.append(title['title']) titles = sorted(titles, key=lambda x: (self.generate_sort_title(x), self.generate_sort_title(x))) - titles_list = self.generate_short_description(u" • ".join(titles), dest="description") + titles_list = self.generate_short_description(" • ".join(titles), dest="description") cmTag.insert(0, NavigableString(self.format_ncx_text(titles_list, dest='description'))) navPointVolumeTag.insert(3, cmTag) @@ -4205,7 +4204,7 @@ class CatalogBuilder(object): rating = '' try: if 'rating' in book: - stars = int(book['rating']) / 2 + stars = int(book['rating']) // 2 if stars: star_string = self.SYMBOL_FULL_RATING * stars empty_stars = self.SYMBOL_EMPTY_RATING * (5 - stars) @@ -4489,7 +4488,7 @@ class CatalogBuilder(object): Return: (str): legal XHTML anchor string of unicode character name """ - fullname = u''.join(unicodedata.name(unicode_type(cc)) for cc in c) + fullname = ''.join(unicodedata.name(unicode_type(cc)) for cc in c) terms = fullname.split() return "_".join(terms) @@ -4521,7 +4520,7 @@ class CatalogBuilder(object): matched = list(set(record['tags']) & set(excluded_tags)) if matched: for rule in self.opts.exclusion_rules: - if rule[1] == _('Tags') and rule[2] == str(matched[0]): + if rule[1] == _('Tags') and rule[2] == unicode_type(matched[0]): self.opts.log.info(" - '%s' by %s (Exclusion rule '%s')" % (record['title'], record['authors'][0], rule[0])) @@ -4808,12 +4807,12 @@ class CatalogBuilder(object): self.progress_int = 0.01 self.reporter(self.progress_int, self.progress_string) if self.opts.cli_environment: - log_msg = u"%3.0f%% %s" % (self.progress_int * 100, self.progress_string) + log_msg = "%3.0f%% %s" % (self.progress_int * 100, self.progress_string) if self.opts.verbose: - log_msg += " (%s)" % str(datetime.timedelta(seconds=int(time.time() - self.opts.start_time))) + log_msg += " (%s)" % unicode_type(datetime.timedelta(seconds=int(time.time() - self.opts.start_time))) else: log_msg = ("%s (%s)" % (self.progress_string, - str(datetime.timedelta(seconds=int(time.time() - self.opts.start_time))))) + unicode_type(datetime.timedelta(seconds=int(time.time() - self.opts.start_time))))) self.opts.log(log_msg) def update_progress_micro_step(self, description, micro_step_pct): diff --git a/src/calibre/library/catalogs/utils.py b/src/calibre/library/catalogs/utils.py index 09d4ad84bc..593b96cef7 100644 --- a/src/calibre/library/catalogs/utils.py +++ b/src/calibre/library/catalogs/utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import absolute_import, division, print_function, unicode_literals -from __future__ import print_function __license__ = 'GPL v3' __copyright__ = '2010, Greg Riker' __docformat__ = 'restructuredtext en' @@ -44,7 +44,7 @@ class NumberToText(object): # {{{ # Build the hundreds component if hundredsComponent: - hundredsComponentString = "%s hundred" % self.hundreds[hundredsComponent/100] + hundredsComponentString = "%s hundred" % self.hundreds[hundredsComponent//100] else: hundredsComponentString = "" @@ -56,7 +56,7 @@ class NumberToText(object): # {{{ onesPart = "" # Get the tens part - tensPart = self.tens[tensComponent / 10] + tensPart = self.tens[tensComponent // 10] onesPart = self.lessThanTwenty[tensComponent % 10] if intToTranslate % 10: @@ -183,8 +183,8 @@ class NumberToText(object): # {{{ self.text = "one billion" else : # Isolate the three-digit number groups - millionsNumber = number/10**6 - thousandsNumber = (number - (millionsNumber * 10**6))/10**3 + millionsNumber = number//10**6 + thousandsNumber = (number - (millionsNumber * 10**6))//10**3 hundredsNumber = number - (millionsNumber * 10**6) - (thousandsNumber * 10**3) if self.verbose: print("Converting %s %s %s" % (millionsNumber, thousandsNumber, hundredsNumber)) @@ -196,7 +196,7 @@ class NumberToText(object): # {{{ # Convert thousandsNumber if thousandsNumber: if number > 1099 and number < 2000: - resultString = '%s %s' % (self.lessThanTwenty[number/100], + resultString = '%s %s' % (self.lessThanTwenty[number//100], self.stringFromInt(number % 100)) self.text = resultString.strip().capitalize() return @@ -222,6 +222,6 @@ class NumberToText(object): # {{{ resultString = "zero" if self.verbose: - self.log(u'resultString: %s' % resultString) + self.log('resultString: %s' % resultString) self.text = resultString.strip().capitalize() # }}} diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index e2fd47390a..01a29e3813 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -1,9 +1,12 @@ -from __future__ import print_function +from __future__ import absolute_import, division, print_function, unicode_literals + __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' + ''' Backend that implements storage of ebooks in an sqlite database. ''' + import sqlite3 as sqlite import datetime, re, sre_constants from zlib import compress, decompress diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index c31de99a00..02b348f3e6 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1,4 +1,5 @@ -from __future__ import with_statement +from __future__ import absolute_import, division, print_function, unicode_literals + __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' @@ -6,6 +7,7 @@ __docformat__ = 'restructuredtext en' ''' The database used to store ebook metadata ''' + import os, sys, shutil, glob, time, functools, traceback, re, \ json, uuid, hashlib, copy, numbers from collections import defaultdict, namedtuple @@ -91,7 +93,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if self._library_id_ is None: ans = self.conn.get('SELECT uuid FROM library_id', all=False) if ans is None: - ans = str(uuid.uuid4()) + ans = unicode_type(uuid.uuid4()) self.library_id = ans else: self._library_id_ = ans @@ -256,11 +258,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): from calibre.library.coloring import migrate_old_rule old_rules = [] for i in range(1, 6): - col = self.prefs.get('column_color_name_'+str(i), None) - templ = self.prefs.get('column_color_template_'+str(i), None) + col = self.prefs.get('column_color_name_'+unicode_type(i), None) + templ = self.prefs.get('column_color_template_'+unicode_type(i), None) if col and templ: try: - del self.prefs['column_color_name_'+str(i)] + del self.prefs['column_color_name_'+unicode_type(i)] rules = migrate_old_rule(self.field_metadata, templ) for templ in rules: old_rules.append((col, templ)) @@ -359,7 +361,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ''') self.conn.execute( 'UPDATE authors SET sort=author_to_author_sort(name) WHERE sort IS NULL') - self.conn.executescript(u''' + self.conn.executescript(''' CREATE TEMP VIEW IF NOT EXISTS tag_browser_news AS SELECT DISTINCT id, name, @@ -372,7 +374,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): (SELECT id FROM tags WHERE name="{0}"))); '''.format(_('News'))) - self.conn.executescript(u''' + self.conn.executescript(''' CREATE TEMP VIEW IF NOT EXISTS tag_browser_filtered_news AS SELECT DISTINCT id, name, @@ -450,7 +452,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # account for the series index column. Field_metadata knows that # the series index is one larger than the series. If you change # it here, be sure to change it there as well. - self.FIELD_MAP[str(col)+'_index'] = base = base+1 + self.FIELD_MAP[unicode_type(col)+'_index'] = base = base+1 self.field_metadata.set_field_record_index( self.custom_column_num_map[col]['label']+'_index', base, @@ -487,7 +489,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # There is a chance that these can be duplicates of an existing # user category. Print the exception and continue. try: - self.field_metadata.add_user_category(label=u'@' + cat, name=cat) + self.field_metadata.add_user_category(label='@' + cat, name=cat) except: traceback.print_exc() @@ -1127,7 +1129,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): try: quathors = mi.authors[:10] # Too many authors causes parsing of # the search expression to fail - query = u' and '.join([u'author:"=%s"'%(a.replace('"', '')) for a in + query = ' and '.join(['author:"=%s"'%(a.replace('"', '')) for a in quathors]) qauthors = mi.authors[10:] except ValueError: @@ -1747,7 +1749,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.id = id def __unicode_representation__(self): - return u'n=%s s=%s c=%d rt=%d rc=%d id=%s' % ( + return 'n=%s s=%s c=%d rt=%d rc=%d id=%s' % ( self.n, self.s, self.c, self.rt, self.rc, self.id) if ispy3: @@ -1977,7 +1979,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # Duplicate the build of items below to avoid using a lambda func # in the main Tag loop. Saves a few % if datatype == 'rating': - formatter = (lambda x:u'\u2605'*int(x/2)) + formatter = (lambda x:'\u2605'*int(x//2)) avgr = lambda x: x.n # eliminate the zero ratings line as well as count == 0 items = [v for v in tcategories[category].values() if v.c > 0 and v.n != 0] @@ -2613,7 +2615,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if commit: self.conn.commit() self.data.set(book_id, self.FIELD_MAP['languages'], - u','.join(final_languages), row_is_id=True) + ','.join(final_languages), row_is_id=True) if notify: self.notify('metadata', [book_id]) return books_to_refresh @@ -2992,7 +2994,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tags = [x.strip().replace(',', ';') for x in tags if x.strip()] tags = [x.decode(preferred_encoding, 'replace') if isbytestring(x) else x for x in tags] - tags = [u' '.join(x.split()) for x in tags] + tags = [' '.join(x.split()) for x in tags] ans, seen = [], set() for tag in tags: if tag.lower() not in seen: @@ -3068,7 +3070,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.commit() for x in ids: - tags = u','.join(self.get_tags(x)) + tags = ','.join(self.get_tags(x)) self.data.set(x, self.FIELD_MAP['tags'], tags, row_is_id=True) if notify: self.notify('metadata', ids) @@ -3124,7 +3126,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.dirtied({id}|books_to_refresh, commit=False) if commit: self.conn.commit() - tags = u','.join(self.get_tags(id)) + tags = ','.join(self.get_tags(id)) self.data.set(id, self.FIELD_MAP['tags'], tags, row_is_id=True) if notify: self.notify('metadata', [id]) @@ -3178,7 +3180,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if not isinstance(series, unicode_type): series = series.decode(preferred_encoding, 'replace') series = series.strip() - series = u' '.join(series.split()) + series = ' '.join(series.split()) sx = self.conn.get('SELECT id,name from series WHERE name=?', (series,)) if sx: aid, cur_name = sx[0] diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index 47df5d20a4..026c9526df 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -1,5 +1,6 @@ #!/usr/bin/env python2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import absolute_import, division, print_function, unicode_literals __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 338903af0b..a5640c7484 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -1,6 +1,6 @@ #!/usr/bin/env python2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement +from __future__ import absolute_import, division, print_function, unicode_literals __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' @@ -217,7 +217,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, if hasattr(mi, 'last_modified') and hasattr(mi.last_modified, 'timetuple'): format_args['last_modified'] = strftime(timefmt, mi.last_modified.timetuple()) - format_args['id'] = str(id) + format_args['id'] = unicode_type(id) # Now format the custom fields custom_metadata = mi.get_all_user_metadata(make_copy=False) for key in custom_metadata: @@ -247,7 +247,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, components = [x.strip() for x in components.split('/')] components = [sanitize_func(x) for x in components if x] if not components: - components = [str(id)] + components = [unicode_type(id)] if to_lowercase: components = [x.lower() for x in components] if replace_whitespace: @@ -360,7 +360,7 @@ def do_save_book_to_disk(db, book_id, mi, plugboards, return not formats_written, book_id, mi.title for fmt in formats: - fmt_path = base_path+'.'+str(fmt) + fmt_path = base_path+'.'+unicode_type(fmt) try: db.copy_format_to(book_id, fmt, fmt_path) formats_written = True diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index 47045156ef..785ddf0663 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -1,6 +1,6 @@ #!/usr/bin/env python2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import print_function, with_statement +from __future__ import absolute_import, division, print_function, unicode_literals __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' @@ -593,7 +593,7 @@ class SchemaUpgrade(object): existing = frozenset(map(int, custom_recipes)) if id_ in existing: id_ = max(existing) + 1000 - id_ = str(id_) + id_ = unicode_type(id_) fname = custom_recipe_filename(id_, title) custom_recipes[id_] = (title, fname) if isinstance(script, unicode_type): diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 95ad9c2898..b237b013ff 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -1,4 +1,5 @@ -from __future__ import print_function, with_statement +from __future__ import absolute_import, division, print_function, unicode_literals + __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' @@ -7,6 +8,7 @@ __docformat__ = 'restructuredtext en' Wrapper for multi-threaded access to a single sqlite database connection. Serializes all calls. ''' + import sqlite3 as sqlite, traceback, time, uuid, os from sqlite3 import IntegrityError, OperationalError from threading import Thread @@ -20,7 +22,7 @@ from calibre import isbytestring, force_unicode from calibre.constants import iswindows, DEBUG, plugins, plugins_loc from calibre.utils.icu import sort_key from calibre import prints -from polyglot.builtins import unicode_type, cmp +from polyglot.builtins import cmp, native_string_type, unicode_type from polyglot import reprlib from polyglot.queue import Queue @@ -78,7 +80,7 @@ def adapt_datetime(dt): sqlite.register_adapter(datetime, adapt_datetime) -sqlite.register_converter('timestamp', convert_timestamp) +sqlite.register_converter(native_string_type('timestamp'), convert_timestamp) def convert_bool(val): @@ -86,8 +88,8 @@ def convert_bool(val): sqlite.register_adapter(bool, lambda x : 1 if x else 0) -sqlite.register_converter('bool', convert_bool) -sqlite.register_converter('BOOL', convert_bool) +sqlite.register_converter(native_string_type('bool'), convert_bool) +sqlite.register_converter(native_string_type('BOOL'), convert_bool) class DynamicFilter(object): @@ -162,7 +164,7 @@ class IdentifiersConcat(object): self.ans = [] def step(self, key, val): - self.ans.append(u'%s:%s'%(key, val)) + self.ans.append('%s:%s'%(key, val)) def finalize(self): try: @@ -262,15 +264,15 @@ def do_connect(path, row_factory=None): conn.row_factory = sqlite.Row if row_factory else (lambda cursor, row : list(row)) conn.create_aggregate('concat', 1, Concatenate) conn.create_aggregate('aum_sortconcat', 4, AumSortedConcatenate) - conn.create_collation('PYNOCASE', partial(pynocase, + conn.create_collation(native_string_type('PYNOCASE'), partial(pynocase, encoding=encoding)) conn.create_function('title_sort', 1, title_sort) conn.create_function('author_to_author_sort', 1, _author_to_author_sort) - conn.create_function('uuid4', 0, lambda : str(uuid.uuid4())) + conn.create_function('uuid4', 0, lambda : unicode_type(uuid.uuid4())) # Dummy functions for dynamically created filters conn.create_function('books_list_filter', 1, lambda x: 1) - conn.create_collation('icucollate', icu_collator) + conn.create_collation(native_string_type('icucollate'), icu_collator) return conn @@ -320,7 +322,7 @@ class DBThread(Thread): break except OperationalError as err: # Retry if unable to open db file - e = str(err) + e = unicode_type(err) if 'unable to open' not in e or i == 2: if 'unable to open' in e: prints('Unable to open database for func', From cdc2e21a77126cda8ff0b78abdf8a57a83a4ac91 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 9 Sep 2019 20:21:23 -0400 Subject: [PATCH 8/8] use raw strings where possible to avoid escaping --- src/calibre/library/catalogs/bibtex.py | 6 +++--- src/calibre/library/catalogs/csv_xml.py | 4 ++-- .../library/catalogs/epub_mobi_builder.py | 16 ++++++++-------- src/calibre/library/catalogs/utils.py | 8 ++++---- src/calibre/library/database.py | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/calibre/library/catalogs/bibtex.py b/src/calibre/library/catalogs/bibtex.py index e352900e65..7d3abd41c0 100644 --- a/src/calibre/library/catalogs/bibtex.py +++ b/src/calibre/library/catalogs/bibtex.py @@ -242,7 +242,7 @@ class BIBTEX(CatalogPlugin): # define a function to replace the template entry by its value def tpl_replace(objtplname) : - tpl_field = re.sub('[\\{\\}]', '', objtplname.group()) + tpl_field = re.sub(r'[\{\}]', '', objtplname.group()) if tpl_field in TEMPLATE_ALLOWED_FIELDS : if tpl_field in ['pubdate', 'timestamp'] : @@ -259,14 +259,14 @@ class BIBTEX(CatalogPlugin): if len(template_citation) >0 : tpl_citation = bibtexclass.utf8ToBibtex( - bibtexclass.ValidateCitationKey(re.sub('\\{[^{}]*\\}', + bibtexclass.ValidateCitationKey(re.sub(r'\{[^{}]*\}', tpl_replace, template_citation))) if len(tpl_citation) >0 : return tpl_citation if len(entry["isbn"]) > 0 : - template_citation = '%s' % re.sub('[\\D]','', entry["isbn"]) + template_citation = '%s' % re.sub(r'[\D]','', entry["isbn"]) else : template_citation = '%s' % unicode_type(entry["id"]) diff --git a/src/calibre/library/catalogs/csv_xml.py b/src/calibre/library/catalogs/csv_xml.py index 382b55ac44..e80c027384 100644 --- a/src/calibre/library/catalogs/csv_xml.py +++ b/src/calibre/library/catalogs/csv_xml.py @@ -156,9 +156,9 @@ class CSV_XML(CatalogPlugin): # Convert HTML to markdown text if isinstance(item, unicode_type): - opening_tag = re.search('<(\\w+)(\x20|>)', item) + opening_tag = re.search(r'<(\w+)( |>)', item) if opening_tag: - closing_tag = re.search('<\\/%s>$' % opening_tag.group(1), item) + closing_tag = re.search(r'<\/%s>$' % opening_tag.group(1), item) if closing_tag: item = html2text(item) diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index a12db3641f..3e8bfa5097 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -1228,11 +1228,11 @@ class CatalogBuilder(object): clipped to max_len """ - normalized = massaged = re.sub('\\s', '', ascii_text(tag).lower()) - if re.search('\\W', normalized): + normalized = massaged = re.sub(r'\s', '', ascii_text(tag).lower()) + if re.search(r'\W', normalized): normalized = '' for c in massaged: - if re.search('\\W', c): + if re.search(r'\W', c): normalized += self.generate_unicode_name(c) else: normalized += c @@ -1395,7 +1395,7 @@ class CatalogBuilder(object): Return: (str): asciized version of author """ - return re.sub("\\W", "", ascii_text(author)) + return re.sub(r"\W", "", ascii_text(author)) def generate_format_args(self, book): """ Generate the format args for template substitution. @@ -4228,9 +4228,9 @@ class CatalogBuilder(object): # Generate a legal XHTML id/href string if self.letter_or_symbol(series) == self.SYMBOLS: - return "symbol_%s_series" % re.sub('\\W', '', series).lower() + return "symbol_%s_series" % re.sub(r'\W', '', series).lower() else: - return "%s_series" % re.sub('\\W', '', ascii_text(series)).lower() + return "%s_series" % re.sub(r'\W', '', ascii_text(series)).lower() def generate_short_description(self, description, dest=None): """ Generate a truncated version of the supplied string. @@ -4311,7 +4311,7 @@ class CatalogBuilder(object): else: if re.match('[0-9]+', word[0]): word = word.replace(',', '') - suffix = re.search('[\\D]', word) + suffix = re.search(r'[\D]', word) if suffix: word = '%10.0f%s' % (float(word[:suffix.start()]), word[suffix.start():]) else: @@ -4327,7 +4327,7 @@ class CatalogBuilder(object): else: if re.search('[0-9]+', word[0]): word = word.replace(',', '') - suffix = re.search('[\\D]', word) + suffix = re.search(r'[\D]', word) if suffix: word = '%10.0f%s' % (float(word[:suffix.start()]), word[suffix.start():]) else: diff --git a/src/calibre/library/catalogs/utils.py b/src/calibre/library/catalogs/utils.py index 593b96cef7..b9e5d69200 100644 --- a/src/calibre/library/catalogs/utils.py +++ b/src/calibre/library/catalogs/utils.py @@ -90,8 +90,8 @@ class NumberToText(object): # {{{ # Special case ordinals if re.search('[st|nd|rd|th]',self.number): self.number = re.sub(',','',self.number) - ordinal_suffix = re.search('[\\D]', self.number) - ordinal_number = re.sub('\\D','',re.sub(',','',self.number)) + ordinal_suffix = re.search(r'[\D]', self.number) + ordinal_number = re.sub(r'\D','',re.sub(',','',self.number)) if self.verbose: self.log("Ordinal: %s" % ordinal_number) self.number_as_float = ordinal_number @@ -155,8 +155,8 @@ class NumberToText(object): # {{{ if self.verbose: self.log("Hybrid: %s" % self.number) # Split the token into number/text - number_position = re.search('\\d',self.number).start() - text_position = re.search('\\D',self.number).start() + number_position = re.search(r'\d',self.number).start() + text_position = re.search(r'\D',self.number).start() if number_position < text_position: number = self.number[:text_position] text = self.number[text_position:] diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index 01a29e3813..48b13667b1 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -60,7 +60,7 @@ def _connect(path): conn = sqlite.connect(path, factory=Connection, detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES) conn.row_factory = lambda cursor, row : list(row) conn.create_aggregate('concat', 1, Concatenate) - title_pat = re.compile('^(A|The|An)\\s+', re.IGNORECASE) + title_pat = re.compile(r'^(A|The|An)\s+', re.IGNORECASE) def title_sort(title): match = title_pat.search(title)