From 37eceaf02fd0ad4ccd29bba11186a6c95fe4066d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 15 Jul 2008 18:26:42 -0700 Subject: [PATCH 01/11] Fix linux binary installer to work with dumb terminals --- src/calibre/linux_installer.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/calibre/linux_installer.py b/src/calibre/linux_installer.py index db9c8dd1b1..2ae669fdf0 100644 --- a/src/calibre/linux_installer.py +++ b/src/calibre/linux_installer.py @@ -240,13 +240,21 @@ def do_postinstall(destdir): os.chdir(cwd) def download_tarball(): - pb = ProgressBar(TerminalController(sys.stdout), 'Downloading calibre...') + try: + pb = ProgressBar(TerminalController(sys.stdout), 'Downloading calibre...') + except ValueError: + print 'Downloading calibre...' + pb = None src = urllib2.urlopen(MOBILEREAD+'calibre-%version-i686.tar.bz2') size = int(src.info()['content-length']) f = tempfile.NamedTemporaryFile() while f.tell() < size: f.write(src.read(4*1024)) - pb.update(f.tell()/float(size)) + percent = f.tell()/float(size) + if pb is not None: + pb.update(percent) + else: + print '%d%, '%int(percent*100) f.seek(0) return f From 97c15c0993377bb0276164270ae2e437b2323f77 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 15 Jul 2008 19:34:27 -0700 Subject: [PATCH 02/11] Fix linux binary installer on dumb terminals --- src/calibre/trac/plugins/Changelog.py | 2 +- src/calibre/trac/plugins/templates/pyinstaller.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/trac/plugins/Changelog.py b/src/calibre/trac/plugins/Changelog.py index 35c54b1bda..ff15329859 100644 --- a/src/calibre/trac/plugins/Changelog.py +++ b/src/calibre/trac/plugins/Changelog.py @@ -69,4 +69,4 @@ class ChangeLogMacro(WikiMacroBase): if __name__ == '__main__': print bzr_log_to_txt() - \ No newline at end of file + diff --git a/src/calibre/trac/plugins/templates/pyinstaller.html b/src/calibre/trac/plugins/templates/pyinstaller.html index c04fd13da4..4061b0aadc 100644 --- a/src/calibre/trac/plugins/templates/pyinstaller.html +++ b/src/calibre/trac/plugins/templates/pyinstaller.html @@ -44,4 +44,4 @@ - \ No newline at end of file + From c4eb7fea8b44425b3c3051efcc70d594b6f2cf56 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 15 Jul 2008 21:42:18 -0700 Subject: [PATCH 03/11] IGN:... --- src/calibre/linux_installer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/linux_installer.py b/src/calibre/linux_installer.py index 2ae669fdf0..8b012339f2 100644 --- a/src/calibre/linux_installer.py +++ b/src/calibre/linux_installer.py @@ -254,7 +254,7 @@ def download_tarball(): if pb is not None: pb.update(percent) else: - print '%d%, '%int(percent*100) + print '%d%%, '%int(percent*100) f.seek(0) return f @@ -277,4 +277,4 @@ def main(args=sys.argv): return 0 if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file + sys.exit(main()) From 28a93ca9c7d2fe85be259b926115fe56ee8f2b93 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 16 Jul 2008 13:10:03 -0700 Subject: [PATCH 04/11] IGN:Fix localization of timestamp for device files --- src/calibre/__init__.py | 7 ++++++- src/calibre/gui2/library.py | 4 ++-- src/calibre/linux_installer.py | 2 +- src/calibre/utils/fontconfig.py | 8 ++++---- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index acd5d2e930..5c55632da9 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -30,7 +30,12 @@ islinux = not(iswindows or isosx) try: locale.setlocale(locale.LC_ALL, '') except: - pass + dl = locale.getdefaultlocale() + try: + if dl: + locale.setlocale(dl[0]) + except: + pass try: preferred_encoding = locale.getpreferredencoding() diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index 70f92e2805..76527c3034 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -574,7 +574,7 @@ class DeviceBooksModel(BooksModel): for row in rows: if not succeeded: indices = self.row_indices(self.index(row, 0)) - self.emit(SIGNAL('dataChanged(QModelIndex, QModelIndex)'), indices[0], indices[-1]) + self.emit(SIGNAL('dataChanged(QModelIndex, QModelIndex)'), indices[0], indices[-1]) def paths_deleted(self, paths): self.map = list(range(0, len(self.db))) @@ -691,7 +691,7 @@ class DeviceBooksModel(BooksModel): dt = item.datetime dt = datetime(*dt[0:6]) dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight) - data[_('Timestamp')] = dt.ctime() + data[_('Timestamp')] = dt.strftime('%a %b %d %H:%M:%S %Y') data[_('Tags')] = ', '.join(item.tags) self.emit(SIGNAL('new_bookdisplay_data(PyQt_PyObject)'), data) diff --git a/src/calibre/linux_installer.py b/src/calibre/linux_installer.py index 8b012339f2..145b255b05 100644 --- a/src/calibre/linux_installer.py +++ b/src/calibre/linux_installer.py @@ -254,7 +254,7 @@ def download_tarball(): if pb is not None: pb.update(percent) else: - print '%d%%, '%int(percent*100) + print '%d%%, '%int(percent*100), f.seek(0) return f diff --git a/src/calibre/utils/fontconfig.py b/src/calibre/utils/fontconfig.py index 3a92a76670..0f6357609c 100644 --- a/src/calibre/utils/fontconfig.py +++ b/src/calibre/utils/fontconfig.py @@ -33,7 +33,7 @@ except: preferred_encoding = 'utf-8' iswindows = 'win32' in sys.platform or 'win64' in sys.platform -isosx = 'darwin' in sys.platform +isosx = 'darwin' in sys.platform def load_library(): if isosx: @@ -94,7 +94,7 @@ class FcValue(Structure): _fields_ = [ ('type', c_int), ('u', _FcValue) - ] + ] lib = load_library() lib.FcPatternCreate.restype = c_void_p @@ -145,12 +145,12 @@ if hasattr(sys, 'frameworks_dir'): elif not lib.FcInit(): raise RuntimeError(_('Could not initialize the fontconfig library')) -def find_font_families(allowed_extensions=['ttf']): +def find_font_families(allowed_extensions=['ttf', 'otf']): ''' Return an alphabetically sorted list of font families available on the system. `allowed_extensions`: A list of allowed extensions for font file types. Defaults to - `['ttf']`. If it is empty, it is ignored. + `['ttf', 'otf']`. If it is empty, it is ignored. ''' allowed_extensions = [i.lower() for i in allowed_extensions] From e6bfebed5a8070105a3b9ffcd75920e5150005b9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 16 Jul 2008 23:42:41 -0700 Subject: [PATCH 05/11] Move scanning of system fonts into a separate thread. This will greatly reduce first run startup times. --- src/calibre/gui2/main.py | 3 ++ src/calibre/utils/fontconfig.py | 51 +++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index d09922848f..5dafe3c683 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -59,6 +59,9 @@ class Main(MainWindow, Ui_MainWindow): def __init__(self, single_instance, opts, parent=None): MainWindow.__init__(self, opts, parent) + # Initialize fontconfig in a separate thread as this can be a lengthy + # process if run for the first time on this machine + self.fc = __import__('calibre.utils.fontconfig', fromlist=1) self.single_instance = single_instance if self.single_instance is not None: self.connect(self.single_instance, SIGNAL('message_received(PyQt_PyObject)'), diff --git a/src/calibre/utils/fontconfig.py b/src/calibre/utils/fontconfig.py index 0f6357609c..3e74362720 100644 --- a/src/calibre/utils/fontconfig.py +++ b/src/calibre/utils/fontconfig.py @@ -128,22 +128,40 @@ lib.FcConfigParseAndLoad.restype = c_int lib.FcConfigBuildFonts.argtypes = [c_void_p] lib.FcConfigBuildFonts.restype = c_int +_init_error = None +_initialized = False +from threading import Timer +def _do_init(): + # Initialize the fontconfig library. This has to be done manually + # for the OS X bundle as it may have its own private fontconfig. + if hasattr(sys, 'frameworks_dir'): + config_dir = os.path.join(os.path.dirname(getattr(sys, 'frameworks_dir')), 'Resources', 'fonts') + if isinstance(config_dir, unicode): + config_dir = config_dir.encode(sys.getfilesystemencoding()) + config = lib.FcConfigCreate() + if not lib.FcConfigParseAndLoad(config, os.path.join(config_dir, 'fonts.conf'), 1): + _init_error = 'Could not parse the fontconfig configuration' + return + if not lib.FcConfigBuildFonts(config): + _init_error = 'Could not build fonts' + return + if not lib.FcConfigSetCurrent(config): + _init_error = 'Could not set font config' + return + elif not lib.FcInit(): + _init_error = _('Could not initialize the fontconfig library') + return + global _initialized + _initialized = True + + +_init_timer = Timer(0.1, _do_init) +_init_timer.start() -# Initialize the fontconfig library. This has to be done manually -# for the OS X bundle as it may have its own private fontconfig. -if hasattr(sys, 'frameworks_dir'): - config_dir = os.path.join(os.path.dirname(getattr(sys, 'frameworks_dir')), 'Resources', 'fonts') - if isinstance(config_dir, unicode): - config_dir = config_dir.encode(sys.getfilesystemencoding()) - config = lib.FcConfigCreate() - if not lib.FcConfigParseAndLoad(config, os.path.join(config_dir, 'fonts.conf'), 1): - raise RuntimeError('Could not parse the fontconfig configuration') - if not lib.FcConfigBuildFonts(config): - raise RuntimeError('Could not build fonts') - if not lib.FcConfigSetCurrent(config): - raise RuntimeError('Could not set font config') -elif not lib.FcInit(): - raise RuntimeError(_('Could not initialize the fontconfig library')) +def join(): + _init_timer.join() + if _init_error is not None: + raise RuntimeError(_init_error) def find_font_families(allowed_extensions=['ttf', 'otf']): ''' @@ -152,6 +170,7 @@ def find_font_families(allowed_extensions=['ttf', 'otf']): `allowed_extensions`: A list of allowed extensions for font file types. Defaults to `['ttf', 'otf']`. If it is empty, it is ignored. ''' + join() allowed_extensions = [i.lower() for i in allowed_extensions] empty_pattern = lib.FcPatternCreate() @@ -193,6 +212,7 @@ def files_for_family(family, normalize=True): they are a tuple (slant, weight) otherwise they are strings from the set `('normal', 'bold', 'italic', 'bi', 'light', 'li')` ''' + join() if isinstance(family, unicode): family = family.encode(preferred_encoding) family_pattern = lib.FcPatternBuild(0, 'family', FcTypeString, family, 0) @@ -268,6 +288,7 @@ def match(name, sort=False, verbose=False): decreasing closeness of matching. `verbose`: If `True` print debugging information to stdout ''' + join() if isinstance(name, unicode): name = name.encode(preferred_encoding) pat = lib.FcNameParse(name) From 25310eb14605d23550f573a2d307e87da4bc5884 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 17 Jul 2008 09:07:17 -0700 Subject: [PATCH 06/11] IGN:... --- src/calibre/library/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 6e4ceb95b8..0e972cb20b 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -170,7 +170,8 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates): for mi, formats in dir_dups: db.import_book(mi, formats) else: - print >>sys.stderr, _('The following books were not added as they already exist in the database (see --duplicates option):') + if dir_dups or file_duplicates: + print >>sys.stderr, _('The following books were not added as they already exist in the database (see --duplicates option):') for mi, formats in dir_dups: title = mi.title if isinstance(title, unicode): From 2960781bb022dd7757def1b966091402d1e77393 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 17 Jul 2008 09:46:36 -0700 Subject: [PATCH 07/11] Add support for extracting cover images to fb2-meta --- src/calibre/ebooks/metadata/fb2.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/metadata/fb2.py b/src/calibre/ebooks/metadata/fb2.py index 5bffb47409..672fc3e9ee 100644 --- a/src/calibre/ebooks/metadata/fb2.py +++ b/src/calibre/ebooks/metadata/fb2.py @@ -5,7 +5,8 @@ __copyright__ = '2008, Anatoly Shipitsin ' '''Read meta information from fb2 files''' -import sys, os +import sys, os, mimetypes +from base64 import b64decode from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup from calibre.ebooks.metadata import MetaInformation @@ -18,15 +19,30 @@ def get_metadata(stream): author= [firstname+" "+lastname] title = soup.find("book-title").string comments = soup.find("annotation") + cp = soup.find('coverpage') + cdata = None + if cp: + cimage = cp.find('image', attrs={'l:href':True}) + if cimage: + id = cimage['l:href'].replace('#', '') + binary = soup.find('binary', id=id, attrs={'content-type':True}) + if binary: + mt = binary['content-type'] + exts = mimetypes.guess_all_extensions(mt) + if not exts: + exts = ['.jpg'] + cdata = (exts[0][1:], b64decode(binary.string.strip())) + if comments and len(comments) > 1: - comments = comments.p.contents[0] + comments = comments.p.contents[0] series = soup.find("sequence") - # series_index = series.index mi = MetaInformation(title, author) mi.comments = comments + mi.author_sort = lastname+'; '+firstname if series: mi.series = series.get('name', None) - # mi.series_index = series_index + if cdata: + mi.cover_data = cdata return mi def main(args=sys.argv): From 27a0a596ddd87a0fdf60aafc21437812d0bf4637 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 17 Jul 2008 11:54:18 -0700 Subject: [PATCH 08/11] Fix #882 --- src/calibre/ebooks/lrf/fb2/fb2.xsl | 32 ++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/calibre/ebooks/lrf/fb2/fb2.xsl b/src/calibre/ebooks/lrf/fb2/fb2.xsl index 75f3c245ed..f67b05b544 100644 --- a/src/calibre/ebooks/lrf/fb2/fb2.xsl +++ b/src/calibre/ebooks/lrf/fb2/fb2.xsl @@ -128,21 +128,35 @@ - - - - - - - + + + + None + + + + + + + + + + + + + + + TOC_ + + @@ -166,7 +180,9 @@ -

+ + + From 1cf4f8661a06bd405c86f9953e860059cb2ea1f7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 17 Jul 2008 11:55:01 -0700 Subject: [PATCH 09/11] Fix #883 --- src/calibre/ebooks/lrf/fb2/convert_from.py | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/calibre/ebooks/lrf/fb2/convert_from.py b/src/calibre/ebooks/lrf/fb2/convert_from.py index 29ceaa11cf..27c55757be 100644 --- a/src/calibre/ebooks/lrf/fb2/convert_from.py +++ b/src/calibre/ebooks/lrf/fb2/convert_from.py @@ -3,15 +3,13 @@ __copyright__ = '2008, Anatoly Shipitsin ' """ Convert .fb2 files to .lrf """ -import os, sys, tempfile, subprocess, shutil, logging, glob +import os, sys, tempfile, shutil, logging +from base64 import b64decode -from calibre.ptempfile import PersistentTemporaryFile from calibre.ebooks.lrf import option_parser as lrf_option_parser from calibre.ebooks.metadata.meta import get_metadata -from calibre.ebooks import ConversionError from calibre.ebooks.lrf.html.convert_from import process_file as html_process_file from calibre import setup_cli_handlers, __appname__ -from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup from calibre.resources import fb2_xsl def option_parser(): @@ -22,25 +20,27 @@ _('''%prog [options] mybook.fb2 %prog converts mybook.fb2 to mybook.lrf''')) parser.add_option('--debug-html-generation', action='store_true', default=False, dest='debug_html_generation', help=_('Print generated HTML to stdout and quit.')) + parser.add_option('--keep-intermediate-files', action='store_true', default=False, + help=_('Keep generated HTML files after completing conversion to LRF.')) return parser +def extract_embedded_content(doc): + for elem in doc.xpath('./*'): + if 'binary' in elem.tag and elem.attrib.has_key('id'): + fname = elem.attrib['id'] + data = b64decode(elem.text.strip()) + open(fname, 'wb').write(data) def generate_html(fb2file, encoding, logger): from lxml import etree - tdir = tempfile.mkdtemp(prefix=__appname__+'_') - ofile = os.path.join(tdir, 'index.xml') + tdir = tempfile.mkdtemp(prefix=__appname__+'_fb2_') cwd = os.getcwdu() os.chdir(tdir) try: logger.info('Parsing XML...') parser = etree.XMLParser(recover=True, no_network=True) - try: - doc = etree.parse(fb2file, parser) - except: - raise - logger.info('Parsing failed. Trying to clean up XML...') - soup = BeautifulStoneSoup(open(fb2file, 'rb').read()) - doc = etree.fromstring(str(soup)) + doc = etree.parse(fb2file, parser) + extract_embedded_content(doc) logger.info('Converting XML to HTML...') styledoc = etree.fromstring(fb2_xsl) @@ -72,7 +72,7 @@ def process_file(path, options, logger=None): options.output = os.path.abspath(os.path.basename(os.path.splitext(path)[0]) + ext) options.output = os.path.abspath(os.path.expanduser(options.output)) if not mi.title: - mi.title = os.path.splitext(os.path.basename(rtf))[0] + mi.title = os.path.splitext(os.path.basename(fb2))[0] if (not options.title or options.title == 'Unknown'): options.title = mi.title if (not options.author or options.author == 'Unknown') and mi.authors: @@ -85,7 +85,7 @@ def process_file(path, options, logger=None): html_process_file(htmlfile, options, logger) finally: os.chdir(cwd) - if hasattr(options, 'keep_intermediate_files') and options.keep_intermediate_files: + if getattr(options, 'keep_intermediate_files', False): logger.debug('Intermediate files in '+ tdir) else: shutil.rmtree(tdir) From 6c339c4d4b17757757a06a6be09273e2084cba97 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 17 Jul 2008 12:20:49 -0700 Subject: [PATCH 10/11] IGN:... --- src/calibre/ebooks/lrf/fb2/fb2.xsl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/ebooks/lrf/fb2/fb2.xsl b/src/calibre/ebooks/lrf/fb2/fb2.xsl index f67b05b544..7a977aee76 100644 --- a/src/calibre/ebooks/lrf/fb2/fb2.xsl +++ b/src/calibre/ebooks/lrf/fb2/fb2.xsl @@ -136,6 +136,11 @@ + + + + + From 669b983d757c4217f7142af7b727153d730b8a0f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 17 Jul 2008 14:42:15 -0700 Subject: [PATCH 11/11] Fix #884 --- src/calibre/ebooks/metadata/toc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/toc.py b/src/calibre/ebooks/metadata/toc.py index a966dd6fae..dc039a7f80 100644 --- a/src/calibre/ebooks/metadata/toc.py +++ b/src/calibre/ebooks/metadata/toc.py @@ -70,7 +70,7 @@ class TOC(list): break if toc is not None: - if toc.lower() != 'ncx': + if toc.lower() not in ('ncx', 'ncxtoc'): toc = urlparse(unquote(toc))[2] toc = toc.replace('/', os.sep) if not os.path.isabs(toc): @@ -88,6 +88,10 @@ class TOC(list): traceback.print_exc(file=sys.stdout) print 'Continuing anyway' else: + path = opfreader.manifest.item(toc.lower()) + if path and os.access(path, os.R_OK): + self.read_ncx_toc(path) + return cwd = os.path.abspath(self.base_path) m = glob.glob(os.path.join(cwd, '*.ncx')) if m: