From e7bbd3f14f85a627f27394b423530b370d42c201 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Aug 2009 14:16:39 -0600 Subject: [PATCH 1/7] Fix #3110 (Book genre) --- src/calibre/ebooks/metadata/fb2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/ebooks/metadata/fb2.py b/src/calibre/ebooks/metadata/fb2.py index e81f8fe108..eded2dc056 100644 --- a/src/calibre/ebooks/metadata/fb2.py +++ b/src/calibre/ebooks/metadata/fb2.py @@ -19,6 +19,8 @@ def get_metadata(stream): author= [firstname+" "+lastname] title = soup.find("book-title").string comments = soup.find("annotation") + tags = soup.findAll('genre') + tags = [t.contents[0] for t in tags] cp = soup.find('coverpage') cdata = None if cp: @@ -39,6 +41,8 @@ def get_metadata(stream): mi = MetaInformation(title, author) mi.comments = comments mi.author_sort = lastname+'; '+firstname + if tags: + mi.tags = tags if series: mi.series = series.get('name', None) try: From 3219cb3aa57d1fba8ed1e67c8e84a21ddde839db Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Aug 2009 14:47:14 -0600 Subject: [PATCH 2/7] Fix #3106 (Mobi book will not convert to ePub) --- src/calibre/ebooks/oeb/base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index 993edea279..6ef95f62d7 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -795,14 +795,18 @@ class Manifest(object): def first_pass(data): try: data = etree.fromstring(data) - except etree.XMLSyntaxError: + except etree.XMLSyntaxError, err: repl = lambda m: ENTITYDEFS.get(m.group(1), m.group(0)) data = ENTITY_RE.sub(repl, data) try: data = etree.fromstring(data) - except etree.XMLSyntaxError: + except etree.XMLSyntaxError, err: self.oeb.logger.warn('Parsing file %r as HTML' % self.href) - data = html.fromstring(data) + if err.args and err.args[0].startswith('Excessive depth'): + from lxml.html import soupparser + data = soupparser.fromstring(data) + else: + data = html.fromstring(data) data.attrib.pop('xmlns', None) for elem in data.iter(tag=etree.Comment): if elem.text: From c363b4361d23aeebf05147ce20134c67dc11ba1c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Aug 2009 15:16:08 -0600 Subject: [PATCH 3/7] IGN:Misc. fixes --- src/calibre/ebooks/oeb/transforms/flatcss.py | 7 ++++++- src/calibre/gui2/add.py | 7 ++++++- src/calibre/manual/templates/layout.html | 4 +++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 6ffd41ed9c..464acbe0e0 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -183,7 +183,12 @@ class CSSFlattener(object): elif value <= slineh: cssdict[property] = "%0.5fem" % (dlineh / fsize) else: - value = round(value / slineh) * dlineh + try: + value = round(value / slineh) * dlineh + except: + self.oeb.logger.warning( + 'Invalid length:', value) + value = 0.0 cssdict[property] = "%0.5fem" % (value / fsize) def flatten_node(self, node, stylizer, names, styles, psize, left=0): diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index b872b033d3..f976d72fee 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -129,7 +129,12 @@ class Adder(QObject): mi = MetaInformation('', [_('Unknown')]) self.critical[name] = open(opf, 'rb').read().decode('utf-8', 'replace') else: - mi = MetaInformation(OPF(opf)) + try: + mi = MetaInformation(OPF(opf)) + except: + import traceback + mi = MetaInformation('', [_('Unknown')]) + self.critical[name] = traceback.format_exc() if not mi.title: mi.title = os.path.splitext(name)[0] mi.title = mi.title if isinstance(mi.title, unicode) else \ diff --git a/src/calibre/manual/templates/layout.html b/src/calibre/manual/templates/layout.html index 3564357684..606fc69386 100644 --- a/src/calibre/manual/templates/layout.html +++ b/src/calibre/manual/templates/layout.html @@ -1,6 +1,8 @@ {% extends "!layout.html" %} {% block sidebarlogo %} -{{ super() }} +
From a52d470f20e1cb9160d61feee36a230f939acfce Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Aug 2009 16:33:28 -0600 Subject: [PATCH 4/7] EPUB output: Now works when launched from the command line with no X server on linux --- src/calibre/ebooks/oeb/transforms/rescale.py | 52 +++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/rescale.py b/src/calibre/ebooks/oeb/transforms/rescale.py index 5b62e5fda5..7ce3b5a588 100644 --- a/src/calibre/ebooks/oeb/transforms/rescale.py +++ b/src/calibre/ebooks/oeb/transforms/rescale.py @@ -6,32 +6,60 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import cStringIO + from calibre import fit_image class RescaleImages(object): 'Rescale all images to fit inside given screen size' def __call__(self, oeb, opts): - from PyQt4.Qt import QApplication, QImage, Qt - from calibre.gui2 import pixmap_to_data self.oeb, self.opts, self.log = oeb, opts, oeb.log - page_width, page_height = opts.dest.width, opts.dest.height - for item in oeb.manifest: + from calibre.gui2 import is_ok_to_use_qt + self.rescale(qt=is_ok_to_use_qt()) + + def rescale(self, qt=True): + from PyQt4.Qt import QImage, Qt + from calibre.gui2 import pixmap_to_data + try: + from PIL import Image as PILImage + PILImage + except ImportError: + import Image as PILImage + + + page_width, page_height = self.opts.dest.width, self.opts.dest.height + for item in self.oeb.manifest: if item.media_type.startswith('image'): raw = item.data if not raw: continue - if QApplication.instance() is None: - QApplication([]) + if qt: + img = QImage(10, 10, QImage.Format_ARGB32_Premultiplied) + if not img.loadFromData(raw): continue + width, height = img.width(), img.height() + else: + f = cStringIO.StringIO(raw) + try: + im = PILImage.open(f) + except IOError: + continue + width, height = im.size + + - img = QImage(10, 10, QImage.Format_ARGB32_Premultiplied) - if not img.loadFromData(raw): continue - width, height = img.width(), img.height() scaled, new_width, new_height = fit_image(width, height, page_width, page_height) if scaled: self.log('Rescaling image', item.href) - img = img.scaled(new_width, new_height, - Qt.IgnoreAspectRatio, Qt.SmoothTransformation) - item.data = pixmap_to_data(img) + if qt: + img = img.scaled(new_width, new_height, + Qt.IgnoreAspectRatio, Qt.SmoothTransformation) + item.data = pixmap_to_data(img) + else: + im = im.resize((int(new_width), int(new_height)), PILImage.ANTIALIAS) + of = cStringIO.StringIO() + im.convert('RGB').save(of, 'JPEG') + item.data = of.getvalue() + From d55a37c41f77127f141975369e149ff219bcf1e4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Aug 2009 17:11:42 -0600 Subject: [PATCH 5/7] Implement #3112 (FB2 icon) --- src/calibre/gui2/__init__.py | 1 + src/calibre/gui2/images/mimetypes/fb2.svg | 26 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 src/calibre/gui2/images/mimetypes/fb2.svg diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index ca676133b8..cca9680207 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -267,6 +267,7 @@ class FileIconProvider(QFileIconProvider): 'azw' : 'mobi', 'mobi' : 'mobi', 'epub' : 'epub', + 'fb2' : 'fb2', } def __init__(self): diff --git a/src/calibre/gui2/images/mimetypes/fb2.svg b/src/calibre/gui2/images/mimetypes/fb2.svg new file mode 100644 index 0000000000..009ca7d109 --- /dev/null +++ b/src/calibre/gui2/images/mimetypes/fb2.svg @@ -0,0 +1,26 @@ + + + + + + + From d668f536d67297e5e69200629d3c51334f995fec Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Aug 2009 08:53:54 -0600 Subject: [PATCH 6/7] Fix #3117 (Calibre inverts author names even when it doesn't need to) --- src/calibre/ebooks/mobi/reader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index d8947568ca..064751a878 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -78,7 +78,10 @@ class EXTHHeader(object): if id == 100: if self.mi.authors == [_('Unknown')]: self.mi.authors = [] - self.mi.authors.append(content.decode(codec, 'ignore').strip()) + au = content.decode(codec, 'ignore').strip() + self.mi.authors.append(au) + if re.match(r'\S+?\s*,\s+\S+', au.strip()): + self.mi.author_sort = au.strip() elif id == 101: self.mi.publisher = content.decode(codec, 'ignore').strip() elif id == 103: From fbaa3217388fef46c62bcc9769a94286475847f9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Aug 2009 09:40:38 -0600 Subject: [PATCH 7/7] Linux automounting: Remove dependency on pmount. Instead use custom mount helper. Linux distributors: Make sure calibre-mount-helper is installed setuid and setgid root --- installer/linux/freeze.py | 1 + setup.py | 30 ++++- src/calibre/devices/linux_mount_helper.c | 141 +++++++++++++++++++++++ src/calibre/devices/usbms/device.py | 41 +++---- src/calibre/library/cli.py | 2 +- src/calibre/trac/plugins/download.py | 10 +- 6 files changed, 197 insertions(+), 28 deletions(-) create mode 100644 src/calibre/devices/linux_mount_helper.c diff --git a/installer/linux/freeze.py b/installer/linux/freeze.py index 193d8183ed..3c64cf52c2 100644 --- a/installer/linux/freeze.py +++ b/installer/linux/freeze.py @@ -28,6 +28,7 @@ def freeze(): binary_includes = [ '/usr/bin/pdftohtml', + '/usr/bin/calibre-mount-helper', '/usr/lib/libunrar.so', '/usr/lib/libsqlite3.so.0', '/usr/lib/libsqlite3.so.0', diff --git a/setup.py b/setup.py index 23db3940f4..9054e9ea9a 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,24 @@ main_functions = { 'gui' : [_ep_to_function(i) for i in entry_points['gui_scripts']], } +def setup_mount_helper(): + def warn(): + print 'WARNING: Failed to compile mount helper. Auto mounting of', + print 'devices will not work' + + if os.geteuid() != 0: + return warn() + import stat + src = os.path.join('src', 'calibre', 'devices', 'linux_mount_helper.c') + dest = '/usr/bin/calibre-mount-helper' + p = subprocess.Popen(['gcc', '-Wall', src, '-o', dest]) + ret = p.wait() + if ret != 0: + return warn() + os.chown(dest, 0, 0) + os.chmod(dest, + stat.S_ISUID|stat.S_ISGID|stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH) + if __name__ == '__main__': from setuptools import setup, find_packages from pyqtdistutils import PyQtExtension, build_ext, Extension @@ -68,7 +86,11 @@ if __name__ == '__main__': qt_inc = qt_inc if qt_inc not in ('', '**Unknown**') and os.path.isdir(qt_inc) else None qt_lib = qmake_query('QT_INSTALL_LIBS').splitlines()[0] qt_lib = qt_lib if qt_lib not in ('', '**Unknown**') and os.path.isdir(qt_lib) else None - + if qt_lib is None or qt_inc is None: + print 'WARNING: Could not find QT librariers and headers.', + print 'Is qmake in your PATH?' + + if iswindows: optional.append(Extension('calibre.plugins.winutil', sources=['src/calibre/utils/windows/winutil.c'], @@ -90,7 +112,8 @@ if __name__ == '__main__': poppler_inc = '/Volumes/sw/build/poppler-0.10.7/qt4/src' poppler_lib = '/Users/kovid/poppler/lib' poppler_inc = os.environ.get('POPPLER_INC_DIR', poppler_inc) - if os.path.exists(os.path.join(poppler_inc, 'poppler-qt4.h')): + if os.path.exists(os.path.join(poppler_inc, 'poppler-qt4.h'))\ + and qt_lib is not None and qt_inc is not None: optional.append(Extension('calibre.plugins.calibre_poppler', sources=['src/calibre/utils/poppler/poppler.cpp'], libraries=(['poppler', 'poppler-qt4']+poppler_libs), @@ -253,6 +276,7 @@ if __name__ == '__main__': if 'develop' in ' '.join(sys.argv) and islinux: subprocess.check_call('calibre_postinstall --do-not-reload-udev-hal', shell=True) + setup_mount_helper() if 'install' in sys.argv and islinux: subprocess.check_call('calibre_postinstall', shell=True) - + setup_mount_helper() diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c new file mode 100644 index 0000000000..41dab3fada --- /dev/null +++ b/src/calibre/devices/linux_mount_helper.c @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MARKER ".created_by_calibre_mount_helper" + +int exists(char *path) { + struct stat file_info; + if (stat(path, &file_info) == 0) return 1; + return 0; +} + +int get_root() { + int res; + res = setreuid(0, 0); + if (res != 0) return 1; + if (setregid(0, 0) != 0) return 1; + return 0; +} + +int do_mount(char *dev, char *mp) { + char options[1000]; + char marker[2000]; + if (exists(dev) == 0) { + fprintf(stderr, "Specified device node does not exist\n"); + return EXIT_FAILURE; + } + if (exists(mp) == 0) { + if (mkdir(mp, S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH) != 0) { + int errsv = errno; + fprintf(stderr, "Failed to create mount point with error: %s\n", strerror(errsv)); + } + } + strncat(marker, mp, strlen(mp)); + strncat(marker, "/", 1); + strncat(marker, MARKER, strlen(MARKER)); + if (exists(marker) == 0) { + int fd = creat(marker, S_IRUSR|S_IWUSR); + if (fd == -1) { + int errsv = errno; + fprintf(stderr, "Failed to create marker with error: %s\n", strerror(errsv)); + return EXIT_FAILURE; + } + close(fd); + } + snprintf(options, 1000, "rw,noexec,nosuid,sync,nodev,quiet,shortname=mixed,uid=%d,gid=%d,umask=077,fmask=0177,dmask=0077,utf8,iocharset=iso8859-1", getuid(), getgid()); + if (get_root() != 0) { + fprintf(stderr, "Failed to elevate to root privileges\n"); + return EXIT_FAILURE; + } + execlp("mount", "mount", "-t", "vfat", "-o", options, dev, mp, NULL); + int errsv = errno; + fprintf(stderr, "Failed to mount with error: %s\n", strerror(errsv)); + return EXIT_FAILURE; +} + +int do_eject(char *dev, char*mp) { + char marker[2000]; + int status = EXIT_FAILURE, ret; + if (get_root() != 0) { + fprintf(stderr, "Failed to elevate to root privileges\n"); + return EXIT_FAILURE; + } + int pid = fork(); + if (pid == -1) { + fprintf(stderr, "Failed to fork\n"); + return EXIT_FAILURE; + } + if (pid == 0) { + if (get_root() != 0) { + fprintf(stderr, "Failed to elevate to root privileges\n"); + return EXIT_FAILURE; + } + execlp("eject", "eject", "-s", dev, NULL); + int errsv = errno; + fprintf(stderr, "Failed to eject with error: %s\n", strerror(errsv)); + return EXIT_FAILURE; + } else { + int i; + for (i =0; i < 7; i++) { + sleep(1); + ret = waitpid(pid, &status, WNOHANG); + if (ret == -1) return EXIT_FAILURE; + if (ret > 0) break; + } + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { + strncat(marker, mp, strlen(mp)); + strncat(marker, "/", 1); + strncat(marker, MARKER, strlen(MARKER)); + if (exists(marker)) { + int urt = unlink(marker); + if (urt == -1) { + fprintf(stderr, "Failed to unlink marker: %s\n", strerror(errno)); + return EXIT_FAILURE; + } + int rmd = rmdir(mp); + if (rmd == -1) { + fprintf(stderr, "Failed to remove mount point: %s\n", strerror(errno)); + return EXIT_FAILURE; + } + } + } + } + return EXIT_SUCCESS; +} + +int main(int argc, char** argv) +{ + char *action, *dev, *mp; + + /*printf("Real UID\t= %d\n", getuid()); + printf("Effective UID\t= %d\n", geteuid()); + printf("Real GID\t= %d\n", getgid()); + printf("Effective GID\t= %d\n", getegid());*/ + + if (argc != 4) { + fprintf(stderr, "Needs 3 arguments: action, device node and mount point\n"); + return EXIT_FAILURE; + } + action = argv[1]; dev = argv[2]; mp = argv[3]; + + if (strncmp(action, "mount", 5) == 0) { + return do_mount(dev, mp); + } + else if (strncmp(action, "eject", 7) == 0) { + return do_eject(dev, mp); + } else { + fprintf(stderr, "Unrecognized action: must be mount or eject\n"); + return EXIT_FAILURE; + } + + + return EXIT_SUCCESS; +} + diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 50f183a83e..5504136e0c 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -17,7 +17,6 @@ import time import re import sys import glob -import shutil from itertools import repeat from math import ceil @@ -489,7 +488,7 @@ class Device(DeviceConfig, DevicePlugin): label = self.STORAGE_CARD_VOLUME_LABEL if type == 'cardb': label = self.STORAGE_CARD2_VOLUME_LABEL - if label is None: + if not label: label = self.STORAGE_CARD_VOLUME_LABEL + ' 2' extra = 0 while True: @@ -501,11 +500,15 @@ class Device(DeviceConfig, DevicePlugin): label += ' (%d)'%extra def do_mount(node, label): - cmd = ['pmount', '-w', '-s'] + cmd = 'calibre-mount-helper' + if getattr(sys, 'frozen_path', False): + cmd = os.path.join(sys.frozen_path, cmd) + cmd = [cmd, 'mount'] try: - p = subprocess.Popen(cmd + [node, label]) + p = subprocess.Popen(cmd + [node, '/media/'+label]) except OSError: - raise DeviceError(_('You must install the pmount package.')) + raise DeviceError( + _('Could not find mount helper: %s.')%cmd[0]) while p.poll() is None: time.sleep(0.1) return p.returncode @@ -520,11 +523,13 @@ class Device(DeviceConfig, DevicePlugin): raise DeviceError(_('Unable to detect the %s disk drive.') %self.__class__.__name__) + self._linux_mount_map = {} mp, ret = mount(main, 'main') if mp is None: raise DeviceError( _('Unable to mount main memory (Error code: %d)')%ret) if not mp.endswith('/'): mp += '/' + self._linux_mount_map[main] = mp self._main_prefix = mp cards = [(carda, '_card_a_prefix', 'carda'), (cardb, '_card_b_prefix', 'cardb')] @@ -536,6 +541,7 @@ class Device(DeviceConfig, DevicePlugin): else: if not mp.endswith('/'): mp += '/' setattr(self, prefix, mp) + self._linux_mount_map[card] = mp def open(self): time.sleep(5) @@ -595,27 +601,16 @@ class Device(DeviceConfig, DevicePlugin): success = False for drive in drives: if drive: - cmd = ['pumount', '-l'] + cmd = 'calibre-mount-helper' + if getattr(sys, 'frozen_path', False): + cmd = os.path.join(sys.frozen_path, cmd) + cmd = [cmd, 'eject'] + mp = getattr(self, "_linux_mount_map", {}).get(drive, + 'dummy/')[:-1] try: - p = subprocess.Popen(cmd + [drive]) + subprocess.Popen(cmd + [drive, mp]).wait() except: pass - while p.poll() is None: - time.sleep(0.1) - success = success or p.returncode == 0 - try: - subprocess.Popen(['sudo', 'eject', drive]) - except: - pass - for x in ('_main_prefix', '_card_a_prefix', '_card_b_prefix'): - x = getattr(self, x, None) - if x is not None: - if x.startswith('/media/') and os.path.exists(x) \ - and not os.listdir(x): - try: - shutil.rmtree(x) - except: - pass def eject(self): if islinux: diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 8bffd05764..75f72297a4 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -153,7 +153,7 @@ def do_list(db, fields, sort_by, ascending, search_text, line_width, separator, break widths = list(base_widths) - titles = map(lambda x, y: '%-*s'%(x, y), widths, fields) + titles = map(lambda x, y: '%-*s%s'%(x-len(separator), y, separator), widths, fields) print terminal_controller.GREEN + ''.join(titles)+terminal_controller.NORMAL wrappers = map(lambda x: TextWrapper(x-1), widths) diff --git a/src/calibre/trac/plugins/download.py b/src/calibre/trac/plugins/download.py index 6ea50c497b..b25d06155f 100644 --- a/src/calibre/trac/plugins/download.py +++ b/src/calibre/trac/plugins/download.py @@ -21,7 +21,6 @@ DEPENDENCIES = [ ('dnspython', '1.6.0', 'dnspython', 'dnspython', 'dnspython', 'dnspython'), ('poppler-qt4', '0.10.6', 'poppler-qt4', 'poppler-qt4', 'poppler-qt4', 'poppler-qt4'), ('podofo', '0.7', 'podofo', 'podofo', 'podofo', 'podofo'), - ('pmount', '0.9.19', 'pmount', 'pmount', 'pmount', 'pmount'), ] @@ -385,6 +384,15 @@ else: print 'Extracting files to %s ...'%destdir extract_tarball(f, destdir) + mh = os.path.join(destdir, 'calibre-mount-helper') + if os.geteuid() == 0: + os.chown(mh, 0, 0) + os.chmod(mh, + stat.S_ISUID|stat.S_ISGID|stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH) + else: + print 'WARNING: Not running as root. Cannot install mount helper.', + print 'Device automounting may not work.' + pi = os.path.join(destdir, 'calibre_postinstall') subprocess.call(pi, shell=True) return 0