From 1f9302436ccf67cedf134567e7b6c7c278cbf208 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 7 Sep 2012 17:53:51 +0530 Subject: [PATCH 01/45] ... --- src/calibre/devices/android/driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 2d5e73bece..10fdb50df9 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -208,7 +208,7 @@ class ANDROID(USBMS): 'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP', 'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD', 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0', - 'COBY_MID'] + 'COBY_MID', 'VS'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', @@ -227,7 +227,7 @@ class ANDROID(USBMS): 'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX', 'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE', 'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', - 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VS'] + 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', From 6f13b354e48f16e5e58b8f4f356d143071ed2088 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 7 Sep 2012 17:57:35 +0530 Subject: [PATCH 02/45] ... --- Changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index d1119e1bbb..b0a9bd68d4 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -35,7 +35,7 @@ - title: "Add an option under Preferences->Look & Feel->Book Details to hide the cover in the book details panel" - - title: "The Calibre Companion Android app that allows wireless connection of Android device to calibre is out of beta. See https://play.google.com/stor/apps/details?id=com.multipie.calibreandroid" + - title: "The Calibre Companion Android app that allows wireless connection of Android device to calibre is out of beta. See https://play.google.com/store/apps/details?id=com.multipie.calibreandroid" bug fixes: - title: "Fix sorting by author not working in the device view in calibre when connected to iTunes" From 3b989ae2a699c7095018dc3ebaab46e0f5cd459b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 7 Sep 2012 20:17:02 +0530 Subject: [PATCH 03/45] ... --- recipes/arcamax.recipe | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/recipes/arcamax.recipe b/recipes/arcamax.recipe index 0f144466d7..924f5ad088 100644 --- a/recipes/arcamax.recipe +++ b/recipes/arcamax.recipe @@ -43,38 +43,38 @@ class Arcamax(BasicNewsRecipe): feeds = [] for title, url in [ ######## COMICS - GENERAL ######## - #(u"9 Chickweed Lane", u"http://www.arcamax.com/ninechickweedlane"), - #(u"Agnes", u"http://www.arcamax.com/agnes"), - #(u"Andy Capp", u"http://www.arcamax.com/andycapp"), + #(u"9 Chickweed Lane", #u"http://www.arcamax.com/thefunnies/ninechickweedlane"), + #(u"Agnes", u"http://www.arcamax.com/thefunnies/agnes"), + #(u"Andy Capp", #u"http://www.arcamax.com/thefunnies/andycapp"), (u"BC", u"http://www.arcamax.com/thefunnies/bc"), - #(u"Baby Blues", u"http://www.arcamax.com/babyblues"), - #(u"Beetle Bailey", u"http://www.arcamax.com/beetlebailey"), + #(u"Baby Blues", #u"http://www.arcamax.com/thefunnies/babyblues"), + #(u"Beetle Bailey", #u"http://www.arcamax.com/thefunnies/beetlebailey"), (u"Blondie", u"http://www.arcamax.com/thefunnies/blondie"), - #u"Boondocks", u"http://www.arcamax.com/boondocks"), - #(u"Cathy", u"http://www.arcamax.com/cathy"), - #(u"Daddys Home", u"http://www.arcamax.com/daddyshome"), + #u"Boondocks", u"http://www.arcamax.com/thefunnies/boondocks"), + #(u"Cathy", u"http://www.arcamax.com/thefunnies/cathy"), + #(u"Daddys Home", #u"http://www.arcamax.com/thefunnies/daddyshome"), (u"Dilbert", u"http://www.arcamax.com/thefunnies/dilbert"), - #(u"Dinette Set", u"http://www.arcamax.com/thedinetteset"), + #(u"Dinette Set", #u"http://www.arcamax.com/thefunnies/thedinetteset"), (u"Dog Eat Doug", u"http://www.arcamax.com/thefunnies/dogeatdoug"), (u"Doonesbury", u"http://www.arcamax.com/thefunnies/doonesbury"), - #(u"Dustin", u"http://www.arcamax.com/dustin"), + #(u"Dustin", u"http://www.arcamax.com/thefunnies/dustin"), (u"Family Circus", u"http://www.arcamax.com/thefunnies/familycircus"), (u"Garfield", u"http://www.arcamax.com/thefunnies/garfield"), - #(u"Get Fuzzy", u"http://www.arcamax.com/getfuzzy"), - #(u"Girls and Sports", u"http://www.arcamax.com/girlsandsports"), - #(u"Hagar the Horrible", u"http://www.arcamax.com/hagarthehorrible"), - #(u"Heathcliff", u"http://www.arcamax.com/heathcliff"), - #(u"Jerry King Cartoons", u"http://www.arcamax.com/humorcartoon"), - #(u"Luann", u"http://www.arcamax.com/luann"), - #(u"Momma", u"http://www.arcamax.com/momma"), - #(u"Mother Goose and Grimm", u"http://www.arcamax.com/mothergooseandgrimm"), + #(u"Get Fuzzy", #u"http://www.arcamax.com/thefunnies/getfuzzy"), + #(u"Girls and Sports", #u"http://www.arcamax.com/thefunnies/girlsandsports"), + #(u"Hagar the Horrible", #u"http://www.arcamax.com/thefunnies/hagarthehorrible"), + #(u"Heathcliff", #u"http://www.arcamax.com/thefunnies/heathcliff"), + #(u"Jerry King Cartoons", #u"http://www.arcamax.com/thefunnies/humorcartoon"), + #(u"Luann", u"http://www.arcamax.com/thefunnies/luann"), + #(u"Momma", u"http://www.arcamax.com/thefunnies/momma"), + #(u"Mother Goose and Grimm", #u"http://www.arcamax.com/thefunnies/mothergooseandgrimm"), (u"Mutts", u"http://www.arcamax.com/thefunnies/mutts"), - #(u"Non Sequitur", u"http://www.arcamax.com/nonsequitur"), - #(u"Pearls Before Swine", u"http://www.arcamax.com/pearlsbeforeswine"), - #(u"Pickles", u"http://www.arcamax.com/pickles"), - #(u"Red and Rover", u"http://www.arcamax.com/redandrover"), - #(u"Rubes", u"http://www.arcamax.com/rubes"), - #(u"Rugrats", u"http://www.arcamax.com/rugrats"), + #(u"Non Sequitur", #u"http://www.arcamax.com/thefunnies/nonsequitur"), + #(u"Pearls Before Swine", #u"http://www.arcamax.com/thefunnies/pearlsbeforeswine"), + #(u"Pickles", u"http://www.arcamax.com/thefunnies/pickles"), + #(u"Red and Rover", #u"http://www.arcamax.com/thefunnies/redandrover"), + #(u"Rubes", u"http://www.arcamax.com/thefunnies/rubes"), + #(u"Rugrats", u"http://www.arcamax.com/thefunnies/rugrats"), (u"Speed Bump", u"http://www.arcamax.com/thefunnies/speedbump"), (u"Wizard of Id", u"http://www.arcamax.com/thefunnies/wizardofid"), (u"Zits", u"http://www.arcamax.com/thefunnies/zits"), From 0a3f3682804a6ccbdf6bf1b7de2279fc08d624ed Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Sep 2012 09:25:59 +0530 Subject: [PATCH 04/45] Fix #1047691 (Drag n drop to Book detail failure) --- src/calibre/library/database2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 17c01a6f56..0b23e3f0a4 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1432,6 +1432,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): pdir = os.path.dirname(dest) if not os.path.exists(pdir): os.makedirs(pdir) + size = 0 if copy_function is not None: copy_function(dest) size = os.path.getsize(dest) @@ -1441,6 +1442,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): with lopen(dest, 'wb') as f: shutil.copyfileobj(stream, f) size = f.tell() + elif os.path.exists(dest): + size = os.path.getsize(dest) self.conn.execute('INSERT OR REPLACE INTO data (book,format,uncompressed_size,name) VALUES (?,?,?,?)', (id, format.upper(), size, name)) self.update_last_modified([id], commit=False) From a06685fd5c304fbb8b8a4ada47ca715f363eb5a8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Sep 2012 09:39:23 +0530 Subject: [PATCH 05/45] Fix #1047721 ([Enhancement] Change new version notification message) --- src/calibre/gui2/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/update.py b/src/calibre/gui2/update.py index a7bc341a96..0b685e2fd2 100644 --- a/src/calibre/gui2/update.py +++ b/src/calibre/gui2/update.py @@ -75,7 +75,7 @@ class UpdateNotification(QDialog): self.logo.setPixmap(QPixmap(I('lt.png')).scaled(100, 100, Qt.IgnoreAspectRatio, Qt.SmoothTransformation)) self.label = QLabel(('

'+ - _('%(app)s has been updated to version %(ver)s. ' + _('New version %(ver)s of %(app)s is available for download. ' 'See the new features.'))%dict( app=__appname__, ver=calibre_version)) From ceebebf54f38dcfcb0756879cbfca08c7e97ebe8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Sep 2012 17:54:20 +0530 Subject: [PATCH 06/45] MTP: Implement user specified ignoring of devices --- src/calibre/devices/errors.py | 6 ++ src/calibre/devices/mtp/base.py | 3 + src/calibre/devices/mtp/driver.py | 9 +- src/calibre/devices/mtp/unix/driver.py | 42 +++++--- src/calibre/devices/mtp/windows/driver.py | 17 ++- src/calibre/gui2/device.py | 6 +- src/calibre/gui2/device_drivers/mtp_config.py | 102 ++++++++++++++---- 7 files changed, 145 insertions(+), 40 deletions(-) diff --git a/src/calibre/devices/errors.py b/src/calibre/devices/errors.py index d906bb86c8..56f9b1460d 100644 --- a/src/calibre/devices/errors.py +++ b/src/calibre/devices/errors.py @@ -110,3 +110,9 @@ class WrongDestinationError(PathError): trying to send books to a non existant storage card.''' pass +class BlacklistedDevice(OpenFailed): + ''' Raise this error during open() when the device being opened has been + blacklisted by the user. Only used in drivers that manage device presence, + like the MTP driver. ''' + pass + diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index a5885ca964..4ada58ecef 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -59,4 +59,7 @@ class MTPDeviceBase(DevicePlugin): from calibre.devices.utils import build_template_regexp return build_template_regexp(self.save_template) + def is_customizable(self): + return True + diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 92184af8ff..55472d3d44 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -16,7 +16,7 @@ from calibre.constants import iswindows, numeric_version from calibre.devices.mtp.base import debug from calibre.ptempfile import SpooledTemporaryFile, PersistentTemporaryDirectory from calibre.utils.config import from_json, to_json, JSONConfig -from calibre.utils.date import now, isoformat +from calibre.utils.date import now, isoformat, utcnow BASE = importlib.import_module('calibre.devices.mtp.%s.driver'%( 'windows' if iswindows else 'unix')).MTP_DEVICE @@ -51,6 +51,8 @@ class MTP_DEVICE(BASE): 'wordplayer/calibretransfer', 'Books', 'sdcard/ebooks', 'eBooks', 'kindle'] p.defaults['send_template'] = config().parse().send_template + p.defaults['blacklist'] = [] + p.defaults['history'] = {} return self._prefs @@ -74,6 +76,11 @@ class MTP_DEVICE(BASE): self.current_library_uuid = library_uuid self.location_paths = None BASE.open(self, devices, library_uuid) + h = self.prefs['history'] + if self.current_serial_num: + h[self.current_serial_num] = (self.current_friendly_name, + isoformat(utcnow())) + self.prefs['history'] = h # Device information {{{ def _update_drive_info(self, storage, location_code, name=None): diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index 3792bb2fcc..31f886b875 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -15,7 +15,7 @@ from functools import partial from calibre import prints, as_unicode from calibre.constants import plugins from calibre.ptempfile import SpooledTemporaryFile -from calibre.devices.errors import OpenFailed, DeviceError +from calibre.devices.errors import OpenFailed, DeviceError, BlacklistedDevice from calibre.devices.mtp.base import MTPDeviceBase, synchronous MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id ' @@ -99,19 +99,25 @@ class MTP_DEVICE(MTPDeviceBase): return False p('Known MTP devices connected:') for d in devs: p(d) - d = devs[0] - p('\nTrying to open:', d) - try: - self.open(d, 'debug') - except: - p('Opening device failed:') - p(traceback.format_exc()) - return False - p('Opened', self.current_friendly_name, 'successfully') - p('Storage info:') - p(pprint.pformat(self.dev.storage_info)) - self.eject() - return True + + for d in devs: + p('\nTrying to open:', d) + try: + self.open(d, 'debug') + except BlacklistedDevice: + p('This device has been blacklisted by the user') + continue + except: + p('Opening device failed:') + p(traceback.format_exc()) + return False + else: + p('Opened', self.current_friendly_name, 'successfully') + p('Storage info:') + p(pprint.pformat(self.dev.storage_info)) + self.post_yank_cleanup() + return True + return False @synchronous def create_device(self, connected_device): @@ -167,6 +173,12 @@ class MTP_DEVICE(MTPDeviceBase): if not storage: self.blacklisted_devices.add(connected_device) raise OpenFailed('No storage found for device %s'%(connected_device,)) + snum = self.dev.serial_number + if snum in self.prefs.get('blacklist', []): + self.blacklisted_devices.add(connected_device) + self.dev = None + raise BlacklistedDevice( + 'The %s device has been blacklisted by the user'%(connected_device,)) self._main_id = storage[0]['id'] self._carda_id = self._cardb_id = None if len(storage) > 1: @@ -176,7 +188,7 @@ class MTP_DEVICE(MTPDeviceBase): self.current_friendly_name = self.dev.friendly_name if not self.current_friendly_name: self.current_friendly_name = self.dev.model_name or _('Unknown MTP device') - self.current_serial_num = self.dev.serial_number + self.current_serial_num = snum @property def filesystem_cache(self): diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 2f606b42d1..50638496d1 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -15,7 +15,7 @@ from itertools import chain from calibre import as_unicode, prints from calibre.constants import plugins, __appname__, numeric_version from calibre.ptempfile import SpooledTemporaryFile -from calibre.devices.errors import OpenFailed, DeviceError +from calibre.devices.errors import OpenFailed, DeviceError, BlacklistedDevice from calibre.devices.mtp.base import MTPDeviceBase class ThreadingViolation(Exception): @@ -163,6 +163,9 @@ class MTP_DEVICE(MTPDeviceBase): p('\nTrying to open:', pnp_id) try: self.open(pnp_id, 'debug-detection') + except BlacklistedDevice: + p('This device has been blacklisted by the user') + continue except: p('Open failed:') p(traceback.format_exc()) @@ -172,7 +175,7 @@ class MTP_DEVICE(MTPDeviceBase): p('Opened', self.current_friendly_name, 'successfully') p('Device info:') p(pprint.pformat(self.dev.data)) - self.eject() + self.post_yank_cleanup() return True p('No suitable MTP devices found') return False @@ -225,7 +228,6 @@ class MTP_DEVICE(MTPDeviceBase): self._main_id = self._carda_id = self._cardb_id = None self.dev = self._filesystem_cache = None - @same_thread def post_yank_cleanup(self): self.currently_connected_pnp_id = self.current_friendly_name = None @@ -256,6 +258,13 @@ class MTP_DEVICE(MTPDeviceBase): if not storage: self.blacklisted_devices.add(connected_device) raise OpenFailed('No storage found for device %s'%(connected_device,)) + snum = devdata.get('serial_number', None) + if snum in self.prefs.get('blacklist', []): + self.blacklisted_devices.add(connected_device) + self.dev = None + raise BlacklistedDevice( + 'The %s device has been blacklisted by the user'%(connected_device,)) + self._main_id = storage[0]['id'] if len(storage) > 1: self._carda_id = storage[1]['id'] @@ -266,7 +275,7 @@ class MTP_DEVICE(MTPDeviceBase): self.current_friendly_name = devdata.get('model_name', _('Unknown MTP device')) self.currently_connected_pnp_id = connected_device - self.current_serial_num = devdata.get('serial_number', None) + self.current_serial_num = snum @same_thread def get_basic_device_information(self): diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 6d638ef9c2..8466fe9320 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -24,7 +24,8 @@ from calibre.gui2 import (config, error_dialog, Dispatcher, dynamic, from calibre.ebooks.metadata import authors_to_string from calibre import preferred_encoding, prints, force_unicode, as_unicode from calibre.utils.filenames import ascii_filename -from calibre.devices.errors import FreeSpaceError, WrongDestinationError +from calibre.devices.errors import (FreeSpaceError, WrongDestinationError, + BlacklistedDevice) from calibre.devices.apple.driver import ITUNES_ASYNC from calibre.devices.folder_device.driver import FOLDER_DEVICE from calibre.devices.bambook.driver import BAMBOOK, BAMBOOKWifi @@ -252,6 +253,9 @@ class DeviceManager(Thread): # {{{ if cd is not None: try: dev.open(cd, self.current_library_uuid) + except BlacklistedDevice as e: + prints('Ignoring blacklisted device: %s'% + as_unicode(e)) except: prints('Error while trying to open %s (Driver: %s)'% (cd, dev)) diff --git a/src/calibre/gui2/device_drivers/mtp_config.py b/src/calibre/gui2/device_drivers/mtp_config.py index b6628f4e65..261fea3df2 100644 --- a/src/calibre/gui2/device_drivers/mtp_config.py +++ b/src/calibre/gui2/device_drivers/mtp_config.py @@ -16,6 +16,7 @@ from PyQt4.Qt import (QWidget, QListWidgetItem, Qt, QToolButton, QLabel, from calibre.ebooks import BOOK_EXTENSIONS from calibre.gui2 import error_dialog from calibre.gui2.dialogs.template_dialog import TemplateDialog +from calibre.utils.date import parse_date class FormatsConfig(QWidget): # {{{ @@ -136,6 +137,45 @@ class SendToConfig(QWidget): # {{{ # }}} +class IgnoredDevices(QWidget): # {{{ + + def __init__(self, devs, blacklist): + QWidget.__init__(self) + self.l = l = QVBoxLayout() + self.setLayout(l) + self.la = la = QLabel('

'+_( + '''Select the devices to be ignored. calibre will not + connect to devices with a checkmark next to their names.''')) + la.setWordWrap(True) + l.addWidget(la) + self.f = f = QListWidget(self) + l.addWidget(f) + + devs = [(snum, (x[0], parse_date(x[1]))) for snum, x in + devs.iteritems()] + for dev, x in sorted(devs, key=lambda x:x[1][1], reverse=True): + name = x[0] + name = '%s [%s]'%(name, dev) + item = QListWidgetItem(name, f) + item.setData(Qt.UserRole, dev) + item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsUserCheckable|Qt.ItemIsSelectable) + item.setCheckState(Qt.Checked if dev in blacklist else Qt.Unchecked) + + @property + def blacklist(self): + return [unicode(self.f.item(i).data(Qt.UserRole).toString()) for i in + xrange(self.f.count()) if self.f.item(i).checkState()==Qt.Checked] + + def ignore_device(self, snum): + for i in xrange(self.f.count()): + i = self.f.item(i) + c = unicode(i.data(Qt.UserRole).toString()) + if c == snum: + i.setCheckState(Qt.Checked) + break + +# }}} + class MTPConfig(QTabWidget): def __init__(self, device, parent=None): @@ -162,6 +202,8 @@ class MTPConfig(QTabWidget): l = QLabel(msg) l.setWordWrap(True) l.setStyleSheet('QLabel { margin-left: 2em }') + l.setMinimumWidth(500) + l.setMinimumHeight(400) self.insertTab(0, l, _('Cannot configure')) else: self.base = QWidget(self) @@ -173,16 +215,34 @@ class MTPConfig(QTabWidget): self.get_pref('format_map')) self.send_to = SendToConfig(self.get_pref('send_to')) self.template = TemplateConfig(self.get_pref('send_template')) - self.base.la = la = QLabel(_('Choose the formats to send to the %s')%self.device.current_friendly_name) + self.base.la = la = QLabel(_( + 'Choose the formats to send to the %s')%self.device.current_friendly_name) la.setWordWrap(True) l.addWidget(la, 0, 0, 1, 1) - l.addWidget(self.formats, 1, 0, 3, 1) + l.addWidget(self.formats, 1, 0, 2, 1) l.addWidget(self.send_to, 1, 1, 1, 1) l.addWidget(self.template, 2, 1, 1, 1) l.setRowStretch(2, 10) + self.base.b = b = QPushButton(QIcon(I('minus.png')), + _('Ignore the %s in calibre')%device.current_friendly_name, + self.base) + l.addWidget(b, 3, 0, 1, 2) + b.clicked.connect(self.ignore_device) + + self.igntab = IgnoredDevices(self.device.prefs['history'], + self.device.prefs['blacklist']) + self.addTab(self.igntab, _('Ignored devices')) self.setCurrentIndex(0) + def ignore_device(self): + self.igntab.ignore_device(self.device.current_serial_num) + self.base.b.setEnabled(False) + self.base.b.setText(_('The %s will be ignored in calibre')% + self.device.current_friendly_name) + self.base.b.setStyleSheet('QPushButton { font-weight: bold }') + self.base.setEnabled(False) + def get_pref(self, key): p = self.device.prefs.get(self.current_device_key, {}) if not p: @@ -194,31 +254,35 @@ class MTPConfig(QTabWidget): return self._device() def validate(self): - if not self.formats.validate(): - return False - if not self.template.validate(): - return False + if hasattr(self, 'formats'): + if not self.formats.validate(): + return False + if not self.template.validate(): + return False return True def commit(self): p = self.device.prefs.get(self.current_device_key, {}) - p.pop('format_map', None) - f = self.formats.format_map - if f and f != self.device.prefs['format_map']: - p['format_map'] = f + if hasattr(self, 'formats'): + p.pop('format_map', None) + f = self.formats.format_map + if f and f != self.device.prefs['format_map']: + p['format_map'] = f - p.pop('send_template', None) - t = self.template.template - if t and t != self.device.prefs['send_template']: - p['send_template'] = t + p.pop('send_template', None) + t = self.template.template + if t and t != self.device.prefs['send_template']: + p['send_template'] = t - p.pop('send_to', None) - s = self.send_to.value - if s and s != self.device.prefs['send_to']: - p['send_to'] = s + p.pop('send_to', None) + s = self.send_to.value + if s and s != self.device.prefs['send_to']: + p['send_to'] = s - self.device.prefs[self.current_device_key] = p + self.device.prefs[self.current_device_key] = p + + self.device.prefs['blacklist'] = self.igntab.blacklist if __name__ == '__main__': from calibre.gui2 import Application From 00c5e5e2f2be0b81e875c6db28e5493842090af6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Sep 2012 20:19:32 +0530 Subject: [PATCH 07/45] Fix #1047865 (Calibre not connecting showing android device) --- src/calibre/devices/android/driver.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 10fdb50df9..e261fe71da 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -208,7 +208,7 @@ class ANDROID(USBMS): 'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP', 'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD', 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0', - 'COBY_MID', 'VS'] + 'COBY_MID', 'VS', 'AINOL'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', @@ -227,7 +227,8 @@ class ANDROID(USBMS): 'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX', 'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE', 'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', - 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E'] + 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E', + 'NOVO7'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', @@ -237,7 +238,8 @@ class ANDROID(USBMS): 'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC', 'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875', 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', - 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E'] + 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E', + 'NOVO7'] OSX_MAIN_MEM = 'Android Device Main Memory' From 88821514f95d167448fe33d9481367b296a4f0ec Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Sep 2012 21:51:42 +0530 Subject: [PATCH 08/45] Update Birmingham post --- recipes/birmingham_post.recipe | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/recipes/birmingham_post.recipe b/recipes/birmingham_post.recipe index ae5d2c9ce9..b9b3c3fc57 100644 --- a/recipes/birmingham_post.recipe +++ b/recipes/birmingham_post.recipe @@ -1,14 +1,17 @@ from calibre.web.feeds.news import BasicNewsRecipe class AdvancedUserRecipe1306097511(BasicNewsRecipe): title = u'Birmingham post' - description = 'News for Birmingham UK' - timefmt = '' + description = 'Author D.Asbury. News for Birmingham UK' + #timefmt = '' + # last update 8/9/12 __author__ = 'Dave Asbury' - cover_url = 'http://1.bp.blogspot.com/_GwWyq5eGw9M/S9BHPHxW55I/AAAAAAAAB6Q/iGCWl0egGzg/s320/Birmingham+post+Lite+front.JPG' + cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/161987_9010212100_2035706408_n.jpg' oldest_article = 2 max_articles_per_feed = 12 + linearize_tables = True remove_empty_feeds = True remove_javascript = True + no_stylesheets = True #auto_cleanup = True language = 'en_GB' @@ -17,11 +20,12 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe): keep_only_tags = [ - dict(name='h1',attrs={'id' : 'article-headline'}), + dict(attrs={'id' : 'article-header'}), + #dict(name='h1',attrs={'id' : 'article-header'}), dict(attrs={'class':['article-meta-author','article-meta-date','article main','art-o art-align-center otm-1 ']}), - dict(name='div',attrs={'class' : 'article-image full'}), - dict(attrs={'clas' : 'art-o art-align-center otm-1 '}), - dict(name='div',attrs={'class' : 'article main'}), + dict(name='div',attrs={'class' : 'article-image full'}), + dict(attrs={'clas' : 'art-o art-align-center otm-1 '}), + dict(name='div',attrs={'class' : 'article main'}), #dict(name='p') #dict(attrs={'id' : 'three-col'}) ] @@ -37,11 +41,9 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe): (u'Bloggs & Comments',u'http://www.birminghampost.net/comment/rss.xml') ] - extra_css = ''' - body {font: sans-serif medium;}' - h1 {text-align : center; font-family:Arial,Helvetica,sans-serif; font-size:20px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold;} - h2 {text-align : center;color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:15px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; } - span{ font-size:9.5px; font-weight:bold;font-style:italic} - p { text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;} - - ''' + extra_css = ''' + h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;text-align:center;} + h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} + p{font-family:Arial,Helvetica,sans-serif;font-size:small;} + body{font-family:Helvetica,Arial,sans-serif;font-size:small;} + ''' From 90be6ee35d59b3ca9aafdac1f0e615b3ecc0097e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 00:47:28 +0530 Subject: [PATCH 09/45] Fix #1047947 (recipe for Die Zeit (subscription only) doesn't work after URL change) --- recipes/zeitde_sub.recipe | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/recipes/zeitde_sub.recipe b/recipes/zeitde_sub.recipe index dfa52e8504..b22e9793ed 100644 --- a/recipes/zeitde_sub.recipe +++ b/recipes/zeitde_sub.recipe @@ -118,13 +118,13 @@ class ZeitEPUBAbo(BasicNewsRecipe): def build_index(self): domain = "https://premium.zeit.de" - url = domain + "/abo/zeit_digital" + url = domain + "/abo/digitalpaket" browser = self.get_browser() # new login process response = browser.open(url) # Get rid of nested form - response.set_data(response.get_data().replace('

', '')) + response.set_data(response.get_data().replace('
', '')) browser.set_response(response) browser.select_form(nr=2) browser.form['name']=self.username @@ -177,13 +177,13 @@ class ZeitEPUBAbo(BasicNewsRecipe): try: self.log.warning('Trying PDF-based cover') domain = "https://premium.zeit.de" - url = domain + "/abo/zeit_digital" + url = domain + "/abo/digitalpaket" browser = self.get_browser() # new login process response=browser.open(url) # Get rid of nested form - response.set_data(response.get_data().replace('
', '')) + response.set_data(response.get_data().replace('
', '')) browser.set_response(response) browser.select_form(nr=2) From 65f9b902fdbc4db09427adfa9112981c89d38a26 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 09:10:42 +0530 Subject: [PATCH 10/45] Fix #1047992 (Sony problem sending to SD card) --- src/calibre/devices/usbms/device.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 02991c4f16..49d766c67f 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -974,6 +974,9 @@ class Device(DeviceConfig, DevicePlugin): def get_carda_ebook_dir(self, for_upload=False): return self.EBOOK_DIR_CARD_A + def get_cardb_ebook_dir(self, for_upload=False): + return self.EBOOK_DIR_CARD_B + def _sanity_check(self, on_card, files): from calibre.devices.utils import sanity_check sanity_check(on_card, files, self.card_prefix(), self.free_space()) From 2bafa6a5116f189629bbfc2a2c64719e630d6348 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 09:51:51 +0530 Subject: [PATCH 11/45] ... --- resources/compiled_coffeescript.zip | Bin 56964 -> 57017 bytes src/calibre/ebooks/oeb/display/mathjax.coffee | 1 + 2 files changed, 1 insertion(+) diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index 573a8128ac70556e1a7468ad34ee82f7d3157c76..0f6b8c3e7a522839ce146a7c6211d81b23512310 100644 GIT binary patch delta 218 zcmZqK%e-?hvrK?DGm8iV2srv_I#$^&_FK)vzyQLs43qAOZq~mrlT}Y&Um-ZmUBODB zS|KMhFEyzsH8Hyw%uOsU$1e~Hy^zEfo<}I5RQ#p-_A=jGKnzb zuwe3;J1Ud)FK|s>e@Bby84u99$sg_*F|me#SOM-FlO6ABF>Of%@&1Q!PwoZs{uhFH e=k6LX@n4>FPgR7K4diq-Al%Q$!0_Y_hz9_&qDUS9 delta 191 zcmdnFm$_vx^WtYb7bKZkL>NGzIzYm4v%uuEG9Cs75SE?H$Sbzda3<^K|Ew?BCL7+> zn9O%qd~(5EBPMzG$%-MelMmk2VmcQBWQtB^x@XEHnF?ka6tPVXxu?aXc?`^TxXh;$ w;LXS+!i>Wju;nn~-+7QGh1|rFjI6{8y{zH@Z&o&tBpVR!XJlYle+R?^07IHG^Z)<= diff --git a/src/calibre/ebooks/oeb/display/mathjax.coffee b/src/calibre/ebooks/oeb/display/mathjax.coffee index ad893baa7e..cd130c85c8 100644 --- a/src/calibre/ebooks/oeb/display/mathjax.coffee +++ b/src/calibre/ebooks/oeb/display/mathjax.coffee @@ -39,6 +39,7 @@ class MathJax showMathMenu: false, extensions: ["tex2jax.js", "asciimath2jax.js", "mml2jax.js"], jax: ["input/TeX","input/MathML","input/AsciiMath","output/SVG"], + // SVG : { linebreaks : { automatic : true } }, TeX: { extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"] } From 8a5ca1df87e3a37af06c20ad10a9f1c180786eac Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 10:41:32 +0530 Subject: [PATCH 12/45] History Today by Rick Shang --- recipes/history_today.recipe | 87 ++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 recipes/history_today.recipe diff --git a/recipes/history_today.recipe b/recipes/history_today.recipe new file mode 100644 index 0000000000..43adf7a358 --- /dev/null +++ b/recipes/history_today.recipe @@ -0,0 +1,87 @@ +import re +from calibre.web.feeds.recipes import BasicNewsRecipe +from collections import OrderedDict + +class HistoryToday(BasicNewsRecipe): + + title = 'History Today' + __author__ = 'Rick Shang' + + description = 'UK-based magazine, publishing articles and book reviews covering all types and periods of history.' + language = 'en' + category = 'news' + encoding = 'UTF-8' + + remove_tags = [dict(name='div',attrs={'class':['print-logo','print-site_name','print-breadcrumb']}), + dict(name='div', attrs={'id':['ht-tools','ht-tools2','ht-tags']})] + no_javascript = True + no_stylesheets = True + + + needs_subscription = True + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + if self.username is not None and self.password is not None: + br.open('http://www.historytoday.com/user/login') + br.select_form(nr=1) + br['name'] = self.username + br['pass'] = self.password + res = br.submit() + raw = res.read() + if 'Session limit exceeded' in raw: + br.select_form(nr=1) + control=br.find_control('sid').items[1] + sid = [] + br['sid']=sid.join(control) + br.submit() + return br + + def parse_index(self): + + #Find date + soup0 = self.index_to_soup('http://www.historytoday.com/') + dates = self.tag_to_string(soup0.find('div',attrs={'id':'block-block-226'}).span) + self.timefmt = u' [%s]'%dates + + #Go to issue + soup = self.index_to_soup('http://www.historytoday.com/contents') + cover = soup.find('div',attrs={'id':'content-area'}).find('img')['src'] + self.cover_url=cover + + #Go to the main body + + div = soup.find ('div', attrs={'class':'region region-content-bottom'}) + + feeds = OrderedDict() + section_title = '' + for section in div.findAll('div', attrs={'id':re.compile("block\-views\-contents.*")}): + section_title = self.tag_to_string(section.find('h2',attrs={'class':'title'})) + sectionbody=section.find('div', attrs={'class':'view-content'}) + for article in sectionbody.findAll('div',attrs={'class':re.compile("views\-row.*")}): + articles = [] + subarticle = [] + subarticle = article.findAll('div') + if len(subarticle) < 2: + continue + title=self.tag_to_string(subarticle[0]) + originalurl="http://www.historytoday.com" + subarticle[0].span.a['href'].strip() + originalpage=self.index_to_soup(originalurl) + printurl=originalpage.find('div',attrs = {'id':'ht-tools'}).a['href'].strip() + url="http://www.historytoday.com" + printurl + desc=self.tag_to_string(subarticle[1]) + articles.append({'title':title, 'url':url, 'description':desc, 'date':''}) + + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles + + + ans = [(key, val) for key, val in feeds.iteritems()] + return ans + + + def cleanup(self): + self.browser.open('http://www.historytoday.com/logout') + From e72a0f2482cce158fdcefa8f770cb0e92aabf023 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 12:46:36 +0530 Subject: [PATCH 13/45] Update countryfile --- recipes/countryfile.recipe | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/recipes/countryfile.recipe b/recipes/countryfile.recipe index 0502129791..71977048c7 100644 --- a/recipes/countryfile.recipe +++ b/recipes/countryfile.recipe @@ -1,12 +1,11 @@ from calibre import browser from calibre.web.feeds.news import BasicNewsRecipe - class AdvancedUserRecipe1325006965(BasicNewsRecipe): title = u'Countryfile.com' #cover_url = 'http://www.countryfile.com/sites/default/files/imagecache/160px_wide/cover/2_1.jpg' __author__ = 'Dave Asbury' description = 'The official website of Countryfile Magazine' - # last updated 15/4/12 + # last updated 9/9//12 language = 'en_GB' oldest_article = 30 max_articles_per_feed = 25 @@ -17,13 +16,14 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe): def get_cover_url(self): soup = self.index_to_soup('http://www.countryfile.com/') cov = soup.find(attrs={'class' : 'imagecache imagecache-160px_wide imagecache-linked imagecache-160px_wide_linked'}) - #print '******** ',cov,' ***' + print '******** ',cov,' ***' cov2 = str(cov) - cov2=cov2[124:-90] - #print '******** ',cov2,' ***' - + cov2=cov2[140:223] + print '******** ',cov2,' ***' + #cov2='http://www.countryfile.com/sites/default/files/imagecache/160px_wide/cover/1b_0.jpg' # try to get cover - if can't get known cover br = browser() + br.set_handle_redirect(False) try: br.open_novisit(cov2) From f967610679ae80e79dfe29c20c0ab3fa7f5c6e97 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 9 Sep 2012 13:17:15 +0200 Subject: [PATCH 14/45] Set default save template to one without subfolders. Helps with readers that don't support them, such as K4A --- src/calibre/devices/smart_device_app/driver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index b1cd1e635b..1dcf74450d 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -84,6 +84,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): PREFIX = '' BACKLOADING_ERROR_MESSAGE = None + SAVE_TEMPLATE = '{title} - {authors} ({id})' + # Some network protocol constants BASE_PACKET_LEN = 4096 PROTOCOL_VERSION = 1 From b5109cbdf06ff0194a12b11547595c59a75edf56 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 17:44:29 +0530 Subject: [PATCH 15/45] Possibly fix a mem leak in device scanning on windows --- src/calibre/utils/windows/winutil.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/calibre/utils/windows/winutil.c b/src/calibre/utils/windows/winutil.c index 42b6462313..0134bf21ed 100644 --- a/src/calibre/utils/windows/winutil.c +++ b/src/calibre/utils/windows/winutil.c @@ -168,9 +168,9 @@ winutil_set_debug(PyObject *self, PyObject *args) { return Py_None; } -static LPTSTR +static LPWSTR get_registry_property(HDEVINFO hDevInfo, DWORD index, DWORD property, BOOL *iterate) { - /* Get a the property specified by `property` from the registry for the + /* Get the property specified by `property` from the registry for the * device enumerated by `index` in the collection `hDevInfo`. `iterate` * will be set to `FALSE` if `index` points outside `hDevInfo`. * :return: A string allocated on the heap containing the property or @@ -178,7 +178,7 @@ get_registry_property(HDEVINFO hDevInfo, DWORD index, DWORD property, BOOL *iter */ SP_DEVINFO_DATA DeviceInfoData; DWORD DataT; - LPTSTR buffer = NULL; + LPWSTR buffer = NULL; DWORD buffersize = 0; DeviceInfoData.cbSize = sizeof(SP_DEVINFO_DATA); @@ -187,7 +187,7 @@ get_registry_property(HDEVINFO hDevInfo, DWORD index, DWORD property, BOOL *iter return NULL; } - while(!SetupDiGetDeviceRegistryProperty( + while(!SetupDiGetDeviceRegistryPropertyW( hDevInfo, &DeviceInfoData, property, @@ -209,7 +209,7 @@ get_registry_property(HDEVINFO hDevInfo, DWORD index, DWORD property, BOOL *iter } static BOOL -check_device_id(LPTSTR buffer, unsigned int vid, unsigned int pid) { +check_device_id(LPWSTR buffer, unsigned int vid, unsigned int pid) { WCHAR xVid[9], dVid[9], xPid[9], dPid[9]; unsigned int j; _snwprintf_s(xVid, 9, _TRUNCATE, L"vid_%4.4x", vid); @@ -672,7 +672,8 @@ winutil_get_usb_devices(PyObject *self, PyObject *args) { HDEVINFO hDevInfo; DWORD i; BOOL iterate = TRUE; PyObject *devices, *temp = (PyObject *)1; - LPTSTR buffer; + LPWSTR buffer; + BOOL ok = 1; if (!PyArg_ParseTuple(args, "")) return NULL; @@ -691,16 +692,17 @@ winutil_get_usb_devices(PyObject *self, PyObject *args) { PyErr_Print(); continue; } buffersize = wcslen(buffer); - for (j = 0; j < buffersize; j++) buffer[j] = tolower(buffer[j]); + for (j = 0; j < buffersize; j++) buffer[j] = towlower(buffer[j]); temp = PyUnicode_FromWideChar(buffer, buffersize); PyMem_Free(buffer); if (temp == NULL) { PyErr_NoMemory(); + ok = 0; break; } - PyList_Append(devices, temp); + PyList_Append(devices, temp); Py_DECREF(temp); temp = NULL; } //for - if (temp == NULL) { Py_DECREF(devices); devices = NULL; } + if (!ok) { Py_DECREF(devices); devices = NULL; } SetupDiDestroyDeviceInfoList(hDevInfo); return devices; } @@ -711,7 +713,7 @@ winutil_is_usb_device_connected(PyObject *self, PyObject *args) { unsigned int vid, pid; HDEVINFO hDevInfo; DWORD i; BOOL iterate = TRUE; - LPTSTR buffer; + LPWSTR buffer; int found = FALSE; PyObject *ans; From 113269d00c6e76725a69d354c9c0359eaf032eb1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 20:38:14 +0530 Subject: [PATCH 16/45] Another leak? --- src/calibre/utils/windows/winutil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/windows/winutil.c b/src/calibre/utils/windows/winutil.c index 0134bf21ed..c00bcc4b41 100644 --- a/src/calibre/utils/windows/winutil.c +++ b/src/calibre/utils/windows/winutil.c @@ -196,7 +196,8 @@ get_registry_property(HDEVINFO hDevInfo, DWORD index, DWORD property, BOOL *iter buffersize, &buffersize)) { if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - buffer = (LPTSTR)PyMem_Malloc(2*buffersize); // Twice for bug in Win2k + if (buffer != NULL) { PyMem_Free(buffer); buffer = NULL; } + buffer = (LPWSTR)PyMem_Malloc(2*buffersize); // Twice for bug in Win2k } else { PyMem_Free(buffer); PyErr_SetFromWindowsErr(0); From 814d92d56ca5ac08eeca285c9fe5cbc4c8a6f68b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 20:39:53 +0530 Subject: [PATCH 17/45] ... --- src/calibre/utils/windows/winutil.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/calibre/utils/windows/winutil.c b/src/calibre/utils/windows/winutil.c index c00bcc4b41..384e2d1e28 100644 --- a/src/calibre/utils/windows/winutil.c +++ b/src/calibre/utils/windows/winutil.c @@ -199,9 +199,8 @@ get_registry_property(HDEVINFO hDevInfo, DWORD index, DWORD property, BOOL *iter if (buffer != NULL) { PyMem_Free(buffer); buffer = NULL; } buffer = (LPWSTR)PyMem_Malloc(2*buffersize); // Twice for bug in Win2k } else { - PyMem_Free(buffer); + if (buffer != NULL) { PyMem_Free(buffer); buffer = NULL; } PyErr_SetFromWindowsErr(0); - buffer = NULL; break; } } //while From 8f095aa63b5e7a2b99f8d42b6af06ea23fcfca81 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 20:45:47 +0530 Subject: [PATCH 18/45] And another potential leak --- src/calibre/utils/windows/winutil.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/windows/winutil.c b/src/calibre/utils/windows/winutil.c index 384e2d1e28..242fe99fa1 100644 --- a/src/calibre/utils/windows/winutil.c +++ b/src/calibre/utils/windows/winutil.c @@ -683,8 +683,10 @@ winutil_get_usb_devices(PyObject *self, PyObject *args) { // Create a Device information set with all USB devices hDevInfo = create_device_info_set(NULL, L"USB", 0, DIGCF_PRESENT | DIGCF_ALLCLASSES); - if (hDevInfo == INVALID_HANDLE_VALUE) + if (hDevInfo == INVALID_HANDLE_VALUE) { + Py_DECREF(devices); return NULL; + } // Enumerate through the set for (i=0; iterate; i++) { buffer = get_registry_property(hDevInfo, i, SPDRP_HARDWAREID, &iterate); From ae34d417f6bd827a2ec6ff72f9c579398900aefa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 21:10:30 +0530 Subject: [PATCH 19/45] Add function to test for mem leaks in the scanner --- src/calibre/devices/scanner.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index e0bb74fa2a..00eae7b09f 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -329,8 +329,30 @@ class DeviceScanner(object): return device.is_usb_connected(self.devices, debug=debug, only_presence=only_presence) +def test_for_mem_leak(): + from calibre.utils.mem import memory, gc_histogram, diff_hists + import gc + gc.disable() + scanner = DeviceScanner() + scanner.scan() + for i in xrange(3): gc.collect() + + for reps in (10, 100, 1000, 10000): + for i in xrange(3): gc.collect() + h1 = gc_histogram() + startmem = memory() + for i in xrange(reps): + scanner.scan() + for i in xrange(3): gc.collect() + usedmem = memory(startmem) + prints('Memory used in %d repetitions of scan(): %.6f KB'%(reps, + 1024*usedmem)) + prints('Differences in python object counts:') + diff_hists(h1, gc_histogram()) + def main(args=sys.argv): + test_for_mem_leak() return 0 if __name__ == '__main__': From b9f1084b5478c57b78cf524250fa2faca45fa8d6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 21:17:30 +0530 Subject: [PATCH 20/45] ... --- src/calibre/devices/scanner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index 00eae7b09f..aea0c94740 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -335,9 +335,10 @@ def test_for_mem_leak(): gc.disable() scanner = DeviceScanner() scanner.scan() + memory() # load the psutil library for i in xrange(3): gc.collect() - for reps in (10, 100, 1000, 10000): + for reps in (1, 10, 100, 1000, 10000): for i in xrange(3): gc.collect() h1 = gc_histogram() startmem = memory() @@ -345,10 +346,11 @@ def test_for_mem_leak(): scanner.scan() for i in xrange(3): gc.collect() usedmem = memory(startmem) - prints('Memory used in %d repetitions of scan(): %.6f KB'%(reps, + prints('Memory used in %d repetitions of scan(): %.10f KB'%(reps, 1024*usedmem)) prints('Differences in python object counts:') diff_hists(h1, gc_histogram()) + prints() def main(args=sys.argv): From bb16a23be84033924fa400bc98d0143fb874e612 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 21:19:26 +0530 Subject: [PATCH 21/45] ... --- src/calibre/devices/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index aea0c94740..288e9a77a1 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -346,7 +346,7 @@ def test_for_mem_leak(): scanner.scan() for i in xrange(3): gc.collect() usedmem = memory(startmem) - prints('Memory used in %d repetitions of scan(): %.10f KB'%(reps, + prints('Memory used in %d repetitions of scan(): %.5f KB'%(reps, 1024*usedmem)) prints('Differences in python object counts:') diff_hists(h1, gc_histogram()) From bcf5a4acfbad6546fa1885db2dec2708548a05ce Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 21:26:25 +0530 Subject: [PATCH 22/45] Update Metro UK --- recipes/metro_uk.recipe | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/recipes/metro_uk.recipe b/recipes/metro_uk.recipe index 5b7b3a64ed..fcceba4ce7 100644 --- a/recipes/metro_uk.recipe +++ b/recipes/metro_uk.recipe @@ -1,10 +1,10 @@ from calibre.web.feeds.news import BasicNewsRecipe class AdvancedUserRecipe1306097511(BasicNewsRecipe): title = u'Metro UK' - description = 'Author Dave Asbury : News as provide by The Metro -UK' + description = 'Author Dave Asbury : News from The Metro - UK' #timefmt = '' __author__ = 'Dave Asbury' - #last update 4/8/12 + #last update 9/9/12 cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/276636_117118184990145_2132092232_n.jpg' no_stylesheets = True oldest_article = 1 @@ -17,23 +17,24 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe): language = 'en_GB' masthead_url = 'http://e-edition.metro.co.uk/images/metro_logo.gif' extra_css = ''' - h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:1.6em;} + h1{font-family:Arial,Helvetica,sans-serif; font-weight:900;font-size:1.6em;} h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:1.2em;} p{font-family:Arial,Helvetica,sans-serif;font-size:1.0em;} body{font-family:Helvetica,Arial,sans-serif;font-size:1.0em;} - ''' + ''' keep_only_tags = [ - #dict(name='h1'), - #dict(name='h2'), - #dict(name='div', attrs={'class' : ['row','article','img-cnt figure','clrd']}) - #dict(name='h3'), - #dict(attrs={'class' : 'BText'}), - ] + #dict(name='h1'), + #dict(name='h2'), + #dict(name='div', attrs={'class' : ['row','article','img-cnt figure','clrd']}) + #dict(name='h3'), + #dict(attrs={'class' : 'BText'}), + ] remove_tags = [ + dict(name='div',attrs={'class' : 'art-fd fd-gr1-b clrd'}), dict(name='span',attrs={'class' : 'share'}), - dict(name='li'), - dict(attrs={'class' : ['twitter-share-button','header-forms','hdr-lnks','close','art-rgt','fd-gr1-b clrd google-article','news m12 clrd clr-b p5t shareBtm','item-ds csl-3-img news','c-1of3 c-last','c-1of1','pd','item-ds csl-3-img sport']}), - dict(attrs={'id' : ['','sky-left','sky-right','ftr-nav','and-ftr','notificationList','logo','miniLogo','comments-news','metro_extras']}) + dict(name='li'), + dict(attrs={'class' : ['twitter-share-button','header-forms','hdr-lnks','close','art-rgt','fd-gr1-b clrd google-article','news m12 clrd clr-b p5t shareBtm','item-ds csl-3-img news','c-1of3 c-last','c-1of1','pd','item-ds csl-3-img sport']}), + dict(attrs={'id' : ['','sky-left','sky-right','ftr-nav','and-ftr','notificationList','logo','miniLogo','comments-news','metro_extras']}) ] remove_tags_before = dict(name='h1') #remove_tags_after = dict(attrs={'id':['topic-buttons']}) From 7bba2267b0a299ccbc7586fd0f7b8c908fa80255 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 22:19:39 +0530 Subject: [PATCH 23/45] Add testing for leaks in the windows pnp scanner --- src/calibre/devices/scanner.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index 288e9a77a1..156ed981cb 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -338,7 +338,7 @@ def test_for_mem_leak(): memory() # load the psutil library for i in xrange(3): gc.collect() - for reps in (1, 10, 100, 1000, 10000): + for reps in (1, 10, 100, 1000): for i in xrange(3): gc.collect() h1 = gc_histogram() startmem = memory() @@ -352,6 +352,22 @@ def test_for_mem_leak(): diff_hists(h1, gc_histogram()) prints() + if not iswindows: + return + + for reps in (1, 10, 100, 1000): + for i in xrange(3): gc.collect() + h1 = gc_histogram() + startmem = memory() + for i in xrange(reps): + win_pnp_drives() + for i in xrange(3): gc.collect() + usedmem = memory(startmem) + prints('Memory used in %d repetitions of pnp_scan(): %.5f KB'%(reps, + 1024*usedmem)) + prints('Differences in python object counts:') + diff_hists(h1, gc_histogram()) + prints() def main(args=sys.argv): test_for_mem_leak() From d638319b307d97fdef2ccb8a0c31152db2e8c84f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Sep 2012 22:48:22 +0530 Subject: [PATCH 24/45] Fix a mem leak in the windows PNP drive scanner --- src/calibre/utils/windows/winutil.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/calibre/utils/windows/winutil.c b/src/calibre/utils/windows/winutil.c index 242fe99fa1..64c943ffa5 100644 --- a/src/calibre/utils/windows/winutil.c +++ b/src/calibre/utils/windows/winutil.c @@ -607,31 +607,28 @@ winutil_get_removable_drives(PyObject *self, PyObject *args) { return NULL; } - ddebug = PyObject_IsTrue(pdebug); + // Find all removable drives + for (j = 0; j < MAX_DRIVES; j++) g_drives[j].letter = 0; + if (!get_all_removable_disks(g_drives)) return NULL; volumes = PyDict_New(); - if (volumes == NULL) return NULL; - - - for (j = 0; j < MAX_DRIVES; j++) g_drives[j].letter = 0; - - // Find all removable drives - if (!get_all_removable_disks(g_drives)) { - return NULL; - } + if (volumes == NULL) return PyErr_NoMemory(); + ddebug = PyObject_IsTrue(pdebug); hDevInfo = create_device_info_set((LPGUID)&GUID_DEVINTERFACE_VOLUME, NULL, NULL, DIGCF_PRESENT | DIGCF_DEVICEINTERFACE); - if (hDevInfo == INVALID_HANDLE_VALUE) return NULL; + if (hDevInfo == INVALID_HANDLE_VALUE) { Py_DECREF(volumes); return NULL; } // Enumerate through the set for (i=0; iterate; i++) { candidates = PyList_New(0); - if (candidates == NULL) return PyErr_NoMemory(); + if (candidates == NULL) { Py_DECREF(volumes); return PyErr_NoMemory();} interfaceDetailData = get_device_ancestors(hDevInfo, i, candidates, &iterate, ddebug); if (interfaceDetailData == NULL) { - PyErr_Print(); continue; + PyErr_Print(); + Py_DECREF(candidates); candidates = NULL; + continue; } length = wcslen(interfaceDetailData->DevicePath); @@ -653,12 +650,13 @@ winutil_get_removable_drives(PyObject *self, PyObject *args) { key = PyBytes_FromFormat("%c", (char)g_drives[j].letter); if (key == NULL) return PyErr_NoMemory(); PyDict_SetItem(volumes, key, candidates); - Py_DECREF(candidates); + Py_DECREF(key); key = NULL; break; } } } + Py_XDECREF(candidates); candidates = NULL; PyMem_Free(interfaceDetailData); } //for From 2e262a144939cdecb3075167f8f920b7e35cc773 Mon Sep 17 00:00:00 2001 From: Eric Lammerts Date: Sun, 9 Sep 2012 15:47:17 -0400 Subject: [PATCH 25/45] Update RSS feeds for De Volkskrant --- recipes/volksrant.recipe | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/recipes/volksrant.recipe b/recipes/volksrant.recipe index 386cb1e729..b3629ee4e0 100644 --- a/recipes/volksrant.recipe +++ b/recipes/volksrant.recipe @@ -73,14 +73,20 @@ class AdvancedUserRecipe1249039563(BasicNewsRecipe): Change Log: Date: 10/15/2010 Feeds updated by Martin Tarenskeen + Date: 09/09/2012 + Feeds updated by Eric Lammerts ''' feeds = [ - (u'Laatste Nieuws', u'http://www.volkskrant.nl/rss/laatstenieuws.rss'), - (u'Binnenland', u'http://www.volkskrant.nl/rss/nederland.rss'), - (u'Buitenland', u'http://www.volkskrant.nl/rss/internationaal.rss'), - (u'Economie', u'http://www.volkskrant.nl/rss/economie.rss'), - (u'Sport', u'http://www.volkskrant.nl/rss/sport.rss'), - (u'Cultuur', u'http://www.volkskrant.nl/rss/kunst.rss'), - (u'Gezondheid & Wetenschap', u'http://www.volkskrant.nl/rss/wetenschap.rss'), - (u'Internet & Media', u'http://www.volkskrant.nl/rss/media.rss') ] + (u'Nieuws', u'http://www.volkskrant.nl/nieuws/rss.xml'), + (u'Binnenland', u'http://www.volkskrant.nl/nieuws/binnenland/rss.xml'), + (u'Buitenland', u'http://www.volkskrant.nl/buitenland/rss.xml'), + (u'Economie', u'http://www.volkskrant.nl/nieuws/economie/rss.xml'), + (u'Politiek', u'http://www.volkskrant.nl/politiek/rss.xml'), + (u'Sport', u'http://www.volkskrant.nl/sport/rss.xml'), + (u'Cultuur', u'http://www.volkskrant.nl/nieuws/cultuur/rss.xml'), + (u'Gezondheid & wetenschap', u'http://www.volkskrant.nl/nieuws/gezondheid--wetenschap/rss.xml'), + (u'Tech & Media', u'http://www.volkskrant.nl/tech-media/rss.xml'), + (u'Reizen', u'http://www.volkskrant.nl/nieuws/reizen/rss.xml'), + (u'Opinie', u'http://www.volkskrant.nl/opinie/rss.xml'), + (u'Opmerkelijk', u'http://www.volkskrant.nl/nieuws/opmerkelijk/rss.xml') ] From 61e13950a36568fe52ed18aee1596ca6659fc849 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Sep 2012 08:55:39 +0530 Subject: [PATCH 26/45] Fix ebook catalog generation on linux systems where the encoding is not UTF-8. Fixes #1048404 (Catalog generation fails) --- src/calibre/library/catalogs/epub_mobi_builder.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index 3fc0df58b2..f592437916 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -211,15 +211,15 @@ class CatalogBuilder(object): (str): sort key """ if not book['series']: - fs = '{:<%d}!{!s}' % longest_author_sort + fs = u'{:<%d}!{!s}' % longest_author_sort key = fs.format(capitalize(book['author_sort']), capitalize(book['title_sort'])) else: index = book['series_index'] integer = int(index) fraction = index-integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - fs = '{:<%d}~{!s}{!s}' % longest_author_sort + series_index = u'%04d%s' % (integer, str(u'%0.4f' % fraction).lstrip(u'0')) + fs = u'{:<%d}~{!s}{!s}' % longest_author_sort key = fs.format(capitalize(book['author_sort']), self.generate_sort_title(book['series']), series_index) @@ -2464,7 +2464,9 @@ class CatalogBuilder(object): title_str=title_str, xmlns=XHTML_NS, ) - + for k, v in args.iteritems(): + if isbytestring(v): + args[k] = v.decode('utf-8') generated_html = P('catalog/template.xhtml', data=True).decode('utf-8').format(**args) generated_html = substitute_entites(generated_html) From 9aef5216e55e784addc51ad067709c855f6293f4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Sep 2012 09:26:34 +0530 Subject: [PATCH 27/45] ... --- src/calibre/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 2308abb567..d70c186502 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -57,7 +57,7 @@ else: # On linux, unicode arguments to os file functions are coerced to an ascii # bytestring if sys.getfilesystemencoding() == 'ascii', which is # just plain dumb. So issue a warning. - print ('WARNING: You do not have the LANG environment variable set. ' + print ('WARNING: You do not have the LANG environment variable set correctly. ' 'This will cause problems with non-ascii filenames. ' 'Set it to something like en_US.UTF-8.\n') except: From 90f7eb8eec2679c3d8b62c56932b94bdfb981592 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Sep 2012 10:09:14 +0530 Subject: [PATCH 28/45] When downloading metadata for many books, if some of them fail, add an option to the downloaded message to show the failed books in the main book list, so that they can be individually processed easily --- src/calibre/gui2/actions/edit_metadata.py | 36 +++++++++++----- src/calibre/gui2/proceed.py | 52 +++++++++++++++++------ 2 files changed, 63 insertions(+), 25 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index c2558d56ae..26d15d0a83 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -80,7 +80,7 @@ class EditMetadataAction(InterfaceAction): Dispatcher(self.metadata_downloaded), ensure_fields=ensure_fields) - def cleanup_bulk_download(self, tdir): + def cleanup_bulk_download(self, tdir, *args): try: shutil.rmtree(tdir, ignore_errors=True) except: @@ -108,22 +108,26 @@ class EditMetadataAction(InterfaceAction): 'Proceed with updating the metadata in your library?')%len(id_map) show_copy_button = False + checkbox_msg = None if failed_ids or failed_covers: show_copy_button = True num = len(failed_ids.union(failed_covers)) msg += '

'+_('Could not download metadata and/or covers for %d of the books. Click' ' "Show details" to see which books.')%num + checkbox_msg = _('Show the &failed books in the main book list ' + 'after updating metadata') - payload = (id_map, tdir, log_file, lm_map) - self.gui.proceed_question(self.apply_downloaded_metadata, - payload, log_file, - _('Download log'), _('Download complete'), msg, + payload = (id_map, tdir, log_file, lm_map, + failed_ids.union(failed_covers)) + self.gui.proceed_question(self.apply_downloaded_metadata, payload, + log_file, _('Download log'), _('Download complete'), msg, det_msg=det_msg, show_copy_button=show_copy_button, - cancel_callback=lambda x:self.cleanup_bulk_download(tdir), - log_is_file=True) + cancel_callback=partial(self.cleanup_bulk_download, tdir), + log_is_file=True, checkbox_msg=checkbox_msg, + checkbox_checked=False) - def apply_downloaded_metadata(self, payload): - good_ids, tdir, log_file, lm_map = payload + def apply_downloaded_metadata(self, payload, *args): + good_ids, tdir, log_file, lm_map, failed_ids = payload if not good_ids: return @@ -162,8 +166,18 @@ class EditMetadataAction(InterfaceAction): cov = None id_map[bid] = (opf, cov) - self.apply_metadata_changes(id_map, callback=lambda x: - self.cleanup_bulk_download(tdir)) + restrict_to_failed = bool(args and args[0]) + if restrict_to_failed: + db.data.set_marked_ids(failed_ids) + + self.apply_metadata_changes(id_map, + callback=partial(self.downloaded_metadata_applied, tdir, + restrict_to_failed)) + + def downloaded_metadata_applied(self, tdir, restrict_to_failed, *args): + if restrict_to_failed: + self.gui.search.set_search_string('marked:true') + self.cleanup_bulk_download(tdir) # }}} diff --git a/src/calibre/gui2/proceed.py b/src/calibre/gui2/proceed.py index 1074792096..9bdf48e086 100644 --- a/src/calibre/gui2/proceed.py +++ b/src/calibre/gui2/proceed.py @@ -11,18 +11,18 @@ from collections import namedtuple from PyQt4.Qt import (QDialog, Qt, QLabel, QGridLayout, QPixmap, QDialogButtonBox, QApplication, QSize, pyqtSignal, QIcon, - QPlainTextEdit) + QPlainTextEdit, QCheckBox) from calibre.constants import __version__ from calibre.gui2.dialogs.message_box import ViewLog Question = namedtuple('Question', 'payload callback cancel_callback ' 'title msg html_log log_viewer_title log_is_file det_msg ' - 'show_copy_button') + 'show_copy_button checkbox_msg checkbox_checked') class ProceedQuestion(QDialog): - ask_question = pyqtSignal(object, object) + ask_question = pyqtSignal(object, object, object) def __init__(self, parent): QDialog.__init__(self, parent) @@ -62,10 +62,13 @@ class ProceedQuestion(QDialog): self.bb.setStandardButtons(self.bb.Yes|self.bb.No) self.bb.button(self.bb.Yes).setDefault(True) + self.checkbox = QCheckBox('', self) + l.addWidget(ic, 0, 0, 1, 1) l.addWidget(msg, 0, 1, 1, 1) - l.addWidget(self.det_msg, 1, 0, 1, 2) - l.addWidget(self.bb, 2, 0, 1, 2) + l.addWidget(self.checkbox, 1, 0, 1, 2) + l.addWidget(self.det_msg, 2, 0, 1, 2) + l.addWidget(self.bb, 3, 0, 1, 2) self.ask_question.connect(self.do_ask_question, type=Qt.QueuedConnection) @@ -82,19 +85,28 @@ class ProceedQuestion(QDialog): if self.questions: payload, callback, cancel_callback = self.questions[0][:3] self.questions = self.questions[1:] - self.ask_question.emit(callback, payload) + cb = None + if self.checkbox.isVisible(): + cb = bool(self.checkbox.isChecked()) + self.ask_question.emit(callback, payload, cb) self.hide() def reject(self): if self.questions: payload, callback, cancel_callback = self.questions[0][:3] self.questions = self.questions[1:] - self.ask_question.emit(cancel_callback, payload) + cb = None + if self.checkbox.isVisible(): + cb = bool(self.checkbox.isChecked()) + self.ask_question.emit(cancel_callback, payload, cb) self.hide() - def do_ask_question(self, callback, payload): + def do_ask_question(self, callback, payload, checkbox_checked): if callable(callback): - callback(payload) + args = [payload] + if checkbox_checked is not None: + args.append(checkbox_checked) + callback(*args) self.show_question() def toggle_det_msg(self, *args): @@ -122,6 +134,10 @@ class ProceedQuestion(QDialog): self.det_msg.setVisible(False) self.det_msg_toggle.setVisible(bool(question.det_msg)) self.det_msg_toggle.setText(self.show_det_msg) + self.checkbox.setVisible(question.checkbox_msg is not None) + if question.checkbox_msg is not None: + self.checkbox.setText(question.checkbox_msg) + self.checkbox.setChecked(question.checkbox_checked) self.do_resize() self.show() self.bb.button(self.bb.Yes).setDefault(True) @@ -129,10 +145,10 @@ class ProceedQuestion(QDialog): def __call__(self, callback, payload, html_log, log_viewer_title, title, msg, det_msg='', show_copy_button=False, cancel_callback=None, - log_is_file=False): + log_is_file=False, checkbox_msg=None, checkbox_checked=False): ''' A non modal popup that notifies the user that a background task has - been completed. This class guarantees that onlya single popup is + been completed. This class guarantees that only a single popup is visible at any one time. Other requests are queued and displayed after the user dismisses the current popup. @@ -147,11 +163,18 @@ class ProceedQuestion(QDialog): :param msg: The msg to display :param det_msg: Detailed message :param log_is_file: If True the html_log parameter is interpreted as - the path to a file on disk containing the log encoded with utf-8 + the path to a file on disk containing the log + encoded with utf-8 + :param checkbox_msg: If not None, a checkbox is displayed in the + dialog, showing this message. The callback is + called with both the payload and the state of the + checkbox as arguments. + :param checkbox_checked: If True the checkbox is checked by default. + ''' question = Question(payload, callback, cancel_callback, title, msg, html_log, log_viewer_title, log_is_file, det_msg, - show_copy_button) + show_copy_button, checkbox_msg, checkbox_checked) self.questions.append(question) self.show_question() @@ -169,7 +192,8 @@ def main(): from calibre.gui2 import Application app = Application([]) p = ProceedQuestion(None) - p(lambda p:None, None, 'ass', 'ass', 'testing', 'testing') + p(lambda p:None, None, 'ass', 'ass', 'testing', 'testing', + checkbox_msg='testing the ruddy checkbox', det_msg='details') p.exec_() app From 1f7965b96d8aeca1e4e06e5b4431b165231a3264 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 10 Sep 2012 09:41:26 +0200 Subject: [PATCH 29/45] Make bonjour for the content server advertize an IP address set in tweaks. --- src/calibre/gui2/actions/device.py | 9 +++++++-- src/calibre/library/server/base.py | 9 ++++++--- src/calibre/utils/mdns.py | 13 +++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/actions/device.py b/src/calibre/gui2/actions/device.py index 6e215661c8..3ec7738362 100644 --- a/src/calibre/gui2/actions/device.py +++ b/src/calibre/gui2/actions/device.py @@ -11,6 +11,7 @@ from PyQt4.Qt import QToolButton, QMenu, pyqtSignal, QIcon, QTimer from calibre.gui2.actions import InterfaceAction from calibre.utils.smtp import config as email_config +from calibre.utils.config import tweaks from calibre.constants import iswindows, isosx from calibre.customize.ui import is_disabled from calibre.devices.bambook.driver import BAMBOOK @@ -84,10 +85,14 @@ class ShareConnMenu(QMenu): # {{{ action=self.toggle_server_action, group=gr) def server_state_changed(self, running): - from calibre.utils.mdns import get_external_ip + from calibre.utils.mdns import get_external_ip, verify_ipV4_address text = _('Start Content Server') if running: - text = _('Stop Content Server') + ' [%s]'%get_external_ip() + listen_on = verify_ipV4_address(tweaks['server_listen_on']) + if listen_on: + text = _('Stop Content Server') + ' [%s]'%listen_on + else: + text = _('Stop Content Server') + ' [%s]'%get_external_ip() self.toggle_server_action.setText(text) def hide_smartdevice_menus(self): diff --git a/src/calibre/library/server/base.py b/src/calibre/library/server/base.py index 3554268c3b..884d273ea9 100644 --- a/src/calibre/library/server/base.py +++ b/src/calibre/library/server/base.py @@ -5,7 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os +import os, socket import logging from logging.handlers import RotatingFileHandler @@ -17,7 +17,7 @@ from calibre.utils.date import fromtimestamp from calibre.library.server import listen_on, log_access_file, log_error_file from calibre.library.server.utils import expose, AuthController from calibre.utils.mdns import publish as publish_zeroconf, \ - unpublish as unpublish_zeroconf, get_external_ip + unpublish as unpublish_zeroconf, get_external_ip, verify_ipV4_address from calibre.library.server.content import ContentServer from calibre.library.server.mobile import MobileServer from calibre.library.server.xml import XMLServer @@ -78,6 +78,7 @@ class BonJour(SimplePlugin): # {{{ SimplePlugin.__init__(self, engine) self.port = port self.prefix = prefix + self.ip_address = '0.0.0.0' @property def mdns_services(self): @@ -90,9 +91,10 @@ class BonJour(SimplePlugin): # {{{ def start(self): + zeroconf_ip_address = verify_ipV4_address(self.ip_address) try: for s in self.mdns_services: - publish_zeroconf(*s) + publish_zeroconf(*s, use_ip_address=zeroconf_ip_address) except: import traceback cherrypy.log.error('Failed to start BonJour:') @@ -140,6 +142,7 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache, if not opts.url_prefix: opts.url_prefix = '' + cherrypy.engine.bonjour.ip_address = listen_on cherrypy.engine.bonjour.port = opts.port cherrypy.engine.bonjour.prefix = opts.url_prefix diff --git a/src/calibre/utils/mdns.py b/src/calibre/utils/mdns.py index abbd6c2247..706516e32c 100644 --- a/src/calibre/utils/mdns.py +++ b/src/calibre/utils/mdns.py @@ -39,6 +39,19 @@ def _get_external_ip(): #print 'ipaddr: %s' % ipaddr return ipaddr +def verify_ipV4_address(ip_address): + result = None + if ip_address != '0.0.0.0' and ip_address != '::': + # do some more sanity checks on the address + try: + socket.inet_aton(ip_address) + if len(ip_address.split('.')) == 4: + result = ip_address + except socket.error: + # Not legal ip address + pass + return result + _ext_ip = None def get_external_ip(): global _ext_ip From 26648346008800313196eea9cc4806216f6ad14f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 10 Sep 2012 14:00:51 +0200 Subject: [PATCH 30/45] Fix mdns to pass through the desired IP address. --- src/calibre/utils/mdns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/mdns.py b/src/calibre/utils/mdns.py index 706516e32c..48027791ab 100644 --- a/src/calibre/utils/mdns.py +++ b/src/calibre/utils/mdns.py @@ -106,7 +106,8 @@ def publish(desc, type, port, properties=None, add_hostname=True, use_ip_address into the TXT record. ''' server = start_server() - service = create_service(desc, type, port, properties, add_hostname) + service = create_service(desc, type, port, properties, add_hostname, + use_ip_address) server.registerService(service) def unpublish(desc, type, port, properties=None, add_hostname=True): From b808b74add46c5beee2cdb09d680f8f17fcc2e09 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Sep 2012 18:12:15 +0530 Subject: [PATCH 31/45] Ebook-viewer: When displaying amthematics, reflow equations that dont fit on a single line --- resources/compiled_coffeescript.zip | Bin 57017 -> 57014 bytes .../jax/output/SVG/autoload/multiline.js | 2 +- src/calibre/ebooks/oeb/display/mathjax.coffee | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index 0f6b8c3e7a522839ce146a7c6211d81b23512310..8e45c8fa6b584c61f35ac0446cc4f880a5b24df1 100644 GIT binary patch delta 147 zcmdnFmwDS>X6XQLW)=|!5b&I+B)cZ z#4tVPnXDKhI=SGk5ff7gkSRO);9V`Ijfr4UrhBGLe+t1&gCe%cA@{VHcrJsv4ww1F V0=!w-Kz6bL;eJL2hKF}RJOBuBEE50# delta 130 zcmdnCmwD%2W|;tQW)=|!5ODOlGMH(5nQ!ufdjf1v I?tm-=040_%)c^nh diff --git a/resources/viewer/mathjax/jax/output/SVG/autoload/multiline.js b/resources/viewer/mathjax/jax/output/SVG/autoload/multiline.js index 30bbe2c45c..4ee0ce4f7a 100644 --- a/resources/viewer/mathjax/jax/output/SVG/autoload/multiline.js +++ b/resources/viewer/mathjax/jax/output/SVG/autoload/multiline.js @@ -187,7 +187,7 @@ MathJax.Hub.Register.StartupHook("SVG Jax Ready",function () { // fill it with the proper elements, // and clean up the bbox // - line = BBOX(); + var line = BBOX(); state.first = broken; state.last = true; this.SVGmoveLine(start,end,line,state,values); line.Clean(); diff --git a/src/calibre/ebooks/oeb/display/mathjax.coffee b/src/calibre/ebooks/oeb/display/mathjax.coffee index cd130c85c8..14633b8fbc 100644 --- a/src/calibre/ebooks/oeb/display/mathjax.coffee +++ b/src/calibre/ebooks/oeb/display/mathjax.coffee @@ -39,7 +39,7 @@ class MathJax showMathMenu: false, extensions: ["tex2jax.js", "asciimath2jax.js", "mml2jax.js"], jax: ["input/TeX","input/MathML","input/AsciiMath","output/SVG"], - // SVG : { linebreaks : { automatic : true } }, + SVG : { linebreaks : { automatic : true } }, TeX: { extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"] } From a6be50425ad235d9e781407dbd99d0d922ad3837 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Sep 2012 18:32:55 +0530 Subject: [PATCH 32/45] ... --- src/calibre/ebooks/conversion/plumber.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index dcb5add27e..8f7ab10e0e 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -1120,7 +1120,7 @@ OptionRecommendation(name='search_replace', self.log.info('Creating %s...'%self.output_plugin.name) our = CompositeProgressReporter(0.67, 1., self.ui_reporter) self.output_plugin.report_progress = our - our(0., _('Creating')+' %s'%self.output_plugin.name) + our(0., _('Running %s plugin')%self.output_plugin.name) with self.output_plugin: self.output_plugin.convert(self.oeb, self.output, self.input_plugin, self.opts, self.log) From 9b988d50372a3a3bf44ba8ea7419b6c9ecfa795b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Sep 2012 21:31:05 +0530 Subject: [PATCH 33/45] ... --- src/calibre/ebooks/conversion/plugins/recipe_input.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/ebooks/conversion/plugins/recipe_input.py b/src/calibre/ebooks/conversion/plugins/recipe_input.py index 790c63badd..9bd4077528 100644 --- a/src/calibre/ebooks/conversion/plugins/recipe_input.py +++ b/src/calibre/ebooks/conversion/plugins/recipe_input.py @@ -66,6 +66,7 @@ class RecipeInput(InputFormatPlugin): if os.access(recipe_or_file, os.R_OK): self.recipe_source = open(recipe_or_file, 'rb').read() recipe = compile_recipe(self.recipe_source) + log('Using custom recipe') else: from calibre.web.feeds.recipes.collection import \ get_builtin_recipe_by_title @@ -87,12 +88,15 @@ class RecipeInput(InputFormatPlugin): 'back to builtin one') builtin = True if builtin: + log('Using bundled builtin recipe') raw = get_builtin_recipe_by_title(title, log=log, download_recipe=False) if raw is None: raise ValueError('Failed to find builtin recipe: '+title) recipe = compile_recipe(raw) self.recipe_source = raw + else: + log('Using downloaded builtin recipe') if recipe is None: raise ValueError('%r is not a valid recipe file or builtin recipe' % From 042048e93ac2bca4914bd79ab159c776f09553b0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Sep 2012 21:35:09 +0530 Subject: [PATCH 34/45] ... --- recipes/houston_chronicle.recipe | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recipes/houston_chronicle.recipe b/recipes/houston_chronicle.recipe index 639d5c2042..ed430aa45a 100644 --- a/recipes/houston_chronicle.recipe +++ b/recipes/houston_chronicle.recipe @@ -15,11 +15,11 @@ class HoustonChronicle(BasicNewsRecipe): remove_attributes = ['style'] auto_cleanup = True - oldest_article = 2.0 + oldest_article = 3.0 #keep_only_tags = {'class':lambda x: x and ('hst-articletitle' in x or #'hst-articletext' in x or 'hst-galleryitem' in x)} - #remove_attributes = ['xmlns'] + remove_attributes = ['xmlns'] feeds = [ ('News', "http://www.chron.com/rss/feed/News-270.php"), @@ -38,4 +38,4 @@ class HoustonChronicle(BasicNewsRecipe): ] - + From 052ed1010a929bfca14f271b2a9846c9d17c0448 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Sep 2012 10:49:08 +0530 Subject: [PATCH 35/45] ... --- recipes/wsj.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/wsj.recipe b/recipes/wsj.recipe index dc6ec83e60..057da7adf7 100644 --- a/recipes/wsj.recipe +++ b/recipes/wsj.recipe @@ -65,7 +65,7 @@ class WallStreetJournal(BasicNewsRecipe): br['password'] = self.password res = br.submit() raw = res.read() - if '>Log Out<' not in raw: + if 'Welcome,' not in raw and '>Logout<' not in raw and '>Log Out<' not in raw: raise ValueError('Failed to log in to wsj.com, check your ' 'username and password') return br From 9ea57154066bb139d288252ba22a92d66edde35d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Sep 2012 12:49:32 +0530 Subject: [PATCH 36/45] Conversion: Add support for CSS pseudo classes :hover, :link, :visited, :first-line, :focus, :active, :first-letter --- src/calibre/ebooks/conversion/plumber.py | 2 + src/calibre/ebooks/oeb/stylizer.py | 73 +++++++++++++------- src/calibre/ebooks/oeb/transforms/flatcss.py | 70 ++++++++++++++----- 3 files changed, 104 insertions(+), 41 deletions(-) diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 8f7ab10e0e..60cce24121 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -1009,6 +1009,8 @@ OptionRecommendation(name='search_replace', pr(0., _('Running transforms on ebook...')) + self.oeb.plumber_output_format = self.output_fmt or '' + from calibre.ebooks.oeb.transforms.guide import Clean Clean()(self.oeb, self.opts) pr(0.1) diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index 969f7c763a..d558f7f49b 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -268,33 +268,41 @@ class Stylizer(object): self.rules = rules self._styles = {} for _, _, cssdict, text, _ in rules: - fl = ':first-letter' in text - if fl: - text = text.replace(':first-letter', '') + fl = re.search(ur':(first-letter|first-line|link|hover|visited|active|focus)', text) + if fl is not None: + text = text.replace(fl.group(), '') selector = get_css_selector(text) matches = selector(tree, self.logger) - if fl: - from lxml.builder import ElementMaker - E = ElementMaker(namespace=XHTML_NS) - for elem in matches: - for x in elem.iter(): - if x.text: - punctuation_chars = [] - text = unicode(x.text) - while text: - if not unicodedata.category(text[0]).startswith('P'): - break - punctuation_chars.append(text[0]) - text = text[1:] + if fl is not None: + fl = fl.group(1) + if fl == 'first-letter' and getattr(self.oeb, + 'plumber_output_format', '').lower() == u'mobi': + # Fake first-letter + from lxml.builder import ElementMaker + E = ElementMaker(namespace=XHTML_NS) + for elem in matches: + for x in elem.iter(): + if x.text: + punctuation_chars = [] + text = unicode(x.text) + while text: + category = unicodedata.category(text[0]) + if category[0] not in {'P', 'Z'}: + break + punctuation_chars.append(text[0]) + text = text[1:] - special_text = u''.join(punctuation_chars) + \ - (text[0] if text else u'') - span = E.span(special_text) - span.tail = text[1:] - x.text = None - x.insert(0, span) - self.style(span)._update_cssdict(cssdict) - break + special_text = u''.join(punctuation_chars) + \ + (text[0] if text else u'') + span = E.span(special_text) + span.tail = text[1:] + x.text = None + x.insert(0, span) + self.style(span)._update_cssdict(cssdict) + break + else: # Element pseudo-class + for elem in matches: + self.style(elem)._update_pseudo_class(fl, cssdict) else: for elem in matches: self.style(elem)._update_cssdict(cssdict) @@ -495,6 +503,7 @@ class Style(object): self._height = None self._lineHeight = None self._bgcolor = None + self._pseudo_classes = {} stylizer._styles[element] = self def set(self, prop, val): @@ -506,6 +515,11 @@ class Style(object): def _update_cssdict(self, cssdict): self._style.update(cssdict) + def _update_pseudo_class(self, name, cssdict): + orig = self._pseudo_classes.get(name, {}) + orig.update(cssdict) + self._pseudo_classes[name] = orig + def _apply_style_attr(self, url_replacer=None): attrib = self._element.attrib if 'style' not in attrib: @@ -778,3 +792,14 @@ class Style(object): def cssdict(self): return dict(self._style) + + def pseudo_classes(self, filter_css): + if filter_css: + css = copy.deepcopy(self._pseudo_classes) + for psel, cssdict in css.iteritems(): + for k in filter_css: + cssdict.pop(k, None) + else: + css = self._pseudo_classes + return {k:v for k, v in css.iteritems() if v} + diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 72c9dc0d72..6633651a82 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -222,7 +222,7 @@ class CSSFlattener(object): value = 0.0 cssdict[property] = "%0.5fem" % (value / fsize) - def flatten_node(self, node, stylizer, names, styles, psize, item_id): + def flatten_node(self, node, stylizer, names, styles, pseudo_styles, psize, item_id): if not isinstance(node.tag, basestring) \ or namespace(node.tag) != XHTML_NS: return @@ -357,25 +357,51 @@ class CSSFlattener(object): cssdict.get('text-align', None) not in ('center', 'right')): cssdict['text-indent'] = "%1.1fem" % indent_size - if cssdict: - items = cssdict.items() - items.sort() - css = u';\n'.join(u'%s: %s' % (key, val) for key, val in items) - classes = node.get('class', '').strip() or 'calibre' - klass = STRIPNUM.sub('', classes.split()[0].replace('_', '')) - if css in styles: - match = styles[css] - else: - match = klass + str(names[klass] or '') - styles[css] = match - names[klass] += 1 - node.attrib['class'] = match + pseudo_classes = style.pseudo_classes(self.filter_css) + if cssdict or pseudo_classes: + keep_classes = set() + + if cssdict: + items = cssdict.items() + items.sort() + css = u';\n'.join(u'%s: %s' % (key, val) for key, val in items) + classes = node.get('class', '').strip() or 'calibre' + klass = STRIPNUM.sub('', classes.split()[0].replace('_', '')) + if css in styles: + match = styles[css] + else: + match = klass + str(names[klass] or '') + styles[css] = match + names[klass] += 1 + node.attrib['class'] = match + keep_classes.add(match) + + for psel, cssdict in pseudo_classes.iteritems(): + items = sorted(cssdict.iteritems()) + css = u';\n'.join(u'%s: %s' % (key, val) for key, val in items) + pstyles = pseudo_styles[psel] + if css in pstyles: + match = pstyles[css] + else: + # We have to use a different class for each psel as + # otherwise you can have incorrect styles for a situation + # like: a:hover { color: red } a:link { color: blue } a.x:hover { color: green } + # If the pcalibre class for a:hover and a:link is the same, + # then the class attribute for a.x tags will contain both + # that class and the class for a.x:hover, which is wrong. + klass = 'pcalibre' + match = klass + str(names[klass] or '') + pstyles[css] = match + names[klass] += 1 + keep_classes.add(match) + node.attrib['class'] = ' '.join(keep_classes) + elif 'class' in node.attrib: del node.attrib['class'] if 'style' in node.attrib: del node.attrib['style'] for child in node: - self.flatten_node(child, stylizer, names, styles, psize, item_id) + self.flatten_node(child, stylizer, names, styles, pseudo_styles, psize, item_id) def flatten_head(self, item, href, global_href): html = item.data @@ -446,7 +472,7 @@ class CSSFlattener(object): def flatten_spine(self): names = defaultdict(int) - styles = {} + styles, pseudo_styles = {}, defaultdict(dict) for item in self.oeb.spine: html = item.data stylizer = self.stylizers[item] @@ -454,10 +480,20 @@ class CSSFlattener(object): self.specializer(item, stylizer) body = html.find(XHTML('body')) fsize = self.context.dest.fbase - self.flatten_node(body, stylizer, names, styles, fsize, item.id) + self.flatten_node(body, stylizer, names, styles, pseudo_styles, fsize, item.id) items = [(key, val) for (val, key) in styles.items()] items.sort() + # :hover must come after link and :active must come after :hover + psels = sorted(pseudo_styles.iterkeys(), key=lambda x : + {'hover':1, 'active':2}.get(x, 0)) + for psel in psels: + styles = pseudo_styles[psel] + if not styles: continue + x = sorted(((k+':'+psel, v) for v, k in styles.iteritems())) + items.extend(x) + css = ''.join(".%s {\n%s;\n}\n\n" % (key, val) for key, val in items) + href = self.replace_css(css) global_css = self.collect_global_css() for item in self.oeb.spine: From 9a8b9874a49de15727006d2dc7b77df38b4ea3c2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Sep 2012 12:51:27 +0530 Subject: [PATCH 37/45] ... --- src/calibre/ebooks/oeb/stylizer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index d558f7f49b..37641c91f2 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -267,8 +267,9 @@ class Stylizer(object): rules.sort() self.rules = rules self._styles = {} + pseudo_pat = re.compile(ur':(first-letter|first-line|link|hover|visited|active|focus)', re.I) for _, _, cssdict, text, _ in rules: - fl = re.search(ur':(first-letter|first-line|link|hover|visited|active|focus)', text) + fl = pseudo_pat.search(text) if fl is not None: text = text.replace(fl.group(), '') selector = get_css_selector(text) From 556f9c844faa5e40b5339f72dc9ee3c9ccd9b5f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Sep 2012 13:33:49 +0530 Subject: [PATCH 38/45] ... --- src/calibre/devices/mtp/driver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 55472d3d44..b43d3db3df 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -47,9 +47,9 @@ class MTP_DEVICE(BASE): from calibre.library.save_to_disk import config self._prefs = p = JSONConfig('mtp_devices') p.defaults['format_map'] = self.FORMATS - p.defaults['send_to'] = ['eBooks/import', - 'wordplayer/calibretransfer', 'Books', 'sdcard/ebooks', - 'eBooks', 'kindle'] + p.defaults['send_to'] = ['Books', 'eBooks/import', 'eBooks', + 'wordplayer/calibretransfer', 'sdcard/ebooks', + 'kindle'] p.defaults['send_template'] = config().parse().send_template p.defaults['blacklist'] = [] p.defaults['history'] = {} @@ -300,7 +300,7 @@ class MTP_DEVICE(BASE): p = path break if p is None: - p = 'eBooks' + p = 'Books' self.location_paths[loc] = p return self.location_paths[on_card] From 378d13d041c157a8475999a9e1985c9da564f11f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Sep 2012 08:47:53 +0530 Subject: [PATCH 39/45] Turn on sending of azw3 files to kindles by default, since the KK now has azw3 support --- src/calibre/devices/kindle/driver.py | 5 ++--- src/calibre/gui2/convert/mobi_output.ui | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/calibre/devices/kindle/driver.py b/src/calibre/devices/kindle/driver.py index 1971faef60..ac3bcb4bc1 100644 --- a/src/calibre/devices/kindle/driver.py +++ b/src/calibre/devices/kindle/driver.py @@ -288,7 +288,7 @@ class KINDLE2(KINDLE): name = 'Kindle 2/3/4/Touch Device Interface' description = _('Communicate with the Kindle 2/3/4/Touch eBook reader.') - FORMATS = KINDLE.FORMATS + ['pdf', 'azw4', 'pobi'] + FORMATS = ['azw3'] + KINDLE.FORMATS + ['pdf', 'azw4', 'pobi'] DELETE_EXTS = KINDLE.DELETE_EXTS + ['.mbp1', '.mbs', '.sdr'] PRODUCT_ID = [0x0002, 0x0004] @@ -449,7 +449,7 @@ class KINDLE_DX(KINDLE2): name = 'Kindle DX Device Interface' description = _('Communicate with the Kindle DX eBook reader.') - + FORMATS = KINDLE2.FORMATS[1:] PRODUCT_ID = [0x0003] BCD = [0x0100] @@ -462,7 +462,6 @@ class KINDLE_FIRE(KINDLE2): description = _('Communicate with the Kindle Fire') gui_name = 'Fire' FORMATS = list(KINDLE2.FORMATS) - FORMATS.insert(0, 'azw3') PRODUCT_ID = [0x0006] BCD = [0x216, 0x100] diff --git a/src/calibre/gui2/convert/mobi_output.ui b/src/calibre/gui2/convert/mobi_output.ui index 8c1c107620..71c19fb0c4 100644 --- a/src/calibre/gui2/convert/mobi_output.ui +++ b/src/calibre/gui2/convert/mobi_output.ui @@ -7,7 +7,7 @@ 0 0 588 - 342 + 416 @@ -91,23 +91,33 @@ - + Personal Doc tag: - + - + Enable sharing of book content via Facebook, etc. WARNING: Disables last read syncing + + + + <b>WARNING:</b> Various Kindle devices have trouble displaying the new or both MOBI filetypes. If you wish to use the new format on your device, convert to AZW3 instead of MOBI. + + + true + + + From b4772cf0382e0f65de0977e996465e3182e39851 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Sep 2012 10:57:13 +0530 Subject: [PATCH 40/45] Show path on device in the book details panel in the device view --- src/calibre/gui2/book_details.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index f03015f4ad..90284df809 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -152,8 +152,16 @@ def render_data(mi, use_roman_numbers=True, all_fields=False): scheme = u'devpath' if isdevice else u'path' url = prepare_string_for_xml(path if isdevice else unicode(mi.id), True) - link = u'%s' % (scheme, url, - prepare_string_for_xml(path, True), _('Click to open')) + pathstr = _('Click to open') + extra = '' + if isdevice: + durl = url + if durl.startswith('mtp:::'): + durl = ':::'.join( (durl.split(':::'))[2:] ) + extra = '
%s'%( + prepare_string_for_xml(durl)) + link = u'%s%s' % (scheme, url, + prepare_string_for_xml(path, True), pathstr, extra) ans.append((field, u'%s%s'%(name, link))) elif field == 'formats': if isdevice: continue From 3032d61204b45b20c2d50dc4ed13640460e6a243 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Sep 2012 13:38:31 +0530 Subject: [PATCH 41/45] MTP: Return a correct driveinfo dict --- src/calibre/devices/mtp/driver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index b43d3db3df..ed187c33c4 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -53,6 +53,7 @@ class MTP_DEVICE(BASE): p.defaults['send_template'] = config().parse().send_template p.defaults['blacklist'] = [] p.defaults['history'] = {} + p.defaults['rules'] = [] return self._prefs @@ -106,7 +107,7 @@ class MTP_DEVICE(BASE): dinfo['mtp_prefix'] = storage.storage_prefix raw = json.dumps(dinfo, default=to_json) self.put_file(storage, self.DRIVEINFO, BytesIO(raw), len(raw)) - self.driveinfo = dinfo + self.driveinfo[location_code] = dinfo def get_device_information(self, end_session=True): self.report_progress(1.0, _('Get device information...')) From ac9ade0409a485b0e0c23530b3f2fac4a241b2cb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Sep 2012 13:51:40 +0530 Subject: [PATCH 42/45] Add CC directory to the default list of folders to send to --- src/calibre/devices/mtp/driver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index ed187c33c4..c999bd9c1d 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -47,9 +47,9 @@ class MTP_DEVICE(BASE): from calibre.library.save_to_disk import config self._prefs = p = JSONConfig('mtp_devices') p.defaults['format_map'] = self.FORMATS - p.defaults['send_to'] = ['Books', 'eBooks/import', 'eBooks', - 'wordplayer/calibretransfer', 'sdcard/ebooks', - 'kindle'] + p.defaults['send_to'] = ['Calibre_Companion', 'Books', + 'eBooks/import', 'eBooks', 'wordplayer/calibretransfer', + 'sdcard/ebooks', 'kindle'] p.defaults['send_template'] = config().parse().send_template p.defaults['blacklist'] = [] p.defaults['history'] = {} From d26bf70d547da7b27587fa24635451296af89ce6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Sep 2012 14:15:54 +0530 Subject: [PATCH 43/45] UI for creating format routing rules for MTP devices --- src/calibre/gui2/device_drivers/mtp_config.py | 153 ++++++++++++++++-- 1 file changed, 139 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/device_drivers/mtp_config.py b/src/calibre/gui2/device_drivers/mtp_config.py index 261fea3df2..7481cbf19c 100644 --- a/src/calibre/gui2/device_drivers/mtp_config.py +++ b/src/calibre/gui2/device_drivers/mtp_config.py @@ -11,7 +11,8 @@ import weakref from PyQt4.Qt import (QWidget, QListWidgetItem, Qt, QToolButton, QLabel, QTabWidget, QGridLayout, QListWidget, QIcon, QLineEdit, QVBoxLayout, - QPushButton) + QPushButton, QGroupBox, QScrollArea, QHBoxLayout, QComboBox, + pyqtSignal, QSizePolicy, QDialog, QDialogButtonBox) from calibre.ebooks import BOOK_EXTENSIONS from calibre.gui2 import error_dialog @@ -86,7 +87,7 @@ class TemplateConfig(QWidget): # {{{ m.setBuddy(t) l.addWidget(m, 0, 0, 1, 2) l.addWidget(t, 1, 0, 1, 1) - b = self.b = QPushButton(_('Template editor')) + b = self.b = QPushButton(_('&Template editor')) l.addWidget(b, 1, 1, 1, 1) b.clicked.connect(self.edit_template) @@ -176,6 +177,113 @@ class IgnoredDevices(QWidget): # {{{ # }}} +# Rules {{{ + +class Rule(QWidget): + + remove = pyqtSignal(object) + + def __init__(self, rule=None): + QWidget.__init__(self) + + self.l = l = QHBoxLayout() + self.setLayout(l) + + self.l1 = l1 = QLabel(_('Send the ')) + l.addWidget(l1) + self.fmt = f = QComboBox(self) + l.addWidget(f) + self.l2 = l2 = QLabel(_(' format to the folder: ')) + l.addWidget(l2) + self.folder = f = QLineEdit(self) + f.setPlaceholderText(_('Folder on the device')) + l.addWidget(f) + self.rb = rb = QPushButton(QIcon(I('list_remove.png')), + _('&Remove rule'), self) + l.addWidget(rb) + rb.clicked.connect(self.removed) + + for fmt in sorted(BOOK_EXTENSIONS): + self.fmt.addItem(fmt.upper(), fmt.lower()) + + self.fmt.setCurrentIndex(0) + + if rule is not None: + fmt, folder = rule + idx = self.fmt.findText(fmt.upper()) + if idx > -1: + self.fmt.setCurrentIndex(idx) + self.folder.setText(folder) + + self.ignore = False + + def removed(self): + self.remove.emit(self) + + @property + def rule(self): + folder = unicode(self.folder.text()).strip() + if folder: + return ( + unicode(self.fmt.itemData(self.fmt.currentIndex()).toString()), + folder + ) + return None + +class FormatRules(QGroupBox): + + def __init__(self, rules): + QGroupBox.__init__(self, _('Format specific sending')) + self.l = l = QVBoxLayout() + self.setLayout(l) + self.la = la = QLabel('

'+_( + '''You can create rules that control where ebooks of a specific + format are sent to on the device. These will take precedence over + the folders specified above.''')) + la.setWordWrap(True) + l.addWidget(la) + self.sa = sa = QScrollArea(self) + sa.setWidgetResizable(True) + self.w = w = QWidget(self) + w.l = QVBoxLayout() + w.setLayout(w.l) + sa.setWidget(w) + l.addWidget(sa) + self.widgets = [] + for rule in rules: + r = Rule(rule) + self.widgets.append(r) + w.l.addWidget(r) + r.remove.connect(self.remove_rule) + + if not self.widgets: + self.add_rule() + + self.b = b = QPushButton(QIcon(I('plus.png')), _('Add a &new rule')) + l.addWidget(b) + b.clicked.connect(self.add_rule) + self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Ignored) + + def add_rule(self): + r = Rule() + self.widgets.append(r) + self.w.l.addWidget(r) + r.remove.connect(self.remove_rule) + self.sa.verticalScrollBar().setValue(self.sa.verticalScrollBar().maximum()) + + def remove_rule(self, rule): + rule.setVisible(False) + rule.ignore = True + + @property + def rules(self): + for w in self.widgets: + if not w.ignore: + r = w.rule + if r is not None: + yield r +# }}} + class MTPConfig(QTabWidget): def __init__(self, device, parent=None): @@ -185,8 +293,8 @@ class MTPConfig(QTabWidget): cd = msg = None if device.current_friendly_name is not None: if device.current_serial_num is None: - msg = '

' + _('The %s device has no serial number, ' - 'it cannot be configured'%device.current_friendly_name) + msg = '

' + (_('The %s device has no serial number, ' + 'it cannot be configured')%device.current_friendly_name) else: cd = 'device-'+device.current_serial_num else: @@ -211,6 +319,7 @@ class MTPConfig(QTabWidget): l = self.base.l = QGridLayout(self.base) self.base.setLayout(l) + self.rules = r = FormatRules(self.get_pref('rules')) self.formats = FormatsConfig(set(BOOK_EXTENSIONS), self.get_pref('format_map')) self.send_to = SendToConfig(self.get_pref('send_to')) @@ -218,17 +327,20 @@ class MTPConfig(QTabWidget): self.base.la = la = QLabel(_( 'Choose the formats to send to the %s')%self.device.current_friendly_name) la.setWordWrap(True) - l.addWidget(la, 0, 0, 1, 1) - l.addWidget(self.formats, 1, 0, 2, 1) - l.addWidget(self.send_to, 1, 1, 1, 1) - l.addWidget(self.template, 2, 1, 1, 1) - l.setRowStretch(2, 10) - self.base.b = b = QPushButton(QIcon(I('minus.png')), - _('Ignore the %s in calibre')%device.current_friendly_name, + self.base.b = b = QPushButton(QIcon(I('list_remove.png')), + _('&Ignore the %s in calibre')%device.current_friendly_name, self.base) - l.addWidget(b, 3, 0, 1, 2) b.clicked.connect(self.ignore_device) + l.addWidget(b, 0, 0, 1, 2) + l.addWidget(la, 1, 0, 1, 1) + l.addWidget(self.formats, 2, 0, 3, 1) + l.addWidget(self.send_to, 2, 1, 1, 1) + l.addWidget(self.template, 3, 1, 1, 1) + l.setRowStretch(4, 10) + l.addWidget(r, 5, 0, 1, 2) + l.setRowStretch(5, 100) + self.igntab = IgnoredDevices(self.device.prefs['history'], self.device.prefs['blacklist']) self.addTab(self.igntab, _('Ignored devices')) @@ -280,6 +392,11 @@ class MTPConfig(QTabWidget): if s and s != self.device.prefs['send_to']: p['send_to'] = s + p.pop('rules', None) + r = list(self.rules.rules) + if r and r != self.device.prefs['rules']: + p['rules'] = r + self.device.prefs[self.current_device_key] = p self.device.prefs['blacklist'] = self.igntab.blacklist @@ -296,8 +413,16 @@ if __name__ == '__main__': cd = dev.detect_managed_devices(s.devices) dev.open(cd, 'test') cw = dev.config_widget() - cw.show() - app.exec_() + d = QDialog() + d.l = QVBoxLayout() + d.setLayout(d.l) + d.l.addWidget(cw) + bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel) + d.l.addWidget(bb) + bb.accepted.connect(d.accept) + bb.rejected.connect(d.reject) + if d.exec_() == d.Accepted: + cw.commit() dev.shutdown() From 4b8933f9fbb83ea6226487659244213a321361a9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Sep 2012 14:30:50 +0530 Subject: [PATCH 44/45] MTP: Implement format routing --- src/calibre/devices/mtp/driver.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index c999bd9c1d..caf3174a11 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -274,15 +274,18 @@ class MTP_DEVICE(BASE): self.plugboards = plugboards self.plugboard_func = pb_func - def create_upload_path(self, path, mdata, fname): + def create_upload_path(self, path, mdata, fname, routing): from calibre.devices.utils import create_upload_path from calibre.utils.filenames import ascii_filename as sanitize + ext = fname.rpartition('.')[-1].lower() + path = routing.get(ext, path) + filepath = create_upload_path(mdata, fname, self.save_template, sanitize, prefix_path=path, path_type=posixpath, maxlen=self.MAX_PATH_LEN, - use_subdirs = True, - news_in_folder = self.NEWS_IN_FOLDER, + use_subdirs=True, + news_in_folder=self.NEWS_IN_FOLDER, ) return tuple(x for x in filepath.split('/')) @@ -330,8 +333,10 @@ class MTP_DEVICE(BASE): self.report_progress(0, _('Transferring books to device...')) i, total = 0, len(files) + routing = {fmt:dest for fmt,dest in self.get_pref('rules')} + for infile, fname, mi in izip(files, names, metadata): - path = self.create_upload_path(prefix, mi, fname) + path = self.create_upload_path(prefix, mi, fname, routing) parent = self.ensure_parent(storage, path) if hasattr(infile, 'read'): pos = infile.tell() From c0aee6772dc23a6e411c341f2b16ddb377c70df8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Sep 2012 14:50:59 +0530 Subject: [PATCH 45/45] ... --- src/calibre/devices/mtp/unix/driver.py | 7 +++++-- src/calibre/devices/mtp/windows/driver.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index 31f886b875..6e5d91c0a0 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import operator, traceback, pprint, sys +import operator, traceback, pprint, sys, time from threading import RLock from collections import namedtuple from functools import partial @@ -16,7 +16,7 @@ from calibre import prints, as_unicode from calibre.constants import plugins from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed, DeviceError, BlacklistedDevice -from calibre.devices.mtp.base import MTPDeviceBase, synchronous +from calibre.devices.mtp.base import MTPDeviceBase, synchronous, debug MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id ' 'bcd serial manufacturer product') @@ -193,6 +193,8 @@ class MTP_DEVICE(MTPDeviceBase): @property def filesystem_cache(self): if self._filesystem_cache is None: + st = time.time() + debug('Loading filesystem metadata...') from calibre.devices.mtp.filesystem_cache import FilesystemCache with self.lock: storage, all_items, all_errs = [], [], [] @@ -220,6 +222,7 @@ class MTP_DEVICE(MTPDeviceBase): self.current_friendly_name, self.format_errorstack(all_errs))) self._filesystem_cache = FilesystemCache(storage, all_items) + debug('Filesystem metadata loaded in %g seconds'%(time.time()-st)) return self._filesystem_cache @synchronous diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 50638496d1..3290115028 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -16,7 +16,7 @@ from calibre import as_unicode, prints from calibre.constants import plugins, __appname__, numeric_version from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed, DeviceError, BlacklistedDevice -from calibre.devices.mtp.base import MTPDeviceBase +from calibre.devices.mtp.base import MTPDeviceBase, debug class ThreadingViolation(Exception): @@ -199,6 +199,8 @@ class MTP_DEVICE(MTPDeviceBase): @property def filesystem_cache(self): if self._filesystem_cache is None: + debug('Loading filesystem metadata...') + st = time.time() from calibre.devices.mtp.filesystem_cache import FilesystemCache ts = self.total_space() all_storage = [] @@ -218,6 +220,7 @@ class MTP_DEVICE(MTPDeviceBase): all_storage.append(storage) items.append(id_map.itervalues()) self._filesystem_cache = FilesystemCache(all_storage, chain(*items)) + debug('Filesystem metadata loaded in %g seconds'%(time.time()-st)) return self._filesystem_cache @same_thread