diff --git a/recipes/bwmagazine2.recipe b/recipes/bwmagazine2.recipe index 300d71806a..77143bbefc 100644 --- a/recipes/bwmagazine2.recipe +++ b/recipes/bwmagazine2.recipe @@ -33,38 +33,36 @@ class BusinessWeekMagazine(BasicNewsRecipe): div0 = soup.find ('div', attrs={'class':'column left'}) section_title = '' feeds = OrderedDict() - articles = [] - for div in div0.findAll('a'): + for div in div0.findAll('h4'): + articles = [] section_title = self.tag_to_string(div.findPrevious('h3')).strip() - self.log('Processing section:', section_title) - title=self.tag_to_string(div).strip() - url=div['href'] + title=self.tag_to_string(div.a).strip() + url=div.a['href'] soup0 = self.index_to_soup(url) urlprint=soup0.find('li', attrs={'class':'print'}).a['href'] articles.append({'title':title, 'url':urlprint, 'description':'', 'date':''}) - if articles: - if section_title not in feeds: - feeds[section_title] = [] - feeds[section_title] += articles + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles + div1 = soup.find ('div', attrs={'class':'column center'}) section_title = '' - articles = [] - for div in div1.findAll('a'): + for div in div1.findAll('h5'): + articles = [] desc=self.tag_to_string(div.findNext('p')).strip() section_title = self.tag_to_string(div.findPrevious('h3')).strip() - self.log('Processing section:', section_title) - title=self.tag_to_string(div).strip() - url=div['href'] + title=self.tag_to_string(div.a).strip() + url=div.a['href'] soup0 = self.index_to_soup(url) urlprint=soup0.find('li', attrs={'class':'print'}).a['href'] articles.append({'title':title, 'url':urlprint, 'description':desc, 'date':''}) - if articles: - if section_title not in feeds: - feeds[section_title] = [] - feeds[section_title] += articles + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles ans = [(key, val) for key, val in feeds.iteritems()] return ans - diff --git a/recipes/chronicle_higher_ed.recipe b/recipes/chronicle_higher_ed.recipe index 7ed834a4e5..f0188d4d77 100644 --- a/recipes/chronicle_higher_ed.recipe +++ b/recipes/chronicle_higher_ed.recipe @@ -13,13 +13,13 @@ class Chronicle(BasicNewsRecipe): keep_only_tags = [ dict(name='div', attrs={'class':'article'}), ] - remove_tags = [dict(name='div',attrs={'class':'related module1'})] + remove_tags = [dict(name='div',attrs={'class':['related module1','maintitle']}), + dict(name='div', attrs={'id':['section-nav','icon-row']})] no_javascript = True no_stylesheets = True needs_subscription = True - def get_browser(self): br = BasicNewsRecipe.get_browser() if self.username is not None and self.password is not None: @@ -27,7 +27,7 @@ class Chronicle(BasicNewsRecipe): br.select_form(nr=1) br['username'] = self.username br['password'] = self.password - br.submit() + br.submit() return br def parse_index(self): @@ -47,33 +47,35 @@ class Chronicle(BasicNewsRecipe): #Go to the main body soup = self.index_to_soup(issueurl) - div0 = soup.find ('div', attrs={'id':'article-body'}) + div = soup.find ('div', attrs={'id':'article-body'}) feeds = OrderedDict() - for div in div0.findAll('div',attrs={'class':'module1'}): - section_title = self.tag_to_string(div.find('h3')) - for post in div.findAll('li',attrs={'class':'sub-promo'}): - articles = [] - a=post.find('a', href=True) + section_title = '' + for post in div.findAll('li'): + articles = [] + a=post.find('a', href=True) + if a is not None: title=self.tag_to_string(a) url="http://chronicle.com"+a['href'].strip() + sectiontitle=post.findPrevious('h3') + if sectiontitle is None: + sectiontitle=post.findPrevious('h4') + section_title=self.tag_to_string(sectiontitle) desc=self.tag_to_string(post.find('p')) articles.append({'title':title, 'url':url, 'description':desc, 'date':''}) - if articles: - if section_title not in feeds: - feeds[section_title] = [] - feeds[section_title] += articles + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles ans = [(key, val) for key, val in feeds.iteritems()] return ans def preprocess_html(self,soup): #process all the images for div in soup.findAll('div', attrs={'class':'tableauPlaceholder'}): - noscripts=div.find('noscript').a div.replaceWith(noscripts) for div0 in soup.findAll('div',text='Powered by Tableau'): div0.extract() return soup - diff --git a/recipes/financial_times_uk.recipe b/recipes/financial_times_uk.recipe index 4c331f115f..16295905bc 100644 --- a/recipes/financial_times_uk.recipe +++ b/recipes/financial_times_uk.recipe @@ -10,7 +10,7 @@ from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe class FinancialTimes(BasicNewsRecipe): - title = 'Financial Times - UK printed edition' + title = 'Financial Times (UK)' __author__ = 'Darko Miletic' description = "The Financial Times (FT) is one of the world's leading business news and information organisations, recognised internationally for its authority, integrity and accuracy." publisher = 'The Financial Times Ltd.' @@ -101,17 +101,19 @@ class FinancialTimes(BasicNewsRecipe): def parse_index(self): feeds = [] soup = self.index_to_soup(self.INDEX) + dates= self.tag_to_string(soup.find('div', attrs={'class':'btm-links'}).find('div')) + self.timefmt = ' [%s]'%dates wide = soup.find('div',attrs={'class':'wide'}) if not wide: return feeds strest = wide.findAll('h3', attrs={'class':'section'}) if not strest: return feeds - st = wide.find('h4',attrs={'class':'section-no-arrow'}) + st = wide.findAll('h4',attrs={'class':'section-no-arrow'}) if st: - strest.insert(0,st) + st.extend(strest) count = 0 - for item in strest: + for item in st: count = count + 1 if self.test and count > 2: return feeds @@ -151,7 +153,7 @@ class FinancialTimes(BasicNewsRecipe): def get_cover_url(self): cdate = datetime.date.today() if cdate.isoweekday() == 7: - cdate -= datetime.timedelta(days=1) + cdate -= datetime.timedelta(days=1) return cdate.strftime('http://specials.ft.com/vtf_pdf/%d%m%y_FRONT1_LON.pdf') def get_obfuscated_article(self, url): @@ -163,9 +165,8 @@ class FinancialTimes(BasicNewsRecipe): count = 10 except: print "Retrying download..." - count += 1 + count += 1 self.temp_files.append(PersistentTemporaryFile('_fa.html')) self.temp_files[-1].write(html) self.temp_files[-1].close() return self.temp_files[-1].name - \ No newline at end of file diff --git a/setup/extensions.py b/setup/extensions.py index 2f2e2aa9ba..f4ed22687b 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -139,6 +139,7 @@ extensions = [ Extension('podofo', [ 'calibre/utils/podofo/utils.cpp', + 'calibre/utils/podofo/output.cpp', 'calibre/utils/podofo/doc.cpp', 'calibre/utils/podofo/outline.cpp', 'calibre/utils/podofo/podofo.cpp', diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 469195627c..c7dc6a5b95 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -675,7 +675,6 @@ from calibre.devices.bambook.driver import BAMBOOK from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX from calibre.devices.smart_device_app.driver import SMART_DEVICE_APP - # Order here matters. The first matched device is the one used. plugins += [ HANLINV3, @@ -749,6 +748,12 @@ plugins += [ SMART_DEVICE_APP, USER_DEFINED, ] + +from calibre.utils.config_base import tweaks +if tweaks.get('test_mtp_driver', False): + from calibre.devices.mtp.driver import MTP_DEVICE + plugins.append(MTP_DEVICE) + # }}} # New metadata download plugins {{{ diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index 37ec55c149..89d0e4e026 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal ' Device drivers. ''' -import sys, time, pprint, operator +import sys, time, pprint, operator, re, os from functools import partial from StringIO import StringIO @@ -27,6 +27,112 @@ def strftime(epoch, zone=time.gmtime): src[2] = INVERSE_MONTH_MAP[int(src[2])] return ' '.join(src) +def build_template_regexp(template): + from calibre import prints + + def replfunc(match, seen=None): + v = match.group(1) + if v in ['authors', 'author_sort']: + v = 'author' + if v in ('title', 'series', 'series_index', 'isbn', 'author'): + if v not in seen: + seen.add(v) + return '(?P<' + v + '>.+?)' + return '(.+?)' + s = set() + f = partial(replfunc, seen=s) + + try: + template = template.rpartition('/')[2] + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + except: + prints(u'Failed to parse template: %r'%template) + template = u'{title} - {authors}' + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + +def create_upload_path(mdata, fname, template, sanitize, + prefix_path='', + path_type=os.path, + maxlen=250, + use_subdirs=True, + news_in_folder=True, + filename_callback=lambda x, y:x, + sanitize_path_components=lambda x: x + ): + from calibre.library.save_to_disk import get_components, config + from calibre.utils.filenames import shorten_components_to + + special_tag = None + if mdata.tags: + for t in mdata.tags: + if t.startswith(_('News')) or t.startswith('/'): + special_tag = t + break + + if mdata.tags and _('News') in mdata.tags: + try: + p = mdata.pubdate + date = (p.year, p.month, p.day) + except: + today = time.localtime() + date = (today[0], today[1], today[2]) + template = u"{title}_%d-%d-%d" % date + + fname = sanitize(fname) + ext = path_type.splitext(fname)[1] + + opts = config().parse() + if not isinstance(template, unicode): + template = template.decode('utf-8') + app_id = str(getattr(mdata, 'application_id', '')) + id_ = mdata.get('id', fname) + extra_components = get_components(template, mdata, id_, + timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1) + if not extra_components: + extra_components.append(sanitize(filename_callback(fname, + mdata))) + else: + extra_components[-1] = sanitize(filename_callback(extra_components[-1]+ext, mdata)) + + if extra_components[-1] and extra_components[-1][0] in ('.', '_'): + extra_components[-1] = 'x' + extra_components[-1][1:] + + if special_tag is not None: + name = extra_components[-1] + extra_components = [] + tag = special_tag + if tag.startswith(_('News')): + if news_in_folder: + extra_components.append('News') + else: + for c in tag.split('/'): + c = sanitize(c) + if not c: continue + extra_components.append(c) + extra_components.append(name) + + if not use_subdirs: + extra_components = extra_components[-1:] + + def remove_trailing_periods(x): + ans = x + while ans.endswith('.'): + ans = ans[:-1].strip() + if not ans: + ans = 'x' + return ans + + extra_components = list(map(remove_trailing_periods, extra_components)) + components = shorten_components_to(maxlen - len(prefix_path), extra_components) + components = sanitize_path_components(components) + if prefix_path: + filepath = path_type.join(prefix_path, *components) + else: + filepath = path_type.join(*components) + + return filepath + + def get_connected_device(): from calibre.customize.ui import device_plugins from calibre.devices.scanner import DeviceScanner @@ -115,54 +221,65 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None): out('Available plugins:', textwrap.fill(' '.join([x.__class__.__name__ for x in devplugins]))) out(' ') - out('Looking for devices...') + found_dev = False for dev in devplugins: - connected, det = s.is_device_connected(dev, debug=True) - if connected: - out('\t\tDetected possible device', dev.__class__.__name__) - connected_devices.append((dev, det)) - - out(' ') - errors = {} - success = False - out('Devices possibly connected:', end=' ') - for dev, det in connected_devices: - out(dev.name, end=', ') - if not connected_devices: - out('None', end='') - out(' ') - for dev, det in connected_devices: - out('Trying to open', dev.name, '...', end=' ') - try: - dev.reset(detected_device=det) - dev.open(det, None) - out('OK') - except: - import traceback - errors[dev] = traceback.format_exc() - out('failed') - continue - success = True - if hasattr(dev, '_main_prefix'): - out('Main memory:', repr(dev._main_prefix)) - out('Total space:', dev.total_space()) - break - if not success and errors: - out('Opening of the following devices failed') - for dev,msg in errors.items(): - out(dev) - out(msg) - out(' ') - - if ioreg is not None: - ioreg = 'IOREG Output\n'+ioreg + if not dev.MANAGES_DEVICE_PRESENCE: continue + out('Looking for devices of type:', dev.__class__.__name__) + if dev.debug_managed_device_detection(s.devices, buf): + found_dev = True + break out(' ') - if ioreg_to_tmp: - open('/tmp/ioreg.txt', 'wb').write(ioreg) - out('Dont forget to send the contents of /tmp/ioreg.txt') - out('You can open it with the command: open /tmp/ioreg.txt') - else: - out(ioreg) + + if not found_dev: + out('Looking for devices...') + for dev in devplugins: + if dev.MANAGES_DEVICE_PRESENCE: continue + connected, det = s.is_device_connected(dev, debug=True) + if connected: + out('\t\tDetected possible device', dev.__class__.__name__) + connected_devices.append((dev, det)) + + out(' ') + errors = {} + success = False + out('Devices possibly connected:', end=' ') + for dev, det in connected_devices: + out(dev.name, end=', ') + if not connected_devices: + out('None', end='') + out(' ') + for dev, det in connected_devices: + out('Trying to open', dev.name, '...', end=' ') + try: + dev.reset(detected_device=det) + dev.open(det, None) + out('OK') + except: + import traceback + errors[dev] = traceback.format_exc() + out('failed') + continue + success = True + if hasattr(dev, '_main_prefix'): + out('Main memory:', repr(dev._main_prefix)) + out('Total space:', dev.total_space()) + break + if not success and errors: + out('Opening of the following devices failed') + for dev,msg in errors.items(): + out(dev) + out(msg) + out(' ') + + if ioreg is not None: + ioreg = 'IOREG Output\n'+ioreg + out(' ') + if ioreg_to_tmp: + open('/tmp/ioreg.txt', 'wb').write(ioreg) + out('Dont forget to send the contents of /tmp/ioreg.txt') + out('You can open it with the command: open /tmp/ioreg.txt') + else: + out(ioreg) if hasattr(buf, 'getvalue'): return buf.getvalue().decode('utf-8') diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 45672fdbd1..9e8aa5fe17 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -197,7 +197,8 @@ class ANDROID(USBMS): 'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON', 'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP', 'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD', - 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0'] + 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0', + 'COBY_MID'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', @@ -216,7 +217,7 @@ class ANDROID(USBMS): 'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX', 'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE', 'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', - 'S5830I_CARD', 'MID7042', 'LINK-CREATE'] + 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', @@ -226,7 +227,7 @@ class ANDROID(USBMS): 'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC', 'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875', 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', - 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042'] + 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035'] OSX_MAIN_MEM = 'Android Device Main Memory' diff --git a/src/calibre/devices/cli.py b/src/calibre/devices/cli.py index 95181bf639..c7b105998d 100755 --- a/src/calibre/devices/cli.py +++ b/src/calibre/devices/cli.py @@ -9,7 +9,7 @@ For usage information run the script. import StringIO, sys, time, os from optparse import OptionParser -from calibre import __version__, __appname__ +from calibre import __version__, __appname__, human_readable from calibre.devices.errors import PathError from calibre.utils.terminfo import TerminalController from calibre.devices.errors import ArgumentError, DeviceError, DeviceLocked @@ -18,16 +18,6 @@ from calibre.devices.scanner import DeviceScanner MINIMUM_COL_WIDTH = 12 #: Minimum width of columns in ls output -def human_readable(size): - """ Convert a size in bytes into a human readle form """ - if size < 1024: divisor, suffix = 1, "" - elif size < 1024*1024: divisor, suffix = 1024., "K" - elif size < 1024*1024*1024: divisor, suffix = 1024*1024, "M" - elif size < 1024*1024*1024*1024: divisor, suffix = 1024*1024, "G" - size = str(size/divisor) - if size.find(".") > -1: size = size[:size.find(".")+2] - return size + suffix - class FileFormatter(object): def __init__(self, file, term): self.term = term @@ -207,11 +197,19 @@ def main(): scanner = DeviceScanner() scanner.scan() connected_devices = [] + for d in device_plugins(): try: d.startup() except: print ('Startup failed for device plugin: %s'%d) + if d.MANAGES_DEVICE_PRESENCE: + cd = d.detect_managed_devices(scanner.devices) + if cd is not None: + connected_devices.append((cd, d)) + dev = d + break + continue ok, det = scanner.is_device_connected(d) if ok: dev = d diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 2c5f60ecfe..d0b2611ead 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -81,6 +81,19 @@ class DevicePlugin(Plugin): #: by. NUKE_COMMENTS = None + #: If True indicates that this driver completely manages device detection, + #: ejecting and so forth. If you set this to True, you *must* implement the + #: detect_managed_devices and debug_managed_device_detection methods. + #: A driver with this set to true is responsible for detection of devices, + #: managing a blacklist of devices, a list of ejected devices and so forth. + #: calibre will periodically call the detect_managed_devices() method and + #: is it returns a detected device, calibre will call open(). open() will + #: be called every time a device is returned even is previous calls to open() + #: failed, therefore the driver must maintain its own blacklist of failed + #: devices. Similarly, when ejecting, calibre will call eject() and then + #: assuming the next call to detect_managed_devices() returns None, it will + #: call post_yank_cleanup(). + MANAGES_DEVICE_PRESENCE = False @classmethod def get_gui_name(cls): @@ -196,6 +209,37 @@ class DevicePlugin(Plugin): return True, dev return False, None + def detect_managed_devices(self, devices_on_system, force_refresh=False): + ''' + Called only if MANAGES_DEVICE_PRESENCE is True. + + Scan for devices that this driver can handle. Should return a device + object if a device is found. This object will be passed to the open() + method as the connected_device. If no device is found, return None. + + This method is called periodically by the GUI, so make sure it is not + too resource intensive. Use a cache to avoid repeatedly scanning the + system. + + :param devices_on_system: Set of USB devices found on the system. + + :param force_refresh: If True and the driver uses a cache to prevent + repeated scanning, the cache must be flushed. + ''' + raise NotImplementedError() + + def debug_managed_device_detection(self, devices_on_system, output): + ''' + Called only if MANAGES_DEVICE_PRESENCE is True. + + Should write information about the devices detected on the system to + output, which is a file like object. + + Should return True if a device was detected and successfully opened, + otherwise False. + ''' + raise NotImplementedError() + # }}} def reset(self, key='-1', log_packets=False, report_progress=None, @@ -270,6 +314,9 @@ class DevicePlugin(Plugin): ''' Un-mount / eject the device from the OS. This does not check if there are pending GUI jobs that need to communicate with the device. + + NOTE: That this method may not be called on the same thread as the rest + of the device methods. ''' raise NotImplementedError() @@ -496,6 +543,10 @@ class DevicePlugin(Plugin): ''' Given a list of paths, returns another list of paths. These paths point to addable versions of the books. + + If there is an error preparing a book, then instead of a path, the + position in the returned list for that book should be a three tuple: + (original_path, the exception instance, traceback) ''' return paths diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index 4f8bbc991f..516b68ae1e 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -9,8 +9,14 @@ __docformat__ = 'restructuredtext en' from functools import wraps +from calibre import prints +from calibre.constants import DEBUG from calibre.devices.interface import DevicePlugin +def debug(*args, **kwargs): + if DEBUG: + prints('MTP:', *args, **kwargs) + def synchronous(func): @wraps(func) def synchronizer(self, *args, **kwargs): @@ -26,11 +32,6 @@ class MTPDeviceBase(DevicePlugin): author = 'Kovid Goyal' version = (1, 0, 0) - THUMBNAIL_HEIGHT = 128 - CAN_SET_METADATA = [] - - BACKLOADING_ERROR_MESSAGE = None - def __init__(self, *args, **kwargs): DevicePlugin.__init__(self, *args, **kwargs) self.progress_reporter = None @@ -53,4 +54,17 @@ class MTPDeviceBase(DevicePlugin): # return False return False + def build_template_regexp(self): + from calibre.devices import build_template_regexp + return build_template_regexp(self.save_template) + + @property + def default_save_template(cls): + from calibre.library.save_to_disk import config + return config().parse().send_template + + @property + def save_template(self): + # TODO: Use the device specific template here + return self.default_save_template diff --git a/src/calibre/devices/mtp/books.py b/src/calibre/devices/mtp/books.py new file mode 100644 index 0000000000..73e483f19e --- /dev/null +++ b/src/calibre/devices/mtp/books.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os + +from calibre.devices.interface import BookList as BL +from calibre.ebooks.metadata.book.base import Metadata +from calibre.ebooks.metadata.book.json_codec import JsonCodec +from calibre.utils.date import utcnow + +class BookList(BL): + + def __init__(self, storage_id): + self.storage_id = storage_id + + def supports_collections(self): + return False + +class Book(Metadata): + + def __init__(self, storage_id, lpath, other=None): + Metadata.__init__(self, _('Unknown'), other=other) + self.storage_id, self.lpath = storage_id, lpath + self.lpath = self.path = self.lpath.replace(os.sep, '/') + self.mtp_relpath = tuple([icu_lower(x) for x in self.lpath.split('/')]) + self.datetime = utcnow().timetuple() + self.thumbail = None + + def matches_file(self, mtp_file): + return (self.storage_id == mtp_file.storage_id and + self.mtp_relpath == mtp_file.mtp_relpath) + +class JSONCodec(JsonCodec): + pass + diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index c3e34a2be5..8f8f4d119b 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -7,31 +7,43 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import json, pprint +import json, traceback, posixpath, importlib, os from io import BytesIO +from calibre import prints from calibre.constants import iswindows, numeric_version +from calibre.devices.mtp.base import debug +from calibre.ptempfile import SpooledTemporaryFile, PersistentTemporaryDirectory from calibre.utils.config import from_json, to_json from calibre.utils.date import now, isoformat -if iswindows: - from calibre.devices.mtp.windows.driver import MTP_DEVICE as BASE - BASE -else: - from calibre.devices.mtp.unix.driver import MTP_DEVICE as BASE -pprint +BASE = importlib.import_module('calibre.devices.mtp.%s.driver'%( + 'windows' if iswindows else 'unix')).MTP_DEVICE class MTP_DEVICE(BASE): METADATA_CACHE = 'metadata.calibre' DRIVEINFO = 'driveinfo.calibre' + CAN_SET_METADATA = [] + NEWS_IN_FOLDER = True + MAX_PATH_LEN = 230 + THUMBNAIL_HEIGHT = 160 + THUMBNAIL_WIDTH = 120 + CAN_SET_METADATA = [] + BACKLOADING_ERROR_MESSAGE = None + MANAGES_DEVICE_PRESENCE = True + def open(self, devices, library_uuid): + self.current_library_uuid = library_uuid + BASE.open(self, devices, library_uuid) + + # Device information {{{ def _update_drive_info(self, storage, location_code, name=None): import uuid f = storage.find_path((self.DRIVEINFO,)) dinfo = {} if f is not None: - stream = self.get_file(f) + stream = self.get_mtp_file(f) try: dinfo = json.load(stream, object_hook=from_json) except: @@ -51,10 +63,6 @@ class MTP_DEVICE(BASE): self.put_file(storage, self.DRIVEINFO, BytesIO(raw), len(raw)) self.driveinfo = dinfo - def open(self, devices, library_uuid): - self.current_library_uuid = library_uuid - BASE.open(self, devices, library_uuid) - def get_device_information(self, end_session=True): self.report_progress(1.0, _('Get device information...')) self.driveinfo = {} @@ -80,6 +88,154 @@ class MTP_DEVICE(BASE): return self._update_drive_info(self.filesystem_cache.storage(sid), location_code, name=name) + # }}} + + # Get list of books from device, with metadata {{{ + def books(self, oncard=None, end_session=True): + from calibre.devices.mtp.books import JSONCodec + from calibre.devices.mtp.books import BookList, Book + sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(oncard, + self._main_id) + if sid is None: + return BookList(None) + + bl = BookList(sid) + # If True then there is a mismatch between the ebooks on the device and + # the metadata cache + need_sync = False + all_books = list(self.filesystem_cache.iterebooks(sid)) + steps = len(all_books) + 2 + count = 0 + + self.report_progress(0, _('Reading metadata from device')) + # Read the cache if it exists + storage = self.filesystem_cache.storage(sid) + cache = storage.find_path((self.METADATA_CACHE,)) + if cache is not None: + json_codec = JSONCodec() + try: + stream = self.get_mtp_file(cache) + json_codec.decode_from_file(stream, bl, Book, sid) + except: + need_sync = True + + relpath_cache = {b.mtp_relpath:i for i, b in enumerate(bl)} + + for mtp_file in all_books: + count += 1 + relpath = mtp_file.mtp_relpath + idx = relpath_cache.get(relpath, None) + if idx is not None: + cached_metadata = bl[idx] + del relpath_cache[relpath] + if cached_metadata.size == mtp_file.size: + cached_metadata.datetime = mtp_file.last_modified.timetuple() + cached_metadata.path = mtp_file.mtp_id_path + debug('Using cached metadata for', + '/'.join(mtp_file.full_path)) + continue # No need to update metadata + book = cached_metadata + else: + book = Book(sid, '/'.join(relpath)) + bl.append(book) + + need_sync = True + self.report_progress(count/steps, _('Reading metadata from %s')% + ('/'.join(relpath))) + try: + book.smart_update(self.read_file_metadata(mtp_file)) + debug('Read metadata for', '/'.join(mtp_file.full_path)) + except: + prints('Failed to read metadata from', + '/'.join(mtp_file.full_path)) + traceback.print_exc() + book.size = mtp_file.size + book.datetime = mtp_file.last_modified.timetuple() + book.path = mtp_file.mtp_id_path + + # Remove books in the cache that no longer exist + for idx in sorted(relpath_cache.itervalues(), reverse=True): + del bl[idx] + need_sync = True + + if need_sync: + self.report_progress(count/steps, _('Updating metadata cache on device')) + self.write_metadata_cache(storage, bl) + self.report_progress(1, _('Finished reading metadata from device')) + return bl + + def read_file_metadata(self, mtp_file): + from calibre.ebooks.metadata.meta import get_metadata + from calibre.customize.ui import quick_metadata + ext = mtp_file.name.rpartition('.')[-1].lower() + stream = self.get_mtp_file(mtp_file) + with quick_metadata: + return get_metadata(stream, stream_type=ext, + force_read_metadata=True, + pattern=self.build_template_regexp()) + + def write_metadata_cache(self, storage, bl): + from calibre.devices.mtp.books import JSONCodec + + if bl.storage_id != storage.storage_id: + # Just a sanity check, should never happen + return + + json_codec = JSONCodec() + stream = SpooledTemporaryFile(10*(1024**2)) + json_codec.encode_to_file(stream, bl) + size = stream.tell() + stream.seek(0) + self.put_file(storage, self.METADATA_CACHE, stream, size) + + def sync_booklists(self, booklists, end_session=True): + for bl in booklists: + if getattr(bl, 'storage_id', None) is None: + continue + storage = self.filesystem_cache.storage(bl.storage_id) + if storage is None: + continue + self.write_metadata_cache(storage, bl) + + # }}} + + # Get files from the device {{{ + def get_file(self, path, outfile, end_session=True): + f = self.filesystem_cache.resolve_mtp_id_path(path) + self.get_mtp_file(f, outfile) + + def prepare_addable_books(self, paths): + tdir = PersistentTemporaryDirectory('_prepare_mtp') + ans = [] + for path in paths: + try: + f = self.filesystem_cache.resolve_mtp_id_path(path) + except Exception as e: + ans.append((path, e, traceback.format_exc())) + continue + base = os.path.join(tdir, '%s'%f.object_id) + os.mkdir(base) + with open(os.path.join(base, f.name), 'wb') as out: + try: + self.get_mtp_file(f, out) + except Exception as e: + ans.append((path, e, traceback.format_exc())) + else: + ans.append(out.name) + return ans + # }}} + + def create_upload_path(self, path, mdata, fname): + from calibre.devices import create_upload_path + from calibre.utils.filenames import ascii_filename as sanitize + filepath = create_upload_path(mdata, fname, self.save_template, sanitize, + prefix_path=path, + path_type=posixpath, + maxlen=self.MAX_PATH_LEN, + use_subdirs = True, + news_in_folder = self.NEWS_IN_FOLDER, + ) + return tuple(x.lower() for x in filepath.split('/')) if __name__ == '__main__': dev = MTP_DEVICE(None) @@ -92,8 +248,9 @@ if __name__ == '__main__': cd = dev.detect_managed_devices(devs) if cd is None: raise ValueError('Failed to detect MTP device') + dev.set_progress_reporter(prints) dev.open(cd, None) - pprint.pprint(dev.get_device_information()) + dev.books() finally: dev.shutdown() diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index cd97c5c2ed..216e06031f 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -7,17 +7,24 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import weakref, sys +import weakref, sys, json from collections import deque from operator import attrgetter from future_builtins import map +from datetime import datetime from calibre import human_readable, prints, force_unicode +from calibre.utils.date import local_tz, as_utc from calibre.utils.icu import sort_key, lower +from calibre.ebooks import BOOK_EXTENSIONS + +bexts = frozenset(BOOK_EXTENSIONS) class FileOrFolder(object): def __init__(self, entry, fs_cache): + self.all_storage_ids = fs_cache.all_storage_ids + self.object_id = entry['id'] self.is_folder = entry['is_folder'] self.storage_id = entry['storage_id'] @@ -28,7 +35,12 @@ class FileOrFolder(object): self.name = force_unicode(n, 'utf-8') self.persistent_id = entry.get('persistent_id', self.object_id) self.size = entry.get('size', 0) - self.all_storage_ids = fs_cache.all_storage_ids + md = entry.get('modified', 0) + try: + self.last_modified = datetime.fromtimestamp(md, local_tz) + except: + self.last_modified = datetime.fromtimestamp(0, local_tz) + self.last_modified = as_utc(self.last_modified) if self.storage_id not in self.all_storage_ids: raise ValueError('Storage id %s not valid for %s, valid values: %s'%(self.storage_id, @@ -50,6 +62,9 @@ class FileOrFolder(object): if self.storage_id == self.object_id: self.storage_prefix = 'mtp:::%s:::'%self.persistent_id + self.is_ebook = (not self.is_folder and + self.name.rpartition('.')[-1].lower() in bexts) + def __repr__(self): name = 'Folder' if self.is_folder else 'File' try: @@ -147,6 +162,13 @@ class FileOrFolder(object): parent = c return parent + @property + def mtp_relpath(self): + return tuple(x.lower() for x in self.full_path[1:]) + + @property + def mtp_id_path(self): + return 'mtp:::' + json.dumps(self.object_id) + ':::' + '/'.join(self.full_path) class FilesystemCache(object): @@ -192,4 +214,24 @@ class FilesystemCache(object): if e.storage_id == storage_id: return e + def iterebooks(self, storage_id): + for x in self.id_map.itervalues(): + if x.storage_id == storage_id and x.is_ebook: + yield x + + def resolve_mtp_id_path(self, path): + if not path.startswith('mtp:::'): + raise ValueError('%s is not a valid MTP path'%path) + parts = path.split(':::') + if len(parts) < 3: + raise ValueError('%s is not a valid MTP path'%path) + try: + object_id = json.loads(parts[1]) + except: + raise ValueError('%s is not a valid MTP path'%path) + try: + return self.id_map[object_id] + except KeyError: + raise ValueError('No object found with MTP path: %s'%path) + diff --git a/src/calibre/devices/mtp/test.py b/src/calibre/devices/mtp/test.py index 0563708ea4..c273bac5e0 100644 --- a/src/calibre/devices/mtp/test.py +++ b/src/calibre/devices/mtp/test.py @@ -128,7 +128,7 @@ class TestDeviceInteraction(unittest.TestCase): raw2 = io.BytesIO() pc = ProgressCallback() - self.dev.get_file(f, raw2, callback=pc) + self.dev.get_mtp_file(f, raw2, callback=pc) self.assertEqual(raw.getvalue(), raw2.getvalue()) self.assertTrue(pc.end_called, msg='Progress callback not called with equal values (get_file)') @@ -162,7 +162,7 @@ class TestDeviceInteraction(unittest.TestCase): self.assertEqual(f.storage_id, self.storage.storage_id) raw2 = io.BytesIO() - self.dev.get_file(f, raw2) + self.dev.get_mtp_file(f, raw2) self.assertEqual(raw.getvalue(), raw2.getvalue()) def measure_memory_usage(self, repetitions, func, *args, **kwargs): @@ -226,7 +226,7 @@ class TestDeviceInteraction(unittest.TestCase): def get_file(f): raw = io.BytesIO() pc = ProgressCallback() - self.dev.get_file(f, raw, callback=pc) + self.dev.get_mtp_file(f, raw, callback=pc) raw.truncate(0) del raw del pc diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index 5d7d767a9b..338913114f 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -17,7 +17,6 @@ from calibre.constants import plugins from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.mtp.base import MTPDeviceBase, synchronous -from calibre.devices.mtp.filesystem_cache import FilesystemCache MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id ' 'bcd serial manufacturer product') @@ -83,6 +82,8 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def debug_managed_device_detection(self, devices_on_system, output): + if self.currently_connected_dev is not None: + return True p = partial(prints, file=output) if self.libmtp is None: err = plugins['libmtp'][1] @@ -175,6 +176,7 @@ class MTP_DEVICE(MTPDeviceBase): @property def filesystem_cache(self): if self._filesystem_cache is None: + from calibre.devices.mtp.filesystem_cache import FilesystemCache with self.lock: storage, all_items, all_errs = [], [], [] for sid, capacity in zip([self._main_id, self._carda_id, @@ -271,7 +273,7 @@ class MTP_DEVICE(MTPDeviceBase): return parent.add_child(ans) @synchronous - def get_file(self, f, stream=None, callback=None): + def get_mtp_file(self, f, stream=None, callback=None): if f.is_folder: raise ValueError('%s if a folder'%(f.full_path,)) if stream is None: diff --git a/src/calibre/devices/mtp/unix/libmtp.c b/src/calibre/devices/mtp/unix/libmtp.c index 86c9349d20..bf07c73a35 100644 --- a/src/calibre/devices/mtp/unix/libmtp.c +++ b/src/calibre/devices/mtp/unix/libmtp.c @@ -121,12 +121,13 @@ static uint16_t data_from_python(void *params, void *priv, uint32_t wantlen, uns static PyObject* build_file_metadata(LIBMTP_file_t *nf, uint32_t storage_id) { PyObject *ans = NULL; - ans = Py_BuildValue("{s:s, s:k, s:k, s:k, s:K, s:O}", + ans = Py_BuildValue("{s:s, s:k, s:k, s:k, s:K, s:L, s:O}", "name", (unsigned long)nf->filename, "id", (unsigned long)nf->item_id, "parent_id", (unsigned long)nf->parent_id, "storage_id", (unsigned long)storage_id, "size", nf->filesize, + "modified", (PY_LONG_LONG)nf->modificationdate, "is_folder", (nf->filetype == LIBMTP_FILETYPE_FOLDER) ? Py_True : Py_False ); diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp index e1f439926c..7186bbdcdb 100644 --- a/src/calibre/devices/mtp/windows/content_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -34,6 +34,7 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { ADDPROP(WPD_OBJECT_ISHIDDEN); ADDPROP(WPD_OBJECT_CAN_DELETE); ADDPROP(WPD_OBJECT_SIZE); + ADDPROP(WPD_OBJECT_DATE_MODIFIED); return properties; @@ -81,6 +82,16 @@ static void set_size_property(PyObject *dict, REFPROPERTYKEY key, const char *py } } +static void set_date_property(PyObject *dict, REFPROPERTYKEY key, const char *pykey, IPortableDeviceValues *properties) { + FLOAT val = 0; + PyObject *t; + + if (SUCCEEDED(properties->GetFloatValue(key, &val))) { + t = Py_BuildValue("d", (double)val); + if (t != NULL) { PyDict_SetItemString(dict, pykey, t); Py_DECREF(t); } + } +} + static void set_content_type_property(PyObject *dict, IPortableDeviceValues *properties) { GUID guid = GUID_NULL; BOOL is_folder = 0; @@ -103,6 +114,8 @@ static void set_properties(PyObject *obj, IPortableDeviceValues *values) { set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", values); set_size_property(obj, WPD_OBJECT_SIZE, "size", values); + set_date_property(obj, WPD_OBJECT_DATE_MODIFIED, "modified", values); + } // }}} diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 0506f63054..7c15797ef6 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -7,8 +7,8 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import time, threading -from functools import wraps +import time, threading, traceback +from functools import wraps, partial from future_builtins import zip from itertools import chain @@ -17,12 +17,12 @@ from calibre.constants import plugins, __appname__, numeric_version from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.mtp.base import MTPDeviceBase -from calibre.devices.mtp.filesystem_cache import FilesystemCache class ThreadingViolation(Exception): def __init__(self): - Exception.__init__('You cannot use the MTP driver from a thread other than the ' + Exception.__init__(self, + 'You cannot use the MTP driver from a thread other than the ' ' thread in which startup() was called') def same_thread(func): @@ -51,6 +51,7 @@ class MTP_DEVICE(MTPDeviceBase): self._main_id = self._carda_id = self._cardb_id = None self.start_thread = None self._filesystem_cache = None + self.eject_dev_on_next_scan = False def startup(self): self.start_thread = threading.current_thread() @@ -75,6 +76,10 @@ class MTP_DEVICE(MTPDeviceBase): @same_thread def detect_managed_devices(self, devices_on_system, force_refresh=False): if self.wpd is None: return None + if self.eject_dev_on_next_scan: + self.eject_dev_on_next_scan = False + if self.currently_connected_pnp_id is not None: + self.do_eject() devices_on_system = frozenset(devices_on_system) if (force_refresh or @@ -124,6 +129,54 @@ class MTP_DEVICE(MTPDeviceBase): return None + @same_thread + def debug_managed_device_detection(self, devices_on_system, output): + import pprint + p = partial(prints, file=output) + if self.currently_connected_pnp_id is not None: + return True + if self.wpd_error: + p('Cannot detect MTP devices') + p(self.wpd_error) + return False + try: + pnp_ids = frozenset(self.wpd.enumerate_devices()) + except: + p("Failed to get list of PNP ids on system") + p(traceback.format_exc()) + return False + + for pnp_id in pnp_ids: + try: + data = self.wpd.device_info(pnp_id) + except: + p('Failed to get data for device:', pnp_id) + p(traceback.format_exc()) + continue + protocol = data.get('protocol', '').lower() + if not protocol.startswith('mtp:'): continue + p('MTP device:', pnp_id) + p(pprint.pformat(data)) + if not self.is_suitable_wpd_device(data): + p('Not a suitable MTP device, ignoring\n') + continue + p('\nTrying to open:', pnp_id) + try: + self.open(pnp_id, 'debug-detection') + except: + p('Open failed:') + p(traceback.format_exc()) + continue + break + if self.currently_connected_pnp_id: + p('Opened', self.current_friendly_name, 'successfully') + p('Device info:') + p(pprint.pformat(self.dev.data)) + self.eject() + return True + p('No suitable MTP devices found') + return False + def is_suitable_wpd_device(self, devdata): # Check that protocol is MTP protocol = devdata.get('protocol', '').lower() @@ -143,6 +196,7 @@ class MTP_DEVICE(MTPDeviceBase): @property def filesystem_cache(self): if self._filesystem_cache is None: + from calibre.devices.mtp.filesystem_cache import FilesystemCache ts = self.total_space() all_storage = [] items = [] @@ -164,19 +218,24 @@ class MTP_DEVICE(MTPDeviceBase): return self._filesystem_cache @same_thread - def post_yank_cleanup(self): - self.currently_connected_pnp_id = self.current_friendly_name = None - self._main_id = self._carda_id = self._cardb_id = None - self.dev = self._filesystem_cache = None - - @same_thread - def eject(self): + def do_eject(self): if self.currently_connected_pnp_id is None: return self.ejected_devices.add(self.currently_connected_pnp_id) self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None self.dev = self._filesystem_cache = None + + @same_thread + def post_yank_cleanup(self): + self.currently_connected_pnp_id = self.current_friendly_name = None + self._main_id = self._carda_id = self._cardb_id = None + self.dev = self._filesystem_cache = None + + def eject(self): + if self.currently_connected_pnp_id is None: return + self.eject_dev_on_next_scan = True + @same_thread def open(self, connected_device, library_uuid): self.dev = self._filesystem_cache = None @@ -200,7 +259,9 @@ class MTP_DEVICE(MTPDeviceBase): self._carda_id = storage[1]['id'] if len(storage) > 2: self._cardb_id = storage[2]['id'] - self.current_friendly_name = devdata.get('friendly_name', None) + self.current_friendly_name = devdata.get('friendly_name', + _('Unknown MTP device')) + self.currently_connected_pnp_id = connected_device @same_thread def get_basic_device_information(self): @@ -232,7 +293,7 @@ class MTP_DEVICE(MTPDeviceBase): return tuple(ans) @same_thread - def get_file(self, f, stream=None, callback=None): + def get_mtp_file(self, f, stream=None, callback=None): if f.is_folder: raise ValueError('%s if a folder'%(f.full_path,)) if stream is None: diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 600916128d..4d4b198de0 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -19,7 +19,7 @@ from calibre.devices.errors import (DeviceError, FreeSpaceError, WrongDestinationError) from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.constants import iswindows, islinux, isosx, isfreebsd, plugins -from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to +from calibre.utils.filenames import ascii_filename as sanitize if isosx: usbobserver, usbobserver_err = plugins['usbobserver'] @@ -1052,78 +1052,16 @@ class Device(DeviceConfig, DevicePlugin): pass def create_upload_path(self, path, mdata, fname, create_dirs=True): - path = os.path.abspath(path) - maxlen = self.MAX_PATH_LEN - - special_tag = None - if mdata.tags: - for t in mdata.tags: - if t.startswith(_('News')) or t.startswith('/'): - special_tag = t - break - + from calibre.devices import create_upload_path settings = self.settings() - template = self.save_template() - if mdata.tags and _('News') in mdata.tags: - try: - p = mdata.pubdate - date = (p.year, p.month, p.day) - except: - today = time.localtime() - date = (today[0], today[1], today[2]) - template = "{title}_%d-%d-%d" % date - use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs - - fname = sanitize(fname) - ext = os.path.splitext(fname)[1] - - from calibre.library.save_to_disk import get_components - from calibre.library.save_to_disk import config - opts = config().parse() - if not isinstance(template, unicode): - template = template.decode('utf-8') - app_id = str(getattr(mdata, 'application_id', '')) - id_ = mdata.get('id', fname) - extra_components = get_components(template, mdata, id_, - timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1) - if not extra_components: - extra_components.append(sanitize(self.filename_callback(fname, - mdata))) - else: - extra_components[-1] = sanitize(self.filename_callback(extra_components[-1]+ext, mdata)) - - if extra_components[-1] and extra_components[-1][0] in ('.', '_'): - extra_components[-1] = 'x' + extra_components[-1][1:] - - if special_tag is not None: - name = extra_components[-1] - extra_components = [] - tag = special_tag - if tag.startswith(_('News')): - if self.NEWS_IN_FOLDER: - extra_components.append('News') - else: - for c in tag.split('/'): - c = sanitize(c) - if not c: continue - extra_components.append(c) - extra_components.append(name) - - if not use_subdirs: - extra_components = extra_components[-1:] - - def remove_trailing_periods(x): - ans = x - while ans.endswith('.'): - ans = ans[:-1].strip() - if not ans: - ans = 'x' - return ans - - extra_components = list(map(remove_trailing_periods, extra_components)) - components = shorten_components_to(maxlen - len(path), extra_components) - components = self.sanitize_path_components(components) - filepath = os.path.join(path, *components) + filepath = create_upload_path(mdata, fname, self.save_template(), sanitize, + prefix_path=os.path.abspath(path), + maxlen=self.MAX_PATH_LEN, + use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs, + news_in_folder = self.NEWS_IN_FOLDER, + filename_callback=self.filename_callback, + sanitize_path_components=self.sanitize_path_components + ) filedir = os.path.dirname(filepath) if create_dirs and not os.path.exists(filedir): diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 12e30073ac..f6c7556fd8 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -10,7 +10,7 @@ driver. It is intended to be subclassed with the relevant parts implemented for a particular device. ''' -import os, re, time, json, functools, shutil +import os, time, json, shutil from itertools import cycle from calibre.constants import numeric_version @@ -404,25 +404,8 @@ class USBMS(CLI, Device): @classmethod def build_template_regexp(cls): - def replfunc(match, seen=None): - v = match.group(1) - if v in ['authors', 'author_sort']: - v = 'author' - if v in ('title', 'series', 'series_index', 'isbn', 'author'): - if v not in seen: - seen.add(v) - return '(?P<' + v + '>.+?)' - return '(.+?)' - s = set() - f = functools.partial(replfunc, seen=s) - template = None - try: - template = cls.save_template().rpartition('/')[2] - return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') - except: - prints(u'Failed to parse template: %r'%template) - template = u'{title} - {authors}' - return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + from calibre.devices import build_template_regexp + return build_template_regexp(cls.save_template()) @classmethod def path_to_unicode(cls, path): diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index d132fe15d0..7f8a01f8bd 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -510,6 +510,7 @@ class OPF(object): # {{{ tags_path = XPath('descendant::*[re:match(name(), "subject", "i")]') isbn_path = XPath('descendant::*[re:match(name(), "identifier", "i") and '+ '(re:match(@scheme, "isbn", "i") or re:match(@opf:scheme, "isbn", "i"))]') + pubdate_path = XPath('descendant::*[re:match(name(), "date", "i")]') raster_cover_path = XPath('descendant::*[re:match(name(), "meta", "i") and ' + 're:match(@name, "cover", "i") and @content]') identifier_path = XPath('descendant::*[re:match(name(), "identifier", "i")]') @@ -538,8 +539,6 @@ class OPF(object): # {{{ formatter=float, none_is=1) title_sort = TitleSortField('title_sort', is_dc=False) rating = MetadataField('rating', is_dc=False, formatter=float) - pubdate = MetadataField('date', formatter=parse_date, - renderer=isoformat) publication_type = MetadataField('publication_type', is_dc=False) timestamp = MetadataField('timestamp', is_dc=False, formatter=parse_date, renderer=isoformat) @@ -852,6 +851,44 @@ class OPF(object): # {{{ return property(fget=fget, fset=fset) + @dynamic_property + def pubdate(self): + + def fget(self): + ans = None + for match in self.pubdate_path(self.metadata): + try: + val = parse_date(etree.tostring(match, encoding=unicode, + method='text', with_tail=False).strip()) + except: + continue + if ans is None or val < ans: + ans = val + return ans + + def fset(self, val): + least_val = least_elem = None + for match in self.pubdate_path(self.metadata): + try: + cval = parse_date(etree.tostring(match, encoding=unicode, + method='text', with_tail=False).strip()) + except: + match.getparent().remove(match) + else: + if not val: + match.getparent().remove(match) + if least_val is None or cval < least_val: + least_val, least_elem = cval, match + + if val: + if least_val is None: + least_elem = self.create_metadata_element('date') + + least_elem.attrib.clear() + least_elem.text = isoformat(val) + + return property(fget=fget, fset=fset) + @dynamic_property def isbn(self): diff --git a/src/calibre/ebooks/pdf/outline_writer.py b/src/calibre/ebooks/pdf/outline_writer.py index 4b2db84f9e..64d11f0208 100644 --- a/src/calibre/ebooks/pdf/outline_writer.py +++ b/src/calibre/ebooks/pdf/outline_writer.py @@ -35,26 +35,31 @@ class Outline(object): page, ypos = 0, 0 item = getattr(toc, 'outline_item_', None) if item is not None: + # First use the item URL without fragment + page, ypos = self.pos_map.get(item, {}).get(None, (0, 0)) if toc.fragment: amap = self.pos_map.get(item, None) if amap is not None: - page, ypos = amap.get(toc.fragment, (0, 0)) - else: - page, ypos = self.pos_map.get(item, {}).get(None, (0, 0)) + page, ypos = amap.get(toc.fragment, (page, ypos)) return page, ypos def add_children(self, toc, parent): for child in toc: page, ypos = self.get_pos(child) text = child.text or _('Page %d')%page + if page >= self.page_count: + page = self.page_count - 1 cn = parent.create(text, page, True) self.add_children(child, cn) def __call__(self, doc): self.pos_map = dict(self.pos_map) + self.page_count = doc.page_count() for child in self.toc: page, ypos = self.get_pos(child) text = child.text or _('Page %d')%page + if page >= self.page_count: + page = self.page_count - 1 node = doc.create_outline(text, page) self.add_children(child, node) diff --git a/src/calibre/ebooks/pdf/writer.py b/src/calibre/ebooks/pdf/writer.py index d6ffa42107..a9cb951e35 100644 --- a/src/calibre/ebooks/pdf/writer.py +++ b/src/calibre/ebooks/pdf/writer.py @@ -121,7 +121,7 @@ class PDFMetadata(object): # {{{ self.author = force_unicode(self.author) # }}} -class Page(QWebPage): +class Page(QWebPage): # {{{ def __init__(self, opts, log): self.log = log @@ -137,17 +137,22 @@ class Page(QWebPage): std = {'serif':opts.pdf_serif_family, 'sans':opts.pdf_sans_family, 'mono':opts.pdf_mono_family}.get(opts.pdf_standard_font, opts.pdf_serif_family) - settings.setFontFamily(QWebSettings.StandardFont, std) - settings.setFontFamily(QWebSettings.SerifFont, opts.pdf_serif_family) - settings.setFontFamily(QWebSettings.SansSerifFont, - opts.pdf_sans_family) - settings.setFontFamily(QWebSettings.FixedFont, opts.pdf_mono_family) + if std: + settings.setFontFamily(QWebSettings.StandardFont, std) + if opts.pdf_serif_family: + settings.setFontFamily(QWebSettings.SerifFont, opts.pdf_serif_family) + if opts.pdf_sans_family: + settings.setFontFamily(QWebSettings.SansSerifFont, + opts.pdf_sans_family) + if opts.pdf_mono_family: + settings.setFontFamily(QWebSettings.FixedFont, opts.pdf_mono_family) def javaScriptConsoleMessage(self, msg, lineno, msgid): self.log.debug(u'JS:', unicode(msg)) def javaScriptAlert(self, frame, msg): self.log(unicode(msg)) +# }}} class PDFWriter(QObject): # {{{ @@ -192,6 +197,7 @@ class PDFWriter(QObject): # {{{ self.insert_cover() self.render_succeeded = False + self.current_page_num = self.doc.page_count() self.combine_queue.append(os.path.join(self.tmp_path, 'qprinter_out.pdf')) self.first_page = True @@ -279,9 +285,13 @@ class PDFWriter(QObject): # {{{ paged_display.fit_images(); ''') mf = self.view.page().mainFrame() + start_page = self.current_page_num + if not self.first_page: + start_page += 1 while True: if not self.first_page: - self.printer.newPage() + if self.printer.newPage(): + self.current_page_num += 1 self.first_page = False mf.render(self.painter) nsl = evaljs('paged_display.next_screen_location()').toInt() @@ -293,11 +303,10 @@ class PDFWriter(QObject): # {{{ amap = self.bridge_value if not isinstance(amap, dict): amap = {} # Some javascript error occurred - pages = self.doc.page_count() - self.outline.set_pos(self.current_item, None, pages, 0) + self.outline.set_pos(self.current_item, None, start_page, 0) for anchor, x in amap.iteritems(): pagenum, ypos = x - self.outline.set_pos(self.current_item, anchor, pages + pagenum, ypos) + self.outline.set_pos(self.current_item, anchor, start_page + pagenum, ypos) def append_doc(self, outpath): doc = self.podofo.PDFDoc() @@ -342,8 +351,7 @@ class PDFWriter(QObject): # {{{ if self.metadata.tags: self.doc.keywords = self.metadata.tags self.outline(self.doc) - raw = self.doc.write() - self.out_stream.write(raw) + self.doc.save_to_fileobj(self.out_stream) self.render_succeeded = True finally: self._delete_tmpdir() diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 00f5bef03d..8f275ec065 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -101,6 +101,7 @@ gprefs.defaults['auto_add_auto_convert'] = True gprefs.defaults['ui_style'] = 'calibre' if iswindows or isosx else 'system' gprefs.defaults['tag_browser_old_look'] = False gprefs.defaults['book_list_tooltips'] = True +gprefs.defaults['bd_show_cover'] = True # }}} NONE = QVariant() #: Null value to return from the data function of item models diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 9d15fa4ac8..ef7ed7a594 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -10,9 +10,9 @@ from functools import partial from PyQt4.Qt import QPixmap, QTimer - -from calibre.gui2 import error_dialog, choose_files, \ - choose_dir, warning_dialog, info_dialog +from calibre import as_unicode +from calibre.gui2 import (error_dialog, choose_files, choose_dir, + warning_dialog, info_dialog) from calibre.gui2.dialogs.add_empty_book import AddEmptyBookDialog from calibre.gui2.dialogs.progress import ProgressDialog from calibre.gui2.widgets import IMAGE_EXTENSIONS @@ -400,12 +400,45 @@ class AddAction(InterfaceAction): d = error_dialog(self.gui, _('Add to library'), _('No book files found')) d.exec_() return - paths = self.gui.device_manager.device.prepare_addable_books(paths) - from calibre.gui2.add import Adder - self.__adder_func = partial(self._add_from_device_adder, on_card=None, - model=view.model()) - self._adder = Adder(self.gui, self.gui.library_view.model().db, - self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server) - self._adder.add(paths) + + self.gui.device_manager.prepare_addable_books(self.Dispatcher(partial( + self.books_prepared, view)), paths) + self.bpd = ProgressDialog(_('Downloading books'), + msg=_('Downloading books from device'), parent=self.gui, + cancelable=False) + QTimer.singleShot(1000, self.show_bpd) + + def show_bpd(self): + if self.bpd is not None: + self.bpd.show() + + def books_prepared(self, view, job): + self.bpd.hide() + self.bpd = None + if job.exception is not None: + self.gui.device_job_exception(job) + return + paths = job.result + ok_paths = [x for x in paths if isinstance(x, basestring)] + failed_paths = [x for x in paths if isinstance(x, tuple)] + if failed_paths: + if not ok_paths: + msg = _('Could not download files from the device') + typ = error_dialog + else: + msg = _('Could not download some files from the device') + typ = warning_dialog + det_msg = [x[0]+ '\n ' + as_unicode(x[1]) for x in failed_paths] + det_msg = '\n\n'.join(det_msg) + typ(self.gui, _('Could not download files'), msg, det_msg=det_msg, + show=True) + + if ok_paths: + from calibre.gui2.add import Adder + self.__adder_func = partial(self._add_from_device_adder, on_card=None, + model=view.model()) + self._adder = Adder(self.gui, self.gui.library_view.model().db, + self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server) + self._adder.add(ok_paths) diff --git a/src/calibre/gui2/actions/view.py b/src/calibre/gui2/actions/view.py index 43e9dad5c4..5a7a991607 100644 --- a/src/calibre/gui2/actions/view.py +++ b/src/calibre/gui2/actions/view.py @@ -256,6 +256,15 @@ class ViewAction(InterfaceAction): db.prefs['gui_view_history'] = history[:vh] self.build_menus(db) + def view_device_book(self, path): + pt = PersistentTemporaryFile('_view_device_book'+\ + os.path.splitext(path)[1]) + self.persistent_files.append(pt) + pt.close() + self.gui.device_manager.view_book( + Dispatcher(self.book_downloaded_for_viewing), + path, pt.name) + def _view_books(self, rows): if not rows or len(rows) == 0: self._launch_viewer() @@ -270,12 +279,5 @@ class ViewAction(InterfaceAction): else: paths = self.gui.current_view().model().paths(rows) for path in paths: - pt = PersistentTemporaryFile('_viewer_'+\ - os.path.splitext(path)[1]) - self.persistent_files.append(pt) - pt.close() - self.gui.device_manager.view_book(\ - Dispatcher(self.book_downloaded_for_viewing), - path, pt.name) - + self.view_device_book(path) diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 3e20f8c67c..f03015f4ad 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -19,8 +19,8 @@ from calibre.ebooks.metadata import fmt_sidx from calibre.ebooks.metadata.sources.identify import urls_from_identifiers from calibre.constants import filesystem_encoding from calibre.library.comments import comments_to_html -from calibre.gui2 import (config, open_local_file, open_url, pixmap_to_data, - gprefs, rating_font) +from calibre.gui2 import (config, open_url, pixmap_to_data, gprefs, + rating_font) from calibre.utils.icu import sort_key from calibre.utils.formatter import EvalFormatter from calibre.utils.date import is_date_undefined @@ -297,7 +297,8 @@ class CoverView(QWidget): # {{{ self.pixmap = self.default_pixmap self.do_layout() self.update() - if not same_item and not config['disable_animations']: + if (not same_item and not config['disable_animations'] and + self.isVisible()): self.animation.start() def paintEvent(self, event): @@ -512,6 +513,7 @@ class DetailsLayout(QLayout): # {{{ self.do_layout(r) def cover_height(self, r): + if not self._children[0].widget().isVisible(): return 0 mh = min(int(r.height()/2.), int(4/3. * r.width())+1) try: ph = self._children[0].widget().pixmap.height() @@ -522,6 +524,7 @@ class DetailsLayout(QLayout): # {{{ return mh def cover_width(self, r): + if not self._children[0].widget().isVisible(): return 0 mw = 1 + int(3/4. * r.height()) try: pw = self._children[0].widget().pixmap.width() @@ -566,6 +569,7 @@ class BookDetails(QWidget): # {{{ files_dropped = pyqtSignal(object, object) cover_changed = pyqtSignal(object, object) cover_removed = pyqtSignal(object) + view_device_book = pyqtSignal(object) # Drag 'n drop {{{ DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS+BOOK_EXTENSIONS @@ -640,7 +644,7 @@ class BookDetails(QWidget): # {{{ id_, fmt = val.split(':') self.view_specific_format.emit(int(id_), fmt) elif typ == 'devpath': - open_local_file(val) + self.view_device_book.emit(val) else: try: open_url(QUrl(link, QUrl.TolerantMode)) @@ -660,6 +664,7 @@ class BookDetails(QWidget): # {{{ self.update_layout() def update_layout(self): + self.cover_view.setVisible(gprefs['bd_show_cover']) self._layout.do_layout(self.rect()) self.cover_view.update_tooltip(self.current_path) diff --git a/src/calibre/gui2/convert/__init__.py b/src/calibre/gui2/convert/__init__.py index e01238a2e5..38fb641987 100644 --- a/src/calibre/gui2/convert/__init__.py +++ b/src/calibre/gui2/convert/__init__.py @@ -143,7 +143,7 @@ class Widget(QWidget): ans = None return ans elif isinstance(g, QFontComboBox): - ans = unicode(QFontInfo(g.currentFont().family())) + return unicode(QFontInfo(g.currentFont()).family()) elif isinstance(g, EncodingComboBox): ans = unicode(g.currentText()).strip() try: diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 03e1932035..98e42f4178 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -128,6 +128,10 @@ class DeviceManager(Thread): # {{{ self.setDaemon(True) # [Device driver, Showing in GUI, Ejected] self.devices = list(device_plugins()) + self.managed_devices = [x for x in self.devices if + not x.MANAGES_DEVICE_PRESENCE] + self.unmanaged_devices = [x for x in self.devices if + x.MANAGES_DEVICE_PRESENCE] self.sleep_time = sleep_time self.connected_slot = connected_slot self.jobs = Queue.Queue(0) @@ -182,12 +186,15 @@ class DeviceManager(Thread): # {{{ prints('Unable to open device', str(dev)) prints(tb) continue - self.connected_device = dev - self.connected_device_kind = device_kind - self.connected_slot(True, device_kind) + self.after_device_connect(dev, device_kind) return True return False + def after_device_connect(self, dev, device_kind): + self.connected_device = dev + self.connected_device_kind = device_kind + self.connected_slot(True, device_kind) + def connected_device_removed(self): while True: try: @@ -215,22 +222,45 @@ class DeviceManager(Thread): # {{{ def detect_device(self): self.scanner.scan() + if self.is_device_connected: - connected, detected_device = \ - self.scanner.is_device_connected(self.connected_device, - only_presence=True) - if not connected: - if DEBUG: - # Allow the device subsystem to output debugging info about - # why it thinks the device is not connected. Used, for e.g. - # in the can_handle() method of the T1 driver + if self.connected_device.MANAGES_DEVICE_PRESENCE: + cd = self.connected_device.detect_managed_devices(self.scanner.devices) + if cd is None: + self.connected_device_removed() + else: + connected, detected_device = \ self.scanner.is_device_connected(self.connected_device, - only_presence=True, debug=True) - self.connected_device_removed() + only_presence=True) + if not connected: + if DEBUG: + # Allow the device subsystem to output debugging info about + # why it thinks the device is not connected. Used, for e.g. + # in the can_handle() method of the T1 driver + self.scanner.is_device_connected(self.connected_device, + only_presence=True, debug=True) + self.connected_device_removed() else: + for dev in self.unmanaged_devices: + try: + cd = dev.detect_managed_devices(self.scanner.devices) + except: + prints('Error during device detection for %s:'%dev) + traceback.print_exc() + else: + if cd is not None: + try: + dev.open(cd, self.current_library_uuid) + except: + prints('Error while trying to open %s (Driver: %s)'% + (cd, dev)) + traceback.print_exc() + else: + self.after_device_connect(dev, 'unmanaged-device') + return try: possibly_connected_devices = [] - for device in self.devices: + for device in self.managed_devices: if device in self.ejected_devices: continue try: @@ -248,7 +278,7 @@ class DeviceManager(Thread): # {{{ prints('Connect to device failed, retrying in 5 seconds...') time.sleep(5) if not self.do_connect(possibly_connected_devices, - device_kind='usb'): + device_kind='device'): if DEBUG: prints('Device connect failed again, giving up') except OpenFailed as e: @@ -264,9 +294,10 @@ class DeviceManager(Thread): # {{{ # disconnect a device def umount_device(self, *args): if self.is_device_connected and not self.job_manager.has_device_jobs(): - if self.connected_device_kind == 'device': + if self.connected_device_kind in {'unmanaged-device', 'device'}: self.connected_device.eject() - self.ejected_devices.add(self.connected_device) + if self.connected_device_kind != 'unmanaged-device': + self.ejected_devices.add(self.connected_device) self.connected_slot(False, self.connected_device_kind) elif hasattr(self.connected_device, 'unmount_device'): # As we are on the wrong thread, this call must *not* do @@ -412,6 +443,14 @@ class DeviceManager(Thread): # {{{ return self.create_job_step(self._books, done, description=_('Get list of books on device'), to_job=add_as_step_to_job) + def _prepare_addable_books(self, paths): + return self.device.prepare_addable_books(paths) + + def prepare_addable_books(self, done, paths, add_as_step_to_job=None): + return self.create_job_step(self._prepare_addable_books, done, args=[paths], + description=_('Prepare files for transfer from device'), + to_job=add_as_step_to_job) + def _annotations(self, path_map): return self.device.get_annotations(path_map) @@ -525,9 +564,8 @@ class DeviceManager(Thread): # {{{ to_job=add_as_step_to_job) def _view_book(self, path, target): - f = open(target, 'wb') - self.device.get_file(path, f) - f.close() + with open(target, 'wb') as f: + self.device.get_file(path, f) return target def view_book(self, done, path, target, add_as_step_to_job=None): diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index a82dfec7fc..338a558f29 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -269,6 +269,8 @@ class LayoutMixin(object): # {{{ self.iactions['Remove Books'].remove_format_by_id) self.book_details.save_specific_format.connect( self.iactions['Save To Disk'].save_library_format_by_ids) + self.book_details.view_device_book.connect( + self.iactions['View'].view_device_book) m = self.library_view.model() if m.rowCount(None) > 0: diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index 8ca6b96379..65e6ab6d9f 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -106,6 +106,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): 'calibre')]) r('book_list_tooltips', gprefs) r('tag_browser_old_look', gprefs, restart_required=True) + r('bd_show_cover', gprefs) r('cover_flow_queue_length', config, restart_required=True) diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index bb60f3db2a..2a4397b22d 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -212,19 +212,32 @@ Book Details - + + + + Note that <b>comments</b> will always be displayed at the end, regardless of the position you assign here. + + + true + + + + + + + Use &Roman numerals for series + + + true + + + + Select displayed metadata - - - - true - - - @@ -247,6 +260,13 @@ + + + + true + + + @@ -288,23 +308,10 @@ Manage Authors. You can use the values {author} and - - + + - Use &Roman numerals for series - - - true - - - - - - - Note that <b>comments</b> will always be displayed at the end, regardless of the position you assign here. - - - true + Show &cover in the book details panel diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index f1103f57ee..17c01a6f56 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -32,7 +32,7 @@ from calibre.customize.ui import run_plugins_on_import from calibre import isbytestring from calibre.utils.filenames import ascii_filename, samefile from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp, - parse_only_date) + parse_only_date, UNDEFINED_DATE) from calibre.utils.config import prefs, tweaks, from_json, to_json from calibre.utils.icu import sort_key, strcmp, lower from calibre.utils.search_query_parser import saved_searches, set_saved_searches @@ -2498,16 +2498,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.notify('metadata', [id]) def set_pubdate(self, id, dt, notify=True, commit=True): - if dt: - if isinstance(dt, basestring): - dt = parse_only_date(dt) - self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id)) - self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True) - self.dirtied([id], commit=False) - if commit: - self.conn.commit() - if notify: - self.notify('metadata', [id]) + if not dt: + dt = UNDEFINED_DATE + if isinstance(dt, basestring): + dt = parse_only_date(dt) + self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id)) + self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True) + self.dirtied([id], commit=False) + if commit: + self.conn.commit() + if notify: + self.notify('metadata', [id]) def set_publisher(self, id, publisher, notify=True, commit=True, @@ -3344,7 +3345,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if mi.timestamp is None: mi.timestamp = utcnow() if mi.pubdate is None: - mi.pubdate = utcnow() + mi.pubdate = UNDEFINED_DATE self.set_metadata(id, mi, ignore_errors=True, commit=True) if cover is not None: try: @@ -3386,7 +3387,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if mi.timestamp is None: mi.timestamp = utcnow() if mi.pubdate is None: - mi.pubdate = utcnow() + mi.pubdate = UNDEFINED_DATE self.set_metadata(id, mi, commit=True, ignore_errors=True) npath = self.run_import_plugins(path, format) format = os.path.splitext(npath)[-1].lower().replace('.', '').upper() @@ -3426,7 +3427,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if mi.timestamp is None: mi.timestamp = utcnow() if mi.pubdate is None: - mi.pubdate = utcnow() + mi.pubdate = UNDEFINED_DATE self.set_metadata(id, mi, ignore_errors=True, commit=True) if preserve_uuid and mi.uuid: self.set_uuid(id, mi.uuid, commit=False) diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index d9fd12d466..65451dab9c 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -229,6 +229,10 @@ def samefile(src, dst): symlinks, case insensitivity, mapped drives, etc. Returns True iff both paths exist and point to the same file on disk. + + Note: On windows will return True if the two string are identical (upto + case) even if the file does not exist. This is because I have no way of + knowing how reliable the GetFileInformationByHandle method is. ''' if iswindows: return samefile_windows(src, dst) diff --git a/src/calibre/utils/magick/magick.c b/src/calibre/utils/magick/magick.c index 8713d2cebb..6fbee2c77d 100644 --- a/src/calibre/utils/magick/magick.c +++ b/src/calibre/utils/magick/magick.c @@ -1104,6 +1104,41 @@ magick_Image_type_setter(magick_Image *self, PyObject *val, void *closure) { // }}} +// Image.depth {{{ +static PyObject * +magick_Image_depth_getter(magick_Image *self, void *closure) { + NULL_CHECK(NULL) + + return Py_BuildValue("n", MagickGetImageDepth(self->wand)); +} + +static int +magick_Image_depth_setter(magick_Image *self, PyObject *val, void *closure) { + size_t depth; + + NULL_CHECK(-1) + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete image depth"); + return -1; + } + + if (!PyInt_Check(val)) { + PyErr_SetString(PyExc_TypeError, "Depth must be an integer"); + return -1; + } + + depth = (size_t)PyInt_AsSsize_t(val); + if (!MagickSetImageDepth(self->wand, depth)) { + PyErr_Format(PyExc_ValueError, "Could not set image depth to %lu", depth); + return -1; + } + + return 0; +} + +// }}} + // Image.destroy {{{ static PyObject * @@ -1238,7 +1273,7 @@ static PyMethodDef magick_Image_methods[] = { }, {"quantize", (PyCFunction)magick_Image_quantize, METH_VARARGS, - "quantize(number_colors, colorspace, treedepth, dither, measure_error) \n\n nalyzes the colors within a reference image and chooses a fixed number of colors to represent the image. The goal of the algorithm is to minimize the color difference between the input and output image while minimizing the processing time." + "quantize(number_colors, colorspace, treedepth, dither, measure_error) \n\n analyzes the colors within a reference image and chooses a fixed number of colors to represent the image. The goal of the algorithm is to minimize the color difference between the input and output image while minimizing the processing time." }, {NULL} /* Sentinel */ @@ -1260,6 +1295,12 @@ static PyGetSetDef magick_Image_getsetters[] = { (char *)"the image type: UndefinedType, BilevelType, GrayscaleType, GrayscaleMatteType, PaletteType, PaletteMatteType, TrueColorType, TrueColorMatteType, ColorSeparationType, ColorSeparationMatteType, or OptimizeType.", NULL}, + {(char *)"depth", + (getter)magick_Image_depth_getter, (setter)magick_Image_depth_setter, + (char *)"the image depth.", + NULL}, + + {NULL} /* Sentinel */ }; diff --git a/src/calibre/utils/podofo/__init__.py b/src/calibre/utils/podofo/__init__.py index 3134dcd1ba..13c12a9bb3 100644 --- a/src/calibre/utils/podofo/__init__.py +++ b/src/calibre/utils/podofo/__init__.py @@ -94,9 +94,8 @@ def delete_all_but(path, pages): if page not in pages: p.delete_page(page) - raw = p.write() with open(path, 'wb') as f: - f.write(raw) + f.save_to_fileobj(path) def test_outline(src): podofo = get_podofo() @@ -114,7 +113,17 @@ def test_outline(src): f.write(raw) print 'Outlined PDF:', out +def test_save_to(src, dest): + podofo = get_podofo() + p = podofo.PDFDoc() + with open(src, 'rb') as f: + raw = f.read() + p.load(raw) + with open(dest, 'wb') as out: + p.save_to_fileobj(out) + print ('Wrote PDF of size:', out.tell()) + if __name__ == '__main__': import sys - test_outline(sys.argv[-1]) + test_save_to(sys.argv[-2], sys.argv[-1]) diff --git a/src/calibre/utils/podofo/doc.cpp b/src/calibre/utils/podofo/doc.cpp index 7166b2320e..90bfc27921 100644 --- a/src/calibre/utils/podofo/doc.cpp +++ b/src/calibre/utils/podofo/doc.cpp @@ -104,6 +104,15 @@ PDFDoc_write(PDFDoc *self, PyObject *args) { return ans; } + +static PyObject * +PDFDoc_save_to_fileobj(PDFDoc *self, PyObject *args) { + PyObject *f; + + if (!PyArg_ParseTuple(args, "O", &f)) return NULL; + return write_doc(self->doc, f); +} + // }}} // extract_first_page() {{{ @@ -453,6 +462,9 @@ static PyMethodDef PDFDoc_methods[] = { {"write", (PyCFunction)PDFDoc_write, METH_VARARGS, "Return the PDF document as a bytestring." }, + {"save_to_fileobj", (PyCFunction)PDFDoc_save_to_fileobj, METH_VARARGS, + "Write the PDF document to the soecified file-like object." + }, {"extract_first_page", (PyCFunction)PDFDoc_extract_first_page, METH_VARARGS, "extract_first_page() -> Remove all but the first page." }, diff --git a/src/calibre/utils/podofo/global.h b/src/calibre/utils/podofo/global.h index fa9a141b21..4a180d86a0 100644 --- a/src/calibre/utils/podofo/global.h +++ b/src/calibre/utils/podofo/global.h @@ -41,6 +41,7 @@ extern void podofo_set_exception(const PdfError &err); extern PyObject * podofo_convert_pdfstring(const PdfString &s); extern PdfString * podofo_convert_pystring(PyObject *py); extern PdfString * podofo_convert_pystring_single_byte(PyObject *py); +extern PyObject* write_doc(PdfMemDocument *doc, PyObject *f); } diff --git a/src/calibre/utils/podofo/output.cpp b/src/calibre/utils/podofo/output.cpp new file mode 100644 index 0000000000..b0620f7f82 --- /dev/null +++ b/src/calibre/utils/podofo/output.cpp @@ -0,0 +1,177 @@ +/* + * output.cpp + * Copyright (C) 2012 Kovid Goyal + * + * Distributed under terms of the GPL3 license. + */ + +#include "global.h" + +using namespace PoDoFo; + +class pyerr : public std::exception { +}; + +class OutputDevice : public PdfOutputDevice { + + private: + PyObject *file; + size_t written; + + void update_written() { + size_t pos; + pos = Tell(); + if (pos > written) written = pos; + } + + public: + OutputDevice(PyObject *f) : file(f), written(0) { Py_XINCREF(file); } + ~OutputDevice() { Py_XDECREF(file); file = NULL; } + + size_t GetLength() const { return written; } + + long PrintVLen(const char* pszFormat, va_list args) { + + if( !pszFormat ) { PODOFO_RAISE_ERROR( ePdfError_InvalidHandle ); } + +#ifdef _MSC_VER + return _vscprintf(pszFormat, args); +#else + char buf[10]; + int res; + res = vsnprintf(buf, 1, pszFormat, args); + if (res < 0) { + PyErr_SetString(PyExc_Exception, "Something bad happened while calling vsnprintf to get buffer length"); + throw pyerr(); + } + return static_cast(res+1); +#endif + } + + void PrintV( const char* pszFormat, long lBytes, va_list args ) { + char *buf; + int res; + + if( !pszFormat ) { PODOFO_RAISE_ERROR( ePdfError_InvalidHandle ); } + + buf = new (std::nothrow) char[lBytes+1]; + if (buf == NULL) { PyErr_NoMemory(); throw pyerr(); } + + // Note: PyOS_vsnprintf produces broken output on windows + res = vsnprintf(buf, lBytes, pszFormat, args); + + if (res < 0) { + PyErr_SetString(PyExc_Exception, "Something bad happened while calling vsnprintf"); + delete[] buf; + throw pyerr(); + } + + Write(buf, static_cast(res)); + delete[] buf; + } + + void Print( const char* pszFormat, ... ) + { + va_list args; + long lBytes; + + va_start( args, pszFormat ); + lBytes = PrintVLen(pszFormat, args); + va_end( args ); + + va_start( args, pszFormat ); + PrintV(pszFormat, lBytes, args); + va_end( args ); + } + + size_t Read( char* pBuffer, size_t lLen ) { + PyObject *ret; + char *buf = NULL; + Py_ssize_t len = 0; + + ret = PyObject_CallMethod(file, (char*)"read", (char*)"n", static_cast(lLen)); + if (ret != NULL) { + if (PyBytes_AsStringAndSize(ret, &buf, &len) != -1) { + memcpy(pBuffer, buf, len); + Py_DECREF(ret); + return static_cast(len); + } + Py_DECREF(ret); + } + + if (PyErr_Occurred() == NULL) + PyErr_SetString(PyExc_Exception, "Failed to read data from python file object"); + + throw pyerr(); + + } + + void Seek(size_t offset) { + PyObject *ret; + ret = PyObject_CallMethod(file, (char*)"seek", (char*)"n", static_cast(offset)); + if (ret == NULL) { + if (PyErr_Occurred() == NULL) + PyErr_SetString(PyExc_Exception, "Failed to seek in python file object"); + throw pyerr(); + } + Py_DECREF(ret); + } + + size_t Tell() const { + PyObject *ret; + unsigned long ans; + + ret = PyObject_CallMethod(file, (char*)"tell", NULL); + if (ret == NULL) { + if (PyErr_Occurred() == NULL) + PyErr_SetString(PyExc_Exception, "Failed to call tell() on python file object"); + throw pyerr(); + } + if (!PyNumber_Check(ret)) { + Py_DECREF(ret); + PyErr_SetString(PyExc_Exception, "tell() method did not return a number"); + throw pyerr(); + } + ans = PyInt_AsUnsignedLongMask(ret); + Py_DECREF(ret); + if (PyErr_Occurred() != NULL) throw pyerr(); + + return static_cast(ans); + } + + void Write(const char* pBuffer, size_t lLen) { + PyObject *ret; + + ret = PyObject_CallMethod(file, (char*)"write", (char*)"s#", pBuffer, (int)lLen); + if (ret == NULL) { + if (PyErr_Occurred() == NULL) + PyErr_SetString(PyExc_Exception, "Failed to call write() on python file object"); + throw pyerr(); + } + Py_DECREF(ret); + update_written(); + } + + void Flush() { + Py_XDECREF(PyObject_CallMethod(file, (char*)"flush", NULL)); + } + +}; + + +PyObject* pdf::write_doc(PdfMemDocument *doc, PyObject *f) { + OutputDevice d(f); + + try { + doc->Write(&d); + } catch(const PdfError & err) { + podofo_set_exception(err); return NULL; + } catch (...) { + if (PyErr_Occurred() == NULL) + PyErr_SetString(PyExc_Exception, "An unknown error occurred while trying to write the pdf to the file object"); + return NULL; + } + + Py_RETURN_NONE; +} +