diff --git a/recipes/le_monde.recipe b/recipes/le_monde.recipe index afc19e4d86..8693676da9 100644 --- a/recipes/le_monde.recipe +++ b/recipes/le_monde.recipe @@ -9,7 +9,7 @@ from calibre.web.feeds.recipes import BasicNewsRecipe class LeMonde(BasicNewsRecipe): title = 'Le Monde' __author__ = 'veezh' - description = 'Actualités' + description = u'Actualités' oldest_article = 1 max_articles_per_feed = 100 no_stylesheets = True diff --git a/recipes/le_monde_sub.recipe b/recipes/le_monde_sub.recipe new file mode 100644 index 0000000000..6f5c16e3d1 --- /dev/null +++ b/recipes/le_monde_sub.recipe @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2012, Rémi Vanicat ' +''' +Lemonde.fr: Version abonnée +''' + + +import os, zipfile, re, time + +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup +from calibre.ptempfile import PersistentTemporaryFile + +class LeMondeAbonne(BasicNewsRecipe): + + title = u'Le Monde: Édition abonnés' + __author__ = u'Rémi Vanicat' + description = u'Actualités' + category = u'Actualités, France, Monde' + language = 'fr' + needs_subscription = True + + no_stylesheets = True + + extra_css = u''' + h1{font-size:130%;} + .ariane{font-size:xx-small;} + .source{font-size:xx-small;} + .href{font-size:xx-small;} + .LM_caption{color:#666666; font-size:x-small;} + .main-article-info{font-family:Arial,Helvetica,sans-serif;} + #full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} + #match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} + ''' + + zipurl_format = 'http://medias.lemonde.fr/abonnes/editionelectronique/%Y%m%d/html/%y%m%d.zip' + coverurl_format = '/img/%y%m%d01.jpg' + path_format = "%y%m%d" + login_url = 'http://www.lemonde.fr/web/journal_electronique/identification/1,56-0,45-0,0.html' + + keep_only_tags = [ dict(name="div", attrs={ 'class': 'po-prti' }), dict(name=['h1']), dict(name='div', attrs={ 'class': 'photo' }), dict(name='div', attrs={ 'class': 'po-ti2' }), dict(name='div', attrs={ 'class': 'ar-txt' }), dict(name='div', attrs={ 'class': 'po_rtcol' }) ] + + article_id_pattern = re.compile("[0-9]+\\.html") + article_url_format = 'http://www.lemonde.fr/journalelectronique/donnees/protege/%Y%m%d/html/' + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + if self.username is not None and self.password is not None: + br.open(self.login_url) + br.select_form(nr=0) + br['login'] = self.username + br['password'] = self.password + br.submit() + return br + + decalage = 24 * 60 * 60 # today Monde has tomorow date + + def get_cover_url(self): + url = time.strftime(self.coverurl_format, self.ltime) + return self.articles_path + url + + def parse_index(self): + browser = self.get_browser() + + second = time.time() + second += self.decalage + ltime = self.ltime = time.gmtime(second) + url = time.strftime(self.zipurl_format, ltime) + + self.timefmt=strftime(" %A %d %B %Y", ltime) + + response = browser.open(url) + + tmp = PersistentTemporaryFile(suffix='.zip') + self.report_progress(0.1,_('downloading zip file')) + tmp.write(response.read()) + tmp.close() + + zfile = zipfile.ZipFile(tmp.name, 'r') + self.report_progress(0.1,_('extracting zip file')) + + zfile.extractall(self.output_dir) + zfile.close() + + path = os.path.join(self.output_dir, time.strftime(self.path_format, ltime), "data") + + self.articles_path = path + + files = os.listdir(path) + + nb_index_files = len([ name for name in files if re.match("frame_gauche_[0-9]+.html", name) ]) + + flux = [] + + article_url = time.strftime(self.article_url_format, ltime) + + for i in range(nb_index_files): + filename = os.path.join(path, "selection_%d.html" % (i + 1)) + tmp = open(filename,'r') + soup=BeautifulSoup(tmp) + title=soup.find('span').contents[0] + tmp.close() + + filename = os.path.join(path, "frame_gauche_%d.html" % (i + 1)) + tmp = open(filename,'r') + soup = BeautifulSoup(tmp) + articles = [] + for link in soup.findAll("a"): + article_file = link['href'] + article_id=self.article_id_pattern.search(article_file).group() + article = { + 'title': link.contents[0], + 'url': article_url + article_id, + 'descripion': '', + 'content': '' + } + articles.append(article) + tmp.close() + + flux.append((title, articles)) + + return flux + + + +# Local Variables: +# mode: python +# End: + diff --git a/recipes/sunday_times.recipe b/recipes/sunday_times.recipe index 1f20f73cd9..973f1792c7 100644 --- a/recipes/sunday_times.recipe +++ b/recipes/sunday_times.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Darko Miletic ' +__copyright__ = '2010-2012, Darko Miletic ' ''' www.thesundaytimes.co.uk ''' @@ -43,13 +43,14 @@ class TimesOnline(BasicNewsRecipe): def get_browser(self): br = BasicNewsRecipe.get_browser() - br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thesundaytimes.co.uk/sto/') + br.open('http://www.thesundaytimes.co.uk/sto/') if self.username is not None and self.password is not None: - data = urllib.urlencode({ 'userName':self.username + data = urllib.urlencode({ + 'gotoUrl' :self.INDEX + ,'username':self.username ,'password':self.password - ,'keepMeLoggedIn':'false' }) - br.open('https://www.timesplus.co.uk/iam/app/authenticate',data) + br.open('https://acs.thetimes.co.uk/user/login',data) return br remove_tags = [ diff --git a/recipes/times_online.recipe b/recipes/times_online.recipe index 1ae8789cd5..1299c92fa3 100644 --- a/recipes/times_online.recipe +++ b/recipes/times_online.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2009-2010, Darko Miletic ' +__copyright__ = '2009-2012, Darko Miletic ' ''' www.thetimes.co.uk ''' @@ -21,6 +21,7 @@ class TimesOnline(BasicNewsRecipe): encoding = 'utf-8' delay = 1 needs_subscription = True + auto_cleanup = False publication_type = 'newspaper' masthead_url = 'http://www.thetimes.co.uk/tto/public/img/the_times_460.gif' INDEX = 'http://www.thetimes.co.uk' @@ -41,13 +42,14 @@ class TimesOnline(BasicNewsRecipe): def get_browser(self): br = BasicNewsRecipe.get_browser() - br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thetimes.co.uk/tto/news/?lightbox=false') + br.open('http://www.thetimes.co.uk/tto/news/') if self.username is not None and self.password is not None: - data = urllib.urlencode({ 'userName':self.username + data = urllib.urlencode({ + 'gotoUrl' :self.INDEX + ,'username':self.username ,'password':self.password - ,'keepMeLoggedIn':'false' }) - br.open('https://www.timesplus.co.uk/iam/app/authenticate',data) + br.open('https://acs.thetimes.co.uk/user/login',data) return br remove_tags = [ @@ -58,6 +60,7 @@ class TimesOnline(BasicNewsRecipe): keep_only_tags = [ dict(attrs={'class':'heading' }) ,dict(attrs={'class':'f-author'}) + ,dict(attrs={'class':['media','byline-timestamp']}) ,dict(attrs={'id':'bodycopy'}) ] @@ -79,11 +82,6 @@ class TimesOnline(BasicNewsRecipe): ,(u'Arts' , PREFIX + u'arts/?view=list' ) ] - def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - return self.adeify_images(soup) - def parse_index(self): totalfeeds = [] lfeeds = self.get_feeds() diff --git a/recipes/variety.recipe b/recipes/variety.recipe index 35418174e1..4980a55dee 100644 --- a/recipes/variety.recipe +++ b/recipes/variety.recipe @@ -19,7 +19,13 @@ class Variety(BasicNewsRecipe): category = 'Entertainment Industry News, Daily Variety, Movie Reviews, TV, Awards, Oscars, Cannes, Box Office, Hollywood' language = 'en' masthead_url = 'http://images1.variety.com/graphics/variety/Variety_logo_green_tm.gif' - extra_css = ' body{font-family: Georgia,"Times New Roman",Times,Courier,serif } img{margin-bottom: 1em} ' + extra_css = """ + body{font-family: Arial,Helvetica,sans-serif; font-size: 1.275em} + .date{font-size: small; border: 1px dotted rgb(204, 204, 204); font-style: italic; color: rgb(102, 102, 102); margin: 5px 0px; padding: 0.5em;} + .author{margin: 5px 0px 5px 20px; padding: 0.5em; background: none repeat scroll 0% 0% rgb(247, 247, 247);} + .art h2{color: rgb(153, 0, 0); font-size: 1.275em; font-weight: bold;} + img{margin-bottom: 1em} + """ conversion_options = { 'comments' : description @@ -29,7 +35,7 @@ class Variety(BasicNewsRecipe): } remove_tags = [dict(name=['object','link','map'])] - + remove_attributes=['lang','vspace','hspace','xmlns:ms','xmlns:dt'] keep_only_tags = [dict(name='div', attrs={'class':'art control'})] feeds = [(u'News & Articles', u'http://feeds.feedburner.com/variety/headlines' )] @@ -37,3 +43,29 @@ class Variety(BasicNewsRecipe): def print_version(self, url): rpt = url.rpartition('.html')[0] return rpt + '?printerfriendly=true' + + def preprocess_raw_html(self, raw, url): + return ''+raw[raw.find(''):] + + def get_article_url(self, article): + url = BasicNewsRecipe.get_article_url(self, article) + return url.rpartition('?')[0] + + def preprocess_html(self, soup): + for item in soup.findAll('a'): + limg = item.find('img') + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + if limg: + item.name = 'div' + item.attrs = [] + else: + str = self.tag_to_string(item) + item.replaceWith(str) + for item in soup.findAll('img'): + if not item.has_key('alt'): + item['alt'] = 'image' + return soup + \ No newline at end of file diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index cae1c6c63e..3517cf1363 100644 Binary files a/resources/compiled_coffeescript.zip and b/resources/compiled_coffeescript.zip differ diff --git a/resources/templates/html.css b/resources/templates/html.css index 79c80583bf..a8b3ab920c 100644 --- a/resources/templates/html.css +++ b/resources/templates/html.css @@ -402,7 +402,3 @@ img, object, svg|svg { height: auto; } -/* These are needed because ADE renders anchors the same as links */ - -a { text-decoration: inherit; color: inherit; cursor: inherit } -a[href] { text-decoration: underline; color: blue; cursor: pointer } diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index ac8e681fed..58390a314a 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -674,7 +674,7 @@ def get_download_filename(url, cookie_file=None): return filename -def human_readable(size): +def human_readable(size, sep=' '): """ Convert a size in bytes into a human readable form """ divisor, suffix = 1, "B" for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')): @@ -686,7 +686,7 @@ def human_readable(size): size = size[:size.find(".")+2] if size.endswith('.0'): size = size[:-2] - return size + " " + suffix + return size + sep + suffix def remove_bracketed_text(src, brackets={u'(':u')', u'[':u']', u'{':u'}'}): diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 3790a32ea7..45672fdbd1 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -197,7 +197,7 @@ class ANDROID(USBMS): 'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON', 'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP', 'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD', - 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ'] + 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', @@ -212,7 +212,7 @@ class ANDROID(USBMS): 'UMS', '.K080', 'P990', 'LTE', 'MB853', 'GT-S5660_CARD', 'A107', 'GT-I9003_CARD', 'XT912', 'FILE-CD_GADGET', 'RK29_SDK', 'MB855', 'XT910', 'BOOK_A10', 'USB_2.0_DRIVER', 'I9100T', 'P999DW', - 'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0_DRIVER', + 'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0', 'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX', 'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE', 'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', @@ -224,7 +224,7 @@ class ANDROID(USBMS): 'ANDROID_MID', 'P990_SD_CARD', '.K080', 'LTE_CARD', 'MB853', 'A1-07___C0541A4F', 'XT912', 'MB855', 'XT910', 'BOOK_A10_CARD', 'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC', - 'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0_DRIVER', 'XT875', + 'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875', 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042'] diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py new file mode 100644 index 0000000000..cc7d41e09b --- /dev/null +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import weakref, sys +from operator import attrgetter +from future_builtins import map + +from calibre import human_readable, prints, force_unicode +from calibre.utils.icu import sort_key + +class FileOrFolder(object): + + def __init__(self, entry, fs_cache, all_storage_ids): + self.object_id = entry['id'] + self.is_folder = entry['is_folder'] + self.name = force_unicode(entry.get('name', '___'), 'utf-8') + self.persistent_id = entry.get('persistent_id', self.object_id) + self.size = entry.get('size', 0) + # self.parent_id is None for storage objects + self.parent_id = entry.get('parent_id', None) + if self.parent_id == 0: + sid = entry['storage_id'] + if sid not in all_storage_ids: + sid = all_storage_ids[0] + self.parent_id = sid + self.is_hidden = entry.get('is_hidden', False) + self.is_system = entry.get('is_system', False) + self.can_delete = entry.get('can_delete', True) + + self.files = [] + self.folders = [] + fs_cache.id_map[self.object_id] = self + self.fs_cache = weakref.ref(fs_cache) + + @property + def id_map(self): + return self.fs_cache().id_map + + @property + def parent(self): + return None if self.parent_id is None else self.id_map[self.parent_id] + + def __iter__(self): + for e in self.folders: + yield e + for e in self.files: + yield e + + def dump(self, prefix='', out=sys.stdout): + c = '+' if self.is_folder else '-' + data = ('%s children'%(sum(map(len, (self.files, self.folders)))) + if self.is_folder else human_readable(self.size)) + line = '%s%s %s [id:%s %s]'%(prefix, c, self.name, self.object_id, data) + prints(line, file=out) + for c in (self.folders, self.files): + for e in sorted(c, key=lambda x:sort_key(x.name)): + e.dump(prefix=prefix+' ', out=out) + +class FilesystemCache(object): + + def __init__(self, all_storage, entries): + self.entries = [] + self.id_map = {} + + for storage in all_storage: + e = FileOrFolder(storage, self, []) + self.entries.append(e) + + self.entries.sort(key=attrgetter('object_id')) + all_storage_ids = [x.object_id for x in self.entries] + + for entry in entries: + FileOrFolder(entry, self, all_storage_ids) + + for item in self.id_map.itervalues(): + p = item.parent + if p is not None: + t = p.folders if item.is_folder else p.files + t.append(item) + + def dump(self, out=sys.stdout): + for e in self.entries: + e.dump(out=out) + + diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index c94a2e2458..835f2245d0 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -9,77 +9,13 @@ __docformat__ = 'restructuredtext en' import time, operator from threading import RLock -from itertools import chain -from collections import deque, OrderedDict from io import BytesIO -from calibre import prints from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.mtp.base import MTPDeviceBase, synchronous +from calibre.devices.mtp.filesystem_cache import FilesystemCache from calibre.devices.mtp.unix.detect import MTPDetect -class FilesystemCache(object): - - def __init__(self, files, folders): - self.files = files - self.folders = folders - self.file_id_map = {f['id']:f for f in files} - self.folder_id_map = {f['id']:f for f in self.iterfolders(set_level=0)} - - # Set the parents of each file - self.files_in_root = OrderedDict() - for f in files: - parents = deque() - pid = f['parent_id'] - while pid is not None and pid > 0: - try: - parent = self.folder_id_map[pid] - except KeyError: - break - parents.appendleft(pid) - pid = parent['parent_id'] - f['parents'] = parents - if not parents: - self.files_in_root[f['id']] = f - - # Set the files in each folder - for f in self.iterfolders(): - f['files'] = [i for i in files if i['parent_id'] == - f['id']] - - # Decode the file and folder names - for f in chain(files, folders): - try: - name = f['name'].decode('utf-8') - except UnicodeDecodeError: - name = 'undecodable_%d'%f['id'] - f['name'] = name - - def iterfolders(self, folders=None, set_level=None): - clevel = None if set_level is None else set_level + 1 - if folders is None: - folders = self.folders - for f in folders: - if set_level is not None: - f['level'] = set_level - yield f - for c in f['children']: - for child in self.iterfolders([c], set_level=clevel): - yield child - - def dump_filesystem(self): - indent = 2 - for f in self.iterfolders(): - prefix = ' '*(indent*f['level']) - prints(prefix, '+', f['name'], 'id=%s'%f['id']) - for leaf in f['files']: - prints(prefix, ' '*indent, '-', leaf['name'], - 'id=%d'%leaf['id'], 'size=%d'%leaf['size'], - 'modtime=%d'%leaf['modtime']) - for leaf in self.files_in_root.itervalues(): - prints('-', leaf['name'], 'id=%d'%leaf['id'], - 'size=%d'%leaf['size'], 'modtime=%d'%leaf['modtime']) - class MTP_DEVICE(MTPDeviceBase): supported_platforms = ['linux'] @@ -87,7 +23,7 @@ class MTP_DEVICE(MTPDeviceBase): def __init__(self, *args, **kwargs): MTPDeviceBase.__init__(self, *args, **kwargs) self.dev = None - self.filesystem_cache = None + self._filesystem_cache = None self.lock = RLock() self.blacklisted_devices = set() @@ -129,7 +65,7 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def post_yank_cleanup(self): - self.dev = self.filesystem_cache = self.current_friendly_name = None + self.dev = self._filesystem_cache = self.current_friendly_name = None @synchronous def startup(self): @@ -140,7 +76,7 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def shutdown(self): - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None def format_errorstack(self, errs): return '\n'.join(['%d:%s'%(code, msg.decode('utf-8', 'replace')) for @@ -148,7 +84,7 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def open(self, connected_device, library_uuid): - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None def blacklist_device(): d = connected_device self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id, @@ -179,23 +115,41 @@ class MTP_DEVICE(MTPDeviceBase): self._carda_id = storage[1]['id'] if len(storage) > 2: self._cardb_id = storage[2]['id'] - self.current_friendly_name = self.dev.name + self.current_friendly_name = self.dev.friendly_name - @synchronous - def read_filesystem_cache(self): - try: - files, errs = self.dev.get_filelist(self) - if errs and not files: - raise DeviceError('Failed to read files from device. Underlying errors:\n' - +self.format_errorstack(errs)) - folders, errs = self.dev.get_folderlist() - if errs and not folders: - raise DeviceError('Failed to read folders from device. Underlying errors:\n' - +self.format_errorstack(errs)) - self.filesystem_cache = FilesystemCache(files, folders) - except: - self.dev = self._main_id = self._carda_id = self._cardb_id = None - raise + @property + def filesystem_cache(self): + if self._filesystem_cache is None: + with self.lock: + files, errs = self.dev.get_filelist(self) + if errs and not files: + raise DeviceError('Failed to read files from device. Underlying errors:\n' + +self.format_errorstack(errs)) + folders, errs = self.dev.get_folderlist() + if errs and not folders: + raise DeviceError('Failed to read folders from device. Underlying errors:\n' + +self.format_errorstack(errs)) + storage = [] + for sid, capacity in zip([self._main_id, self._carda_id, + self._cardb_id], self.total_space()): + if sid is not None: + name = _('Unknown') + for x in self.dev.storage_info: + if x['id'] == sid: + name = x['name'] + break + storage.append({'id':sid, 'size':capacity, + 'is_folder':True, 'name':name}) + all_folders = [] + def recurse(f): + all_folders.append(f) + for c in f['children']: + recurse(c) + + for f in folders: recurse(f) + self._filesystem_cache = FilesystemCache(storage, + all_folders+files) + return self._filesystem_cache @synchronous def get_device_information(self, end_session=True): @@ -246,7 +200,6 @@ if __name__ == '__main__': devs = linux_scanner() mtp_devs = dev.detect(devs) dev.open(list(mtp_devs)[0], 'xxx') - dev.read_filesystem_cache() d = dev.dev print ("Opened device:", dev.get_gui_name()) print ("Storage info:") @@ -257,7 +210,7 @@ if __name__ == '__main__': # fname = b'moose.txt' # src = BytesIO(raw) # print (d.put_file(dev._main_id, 0, fname, src, len(raw), PR())) - dev.filesystem_cache.dump_filesystem() + dev.filesystem_cache.dump() # with open('/tmp/flint.epub', 'wb') as f: # print(d.get_file(786, f, PR())) # print() diff --git a/src/calibre/devices/mtp/unix/libmtp.c b/src/calibre/devices/mtp/unix/libmtp.c index 7ea987782a..ffab2e8b30 100644 --- a/src/calibre/devices/mtp/unix/libmtp.c +++ b/src/calibre/devices/mtp/unix/libmtp.c @@ -55,7 +55,7 @@ static int report_progress(uint64_t const sent, uint64_t const total, void const cb = (ProgressCallback *)data; if (cb->obj != NULL) { PyEval_RestoreThread(cb->state); - res = PyObject_CallMethod(cb->obj, "report_progress", "KK", sent, total); + res = PyObject_CallFunction(cb->obj, "KK", sent, total); Py_XDECREF(res); cb->state = PyEval_SaveThread(); } @@ -315,7 +315,7 @@ libmtp_Device_storage_info(libmtp_Device *self, void *closure) { "capacity", storage->MaxCapacity, "freespace_bytes", storage->FreeSpaceInBytes, "freespace_objects", storage->FreeSpaceInObjects, - "storage_desc", storage->StorageDescription, + "name", storage->StorageDescription, "volume_id", storage->VolumeIdentifier ); @@ -339,6 +339,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs if (!PyArg_ParseTuple(args, "|O", &callback)) return NULL; + if (callback == NULL || !PyCallable_Check(callback)) callback = NULL; cb.obj = callback; ans = PyList_New(0); @@ -357,13 +358,14 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs } for (f=tf; f != NULL; f=f->next) { - fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}", + fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k,s:O}", "id", f->item_id, "parent_id", f->parent_id, "storage_id", f->storage_id, "name", f->filename, "size", f->filesize, - "modtime", f->modificationdate + "modtime", f->modificationdate, + "is_folder", Py_False ); if (fo == NULL || PyList_Append(ans, fo) != 0) break; Py_DECREF(fo); @@ -377,7 +379,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs if (callback != NULL) { // Bug in libmtp where it does not call callback with 100% - fo = PyObject_CallMethod(callback, "report_progress", "KK", PyList_Size(ans), PyList_Size(ans)); + fo = PyObject_CallFunction(callback, "KK", PyList_Size(ans), PyList_Size(ans)); Py_XDECREF(fo); } @@ -392,11 +394,12 @@ int folderiter(LIBMTP_folder_t *f, PyObject *parent) { children = PyList_New(0); if (children == NULL) { PyErr_NoMemory(); return 1;} - folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}", + folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:O,s:N}", "id", f->folder_id, "parent_id", f->parent_id, "storage_id", f->storage_id, "name", f->name, + "is_folder", Py_True, "children", children); if (folder == NULL) return 1; PyList_Append(parent, folder); @@ -454,6 +457,7 @@ libmtp_Device_get_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) { if (!PyArg_ParseTuple(args, "kO|O", &fileid, &stream, &callback)) return NULL; errs = PyList_New(0); if (errs == NULL) { PyErr_NoMemory(); return NULL; } + if (callback == NULL || !PyCallable_Check(callback)) callback = NULL; cb.obj = callback; cb.extra = stream; Py_XINCREF(callback); Py_INCREF(stream); @@ -486,6 +490,7 @@ libmtp_Device_put_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) { if (!PyArg_ParseTuple(args, "kksOK|O", &storage_id, &parent_id, &name, &stream, &filesize, &callback)) return NULL; errs = PyList_New(0); if (errs == NULL) { PyErr_NoMemory(); return NULL; } + if (callback == NULL || !PyCallable_Check(callback)) callback = NULL; cb.obj = callback; cb.extra = stream; f.parent_id = parent_id; f.storage_id = storage_id; f.item_id = 0; f.filename = name; f.filetype = LIBMTP_FILETYPE_UNKNOWN; f.filesize = filesize; @@ -599,7 +604,7 @@ static PyMethodDef libmtp_Device_methods[] = { }, {"get_filelist", (PyCFunction)libmtp_Device_get_filelist, METH_VARARGS, - "get_filelist(callback=None) -> Get the list of files on the device. callback must be an object that has a method named 'report_progress(current, total)'. Returns files, errors." + "get_filelist(callback=None) -> Get the list of files on the device. callback must be callable accepts arguments (current, total)'. Returns files, errors." }, {"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS, diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp index f9026c35f8..70cead4893 100644 --- a/src/calibre/devices/mtp/windows/content_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -14,7 +14,7 @@ namespace wpd { static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { // {{{ - IPortableDeviceKeyCollection *properties; + IPortableDeviceKeyCollection *properties = NULL; HRESULT hr; Py_BEGIN_ALLOW_THREADS; @@ -28,7 +28,7 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { ADDPROP(WPD_OBJECT_PARENT_ID); ADDPROP(WPD_OBJECT_PERSISTENT_UNIQUE_ID); ADDPROP(WPD_OBJECT_NAME); - ADDPROP(WPD_OBJECT_SYNC_ID); + // ADDPROP(WPD_OBJECT_SYNC_ID); ADDPROP(WPD_OBJECT_ISSYSTEM); ADDPROP(WPD_OBJECT_ISHIDDEN); ADDPROP(WPD_OBJECT_CAN_DELETE); @@ -87,8 +87,25 @@ static void set_content_type_property(PyObject *dict, IPortableDeviceValues *pro if (SUCCEEDED(properties->GetGuidValue(WPD_OBJECT_CONTENT_TYPE, &guid)) && IsEqualGUID(guid, WPD_CONTENT_TYPE_FOLDER)) is_folder = 1; PyDict_SetItemString(dict, "is_folder", (is_folder) ? Py_True : Py_False); } + +static void set_properties(PyObject *obj, IPortableDeviceValues *values) { + set_content_type_property(obj, values); + + set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", values); + set_string_property(obj, WPD_OBJECT_NAME, "name", values); + // set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values); + set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", values); + + set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", values); + set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", values); + set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", values); + + set_size_property(obj, WPD_OBJECT_SIZE, "size", values); +} + // }}} +// Bulk get filesystem {{{ class GetBulkCallback : public IPortableDevicePropertiesBulkCallback { public: @@ -154,19 +171,8 @@ public: } Py_DECREF(temp); - set_content_type_property(obj, properties); + set_properties(obj, properties); - set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", properties); - set_string_property(obj, WPD_OBJECT_NAME, "name", properties); - set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", properties); - set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", properties); - - set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", properties); - set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", properties); - set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", properties); - - set_size_property(obj, WPD_OBJECT_SIZE, "size", properties); - properties->Release(); properties = NULL; } } // end for loop @@ -240,6 +246,9 @@ end: return folders; } +// }}} + +// find_all_objects_in() {{{ static BOOL find_all_objects_in(IPortableDeviceContent *content, IPortableDevicePropVariantCollection *object_ids, const wchar_t *parent_id) { /* * Find all children of the object identified by parent_id, recursively. @@ -286,9 +295,82 @@ end: if (children != NULL) children->Release(); PropVariantClear(&pv); return ok; +} // }}} + +// Single get filesystem {{{ + +static PyObject* get_object_properties(IPortableDeviceProperties *devprops, IPortableDeviceKeyCollection *properties, const wchar_t *object_id) { + IPortableDeviceValues *values = NULL; + HRESULT hr; + PyObject *ans = NULL, *temp = NULL; + + Py_BEGIN_ALLOW_THREADS; + hr = devprops->GetValues(object_id, properties, &values); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to get properties for object", hr); goto end; } + + temp = wchar_to_unicode(object_id); + if (temp == NULL) goto end; + + ans = PyDict_New(); + if (ans == NULL) { PyErr_NoMemory(); goto end; } + if (PyDict_SetItemString(ans, "id", temp) != 0) { Py_DECREF(ans); ans = NULL; PyErr_NoMemory(); goto end; } + + set_properties(ans, values); + +end: + Py_XDECREF(temp); + if (values != NULL) values->Release(); + return ans; } -PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { +static PyObject* single_get_filesystem(IPortableDeviceContent *content, const wchar_t *storage_id, IPortableDevicePropVariantCollection *object_ids) { + DWORD num, i; + PROPVARIANT pv; + HRESULT hr; + BOOL ok = 1; + PyObject *ans = NULL, *item = NULL; + IPortableDeviceProperties *devprops = NULL; + IPortableDeviceKeyCollection *properties = NULL; + + hr = content->Properties(&devprops); + if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; } + + properties = create_filesystem_properties_collection(); + if (properties == NULL) goto end; + + hr = object_ids->GetCount(&num); + if (FAILED(hr)) { hresult_set_exc("Failed to get object id count", hr); goto end; } + + ans = PyDict_New(); + if (ans == NULL) goto end; + + for (i = 0; i < num; i++) { + ok = 0; + PropVariantInit(&pv); + hr = object_ids->GetAt(i, &pv); + if (SUCCEEDED(hr) && pv.pwszVal != NULL) { + item = get_object_properties(devprops, properties, pv.pwszVal); + if (item != NULL) { + PyDict_SetItem(ans, PyDict_GetItemString(item, "id"), item); + Py_DECREF(item); item = NULL; + ok = 1; + } + } else hresult_set_exc("Failed to get item from IPortableDevicePropVariantCollection", hr); + + PropVariantClear(&pv); + if (!ok) { Py_DECREF(ans); ans = NULL; break; } + } + +end: + if (devprops != NULL) devprops->Release(); + if (properties != NULL) properties->Release(); + + return ans; +} +// }}} + +PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { // {{{ PyObject *folders = NULL; IPortableDevicePropVariantCollection *object_ids = NULL; IPortableDeviceContent *content = NULL; @@ -310,12 +392,112 @@ PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id if (!ok) goto end; if (bulk_properties != NULL) folders = bulk_get_filesystem(device, bulk_properties, storage_id, object_ids); + else folders = single_get_filesystem(content, storage_id, object_ids); end: if (content != NULL) content->Release(); if (object_ids != NULL) object_ids->Release(); return folders; -} +} // }}} + +PyObject* wpd::get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback) { // {{{ + IPortableDeviceContent *content = NULL; + IPortableDeviceResources *resources = NULL; + IPortableDeviceProperties *devprops = NULL; + IPortableDeviceValues *values = NULL; + IPortableDeviceKeyCollection *properties = NULL; + IStream *stream = NULL; + HRESULT hr; + DWORD bufsize = 4096; + char *buf = NULL; + ULONG bytes_read = 0, total_read = 0; + BOOL ok = FALSE; + PyObject *res = NULL; + ULONGLONG filesize = 0; + + Py_BEGIN_ALLOW_THREADS; + hr = device->Content(&content); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = content->Properties(&devprops); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = CoCreateInstance(CLSID_PortableDeviceKeyCollection, NULL, + CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&properties)); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create filesystem properties collection", hr); goto end; } + hr = properties->Add(WPD_OBJECT_SIZE); + if (FAILED(hr)) { hresult_set_exc("Failed to add filesize property to properties collection", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = devprops->GetValues(object_id, properties, &values); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to get filesize for object", hr); goto end; } + hr = values->GetUnsignedLargeIntegerValue(WPD_OBJECT_SIZE, &filesize); + if (FAILED(hr)) { hresult_set_exc("Failed to get filesize from values collection", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = content->Transfer(&resources); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create resources interface", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = resources->GetStream(object_id, WPD_RESOURCE_DEFAULT, STGM_READ, &bufsize, &stream); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { + if (HRESULT_FROM_WIN32(ERROR_BUSY) == hr) { + PyErr_SetString(WPDFileBusy, "Object is in use"); + } else hresult_set_exc("Failed to create stream interface to read from object", hr); + goto end; + } + + buf = (char *)calloc(bufsize+10, 1); + if (buf == NULL) { PyErr_NoMemory(); goto end; } + + while (TRUE) { + bytes_read = 0; + Py_BEGIN_ALLOW_THREADS; + hr = stream->Read(buf, bufsize, &bytes_read); + Py_END_ALLOW_THREADS; + total_read = total_read + bytes_read; + if (hr == STG_E_ACCESSDENIED) { + PyErr_SetString(PyExc_IOError, "Read access is denied to this object"); break; + } else if (hr == S_OK || hr == S_FALSE) { + if (bytes_read > 0) { + res = PyObject_CallMethod(dest, "write", "s#", buf, bytes_read); + if (res == NULL) break; + Py_DECREF(res); res = NULL; + if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_read, filesize)); + } + } else { hresult_set_exc("Failed to read file from device", hr); break; } + + if (hr == S_FALSE || bytes_read < bufsize) { + ok = TRUE; + Py_XDECREF(PyObject_CallMethod(dest, "flush", NULL)); + break; + } + } + + if (ok && total_read != filesize) { + ok = FALSE; + PyErr_SetString(WPDError, "Failed to read all data from file"); + } + +end: + if (content != NULL) content->Release(); + if (devprops != NULL) devprops->Release(); + if (resources != NULL) resources->Release(); + if (stream != NULL) stream->Release(); + if (values != NULL) values->Release(); + if (properties != NULL) properties->Release(); + if (buf != NULL) free(buf); + if (!ok) return NULL; + Py_RETURN_NONE; +} // }}} } // namespace wpd diff --git a/src/calibre/devices/mtp/windows/device.cpp b/src/calibre/devices/mtp/windows/device.cpp index 0a03b9e735..d79db0a2d3 100644 --- a/src/calibre/devices/mtp/windows/device.cpp +++ b/src/calibre/devices/mtp/windows/device.cpp @@ -78,7 +78,7 @@ update_data(Device *self, PyObject *args, PyObject *kwargs) { // get_filesystem() {{{ static PyObject* py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) { - PyObject *storage_id, *ans = NULL; + PyObject *storage_id; wchar_t *storage; if (!PyArg_ParseTuple(args, "O", &storage_id)) return NULL; @@ -88,6 +88,21 @@ py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) { return wpd::get_filesystem(self->device, storage, self->bulk_properties); } // }}} +// get_file() {{{ +static PyObject* +py_get_file(Device *self, PyObject *args, PyObject *kwargs) { + PyObject *object_id, *stream, *callback = NULL; + wchar_t *object; + + if (!PyArg_ParseTuple(args, "OO|O", &object_id, &stream, &callback)) return NULL; + object = unicode_to_wchar(object_id); + if (object == NULL) return NULL; + + if (callback == NULL || !PyCallable_Check(callback)) callback = NULL; + + return wpd::get_file(self->device, object, stream, callback); +} // }}} + static PyMethodDef Device_methods[] = { {"update_data", (PyCFunction)update_data, METH_VARARGS, "update_data() -> Reread the basic device data from the device (total, space, free space, storage locations, etc.)" @@ -97,6 +112,10 @@ static PyMethodDef Device_methods[] = { "get_filesystem(storage_id) -> Get all files/folders on the storage identified by storage_id. Tries to use bulk operations when possible." }, + {"get_file", (PyCFunction)py_get_file, METH_VARARGS, + "get_file(object_id, stream, callback=None) -> Get the file identified by object_id from the device. The file is written to the stream object, which must be a file like object. If callback is not None, it must be a callable that accepts two arguments: (bytes_read, total_size). It will be called after each chunk is read from the device. Note that it can be called multiple times with the same values." + }, + {NULL} }; diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index fcfc415c90..51f5bfd60d 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -7,13 +7,32 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import time -from threading import RLock +import time, threading +from functools import wraps +from future_builtins import zip +from itertools import chain from calibre import as_unicode, prints from calibre.constants import plugins, __appname__, numeric_version +from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed -from calibre.devices.mtp.base import MTPDeviceBase, synchronous +from calibre.devices.mtp.base import MTPDeviceBase +from calibre.devices.mtp.filesystem_cache import FilesystemCache + +class ThreadingViolation(Exception): + + def __init__(self): + Exception.__init__('You cannot use the MTP driver from a thread other than the ' + ' thread in which startup() was called') + +def same_thread(func): + @wraps(func) + def check_thread(self, *args, **kwargs): + if self.start_thread is not threading.current_thread(): + raise ThreadingViolation() + return func(self, *args, **kwargs) + return check_thread + class MTP_DEVICE(MTPDeviceBase): @@ -22,7 +41,6 @@ class MTP_DEVICE(MTPDeviceBase): def __init__(self, *args, **kwargs): MTPDeviceBase.__init__(self, *args, **kwargs) self.dev = None - self.lock = RLock() self.blacklisted_devices = set() self.ejected_devices = set() self.currently_connected_pnp_id = None @@ -31,9 +49,11 @@ class MTP_DEVICE(MTPDeviceBase): self.last_refresh_devices_time = time.time() self.wpd = self.wpd_error = None self._main_id = self._carda_id = self._cardb_id = None + self.start_thread = None + self._filesystem_cache = None - @synchronous def startup(self): + self.start_thread = threading.current_thread() self.wpd, self.wpd_error = plugins['wpd'] if self.wpd is not None: try: @@ -46,13 +66,13 @@ class MTP_DEVICE(MTPDeviceBase): except Exception as e: self.wpd_error = as_unicode(e) - @synchronous + @same_thread def shutdown(self): - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = self.start_thread = None if self.wpd is not None: self.wpd.uninit() - @synchronous + @same_thread def detect_managed_devices(self, devices_on_system): if self.wpd is None: return None @@ -119,23 +139,45 @@ class MTP_DEVICE(MTPDeviceBase): return True - @synchronous + @property + def filesystem_cache(self): + if self._filesystem_cache is None: + ts = self.total_space() + all_storage = [] + items = [] + for storage_id, capacity in zip([self._main_id, self._carda_id, + self._cardb_id], ts): + if storage_id is None: continue + name = _('Unknown') + for s in self.dev.data['storage']: + if s['id'] == storage_id: + name = s['name'] + break + storage = {'id':storage_id, 'size':capacity, 'name':name, + 'is_folder':True} + id_map = self.dev.get_filesystem(storage_id) + all_storage.append(storage) + items.append(id_map.itervalues()) + self._filesystem_cache = FilesystemCache(all_storage, chain(*items)) + return self._filesystem_cache + + @same_thread def post_yank_cleanup(self): self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None - @synchronous + @same_thread def eject(self): if self.currently_connected_pnp_id is None: return self.ejected_devices.add(self.currently_connected_pnp_id) self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None - @synchronous + @same_thread def open(self, connected_device, library_uuid): - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None try: self.dev = self.wpd.Device(connected_device) except self.wpd.WPDError: @@ -158,13 +200,13 @@ class MTP_DEVICE(MTPDeviceBase): self._cardb_id = storage[2]['id'] self.current_friendly_name = devdata.get('friendly_name', None) - @synchronous + @same_thread def get_device_information(self, end_session=True): d = self.dev.data dv = d.get('device_version', '') return (self.current_friendly_name, dv, dv, '') - @synchronous + @same_thread def card_prefix(self, end_session=True): ans = [None, None] if self._carda_id is not None: @@ -173,7 +215,7 @@ class MTP_DEVICE(MTPDeviceBase): ans[1] = 'mtp:::%s:::'%self._cardb_id return tuple(ans) - @synchronous + @same_thread def total_space(self, end_session=True): ans = [0, 0, 0] dd = self.dev.data @@ -184,7 +226,7 @@ class MTP_DEVICE(MTPDeviceBase): ans[i] = s['capacity'] return tuple(ans) - @synchronous + @same_thread def free_space(self, end_session=True): self.dev.update_data() ans = [0, 0, 0] @@ -196,5 +238,14 @@ class MTP_DEVICE(MTPDeviceBase): ans[i] = s['free_space'] return tuple(ans) - + @same_thread + def get_file(self, object_id, stream=None, callback=None): + if stream is None: + stream = SpooledTemporaryFile(5*1024*1024, '_wpd_receive_file.dat') + try: + self.dev.get_file(object_id, stream, callback) + except self.wpd.WPDFileBusy: + time.sleep(2) + self.dev.get_file(object_id, stream, callback) + return stream diff --git a/src/calibre/devices/mtp/windows/global.h b/src/calibre/devices/mtp/windows/global.h index cbf489d424..47f0786249 100644 --- a/src/calibre/devices/mtp/windows/global.h +++ b/src/calibre/devices/mtp/windows/global.h @@ -20,7 +20,7 @@ namespace wpd { // Module exception types -extern PyObject *WPDError, *NoWPD; +extern PyObject *WPDError, *NoWPD, *WPDFileBusy; // The global device manager extern IPortableDeviceManager *portable_device_manager; @@ -50,13 +50,14 @@ extern PyTypeObject DeviceType; // Utility functions PyObject *hresult_set_exc(const char *msg, HRESULT hr); wchar_t *unicode_to_wchar(PyObject *o); -PyObject *wchar_to_unicode(wchar_t *o); +PyObject *wchar_to_unicode(const wchar_t *o); int pump_waiting_messages(); extern IPortableDeviceValues* get_client_information(); extern IPortableDevice* open_device(const wchar_t *pnp_id, IPortableDeviceValues *client_information); extern PyObject* get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **bulk_properties); extern PyObject* get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties); +extern PyObject* get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback); } diff --git a/src/calibre/devices/mtp/windows/remote.py b/src/calibre/devices/mtp/windows/remote.py index a6502b991b..a3686ce88c 100644 --- a/src/calibre/devices/mtp/windows/remote.py +++ b/src/calibre/devices/mtp/windows/remote.py @@ -7,8 +7,8 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import subprocess, sys, os, pprint, signal, time, glob -pprint +import subprocess, sys, os, pprint, signal, time, glob, io +pprint, io def build(mod='wpd'): master = subprocess.Popen('ssh -MN getafix'.split()) @@ -70,7 +70,10 @@ def main(): print ('Connected to:', dev.get_gui_name()) print ('Total space', dev.total_space()) print ('Free space', dev.free_space()) - pprint.pprint(dev.dev.get_filesystem(dev._main_id)) + dev.filesystem_cache.dump() + # print ('Fetching file: oFF (198214 bytes)') + # stream = dev.get_file('oFF') + # print ("Fetched size: ", stream.tell()) finally: dev.shutdown() diff --git a/src/calibre/devices/mtp/windows/utils.cpp b/src/calibre/devices/mtp/windows/utils.cpp index 243bcc0f59..7592e0d4a9 100644 --- a/src/calibre/devices/mtp/windows/utils.cpp +++ b/src/calibre/devices/mtp/windows/utils.cpp @@ -43,7 +43,7 @@ wchar_t *wpd::unicode_to_wchar(PyObject *o) { return buf; } -PyObject *wpd::wchar_to_unicode(wchar_t *o) { +PyObject *wpd::wchar_to_unicode(const wchar_t *o) { PyObject *ans; if (o == NULL) return NULL; ans = PyUnicode_FromWideChar(o, wcslen(o)); diff --git a/src/calibre/devices/mtp/windows/wpd.cpp b/src/calibre/devices/mtp/windows/wpd.cpp index 561eeb1bbc..51a55b97ac 100644 --- a/src/calibre/devices/mtp/windows/wpd.cpp +++ b/src/calibre/devices/mtp/windows/wpd.cpp @@ -10,7 +10,7 @@ using namespace wpd; // Module exception types -PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL; +PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL, *wpd::WPDFileBusy = NULL; // The global device manager IPortableDeviceManager *wpd::portable_device_manager = NULL; @@ -199,6 +199,9 @@ initwpd(void) { NoWPD = PyErr_NewException("wpd.NoWPD", NULL, NULL); if (NoWPD == NULL) return; + WPDFileBusy = PyErr_NewException("wpd.WPDFileBusy", NULL, NULL); + if (WPDFileBusy == NULL) return; + Py_INCREF(&DeviceType); PyModule_AddObject(m, "Device", (PyObject *)&DeviceType); diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index b51e55b829..8b76255532 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -50,10 +50,10 @@ class PRST1(USBMS): VENDOR_NAME = 'SONY' WINDOWS_MAIN_MEM = re.compile( - r'(PRS-T1&)' + r'(PRS-T(1|2)&)' ) WINDOWS_CARD_A_MEM = re.compile( - r'(PRS-T1__SD&)' + r'(PRS-T(1|2)__SD&)' ) MAIN_MEMORY_VOLUME_LABEL = 'SONY Reader Main Memory' STORAGE_CARD_VOLUME_LABEL = 'SONY Reader Storage Card' diff --git a/src/calibre/ebooks/mobi/writer2/serializer.py b/src/calibre/ebooks/mobi/writer2/serializer.py index 2b38c1e6a6..d3269b2d5e 100644 --- a/src/calibre/ebooks/mobi/writer2/serializer.py +++ b/src/calibre/ebooks/mobi/writer2/serializer.py @@ -235,7 +235,7 @@ class Serializer(object): itemhref = re.sub(r'article_\d+/', '', itemhref) self.href_offsets[itemhref].append(buf.tell()) buf.write('0000000000') - buf.write(' >') + buf.write(' >') t = tocitem.title if isinstance(t, unicode): t = t.encode('utf-8') diff --git a/src/calibre/ebooks/oeb/display/mathjax.coffee b/src/calibre/ebooks/oeb/display/mathjax.coffee index 50a75cca6b..ad893baa7e 100644 --- a/src/calibre/ebooks/oeb/display/mathjax.coffee +++ b/src/calibre/ebooks/oeb/display/mathjax.coffee @@ -23,24 +23,21 @@ class MathJax this.pending_cfi = null this.hub = null - load_mathjax: (script) -> + load_mathjax: (user_config) -> if this.base == null log('You must specify the path to the MathJax installation before trying to load MathJax') return null - created = false - if script == null - script = document.createElement('script') - created = true + script = document.createElement('script') script.type = 'text/javascript' script.src = 'file://' + this.base + '/MathJax.js' - - script.text = script.text + ''' + script.text = user_config + ''' + MathJax.Hub.signal.Interest(function (message) {if (String(message).match(/error/i)) {console.log(message)}}); MathJax.Hub.Config({ positionToHash: false, showMathMenu: false, - extensions: ["tex2jax.js","asciimath2jax.js","mml2jax.js"], + extensions: ["tex2jax.js", "asciimath2jax.js", "mml2jax.js"], jax: ["input/TeX","input/MathML","input/AsciiMath","output/SVG"], TeX: { extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"] @@ -50,9 +47,7 @@ class MathJax MathJax.Hub.Register.StartupHook("End", window.mathjax.load_finished); window.mathjax.hub = MathJax.Hub ''' - - if created - document.head.appendChild(script) + document.head.appendChild(script) load_finished: () => log('MathJax load finished!') @@ -67,14 +62,17 @@ class MathJax this.math_present = false this.math_loaded = false this.pending_cfi = null + user_config = '' for c in document.getElementsByTagName('script') if c.getAttribute('type') == 'text/x-mathjax-config' + if c.text + user_config += c.text script = c - break + c.parentNode.removeChild(c) if script != null or document.getElementsByTagName('math').length > 0 this.math_present = true - this.load_mathjax(script) + this.load_mathjax(user_config) after_resize: () -> if not this.math_present or this.hub == null diff --git a/src/calibre/ebooks/oeb/display/paged.coffee b/src/calibre/ebooks/oeb/display/paged.coffee index de6645ce88..4f912513a9 100644 --- a/src/calibre/ebooks/oeb/display/paged.coffee +++ b/src/calibre/ebooks/oeb/display/paged.coffee @@ -334,6 +334,16 @@ class PagedDisplay elem = elems[0] if not elem return + if window.mathjax?.math_present + # MathJax links to children of SVG tags and scrollIntoView doesn't + # work properly for them, so if this link points to something + # inside an tag we instead scroll the parent of the svg tag + # into view. + parent = elem + while parent and parent?.tagName?.toLowerCase() != 'svg' + parent = parent.parentNode + if parent?.tagName?.toLowerCase() == 'svg' + elem = parent.parentNode elem.scrollIntoView() if this.in_paged_mode # Ensure we are scrolled to the column containing elem @@ -368,7 +378,9 @@ class PagedDisplay # The Conformal Fragment Identifier at the current position, returns # null if it could not be calculated. Requires the cfi.coffee library. ans = null - if not window.cfi? + if not window.cfi? or (window.mathjax?.math_present and not window.mathjax?.math_loaded) + # If MathJax is loading, it is changing the DOM, so we cannot + # reliably generate a CFI return ans if this.in_paged_mode c = this.current_column_location() @@ -402,9 +414,9 @@ class PagedDisplay return ans click_for_page_turn: (event) -> - # Check if the click event event should generate a apge turn. Returns + # Check if the click event should generate a page turn. Returns # null if it should not, true if it is a backwards page turn, false if - # it is a forward apge turn. + # it is a forward page turn. left_boundary = this.current_margin_side right_bondary = this.screen_width - this.current_margin_side if left_boundary > event.clientX diff --git a/src/calibre/gui2/store/stores/bn_plugin.py b/src/calibre/gui2/store/stores/bn_plugin.py index 1f7eb2a91e..65a7eee194 100644 --- a/src/calibre/gui2/store/stores/bn_plugin.py +++ b/src/calibre/gui2/store/stores/bn_plugin.py @@ -34,26 +34,29 @@ class BNStore(BasicStoreConfig, StorePlugin): d.exec_() def search(self, query, max_results=10, timeout=60): - url = 'http://www.barnesandnoble.com/s/%s?keyword=%s&store=ebook' % (query.replace(' ', '-'), urllib.quote_plus(query)) + url = 'http://www.barnesandnoble.com/s/%s?keyword=%s&store=ebook&view=list' % (query.replace(' ', '-'), urllib.quote_plus(query)) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) + raw = f.read() + doc = html.fromstring(raw) for data in doc.xpath('//ul[contains(@class, "result-set")]/li[contains(@class, "result")]'): if counter <= 0: break - id = ''.join(data.xpath('.//div[contains(@class, "image-bounding-box")]/a/@href')) + id = ''.join(data.xpath('.//div[contains(@class, "image-block")]/a/@href')) if not id: continue cover_url = ''.join(data.xpath('.//img[contains(@class, "product-image")]/@src')) - title = ''.join(data.xpath('.//a[@class="title"]//text()')) - author = ', '.join(data.xpath('.//a[@class="contributor"]//text()')) - price = ''.join(data.xpath('.//div[@class="price-format"]//span[contains(@class, "price")]/text()')) + title = ''.join(data.xpath('descendant::p[@class="title"]//span[@class="name"]//text()')).strip() + if not title: continue + + author = ', '.join(data.xpath('.//ul[@class="contributors"]//a[@class="subtle"]//text()')).strip() + price = ''.join(data.xpath('.//a[contains(@class, "bn-price")]//text()')) counter -= 1 diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index 6e4d129eaf..a63fb2c8c6 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -185,6 +185,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer): self.pos.setDecimals(1) self.pos.setSuffix('/'+_('Unknown')+' ') self.pos.setMinimum(1.) + self.splitter.setCollapsible(0, False) + self.splitter.setCollapsible(1, False) self.pos.setMinimumWidth(150) self.tool_bar2.insertWidget(self.action_find_next, self.pos) self.reference = Reference() @@ -1028,6 +1030,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer): av = available_height() - 30 if self.height() > av: self.resize(self.width(), av) + self.splitter.setCollapsible(0, False) + self.splitter.setCollapsible(1, False) def config(defaults=None): desc = _('Options to control the ebook viewer') diff --git a/src/calibre/gui2/viewer/main.ui b/src/calibre/gui2/viewer/main.ui index fe0fa62a79..ddc4cfb776 100644 --- a/src/calibre/gui2/viewer/main.ui +++ b/src/calibre/gui2/viewer/main.ui @@ -27,7 +27,14 @@ false - + + + + 150 + 0 + + + QFrame::StyledPanel