mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Merge from trunk
This commit is contained in:
Charles Haley
commit
b16d35f1f6
@ -9,7 +9,7 @@ from calibre.web.feeds.recipes import BasicNewsRecipe
|
||||
class LeMonde(BasicNewsRecipe):
|
||||
title = 'Le Monde'
|
||||
__author__ = 'veezh'
|
||||
description = 'Actualités'
|
||||
description = u'Actualités'
|
||||
oldest_article = 1
|
||||
max_articles_per_feed = 100
|
||||
no_stylesheets = True
|
||||
|
||||
132
recipes/le_monde_sub.recipe
Normal file
132
recipes/le_monde_sub.recipe
Normal file
@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2012, Rémi Vanicat <vanicat at debian.org>'
|
||||
'''
|
||||
Lemonde.fr: Version abonnée
|
||||
'''
|
||||
|
||||
|
||||
import os, zipfile, re, time
|
||||
|
||||
from calibre import strftime
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
from calibre.ebooks.BeautifulSoup import BeautifulSoup
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
|
||||
class LeMondeAbonne(BasicNewsRecipe):
|
||||
|
||||
title = u'Le Monde: Édition abonnés'
|
||||
__author__ = u'Rémi Vanicat'
|
||||
description = u'Actualités'
|
||||
category = u'Actualités, France, Monde'
|
||||
language = 'fr'
|
||||
needs_subscription = True
|
||||
|
||||
no_stylesheets = True
|
||||
|
||||
extra_css = u'''
|
||||
h1{font-size:130%;}
|
||||
.ariane{font-size:xx-small;}
|
||||
.source{font-size:xx-small;}
|
||||
.href{font-size:xx-small;}
|
||||
.LM_caption{color:#666666; font-size:x-small;}
|
||||
.main-article-info{font-family:Arial,Helvetica,sans-serif;}
|
||||
#full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
|
||||
#match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
|
||||
'''
|
||||
|
||||
zipurl_format = 'http://medias.lemonde.fr/abonnes/editionelectronique/%Y%m%d/html/%y%m%d.zip'
|
||||
coverurl_format = '/img/%y%m%d01.jpg'
|
||||
path_format = "%y%m%d"
|
||||
login_url = 'http://www.lemonde.fr/web/journal_electronique/identification/1,56-0,45-0,0.html'
|
||||
|
||||
keep_only_tags = [ dict(name="div", attrs={ 'class': 'po-prti' }), dict(name=['h1']), dict(name='div', attrs={ 'class': 'photo' }), dict(name='div', attrs={ 'class': 'po-ti2' }), dict(name='div', attrs={ 'class': 'ar-txt' }), dict(name='div', attrs={ 'class': 'po_rtcol' }) ]
|
||||
|
||||
article_id_pattern = re.compile("[0-9]+\\.html")
|
||||
article_url_format = 'http://www.lemonde.fr/journalelectronique/donnees/protege/%Y%m%d/html/'
|
||||
|
||||
def get_browser(self):
|
||||
br = BasicNewsRecipe.get_browser()
|
||||
if self.username is not None and self.password is not None:
|
||||
br.open(self.login_url)
|
||||
br.select_form(nr=0)
|
||||
br['login'] = self.username
|
||||
br['password'] = self.password
|
||||
br.submit()
|
||||
return br
|
||||
|
||||
decalage = 24 * 60 * 60 # today Monde has tomorow date
|
||||
|
||||
def get_cover_url(self):
|
||||
url = time.strftime(self.coverurl_format, self.ltime)
|
||||
return self.articles_path + url
|
||||
|
||||
def parse_index(self):
|
||||
browser = self.get_browser()
|
||||
|
||||
second = time.time()
|
||||
second += self.decalage
|
||||
ltime = self.ltime = time.gmtime(second)
|
||||
url = time.strftime(self.zipurl_format, ltime)
|
||||
|
||||
self.timefmt=strftime(" %A %d %B %Y", ltime)
|
||||
|
||||
response = browser.open(url)
|
||||
|
||||
tmp = PersistentTemporaryFile(suffix='.zip')
|
||||
self.report_progress(0.1,_('downloading zip file'))
|
||||
tmp.write(response.read())
|
||||
tmp.close()
|
||||
|
||||
zfile = zipfile.ZipFile(tmp.name, 'r')
|
||||
self.report_progress(0.1,_('extracting zip file'))
|
||||
|
||||
zfile.extractall(self.output_dir)
|
||||
zfile.close()
|
||||
|
||||
path = os.path.join(self.output_dir, time.strftime(self.path_format, ltime), "data")
|
||||
|
||||
self.articles_path = path
|
||||
|
||||
files = os.listdir(path)
|
||||
|
||||
nb_index_files = len([ name for name in files if re.match("frame_gauche_[0-9]+.html", name) ])
|
||||
|
||||
flux = []
|
||||
|
||||
article_url = time.strftime(self.article_url_format, ltime)
|
||||
|
||||
for i in range(nb_index_files):
|
||||
filename = os.path.join(path, "selection_%d.html" % (i + 1))
|
||||
tmp = open(filename,'r')
|
||||
soup=BeautifulSoup(tmp)
|
||||
title=soup.find('span').contents[0]
|
||||
tmp.close()
|
||||
|
||||
filename = os.path.join(path, "frame_gauche_%d.html" % (i + 1))
|
||||
tmp = open(filename,'r')
|
||||
soup = BeautifulSoup(tmp)
|
||||
articles = []
|
||||
for link in soup.findAll("a"):
|
||||
article_file = link['href']
|
||||
article_id=self.article_id_pattern.search(article_file).group()
|
||||
article = {
|
||||
'title': link.contents[0],
|
||||
'url': article_url + article_id,
|
||||
'descripion': '',
|
||||
'content': ''
|
||||
}
|
||||
articles.append(article)
|
||||
tmp.close()
|
||||
|
||||
flux.append((title, articles))
|
||||
|
||||
return flux
|
||||
|
||||
|
||||
|
||||
# Local Variables:
|
||||
# mode: python
|
||||
# End:
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
|
||||
__copyright__ = '2010-2012, Darko Miletic <darko.miletic at gmail.com>'
|
||||
'''
|
||||
www.thesundaytimes.co.uk
|
||||
'''
|
||||
@ -43,13 +43,14 @@ class TimesOnline(BasicNewsRecipe):
|
||||
|
||||
def get_browser(self):
|
||||
br = BasicNewsRecipe.get_browser()
|
||||
br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thesundaytimes.co.uk/sto/')
|
||||
br.open('http://www.thesundaytimes.co.uk/sto/')
|
||||
if self.username is not None and self.password is not None:
|
||||
data = urllib.urlencode({ 'userName':self.username
|
||||
data = urllib.urlencode({
|
||||
'gotoUrl' :self.INDEX
|
||||
,'username':self.username
|
||||
,'password':self.password
|
||||
,'keepMeLoggedIn':'false'
|
||||
})
|
||||
br.open('https://www.timesplus.co.uk/iam/app/authenticate',data)
|
||||
br.open('https://acs.thetimes.co.uk/user/login',data)
|
||||
return br
|
||||
|
||||
remove_tags = [
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
|
||||
__copyright__ = '2009-2012, Darko Miletic <darko.miletic at gmail.com>'
|
||||
'''
|
||||
www.thetimes.co.uk
|
||||
'''
|
||||
@ -21,6 +21,7 @@ class TimesOnline(BasicNewsRecipe):
|
||||
encoding = 'utf-8'
|
||||
delay = 1
|
||||
needs_subscription = True
|
||||
auto_cleanup = False
|
||||
publication_type = 'newspaper'
|
||||
masthead_url = 'http://www.thetimes.co.uk/tto/public/img/the_times_460.gif'
|
||||
INDEX = 'http://www.thetimes.co.uk'
|
||||
@ -41,13 +42,14 @@ class TimesOnline(BasicNewsRecipe):
|
||||
|
||||
def get_browser(self):
|
||||
br = BasicNewsRecipe.get_browser()
|
||||
br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thetimes.co.uk/tto/news/?lightbox=false')
|
||||
br.open('http://www.thetimes.co.uk/tto/news/')
|
||||
if self.username is not None and self.password is not None:
|
||||
data = urllib.urlencode({ 'userName':self.username
|
||||
data = urllib.urlencode({
|
||||
'gotoUrl' :self.INDEX
|
||||
,'username':self.username
|
||||
,'password':self.password
|
||||
,'keepMeLoggedIn':'false'
|
||||
})
|
||||
br.open('https://www.timesplus.co.uk/iam/app/authenticate',data)
|
||||
br.open('https://acs.thetimes.co.uk/user/login',data)
|
||||
return br
|
||||
|
||||
remove_tags = [
|
||||
@ -58,6 +60,7 @@ class TimesOnline(BasicNewsRecipe):
|
||||
keep_only_tags = [
|
||||
dict(attrs={'class':'heading' })
|
||||
,dict(attrs={'class':'f-author'})
|
||||
,dict(attrs={'class':['media','byline-timestamp']})
|
||||
,dict(attrs={'id':'bodycopy'})
|
||||
]
|
||||
|
||||
@ -79,11 +82,6 @@ class TimesOnline(BasicNewsRecipe):
|
||||
,(u'Arts' , PREFIX + u'arts/?view=list' )
|
||||
]
|
||||
|
||||
def preprocess_html(self, soup):
|
||||
for item in soup.findAll(style=True):
|
||||
del item['style']
|
||||
return self.adeify_images(soup)
|
||||
|
||||
def parse_index(self):
|
||||
totalfeeds = []
|
||||
lfeeds = self.get_feeds()
|
||||
|
||||
@ -19,7 +19,13 @@ class Variety(BasicNewsRecipe):
|
||||
category = 'Entertainment Industry News, Daily Variety, Movie Reviews, TV, Awards, Oscars, Cannes, Box Office, Hollywood'
|
||||
language = 'en'
|
||||
masthead_url = 'http://images1.variety.com/graphics/variety/Variety_logo_green_tm.gif'
|
||||
extra_css = ' body{font-family: Georgia,"Times New Roman",Times,Courier,serif } img{margin-bottom: 1em} '
|
||||
extra_css = """
|
||||
body{font-family: Arial,Helvetica,sans-serif; font-size: 1.275em}
|
||||
.date{font-size: small; border: 1px dotted rgb(204, 204, 204); font-style: italic; color: rgb(102, 102, 102); margin: 5px 0px; padding: 0.5em;}
|
||||
.author{margin: 5px 0px 5px 20px; padding: 0.5em; background: none repeat scroll 0% 0% rgb(247, 247, 247);}
|
||||
.art h2{color: rgb(153, 0, 0); font-size: 1.275em; font-weight: bold;}
|
||||
img{margin-bottom: 1em}
|
||||
"""
|
||||
|
||||
conversion_options = {
|
||||
'comments' : description
|
||||
@ -29,7 +35,7 @@ class Variety(BasicNewsRecipe):
|
||||
}
|
||||
|
||||
remove_tags = [dict(name=['object','link','map'])]
|
||||
|
||||
remove_attributes=['lang','vspace','hspace','xmlns:ms','xmlns:dt']
|
||||
keep_only_tags = [dict(name='div', attrs={'class':'art control'})]
|
||||
|
||||
feeds = [(u'News & Articles', u'http://feeds.feedburner.com/variety/headlines' )]
|
||||
@ -37,3 +43,29 @@ class Variety(BasicNewsRecipe):
|
||||
def print_version(self, url):
|
||||
rpt = url.rpartition('.html')[0]
|
||||
return rpt + '?printerfriendly=true'
|
||||
|
||||
def preprocess_raw_html(self, raw, url):
|
||||
return '<html><head>'+raw[raw.find('</head>'):]
|
||||
|
||||
def get_article_url(self, article):
|
||||
url = BasicNewsRecipe.get_article_url(self, article)
|
||||
return url.rpartition('?')[0]
|
||||
|
||||
def preprocess_html(self, soup):
|
||||
for item in soup.findAll('a'):
|
||||
limg = item.find('img')
|
||||
if item.string is not None:
|
||||
str = item.string
|
||||
item.replaceWith(str)
|
||||
else:
|
||||
if limg:
|
||||
item.name = 'div'
|
||||
item.attrs = []
|
||||
else:
|
||||
str = self.tag_to_string(item)
|
||||
item.replaceWith(str)
|
||||
for item in soup.findAll('img'):
|
||||
if not item.has_key('alt'):
|
||||
item['alt'] = 'image'
|
||||
return soup
|
||||
|
||||
Binary file not shown.
@ -402,7 +402,3 @@ img, object, svg|svg {
|
||||
height: auto;
|
||||
}
|
||||
|
||||
/* These are needed because ADE renders anchors the same as links */
|
||||
|
||||
a { text-decoration: inherit; color: inherit; cursor: inherit }
|
||||
a[href] { text-decoration: underline; color: blue; cursor: pointer }
|
||||
|
||||
@ -674,7 +674,7 @@ def get_download_filename(url, cookie_file=None):
|
||||
|
||||
return filename
|
||||
|
||||
def human_readable(size):
|
||||
def human_readable(size, sep=' '):
|
||||
""" Convert a size in bytes into a human readable form """
|
||||
divisor, suffix = 1, "B"
|
||||
for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
|
||||
@ -686,7 +686,7 @@ def human_readable(size):
|
||||
size = size[:size.find(".")+2]
|
||||
if size.endswith('.0'):
|
||||
size = size[:-2]
|
||||
return size + " " + suffix
|
||||
return size + sep + suffix
|
||||
|
||||
def remove_bracketed_text(src,
|
||||
brackets={u'(':u')', u'[':u']', u'{':u'}'}):
|
||||
|
||||
@ -197,7 +197,7 @@ class ANDROID(USBMS):
|
||||
'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON',
|
||||
'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP',
|
||||
'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD',
|
||||
'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ']
|
||||
'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0']
|
||||
WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
|
||||
'__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
|
||||
'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID',
|
||||
@ -212,7 +212,7 @@ class ANDROID(USBMS):
|
||||
'UMS', '.K080', 'P990', 'LTE', 'MB853', 'GT-S5660_CARD', 'A107',
|
||||
'GT-I9003_CARD', 'XT912', 'FILE-CD_GADGET', 'RK29_SDK', 'MB855',
|
||||
'XT910', 'BOOK_A10', 'USB_2.0_DRIVER', 'I9100T', 'P999DW',
|
||||
'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0_DRIVER',
|
||||
'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0',
|
||||
'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX',
|
||||
'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE',
|
||||
'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID',
|
||||
@ -224,7 +224,7 @@ class ANDROID(USBMS):
|
||||
'ANDROID_MID', 'P990_SD_CARD', '.K080', 'LTE_CARD', 'MB853',
|
||||
'A1-07___C0541A4F', 'XT912', 'MB855', 'XT910', 'BOOK_A10_CARD',
|
||||
'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC',
|
||||
'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0_DRIVER', 'XT875',
|
||||
'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875',
|
||||
'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727',
|
||||
'USB_FLASH_DRIVER', 'ANDROID', 'MID7042']
|
||||
|
||||
|
||||
91
src/calibre/devices/mtp/filesystem_cache.py
Normal file
91
src/calibre/devices/mtp/filesystem_cache.py
Normal file
@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python
|
||||
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
import weakref, sys
|
||||
from operator import attrgetter
|
||||
from future_builtins import map
|
||||
|
||||
from calibre import human_readable, prints, force_unicode
|
||||
from calibre.utils.icu import sort_key
|
||||
|
||||
class FileOrFolder(object):
|
||||
|
||||
def __init__(self, entry, fs_cache, all_storage_ids):
|
||||
self.object_id = entry['id']
|
||||
self.is_folder = entry['is_folder']
|
||||
self.name = force_unicode(entry.get('name', '___'), 'utf-8')
|
||||
self.persistent_id = entry.get('persistent_id', self.object_id)
|
||||
self.size = entry.get('size', 0)
|
||||
# self.parent_id is None for storage objects
|
||||
self.parent_id = entry.get('parent_id', None)
|
||||
if self.parent_id == 0:
|
||||
sid = entry['storage_id']
|
||||
if sid not in all_storage_ids:
|
||||
sid = all_storage_ids[0]
|
||||
self.parent_id = sid
|
||||
self.is_hidden = entry.get('is_hidden', False)
|
||||
self.is_system = entry.get('is_system', False)
|
||||
self.can_delete = entry.get('can_delete', True)
|
||||
|
||||
self.files = []
|
||||
self.folders = []
|
||||
fs_cache.id_map[self.object_id] = self
|
||||
self.fs_cache = weakref.ref(fs_cache)
|
||||
|
||||
@property
|
||||
def id_map(self):
|
||||
return self.fs_cache().id_map
|
||||
|
||||
@property
|
||||
def parent(self):
|
||||
return None if self.parent_id is None else self.id_map[self.parent_id]
|
||||
|
||||
def __iter__(self):
|
||||
for e in self.folders:
|
||||
yield e
|
||||
for e in self.files:
|
||||
yield e
|
||||
|
||||
def dump(self, prefix='', out=sys.stdout):
|
||||
c = '+' if self.is_folder else '-'
|
||||
data = ('%s children'%(sum(map(len, (self.files, self.folders))))
|
||||
if self.is_folder else human_readable(self.size))
|
||||
line = '%s%s %s [id:%s %s]'%(prefix, c, self.name, self.object_id, data)
|
||||
prints(line, file=out)
|
||||
for c in (self.folders, self.files):
|
||||
for e in sorted(c, key=lambda x:sort_key(x.name)):
|
||||
e.dump(prefix=prefix+' ', out=out)
|
||||
|
||||
class FilesystemCache(object):
|
||||
|
||||
def __init__(self, all_storage, entries):
|
||||
self.entries = []
|
||||
self.id_map = {}
|
||||
|
||||
for storage in all_storage:
|
||||
e = FileOrFolder(storage, self, [])
|
||||
self.entries.append(e)
|
||||
|
||||
self.entries.sort(key=attrgetter('object_id'))
|
||||
all_storage_ids = [x.object_id for x in self.entries]
|
||||
|
||||
for entry in entries:
|
||||
FileOrFolder(entry, self, all_storage_ids)
|
||||
|
||||
for item in self.id_map.itervalues():
|
||||
p = item.parent
|
||||
if p is not None:
|
||||
t = p.folders if item.is_folder else p.files
|
||||
t.append(item)
|
||||
|
||||
def dump(self, out=sys.stdout):
|
||||
for e in self.entries:
|
||||
e.dump(out=out)
|
||||
|
||||
|
||||
@ -9,77 +9,13 @@ __docformat__ = 'restructuredtext en'
|
||||
|
||||
import time, operator
|
||||
from threading import RLock
|
||||
from itertools import chain
|
||||
from collections import deque, OrderedDict
|
||||
from io import BytesIO
|
||||
|
||||
from calibre import prints
|
||||
from calibre.devices.errors import OpenFailed, DeviceError
|
||||
from calibre.devices.mtp.base import MTPDeviceBase, synchronous
|
||||
from calibre.devices.mtp.filesystem_cache import FilesystemCache
|
||||
from calibre.devices.mtp.unix.detect import MTPDetect
|
||||
|
||||
class FilesystemCache(object):
|
||||
|
||||
def __init__(self, files, folders):
|
||||
self.files = files
|
||||
self.folders = folders
|
||||
self.file_id_map = {f['id']:f for f in files}
|
||||
self.folder_id_map = {f['id']:f for f in self.iterfolders(set_level=0)}
|
||||
|
||||
# Set the parents of each file
|
||||
self.files_in_root = OrderedDict()
|
||||
for f in files:
|
||||
parents = deque()
|
||||
pid = f['parent_id']
|
||||
while pid is not None and pid > 0:
|
||||
try:
|
||||
parent = self.folder_id_map[pid]
|
||||
except KeyError:
|
||||
break
|
||||
parents.appendleft(pid)
|
||||
pid = parent['parent_id']
|
||||
f['parents'] = parents
|
||||
if not parents:
|
||||
self.files_in_root[f['id']] = f
|
||||
|
||||
# Set the files in each folder
|
||||
for f in self.iterfolders():
|
||||
f['files'] = [i for i in files if i['parent_id'] ==
|
||||
f['id']]
|
||||
|
||||
# Decode the file and folder names
|
||||
for f in chain(files, folders):
|
||||
try:
|
||||
name = f['name'].decode('utf-8')
|
||||
except UnicodeDecodeError:
|
||||
name = 'undecodable_%d'%f['id']
|
||||
f['name'] = name
|
||||
|
||||
def iterfolders(self, folders=None, set_level=None):
|
||||
clevel = None if set_level is None else set_level + 1
|
||||
if folders is None:
|
||||
folders = self.folders
|
||||
for f in folders:
|
||||
if set_level is not None:
|
||||
f['level'] = set_level
|
||||
yield f
|
||||
for c in f['children']:
|
||||
for child in self.iterfolders([c], set_level=clevel):
|
||||
yield child
|
||||
|
||||
def dump_filesystem(self):
|
||||
indent = 2
|
||||
for f in self.iterfolders():
|
||||
prefix = ' '*(indent*f['level'])
|
||||
prints(prefix, '+', f['name'], 'id=%s'%f['id'])
|
||||
for leaf in f['files']:
|
||||
prints(prefix, ' '*indent, '-', leaf['name'],
|
||||
'id=%d'%leaf['id'], 'size=%d'%leaf['size'],
|
||||
'modtime=%d'%leaf['modtime'])
|
||||
for leaf in self.files_in_root.itervalues():
|
||||
prints('-', leaf['name'], 'id=%d'%leaf['id'],
|
||||
'size=%d'%leaf['size'], 'modtime=%d'%leaf['modtime'])
|
||||
|
||||
class MTP_DEVICE(MTPDeviceBase):
|
||||
|
||||
supported_platforms = ['linux']
|
||||
@ -87,7 +23,7 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
def __init__(self, *args, **kwargs):
|
||||
MTPDeviceBase.__init__(self, *args, **kwargs)
|
||||
self.dev = None
|
||||
self.filesystem_cache = None
|
||||
self._filesystem_cache = None
|
||||
self.lock = RLock()
|
||||
self.blacklisted_devices = set()
|
||||
|
||||
@ -129,7 +65,7 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
|
||||
@synchronous
|
||||
def post_yank_cleanup(self):
|
||||
self.dev = self.filesystem_cache = self.current_friendly_name = None
|
||||
self.dev = self._filesystem_cache = self.current_friendly_name = None
|
||||
|
||||
@synchronous
|
||||
def startup(self):
|
||||
@ -140,7 +76,7 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
|
||||
@synchronous
|
||||
def shutdown(self):
|
||||
self.dev = self.filesystem_cache = None
|
||||
self.dev = self._filesystem_cache = None
|
||||
|
||||
def format_errorstack(self, errs):
|
||||
return '\n'.join(['%d:%s'%(code, msg.decode('utf-8', 'replace')) for
|
||||
@ -148,7 +84,7 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
|
||||
@synchronous
|
||||
def open(self, connected_device, library_uuid):
|
||||
self.dev = self.filesystem_cache = None
|
||||
self.dev = self._filesystem_cache = None
|
||||
def blacklist_device():
|
||||
d = connected_device
|
||||
self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
|
||||
@ -179,11 +115,12 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
self._carda_id = storage[1]['id']
|
||||
if len(storage) > 2:
|
||||
self._cardb_id = storage[2]['id']
|
||||
self.current_friendly_name = self.dev.name
|
||||
self.current_friendly_name = self.dev.friendly_name
|
||||
|
||||
@synchronous
|
||||
def read_filesystem_cache(self):
|
||||
try:
|
||||
@property
|
||||
def filesystem_cache(self):
|
||||
if self._filesystem_cache is None:
|
||||
with self.lock:
|
||||
files, errs = self.dev.get_filelist(self)
|
||||
if errs and not files:
|
||||
raise DeviceError('Failed to read files from device. Underlying errors:\n'
|
||||
@ -192,10 +129,27 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
if errs and not folders:
|
||||
raise DeviceError('Failed to read folders from device. Underlying errors:\n'
|
||||
+self.format_errorstack(errs))
|
||||
self.filesystem_cache = FilesystemCache(files, folders)
|
||||
except:
|
||||
self.dev = self._main_id = self._carda_id = self._cardb_id = None
|
||||
raise
|
||||
storage = []
|
||||
for sid, capacity in zip([self._main_id, self._carda_id,
|
||||
self._cardb_id], self.total_space()):
|
||||
if sid is not None:
|
||||
name = _('Unknown')
|
||||
for x in self.dev.storage_info:
|
||||
if x['id'] == sid:
|
||||
name = x['name']
|
||||
break
|
||||
storage.append({'id':sid, 'size':capacity,
|
||||
'is_folder':True, 'name':name})
|
||||
all_folders = []
|
||||
def recurse(f):
|
||||
all_folders.append(f)
|
||||
for c in f['children']:
|
||||
recurse(c)
|
||||
|
||||
for f in folders: recurse(f)
|
||||
self._filesystem_cache = FilesystemCache(storage,
|
||||
all_folders+files)
|
||||
return self._filesystem_cache
|
||||
|
||||
@synchronous
|
||||
def get_device_information(self, end_session=True):
|
||||
@ -246,7 +200,6 @@ if __name__ == '__main__':
|
||||
devs = linux_scanner()
|
||||
mtp_devs = dev.detect(devs)
|
||||
dev.open(list(mtp_devs)[0], 'xxx')
|
||||
dev.read_filesystem_cache()
|
||||
d = dev.dev
|
||||
print ("Opened device:", dev.get_gui_name())
|
||||
print ("Storage info:")
|
||||
@ -257,7 +210,7 @@ if __name__ == '__main__':
|
||||
# fname = b'moose.txt'
|
||||
# src = BytesIO(raw)
|
||||
# print (d.put_file(dev._main_id, 0, fname, src, len(raw), PR()))
|
||||
dev.filesystem_cache.dump_filesystem()
|
||||
dev.filesystem_cache.dump()
|
||||
# with open('/tmp/flint.epub', 'wb') as f:
|
||||
# print(d.get_file(786, f, PR()))
|
||||
# print()
|
||||
|
||||
@ -55,7 +55,7 @@ static int report_progress(uint64_t const sent, uint64_t const total, void const
|
||||
cb = (ProgressCallback *)data;
|
||||
if (cb->obj != NULL) {
|
||||
PyEval_RestoreThread(cb->state);
|
||||
res = PyObject_CallMethod(cb->obj, "report_progress", "KK", sent, total);
|
||||
res = PyObject_CallFunction(cb->obj, "KK", sent, total);
|
||||
Py_XDECREF(res);
|
||||
cb->state = PyEval_SaveThread();
|
||||
}
|
||||
@ -315,7 +315,7 @@ libmtp_Device_storage_info(libmtp_Device *self, void *closure) {
|
||||
"capacity", storage->MaxCapacity,
|
||||
"freespace_bytes", storage->FreeSpaceInBytes,
|
||||
"freespace_objects", storage->FreeSpaceInObjects,
|
||||
"storage_desc", storage->StorageDescription,
|
||||
"name", storage->StorageDescription,
|
||||
"volume_id", storage->VolumeIdentifier
|
||||
);
|
||||
|
||||
@ -339,6 +339,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
|
||||
|
||||
|
||||
if (!PyArg_ParseTuple(args, "|O", &callback)) return NULL;
|
||||
if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
|
||||
cb.obj = callback;
|
||||
|
||||
ans = PyList_New(0);
|
||||
@ -357,13 +358,14 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
|
||||
}
|
||||
|
||||
for (f=tf; f != NULL; f=f->next) {
|
||||
fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
|
||||
fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k,s:O}",
|
||||
"id", f->item_id,
|
||||
"parent_id", f->parent_id,
|
||||
"storage_id", f->storage_id,
|
||||
"name", f->filename,
|
||||
"size", f->filesize,
|
||||
"modtime", f->modificationdate
|
||||
"modtime", f->modificationdate,
|
||||
"is_folder", Py_False
|
||||
);
|
||||
if (fo == NULL || PyList_Append(ans, fo) != 0) break;
|
||||
Py_DECREF(fo);
|
||||
@ -377,7 +379,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
|
||||
|
||||
if (callback != NULL) {
|
||||
// Bug in libmtp where it does not call callback with 100%
|
||||
fo = PyObject_CallMethod(callback, "report_progress", "KK", PyList_Size(ans), PyList_Size(ans));
|
||||
fo = PyObject_CallFunction(callback, "KK", PyList_Size(ans), PyList_Size(ans));
|
||||
Py_XDECREF(fo);
|
||||
}
|
||||
|
||||
@ -392,11 +394,12 @@ int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
|
||||
children = PyList_New(0);
|
||||
if (children == NULL) { PyErr_NoMemory(); return 1;}
|
||||
|
||||
folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
|
||||
folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:O,s:N}",
|
||||
"id", f->folder_id,
|
||||
"parent_id", f->parent_id,
|
||||
"storage_id", f->storage_id,
|
||||
"name", f->name,
|
||||
"is_folder", Py_True,
|
||||
"children", children);
|
||||
if (folder == NULL) return 1;
|
||||
PyList_Append(parent, folder);
|
||||
@ -454,6 +457,7 @@ libmtp_Device_get_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
|
||||
if (!PyArg_ParseTuple(args, "kO|O", &fileid, &stream, &callback)) return NULL;
|
||||
errs = PyList_New(0);
|
||||
if (errs == NULL) { PyErr_NoMemory(); return NULL; }
|
||||
if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
|
||||
|
||||
cb.obj = callback; cb.extra = stream;
|
||||
Py_XINCREF(callback); Py_INCREF(stream);
|
||||
@ -486,6 +490,7 @@ libmtp_Device_put_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
|
||||
if (!PyArg_ParseTuple(args, "kksOK|O", &storage_id, &parent_id, &name, &stream, &filesize, &callback)) return NULL;
|
||||
errs = PyList_New(0);
|
||||
if (errs == NULL) { PyErr_NoMemory(); return NULL; }
|
||||
if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
|
||||
|
||||
cb.obj = callback; cb.extra = stream;
|
||||
f.parent_id = parent_id; f.storage_id = storage_id; f.item_id = 0; f.filename = name; f.filetype = LIBMTP_FILETYPE_UNKNOWN; f.filesize = filesize;
|
||||
@ -599,7 +604,7 @@ static PyMethodDef libmtp_Device_methods[] = {
|
||||
},
|
||||
|
||||
{"get_filelist", (PyCFunction)libmtp_Device_get_filelist, METH_VARARGS,
|
||||
"get_filelist(callback=None) -> Get the list of files on the device. callback must be an object that has a method named 'report_progress(current, total)'. Returns files, errors."
|
||||
"get_filelist(callback=None) -> Get the list of files on the device. callback must be callable accepts arguments (current, total)'. Returns files, errors."
|
||||
},
|
||||
|
||||
{"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS,
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
namespace wpd {
|
||||
|
||||
static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { // {{{
|
||||
IPortableDeviceKeyCollection *properties;
|
||||
IPortableDeviceKeyCollection *properties = NULL;
|
||||
HRESULT hr;
|
||||
|
||||
Py_BEGIN_ALLOW_THREADS;
|
||||
@ -28,7 +28,7 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() {
|
||||
ADDPROP(WPD_OBJECT_PARENT_ID);
|
||||
ADDPROP(WPD_OBJECT_PERSISTENT_UNIQUE_ID);
|
||||
ADDPROP(WPD_OBJECT_NAME);
|
||||
ADDPROP(WPD_OBJECT_SYNC_ID);
|
||||
// ADDPROP(WPD_OBJECT_SYNC_ID);
|
||||
ADDPROP(WPD_OBJECT_ISSYSTEM);
|
||||
ADDPROP(WPD_OBJECT_ISHIDDEN);
|
||||
ADDPROP(WPD_OBJECT_CAN_DELETE);
|
||||
@ -87,8 +87,25 @@ static void set_content_type_property(PyObject *dict, IPortableDeviceValues *pro
|
||||
if (SUCCEEDED(properties->GetGuidValue(WPD_OBJECT_CONTENT_TYPE, &guid)) && IsEqualGUID(guid, WPD_CONTENT_TYPE_FOLDER)) is_folder = 1;
|
||||
PyDict_SetItemString(dict, "is_folder", (is_folder) ? Py_True : Py_False);
|
||||
}
|
||||
|
||||
static void set_properties(PyObject *obj, IPortableDeviceValues *values) {
|
||||
set_content_type_property(obj, values);
|
||||
|
||||
set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", values);
|
||||
set_string_property(obj, WPD_OBJECT_NAME, "name", values);
|
||||
// set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values);
|
||||
set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", values);
|
||||
|
||||
set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", values);
|
||||
set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", values);
|
||||
set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", values);
|
||||
|
||||
set_size_property(obj, WPD_OBJECT_SIZE, "size", values);
|
||||
}
|
||||
|
||||
// }}}
|
||||
|
||||
// Bulk get filesystem {{{
|
||||
class GetBulkCallback : public IPortableDevicePropertiesBulkCallback {
|
||||
|
||||
public:
|
||||
@ -154,18 +171,7 @@ public:
|
||||
}
|
||||
Py_DECREF(temp);
|
||||
|
||||
set_content_type_property(obj, properties);
|
||||
|
||||
set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", properties);
|
||||
set_string_property(obj, WPD_OBJECT_NAME, "name", properties);
|
||||
set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", properties);
|
||||
set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", properties);
|
||||
|
||||
set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", properties);
|
||||
set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", properties);
|
||||
set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", properties);
|
||||
|
||||
set_size_property(obj, WPD_OBJECT_SIZE, "size", properties);
|
||||
set_properties(obj, properties);
|
||||
|
||||
properties->Release(); properties = NULL;
|
||||
}
|
||||
@ -240,6 +246,9 @@ end:
|
||||
return folders;
|
||||
}
|
||||
|
||||
// }}}
|
||||
|
||||
// find_all_objects_in() {{{
|
||||
static BOOL find_all_objects_in(IPortableDeviceContent *content, IPortableDevicePropVariantCollection *object_ids, const wchar_t *parent_id) {
|
||||
/*
|
||||
* Find all children of the object identified by parent_id, recursively.
|
||||
@ -286,9 +295,82 @@ end:
|
||||
if (children != NULL) children->Release();
|
||||
PropVariantClear(&pv);
|
||||
return ok;
|
||||
} // }}}
|
||||
|
||||
// Single get filesystem {{{
|
||||
|
||||
static PyObject* get_object_properties(IPortableDeviceProperties *devprops, IPortableDeviceKeyCollection *properties, const wchar_t *object_id) {
|
||||
IPortableDeviceValues *values = NULL;
|
||||
HRESULT hr;
|
||||
PyObject *ans = NULL, *temp = NULL;
|
||||
|
||||
Py_BEGIN_ALLOW_THREADS;
|
||||
hr = devprops->GetValues(object_id, properties, &values);
|
||||
Py_END_ALLOW_THREADS;
|
||||
if (FAILED(hr)) { hresult_set_exc("Failed to get properties for object", hr); goto end; }
|
||||
|
||||
temp = wchar_to_unicode(object_id);
|
||||
if (temp == NULL) goto end;
|
||||
|
||||
ans = PyDict_New();
|
||||
if (ans == NULL) { PyErr_NoMemory(); goto end; }
|
||||
if (PyDict_SetItemString(ans, "id", temp) != 0) { Py_DECREF(ans); ans = NULL; PyErr_NoMemory(); goto end; }
|
||||
|
||||
set_properties(ans, values);
|
||||
|
||||
end:
|
||||
Py_XDECREF(temp);
|
||||
if (values != NULL) values->Release();
|
||||
return ans;
|
||||
}
|
||||
|
||||
PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) {
|
||||
static PyObject* single_get_filesystem(IPortableDeviceContent *content, const wchar_t *storage_id, IPortableDevicePropVariantCollection *object_ids) {
|
||||
DWORD num, i;
|
||||
PROPVARIANT pv;
|
||||
HRESULT hr;
|
||||
BOOL ok = 1;
|
||||
PyObject *ans = NULL, *item = NULL;
|
||||
IPortableDeviceProperties *devprops = NULL;
|
||||
IPortableDeviceKeyCollection *properties = NULL;
|
||||
|
||||
hr = content->Properties(&devprops);
|
||||
if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
|
||||
|
||||
properties = create_filesystem_properties_collection();
|
||||
if (properties == NULL) goto end;
|
||||
|
||||
hr = object_ids->GetCount(&num);
|
||||
if (FAILED(hr)) { hresult_set_exc("Failed to get object id count", hr); goto end; }
|
||||
|
||||
ans = PyDict_New();
|
||||
if (ans == NULL) goto end;
|
||||
|
||||
for (i = 0; i < num; i++) {
|
||||
ok = 0;
|
||||
PropVariantInit(&pv);
|
||||
hr = object_ids->GetAt(i, &pv);
|
||||
if (SUCCEEDED(hr) && pv.pwszVal != NULL) {
|
||||
item = get_object_properties(devprops, properties, pv.pwszVal);
|
||||
if (item != NULL) {
|
||||
PyDict_SetItem(ans, PyDict_GetItemString(item, "id"), item);
|
||||
Py_DECREF(item); item = NULL;
|
||||
ok = 1;
|
||||
}
|
||||
} else hresult_set_exc("Failed to get item from IPortableDevicePropVariantCollection", hr);
|
||||
|
||||
PropVariantClear(&pv);
|
||||
if (!ok) { Py_DECREF(ans); ans = NULL; break; }
|
||||
}
|
||||
|
||||
end:
|
||||
if (devprops != NULL) devprops->Release();
|
||||
if (properties != NULL) properties->Release();
|
||||
|
||||
return ans;
|
||||
}
|
||||
// }}}
|
||||
|
||||
PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { // {{{
|
||||
PyObject *folders = NULL;
|
||||
IPortableDevicePropVariantCollection *object_ids = NULL;
|
||||
IPortableDeviceContent *content = NULL;
|
||||
@ -310,12 +392,112 @@ PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id
|
||||
if (!ok) goto end;
|
||||
|
||||
if (bulk_properties != NULL) folders = bulk_get_filesystem(device, bulk_properties, storage_id, object_ids);
|
||||
else folders = single_get_filesystem(content, storage_id, object_ids);
|
||||
|
||||
end:
|
||||
if (content != NULL) content->Release();
|
||||
if (object_ids != NULL) object_ids->Release();
|
||||
|
||||
return folders;
|
||||
} // }}}
|
||||
|
||||
PyObject* wpd::get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback) { // {{{
|
||||
IPortableDeviceContent *content = NULL;
|
||||
IPortableDeviceResources *resources = NULL;
|
||||
IPortableDeviceProperties *devprops = NULL;
|
||||
IPortableDeviceValues *values = NULL;
|
||||
IPortableDeviceKeyCollection *properties = NULL;
|
||||
IStream *stream = NULL;
|
||||
HRESULT hr;
|
||||
DWORD bufsize = 4096;
|
||||
char *buf = NULL;
|
||||
ULONG bytes_read = 0, total_read = 0;
|
||||
BOOL ok = FALSE;
|
||||
PyObject *res = NULL;
|
||||
ULONGLONG filesize = 0;
|
||||
|
||||
Py_BEGIN_ALLOW_THREADS;
|
||||
hr = device->Content(&content);
|
||||
Py_END_ALLOW_THREADS;
|
||||
if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; }
|
||||
|
||||
Py_BEGIN_ALLOW_THREADS;
|
||||
hr = content->Properties(&devprops);
|
||||
Py_END_ALLOW_THREADS;
|
||||
if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
|
||||
|
||||
Py_BEGIN_ALLOW_THREADS;
|
||||
hr = CoCreateInstance(CLSID_PortableDeviceKeyCollection, NULL,
|
||||
CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&properties));
|
||||
Py_END_ALLOW_THREADS;
|
||||
if (FAILED(hr)) { hresult_set_exc("Failed to create filesystem properties collection", hr); goto end; }
|
||||
hr = properties->Add(WPD_OBJECT_SIZE);
|
||||
if (FAILED(hr)) { hresult_set_exc("Failed to add filesize property to properties collection", hr); goto end; }
|
||||
|
||||
Py_BEGIN_ALLOW_THREADS;
|
||||
hr = devprops->GetValues(object_id, properties, &values);
|
||||
Py_END_ALLOW_THREADS;
|
||||
if (FAILED(hr)) { hresult_set_exc("Failed to get filesize for object", hr); goto end; }
|
||||
hr = values->GetUnsignedLargeIntegerValue(WPD_OBJECT_SIZE, &filesize);
|
||||
if (FAILED(hr)) { hresult_set_exc("Failed to get filesize from values collection", hr); goto end; }
|
||||
|
||||
Py_BEGIN_ALLOW_THREADS;
|
||||
hr = content->Transfer(&resources);
|
||||
Py_END_ALLOW_THREADS;
|
||||
if (FAILED(hr)) { hresult_set_exc("Failed to create resources interface", hr); goto end; }
|
||||
|
||||
Py_BEGIN_ALLOW_THREADS;
|
||||
hr = resources->GetStream(object_id, WPD_RESOURCE_DEFAULT, STGM_READ, &bufsize, &stream);
|
||||
Py_END_ALLOW_THREADS;
|
||||
if (FAILED(hr)) {
|
||||
if (HRESULT_FROM_WIN32(ERROR_BUSY) == hr) {
|
||||
PyErr_SetString(WPDFileBusy, "Object is in use");
|
||||
} else hresult_set_exc("Failed to create stream interface to read from object", hr);
|
||||
goto end;
|
||||
}
|
||||
|
||||
buf = (char *)calloc(bufsize+10, 1);
|
||||
if (buf == NULL) { PyErr_NoMemory(); goto end; }
|
||||
|
||||
while (TRUE) {
|
||||
bytes_read = 0;
|
||||
Py_BEGIN_ALLOW_THREADS;
|
||||
hr = stream->Read(buf, bufsize, &bytes_read);
|
||||
Py_END_ALLOW_THREADS;
|
||||
total_read = total_read + bytes_read;
|
||||
if (hr == STG_E_ACCESSDENIED) {
|
||||
PyErr_SetString(PyExc_IOError, "Read access is denied to this object"); break;
|
||||
} else if (hr == S_OK || hr == S_FALSE) {
|
||||
if (bytes_read > 0) {
|
||||
res = PyObject_CallMethod(dest, "write", "s#", buf, bytes_read);
|
||||
if (res == NULL) break;
|
||||
Py_DECREF(res); res = NULL;
|
||||
if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_read, filesize));
|
||||
}
|
||||
} else { hresult_set_exc("Failed to read file from device", hr); break; }
|
||||
|
||||
if (hr == S_FALSE || bytes_read < bufsize) {
|
||||
ok = TRUE;
|
||||
Py_XDECREF(PyObject_CallMethod(dest, "flush", NULL));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ok && total_read != filesize) {
|
||||
ok = FALSE;
|
||||
PyErr_SetString(WPDError, "Failed to read all data from file");
|
||||
}
|
||||
|
||||
end:
|
||||
if (content != NULL) content->Release();
|
||||
if (devprops != NULL) devprops->Release();
|
||||
if (resources != NULL) resources->Release();
|
||||
if (stream != NULL) stream->Release();
|
||||
if (values != NULL) values->Release();
|
||||
if (properties != NULL) properties->Release();
|
||||
if (buf != NULL) free(buf);
|
||||
if (!ok) return NULL;
|
||||
Py_RETURN_NONE;
|
||||
} // }}}
|
||||
|
||||
} // namespace wpd
|
||||
|
||||
@ -78,7 +78,7 @@ update_data(Device *self, PyObject *args, PyObject *kwargs) {
|
||||
// get_filesystem() {{{
|
||||
static PyObject*
|
||||
py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) {
|
||||
PyObject *storage_id, *ans = NULL;
|
||||
PyObject *storage_id;
|
||||
wchar_t *storage;
|
||||
|
||||
if (!PyArg_ParseTuple(args, "O", &storage_id)) return NULL;
|
||||
@ -88,6 +88,21 @@ py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) {
|
||||
return wpd::get_filesystem(self->device, storage, self->bulk_properties);
|
||||
} // }}}
|
||||
|
||||
// get_file() {{{
|
||||
static PyObject*
|
||||
py_get_file(Device *self, PyObject *args, PyObject *kwargs) {
|
||||
PyObject *object_id, *stream, *callback = NULL;
|
||||
wchar_t *object;
|
||||
|
||||
if (!PyArg_ParseTuple(args, "OO|O", &object_id, &stream, &callback)) return NULL;
|
||||
object = unicode_to_wchar(object_id);
|
||||
if (object == NULL) return NULL;
|
||||
|
||||
if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
|
||||
|
||||
return wpd::get_file(self->device, object, stream, callback);
|
||||
} // }}}
|
||||
|
||||
static PyMethodDef Device_methods[] = {
|
||||
{"update_data", (PyCFunction)update_data, METH_VARARGS,
|
||||
"update_data() -> Reread the basic device data from the device (total, space, free space, storage locations, etc.)"
|
||||
@ -97,6 +112,10 @@ static PyMethodDef Device_methods[] = {
|
||||
"get_filesystem(storage_id) -> Get all files/folders on the storage identified by storage_id. Tries to use bulk operations when possible."
|
||||
},
|
||||
|
||||
{"get_file", (PyCFunction)py_get_file, METH_VARARGS,
|
||||
"get_file(object_id, stream, callback=None) -> Get the file identified by object_id from the device. The file is written to the stream object, which must be a file like object. If callback is not None, it must be a callable that accepts two arguments: (bytes_read, total_size). It will be called after each chunk is read from the device. Note that it can be called multiple times with the same values."
|
||||
},
|
||||
|
||||
{NULL}
|
||||
};
|
||||
|
||||
|
||||
@ -7,13 +7,32 @@ __license__ = 'GPL v3'
|
||||
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
import time
|
||||
from threading import RLock
|
||||
import time, threading
|
||||
from functools import wraps
|
||||
from future_builtins import zip
|
||||
from itertools import chain
|
||||
|
||||
from calibre import as_unicode, prints
|
||||
from calibre.constants import plugins, __appname__, numeric_version
|
||||
from calibre.ptempfile import SpooledTemporaryFile
|
||||
from calibre.devices.errors import OpenFailed
|
||||
from calibre.devices.mtp.base import MTPDeviceBase, synchronous
|
||||
from calibre.devices.mtp.base import MTPDeviceBase
|
||||
from calibre.devices.mtp.filesystem_cache import FilesystemCache
|
||||
|
||||
class ThreadingViolation(Exception):
|
||||
|
||||
def __init__(self):
|
||||
Exception.__init__('You cannot use the MTP driver from a thread other than the '
|
||||
' thread in which startup() was called')
|
||||
|
||||
def same_thread(func):
|
||||
@wraps(func)
|
||||
def check_thread(self, *args, **kwargs):
|
||||
if self.start_thread is not threading.current_thread():
|
||||
raise ThreadingViolation()
|
||||
return func(self, *args, **kwargs)
|
||||
return check_thread
|
||||
|
||||
|
||||
class MTP_DEVICE(MTPDeviceBase):
|
||||
|
||||
@ -22,7 +41,6 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
def __init__(self, *args, **kwargs):
|
||||
MTPDeviceBase.__init__(self, *args, **kwargs)
|
||||
self.dev = None
|
||||
self.lock = RLock()
|
||||
self.blacklisted_devices = set()
|
||||
self.ejected_devices = set()
|
||||
self.currently_connected_pnp_id = None
|
||||
@ -31,9 +49,11 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
self.last_refresh_devices_time = time.time()
|
||||
self.wpd = self.wpd_error = None
|
||||
self._main_id = self._carda_id = self._cardb_id = None
|
||||
self.start_thread = None
|
||||
self._filesystem_cache = None
|
||||
|
||||
@synchronous
|
||||
def startup(self):
|
||||
self.start_thread = threading.current_thread()
|
||||
self.wpd, self.wpd_error = plugins['wpd']
|
||||
if self.wpd is not None:
|
||||
try:
|
||||
@ -46,13 +66,13 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
except Exception as e:
|
||||
self.wpd_error = as_unicode(e)
|
||||
|
||||
@synchronous
|
||||
@same_thread
|
||||
def shutdown(self):
|
||||
self.dev = self.filesystem_cache = None
|
||||
self.dev = self._filesystem_cache = self.start_thread = None
|
||||
if self.wpd is not None:
|
||||
self.wpd.uninit()
|
||||
|
||||
@synchronous
|
||||
@same_thread
|
||||
def detect_managed_devices(self, devices_on_system):
|
||||
if self.wpd is None: return None
|
||||
|
||||
@ -119,23 +139,45 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
|
||||
return True
|
||||
|
||||
@synchronous
|
||||
@property
|
||||
def filesystem_cache(self):
|
||||
if self._filesystem_cache is None:
|
||||
ts = self.total_space()
|
||||
all_storage = []
|
||||
items = []
|
||||
for storage_id, capacity in zip([self._main_id, self._carda_id,
|
||||
self._cardb_id], ts):
|
||||
if storage_id is None: continue
|
||||
name = _('Unknown')
|
||||
for s in self.dev.data['storage']:
|
||||
if s['id'] == storage_id:
|
||||
name = s['name']
|
||||
break
|
||||
storage = {'id':storage_id, 'size':capacity, 'name':name,
|
||||
'is_folder':True}
|
||||
id_map = self.dev.get_filesystem(storage_id)
|
||||
all_storage.append(storage)
|
||||
items.append(id_map.itervalues())
|
||||
self._filesystem_cache = FilesystemCache(all_storage, chain(*items))
|
||||
return self._filesystem_cache
|
||||
|
||||
@same_thread
|
||||
def post_yank_cleanup(self):
|
||||
self.currently_connected_pnp_id = self.current_friendly_name = None
|
||||
self._main_id = self._carda_id = self._cardb_id = None
|
||||
self.dev = self.filesystem_cache = None
|
||||
self.dev = self._filesystem_cache = None
|
||||
|
||||
@synchronous
|
||||
@same_thread
|
||||
def eject(self):
|
||||
if self.currently_connected_pnp_id is None: return
|
||||
self.ejected_devices.add(self.currently_connected_pnp_id)
|
||||
self.currently_connected_pnp_id = self.current_friendly_name = None
|
||||
self._main_id = self._carda_id = self._cardb_id = None
|
||||
self.dev = self.filesystem_cache = None
|
||||
self.dev = self._filesystem_cache = None
|
||||
|
||||
@synchronous
|
||||
@same_thread
|
||||
def open(self, connected_device, library_uuid):
|
||||
self.dev = self.filesystem_cache = None
|
||||
self.dev = self._filesystem_cache = None
|
||||
try:
|
||||
self.dev = self.wpd.Device(connected_device)
|
||||
except self.wpd.WPDError:
|
||||
@ -158,13 +200,13 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
self._cardb_id = storage[2]['id']
|
||||
self.current_friendly_name = devdata.get('friendly_name', None)
|
||||
|
||||
@synchronous
|
||||
@same_thread
|
||||
def get_device_information(self, end_session=True):
|
||||
d = self.dev.data
|
||||
dv = d.get('device_version', '')
|
||||
return (self.current_friendly_name, dv, dv, '')
|
||||
|
||||
@synchronous
|
||||
@same_thread
|
||||
def card_prefix(self, end_session=True):
|
||||
ans = [None, None]
|
||||
if self._carda_id is not None:
|
||||
@ -173,7 +215,7 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
ans[1] = 'mtp:::%s:::'%self._cardb_id
|
||||
return tuple(ans)
|
||||
|
||||
@synchronous
|
||||
@same_thread
|
||||
def total_space(self, end_session=True):
|
||||
ans = [0, 0, 0]
|
||||
dd = self.dev.data
|
||||
@ -184,7 +226,7 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
ans[i] = s['capacity']
|
||||
return tuple(ans)
|
||||
|
||||
@synchronous
|
||||
@same_thread
|
||||
def free_space(self, end_session=True):
|
||||
self.dev.update_data()
|
||||
ans = [0, 0, 0]
|
||||
@ -196,5 +238,14 @@ class MTP_DEVICE(MTPDeviceBase):
|
||||
ans[i] = s['free_space']
|
||||
return tuple(ans)
|
||||
|
||||
|
||||
@same_thread
|
||||
def get_file(self, object_id, stream=None, callback=None):
|
||||
if stream is None:
|
||||
stream = SpooledTemporaryFile(5*1024*1024, '_wpd_receive_file.dat')
|
||||
try:
|
||||
self.dev.get_file(object_id, stream, callback)
|
||||
except self.wpd.WPDFileBusy:
|
||||
time.sleep(2)
|
||||
self.dev.get_file(object_id, stream, callback)
|
||||
return stream
|
||||
|
||||
|
||||
@ -20,7 +20,7 @@
|
||||
namespace wpd {
|
||||
|
||||
// Module exception types
|
||||
extern PyObject *WPDError, *NoWPD;
|
||||
extern PyObject *WPDError, *NoWPD, *WPDFileBusy;
|
||||
|
||||
// The global device manager
|
||||
extern IPortableDeviceManager *portable_device_manager;
|
||||
@ -50,13 +50,14 @@ extern PyTypeObject DeviceType;
|
||||
// Utility functions
|
||||
PyObject *hresult_set_exc(const char *msg, HRESULT hr);
|
||||
wchar_t *unicode_to_wchar(PyObject *o);
|
||||
PyObject *wchar_to_unicode(wchar_t *o);
|
||||
PyObject *wchar_to_unicode(const wchar_t *o);
|
||||
int pump_waiting_messages();
|
||||
|
||||
extern IPortableDeviceValues* get_client_information();
|
||||
extern IPortableDevice* open_device(const wchar_t *pnp_id, IPortableDeviceValues *client_information);
|
||||
extern PyObject* get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **bulk_properties);
|
||||
extern PyObject* get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties);
|
||||
extern PyObject* get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -7,8 +7,8 @@ __license__ = 'GPL v3'
|
||||
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
import subprocess, sys, os, pprint, signal, time, glob
|
||||
pprint
|
||||
import subprocess, sys, os, pprint, signal, time, glob, io
|
||||
pprint, io
|
||||
|
||||
def build(mod='wpd'):
|
||||
master = subprocess.Popen('ssh -MN getafix'.split())
|
||||
@ -70,7 +70,10 @@ def main():
|
||||
print ('Connected to:', dev.get_gui_name())
|
||||
print ('Total space', dev.total_space())
|
||||
print ('Free space', dev.free_space())
|
||||
pprint.pprint(dev.dev.get_filesystem(dev._main_id))
|
||||
dev.filesystem_cache.dump()
|
||||
# print ('Fetching file: oFF (198214 bytes)')
|
||||
# stream = dev.get_file('oFF')
|
||||
# print ("Fetched size: ", stream.tell())
|
||||
finally:
|
||||
dev.shutdown()
|
||||
|
||||
|
||||
@ -43,7 +43,7 @@ wchar_t *wpd::unicode_to_wchar(PyObject *o) {
|
||||
return buf;
|
||||
}
|
||||
|
||||
PyObject *wpd::wchar_to_unicode(wchar_t *o) {
|
||||
PyObject *wpd::wchar_to_unicode(const wchar_t *o) {
|
||||
PyObject *ans;
|
||||
if (o == NULL) return NULL;
|
||||
ans = PyUnicode_FromWideChar(o, wcslen(o));
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
using namespace wpd;
|
||||
|
||||
// Module exception types
|
||||
PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL;
|
||||
PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL, *wpd::WPDFileBusy = NULL;
|
||||
|
||||
// The global device manager
|
||||
IPortableDeviceManager *wpd::portable_device_manager = NULL;
|
||||
@ -199,6 +199,9 @@ initwpd(void) {
|
||||
NoWPD = PyErr_NewException("wpd.NoWPD", NULL, NULL);
|
||||
if (NoWPD == NULL) return;
|
||||
|
||||
WPDFileBusy = PyErr_NewException("wpd.WPDFileBusy", NULL, NULL);
|
||||
if (WPDFileBusy == NULL) return;
|
||||
|
||||
Py_INCREF(&DeviceType);
|
||||
PyModule_AddObject(m, "Device", (PyObject *)&DeviceType);
|
||||
|
||||
|
||||
@ -50,10 +50,10 @@ class PRST1(USBMS):
|
||||
|
||||
VENDOR_NAME = 'SONY'
|
||||
WINDOWS_MAIN_MEM = re.compile(
|
||||
r'(PRS-T1&)'
|
||||
r'(PRS-T(1|2)&)'
|
||||
)
|
||||
WINDOWS_CARD_A_MEM = re.compile(
|
||||
r'(PRS-T1__SD&)'
|
||||
r'(PRS-T(1|2)__SD&)'
|
||||
)
|
||||
MAIN_MEMORY_VOLUME_LABEL = 'SONY Reader Main Memory'
|
||||
STORAGE_CARD_VOLUME_LABEL = 'SONY Reader Storage Card'
|
||||
|
||||
@ -235,7 +235,7 @@ class Serializer(object):
|
||||
itemhref = re.sub(r'article_\d+/', '', itemhref)
|
||||
self.href_offsets[itemhref].append(buf.tell())
|
||||
buf.write('0000000000')
|
||||
buf.write(' ><font size="+1" color="blue"><b><u>')
|
||||
buf.write(' ><font size="+1"><b><u>')
|
||||
t = tocitem.title
|
||||
if isinstance(t, unicode):
|
||||
t = t.encode('utf-8')
|
||||
|
||||
@ -23,20 +23,17 @@ class MathJax
|
||||
this.pending_cfi = null
|
||||
this.hub = null
|
||||
|
||||
load_mathjax: (script) ->
|
||||
load_mathjax: (user_config) ->
|
||||
if this.base == null
|
||||
log('You must specify the path to the MathJax installation before trying to load MathJax')
|
||||
return null
|
||||
|
||||
created = false
|
||||
if script == null
|
||||
script = document.createElement('script')
|
||||
created = true
|
||||
|
||||
script.type = 'text/javascript'
|
||||
script.src = 'file://' + this.base + '/MathJax.js'
|
||||
|
||||
script.text = script.text + '''
|
||||
script.text = user_config + '''
|
||||
MathJax.Hub.signal.Interest(function (message) {if (String(message).match(/error/i)) {console.log(message)}});
|
||||
MathJax.Hub.Config({
|
||||
positionToHash: false,
|
||||
showMathMenu: false,
|
||||
@ -50,8 +47,6 @@ class MathJax
|
||||
MathJax.Hub.Register.StartupHook("End", window.mathjax.load_finished);
|
||||
window.mathjax.hub = MathJax.Hub
|
||||
'''
|
||||
|
||||
if created
|
||||
document.head.appendChild(script)
|
||||
|
||||
load_finished: () =>
|
||||
@ -67,14 +62,17 @@ class MathJax
|
||||
this.math_present = false
|
||||
this.math_loaded = false
|
||||
this.pending_cfi = null
|
||||
user_config = ''
|
||||
for c in document.getElementsByTagName('script')
|
||||
if c.getAttribute('type') == 'text/x-mathjax-config'
|
||||
if c.text
|
||||
user_config += c.text
|
||||
script = c
|
||||
break
|
||||
c.parentNode.removeChild(c)
|
||||
|
||||
if script != null or document.getElementsByTagName('math').length > 0
|
||||
this.math_present = true
|
||||
this.load_mathjax(script)
|
||||
this.load_mathjax(user_config)
|
||||
|
||||
after_resize: () ->
|
||||
if not this.math_present or this.hub == null
|
||||
|
||||
@ -334,6 +334,16 @@ class PagedDisplay
|
||||
elem = elems[0]
|
||||
if not elem
|
||||
return
|
||||
if window.mathjax?.math_present
|
||||
# MathJax links to children of SVG tags and scrollIntoView doesn't
|
||||
# work properly for them, so if this link points to something
|
||||
# inside an <svg> tag we instead scroll the parent of the svg tag
|
||||
# into view.
|
||||
parent = elem
|
||||
while parent and parent?.tagName?.toLowerCase() != 'svg'
|
||||
parent = parent.parentNode
|
||||
if parent?.tagName?.toLowerCase() == 'svg'
|
||||
elem = parent.parentNode
|
||||
elem.scrollIntoView()
|
||||
if this.in_paged_mode
|
||||
# Ensure we are scrolled to the column containing elem
|
||||
@ -368,7 +378,9 @@ class PagedDisplay
|
||||
# The Conformal Fragment Identifier at the current position, returns
|
||||
# null if it could not be calculated. Requires the cfi.coffee library.
|
||||
ans = null
|
||||
if not window.cfi?
|
||||
if not window.cfi? or (window.mathjax?.math_present and not window.mathjax?.math_loaded)
|
||||
# If MathJax is loading, it is changing the DOM, so we cannot
|
||||
# reliably generate a CFI
|
||||
return ans
|
||||
if this.in_paged_mode
|
||||
c = this.current_column_location()
|
||||
@ -402,9 +414,9 @@ class PagedDisplay
|
||||
return ans
|
||||
|
||||
click_for_page_turn: (event) ->
|
||||
# Check if the click event event should generate a apge turn. Returns
|
||||
# Check if the click event should generate a page turn. Returns
|
||||
# null if it should not, true if it is a backwards page turn, false if
|
||||
# it is a forward apge turn.
|
||||
# it is a forward page turn.
|
||||
left_boundary = this.current_margin_side
|
||||
right_bondary = this.screen_width - this.current_margin_side
|
||||
if left_boundary > event.clientX
|
||||
|
||||
@ -34,26 +34,29 @@ class BNStore(BasicStoreConfig, StorePlugin):
|
||||
d.exec_()
|
||||
|
||||
def search(self, query, max_results=10, timeout=60):
|
||||
url = 'http://www.barnesandnoble.com/s/%s?keyword=%s&store=ebook' % (query.replace(' ', '-'), urllib.quote_plus(query))
|
||||
url = 'http://www.barnesandnoble.com/s/%s?keyword=%s&store=ebook&view=list' % (query.replace(' ', '-'), urllib.quote_plus(query))
|
||||
|
||||
br = browser()
|
||||
|
||||
counter = max_results
|
||||
with closing(br.open(url, timeout=timeout)) as f:
|
||||
doc = html.fromstring(f.read())
|
||||
raw = f.read()
|
||||
doc = html.fromstring(raw)
|
||||
for data in doc.xpath('//ul[contains(@class, "result-set")]/li[contains(@class, "result")]'):
|
||||
if counter <= 0:
|
||||
break
|
||||
|
||||
id = ''.join(data.xpath('.//div[contains(@class, "image-bounding-box")]/a/@href'))
|
||||
id = ''.join(data.xpath('.//div[contains(@class, "image-block")]/a/@href'))
|
||||
if not id:
|
||||
continue
|
||||
|
||||
cover_url = ''.join(data.xpath('.//img[contains(@class, "product-image")]/@src'))
|
||||
|
||||
title = ''.join(data.xpath('.//a[@class="title"]//text()'))
|
||||
author = ', '.join(data.xpath('.//a[@class="contributor"]//text()'))
|
||||
price = ''.join(data.xpath('.//div[@class="price-format"]//span[contains(@class, "price")]/text()'))
|
||||
title = ''.join(data.xpath('descendant::p[@class="title"]//span[@class="name"]//text()')).strip()
|
||||
if not title: continue
|
||||
|
||||
author = ', '.join(data.xpath('.//ul[@class="contributors"]//a[@class="subtle"]//text()')).strip()
|
||||
price = ''.join(data.xpath('.//a[contains(@class, "bn-price")]//text()'))
|
||||
|
||||
counter -= 1
|
||||
|
||||
|
||||
@ -185,6 +185,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
|
||||
self.pos.setDecimals(1)
|
||||
self.pos.setSuffix('/'+_('Unknown')+' ')
|
||||
self.pos.setMinimum(1.)
|
||||
self.splitter.setCollapsible(0, False)
|
||||
self.splitter.setCollapsible(1, False)
|
||||
self.pos.setMinimumWidth(150)
|
||||
self.tool_bar2.insertWidget(self.action_find_next, self.pos)
|
||||
self.reference = Reference()
|
||||
@ -1028,6 +1030,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
|
||||
av = available_height() - 30
|
||||
if self.height() > av:
|
||||
self.resize(self.width(), av)
|
||||
self.splitter.setCollapsible(0, False)
|
||||
self.splitter.setCollapsible(1, False)
|
||||
|
||||
def config(defaults=None):
|
||||
desc = _('Options to control the ebook viewer')
|
||||
|
||||
@ -27,7 +27,14 @@
|
||||
<property name="childrenCollapsible">
|
||||
<bool>false</bool>
|
||||
</property>
|
||||
<widget class="TOCView" name="toc"/>
|
||||
<widget class="TOCView" name="toc">
|
||||
<property name="minimumSize">
|
||||
<size>
|
||||
<width>150</width>
|
||||
<height>0</height>
|
||||
</size>
|
||||
</property>
|
||||
</widget>
|
||||
<widget class="QFrame" name="frame">
|
||||
<property name="frameShape">
|
||||
<enum>QFrame::StyledPanel</enum>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user