From bb70db27558d03d317caf2347da9409ae937c3f3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Aug 2012 11:28:32 +0530 Subject: [PATCH 01/22] Ebook viewer: Prevent using the splitter to hide the ToC --- src/calibre/gui2/viewer/main.py | 4 ++++ src/calibre/gui2/viewer/main.ui | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index 6e4d129eaf..a63fb2c8c6 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -185,6 +185,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer): self.pos.setDecimals(1) self.pos.setSuffix('/'+_('Unknown')+' ') self.pos.setMinimum(1.) + self.splitter.setCollapsible(0, False) + self.splitter.setCollapsible(1, False) self.pos.setMinimumWidth(150) self.tool_bar2.insertWidget(self.action_find_next, self.pos) self.reference = Reference() @@ -1028,6 +1030,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer): av = available_height() - 30 if self.height() > av: self.resize(self.width(), av) + self.splitter.setCollapsible(0, False) + self.splitter.setCollapsible(1, False) def config(defaults=None): desc = _('Options to control the ebook viewer') diff --git a/src/calibre/gui2/viewer/main.ui b/src/calibre/gui2/viewer/main.ui index fe0fa62a79..ddc4cfb776 100644 --- a/src/calibre/gui2/viewer/main.ui +++ b/src/calibre/gui2/viewer/main.ui @@ -27,7 +27,14 @@ false - + + + + 150 + 0 + + + QFrame::StyledPanel From ebbfda816d7247a8c973fb16dfb6b2c3b71d8f3a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Aug 2012 12:21:13 +0530 Subject: [PATCH 02/22] MathJax: Fix equation refs not scrolling to the correct position in flow mode. Also handle multiple mathjax config script tags. --- src/calibre/ebooks/oeb/display/mathjax.coffee | 24 +++++++++---------- src/calibre/ebooks/oeb/display/paged.coffee | 10 ++++++++ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/calibre/ebooks/oeb/display/mathjax.coffee b/src/calibre/ebooks/oeb/display/mathjax.coffee index 50a75cca6b..ad893baa7e 100644 --- a/src/calibre/ebooks/oeb/display/mathjax.coffee +++ b/src/calibre/ebooks/oeb/display/mathjax.coffee @@ -23,24 +23,21 @@ class MathJax this.pending_cfi = null this.hub = null - load_mathjax: (script) -> + load_mathjax: (user_config) -> if this.base == null log('You must specify the path to the MathJax installation before trying to load MathJax') return null - created = false - if script == null - script = document.createElement('script') - created = true + script = document.createElement('script') script.type = 'text/javascript' script.src = 'file://' + this.base + '/MathJax.js' - - script.text = script.text + ''' + script.text = user_config + ''' + MathJax.Hub.signal.Interest(function (message) {if (String(message).match(/error/i)) {console.log(message)}}); MathJax.Hub.Config({ positionToHash: false, showMathMenu: false, - extensions: ["tex2jax.js","asciimath2jax.js","mml2jax.js"], + extensions: ["tex2jax.js", "asciimath2jax.js", "mml2jax.js"], jax: ["input/TeX","input/MathML","input/AsciiMath","output/SVG"], TeX: { extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"] @@ -50,9 +47,7 @@ class MathJax MathJax.Hub.Register.StartupHook("End", window.mathjax.load_finished); window.mathjax.hub = MathJax.Hub ''' - - if created - document.head.appendChild(script) + document.head.appendChild(script) load_finished: () => log('MathJax load finished!') @@ -67,14 +62,17 @@ class MathJax this.math_present = false this.math_loaded = false this.pending_cfi = null + user_config = '' for c in document.getElementsByTagName('script') if c.getAttribute('type') == 'text/x-mathjax-config' + if c.text + user_config += c.text script = c - break + c.parentNode.removeChild(c) if script != null or document.getElementsByTagName('math').length > 0 this.math_present = true - this.load_mathjax(script) + this.load_mathjax(user_config) after_resize: () -> if not this.math_present or this.hub == null diff --git a/src/calibre/ebooks/oeb/display/paged.coffee b/src/calibre/ebooks/oeb/display/paged.coffee index de6645ce88..a26f5931c9 100644 --- a/src/calibre/ebooks/oeb/display/paged.coffee +++ b/src/calibre/ebooks/oeb/display/paged.coffee @@ -334,6 +334,16 @@ class PagedDisplay elem = elems[0] if not elem return + if window.mathjax?.math_present + # MathJax links to children of SVG tags and scrollIntoView doesn't + # work properly for them, so if this link points to something + # inside an tag we instead scroll the parent of the svg tag + # into view. + parent = elem + while parent and parent?.tagName?.toLowerCase() != 'svg' + parent = parent.parentNode + if parent?.tagName?.toLowerCase() == 'svg' + elem = parent.parentNode elem.scrollIntoView() if this.in_paged_mode # Ensure we are scrolled to the column containing elem From a5590458accaee68b6f3bf8694d544495f4be838 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Aug 2012 12:24:58 +0530 Subject: [PATCH 03/22] ... --- resources/compiled_coffeescript.zip | Bin 56255 -> 56865 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index cae1c6c63e5aaeb3bff1ee3892915ea245ed6ade..1b22ec5b04170cc54e829f7a11e5cff2635db40a 100644 GIT binary patch delta 903 zcmaiz&rcIU6vvmgQY{>6XtA-PuR!RoC4+4|sH7-C@E4*66OzVA*6nn=&F&8DP74Vk zXz*aT7Xv_F+aG&) z&mV{g-iRrYQM=lAV|79hRwta%&eE0OAk4`YHSzV^9&~PgTfZE@gRK!fmnq9!F+YG}eCr3Z7UOAbF%EKi$c%WgbAF9vYL2?{{+JMV%&K%{<7n=civ7I3zbem@SYw7){UuJ=)>?A--WVG ztyzVc1x;=KYDKodRxE9fW50hB{ed(-4zQJ$SV%$gzRZhvKwHk`zexl$B|sAVy0NR!+sm&Il}B|2+{EiX&M zPfXkil%mip+lr>S`-+*8D_*jD2f=x8BG-nW+KKbC-D&G+;Y6avuzIo_Zt=N?pG@Mu zL@;;iebA#)2*<>`u4rpeDhsZgE^>VFF}5D+jw>zICKfeVg^syRm=_fZKj+SRJ$p5@ z_M7;|tuxN|&~%#KEnswLO6e7IiDujcc{@K4ZE* pt=2NWb8_AM@w5ZGb}#zWd3PGeoNF%ZVEStxZtjNLC-=NWzW{?1C9wbi delta 431 zcmZ3uhk5^YW|;tQW)=|!5V#v9>=?VE%KaTT0|N-lGE6>rM}G5nw{uL=#mPmP1tppa z)m#c-P?lJvkX)3SSdyAD`HAOBWhh(0Rv|4hr#RIbsw^{2K?9;y!FKY;b;6T-y$m@O z;5z5)h)>?J913ifne^uV2;U}ceR*y#euoL eiQJP*fn3)7$qVngGKpRQi~hYU$M)(b$Q%Fw)|iX{ From d4d2cd508cbd89624a5b02250a78d3dcfa8af19b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Aug 2012 13:07:03 +0530 Subject: [PATCH 04/22] ... --- resources/compiled_coffeescript.zip | Bin 56865 -> 57018 bytes src/calibre/ebooks/oeb/display/paged.coffee | 8 +++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index 1b22ec5b04170cc54e829f7a11e5cff2635db40a..3517cf1363c2d6b808de9b2fa7ecd4e1213d7fdd 100644 GIT binary patch delta 317 zcmZ3uhk4gtW|;tQW)=|!5V)No;#gNwU?tpvl$eR43azaETd3$7jY`b1A?9L{prD z2DfpvL9C8KQD%BZ$>f=bq&Kf|kYZy7TD-ZWkT-F2dGIgU&5f7Na!$?<=Gc79Z6(iS zzdKxfs1BQ)cE?%)B&2>uHu7cy0|N+4f;<2Rq3)9x-c_Ic=8haw>I$$Y<^^+1*1W65 zB(e?6^-biSya&jQKL+Ohy(_@N$Rxr%`GA`g58M<6lM6suMG;mukPsUX?q_6RczOrK F0|1f}WuX87 delta 240 zcmdnBmwDkHX6XQLW)=|!5Rga~aV*}i?9lASzyQM1lLhX|ZM1x9H~D-E$7By3`meCQoFQpZv=~mD@1dAXZ1AC^J2yWb*mB z(wki!rPwyxU0BJpIei6_9CLS#(B>p}-o(wB$Ify}FfxfSW3ztp|2x)`Q}4+!?Qxr2 z=q@^W);*rd<##oha)T!?yz9;+lL!|5e^(r2-elXm(oE@vKq2YLT=%q?$}fRsCmiFO NyyTt$o6jAP_W&RxQosNJ diff --git a/src/calibre/ebooks/oeb/display/paged.coffee b/src/calibre/ebooks/oeb/display/paged.coffee index a26f5931c9..4f912513a9 100644 --- a/src/calibre/ebooks/oeb/display/paged.coffee +++ b/src/calibre/ebooks/oeb/display/paged.coffee @@ -378,7 +378,9 @@ class PagedDisplay # The Conformal Fragment Identifier at the current position, returns # null if it could not be calculated. Requires the cfi.coffee library. ans = null - if not window.cfi? + if not window.cfi? or (window.mathjax?.math_present and not window.mathjax?.math_loaded) + # If MathJax is loading, it is changing the DOM, so we cannot + # reliably generate a CFI return ans if this.in_paged_mode c = this.current_column_location() @@ -412,9 +414,9 @@ class PagedDisplay return ans click_for_page_turn: (event) -> - # Check if the click event event should generate a apge turn. Returns + # Check if the click event should generate a page turn. Returns # null if it should not, true if it is a backwards page turn, false if - # it is a forward apge turn. + # it is a forward page turn. left_boundary = this.current_margin_side right_bondary = this.screen_width - this.current_margin_side if left_boundary > event.clientX From a88d95310fbb686dc8eca0dde24716b94a2acd31 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Aug 2012 17:45:57 +0530 Subject: [PATCH 05/22] Le Monde subscription version by Remi Vanicat --- recipes/le_monde.recipe | 2 +- recipes/le_monde_sub.recipe | 135 ++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 recipes/le_monde_sub.recipe diff --git a/recipes/le_monde.recipe b/recipes/le_monde.recipe index afc19e4d86..8693676da9 100644 --- a/recipes/le_monde.recipe +++ b/recipes/le_monde.recipe @@ -9,7 +9,7 @@ from calibre.web.feeds.recipes import BasicNewsRecipe class LeMonde(BasicNewsRecipe): title = 'Le Monde' __author__ = 'veezh' - description = 'Actualités' + description = u'Actualités' oldest_article = 1 max_articles_per_feed = 100 no_stylesheets = True diff --git a/recipes/le_monde_sub.recipe b/recipes/le_monde_sub.recipe new file mode 100644 index 0000000000..ab75b96ae1 --- /dev/null +++ b/recipes/le_monde_sub.recipe @@ -0,0 +1,135 @@ +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2012, Rémi Vanicat ' +''' +Lemonde.fr: Version abonnée +''' + + +import os, zipfile, re, time + +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup +from calibre.ptempfile import PersistentTemporaryFile + +class LeMondeAbonne(BasicNewsRecipe): + + title = u'Le Monde: Édition abonnés' + __author__ = u'Rémi Vanicat' + description = u'Actualités' + category = u'Actualités, France, Monde' + language = 'fr' + needs_subscription = True + + no_stylesheets = True + + extra_css = u''' + h1{font-size:130%;} + .ariane{font-size:xx-small;} + .source{font-size:xx-small;} + .href{font-size:xx-small;} + .LM_caption{color:#666666; font-size:x-small;} + .main-article-info{font-family:Arial,Helvetica,sans-serif;} + #full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} + #match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} + ''' + + zipurl_format = 'http://medias.lemonde.fr/abonnes/editionelectronique/%Y%m%d/html/%y%m%d.zip' + coverurl_format = '/img/%y%m%d01.jpg' + path_format = "%y%m%d" + login_url = 'http://www.lemonde.fr/web/journal_electronique/identification/1,56-0,45-0,0.html' + + keep_only_tags = [ dict(name="div", attrs={ 'class': 'po-prti' }), dict(name=['h1']), dict(name='div', attrs={ 'class': 'photo' }), dict(name='div', attrs={ 'class': 'po-ti2' }), dict(name='div', attrs={ 'class': 'ar-txt' }), dict(name='div', attrs={ 'class': 'po_rtcol' }) ] + + article_id_pattern = re.compile("[0-9]+\\.html") + article_url_format = 'http://www.lemonde.fr/journalelectronique/donnees/protege/%Y%m%d/html/' + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + if self.username is not None and self.password is not None: + br.open(self.login_url) + br.select_form(nr=0) + br['login'] = self.username + br['password'] = self.password + br.submit() + return br + + decalage = 24 * 60 * 60 # today Monde has tomorow date + + def get_cover_url(self): + second = time.time() + second += self.decalage + ltime = time.localtime(second) + url = time.strftime(self.coverurl_format, ltime) + return self.articles_path + url + + def parse_index(self): + browser = self.get_browser() + + second = time.time() + second += self.decalage + ltime = time.localtime(second) + url = time.strftime(self.zipurl_format, ltime) + + self.timefmt=strftime(" %A %d %B %Y", ltime) + + response = browser.open(url) + + tmp = PersistentTemporaryFile(suffix='.zip') + self.report_progress(0.1,_('downloading zip file')) + tmp.write(response.read()) + tmp.close() + + zfile = zipfile.ZipFile(tmp.name, 'r') + self.report_progress(0.1,_('extracting zip file')) + + zfile.extractall(self.output_dir) + zfile.close() + + path = os.path.join(self.output_dir, time.strftime(self.path_format, ltime), "data") + + self.articles_path = path + + files = os.listdir(path) + + nb_index_files = len([ name for name in files if re.match("frame_gauche_[0-9]+.html", name) ]) + + flux = [] + + article_url = time.strftime(self.article_url_format, ltime) + + for i in range(nb_index_files): + filename = os.path.join(path, "selection_%d.html" % (i + 1)) + tmp = open(filename,'r') + soup=BeautifulSoup(tmp) + title=soup.find('span').contents[0] + tmp.close() + + filename = os.path.join(path, "frame_gauche_%d.html" % (i + 1)) + tmp = open(filename,'r') + soup = BeautifulSoup(tmp) + articles = [] + for link in soup.findAll("a"): + article_file = link['href'] + article_id=self.article_id_pattern.search(article_file).group() + article = { + 'title': link.contents[0], + 'url': article_url + article_id, + 'descripion': '', + 'content': '' + } + articles.append(article) + tmp.close() + + flux.append((title, articles)) + + return flux + + + +# Local Variables: +# mode: python +# End: + From 7a272a05d7a4fb74cf240ed1f3d764ca1a51a8d7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Aug 2012 23:06:50 +0530 Subject: [PATCH 06/22] Fix #1039103 (device not reconized) --- src/calibre/devices/android/driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 3790a32ea7..ef2cffa3a0 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -212,7 +212,7 @@ class ANDROID(USBMS): 'UMS', '.K080', 'P990', 'LTE', 'MB853', 'GT-S5660_CARD', 'A107', 'GT-I9003_CARD', 'XT912', 'FILE-CD_GADGET', 'RK29_SDK', 'MB855', 'XT910', 'BOOK_A10', 'USB_2.0_DRIVER', 'I9100T', 'P999DW', - 'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0_DRIVER', + 'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0', 'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX', 'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE', 'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', @@ -224,7 +224,7 @@ class ANDROID(USBMS): 'ANDROID_MID', 'P990_SD_CARD', '.K080', 'LTE_CARD', 'MB853', 'A1-07___C0541A4F', 'XT912', 'MB855', 'XT910', 'BOOK_A10_CARD', 'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC', - 'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0_DRIVER', 'XT875', + 'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875', 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042'] From 4131aa18da11ee9d6ddbc6b7312b662e5db16f4d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Aug 2012 23:24:31 +0530 Subject: [PATCH 07/22] No longer explicitly set the color of links to blue. This was needed for old versions of ADE which did not display links correctly without it. Since newer versions of ADE no longer have this problem, remove the workaround. --- resources/templates/html.css | 4 ---- src/calibre/ebooks/mobi/writer2/serializer.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/resources/templates/html.css b/resources/templates/html.css index 79c80583bf..a8b3ab920c 100644 --- a/resources/templates/html.css +++ b/resources/templates/html.css @@ -402,7 +402,3 @@ img, object, svg|svg { height: auto; } -/* These are needed because ADE renders anchors the same as links */ - -a { text-decoration: inherit; color: inherit; cursor: inherit } -a[href] { text-decoration: underline; color: blue; cursor: pointer } diff --git a/src/calibre/ebooks/mobi/writer2/serializer.py b/src/calibre/ebooks/mobi/writer2/serializer.py index 2b38c1e6a6..d3269b2d5e 100644 --- a/src/calibre/ebooks/mobi/writer2/serializer.py +++ b/src/calibre/ebooks/mobi/writer2/serializer.py @@ -235,7 +235,7 @@ class Serializer(object): itemhref = re.sub(r'article_\d+/', '', itemhref) self.href_offsets[itemhref].append(buf.tell()) buf.write('0000000000') - buf.write(' >') + buf.write(' >') t = tocitem.title if isinstance(t, unicode): t = t.encode('utf-8') From db26e18d7cef8a1fba709dec781235484b3f49e3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 09:23:42 +0530 Subject: [PATCH 08/22] Fix #1039285 (Updated recipe for The Times UK) --- recipes/times_online.recipe | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/recipes/times_online.recipe b/recipes/times_online.recipe index 1ae8789cd5..1299c92fa3 100644 --- a/recipes/times_online.recipe +++ b/recipes/times_online.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2009-2010, Darko Miletic ' +__copyright__ = '2009-2012, Darko Miletic ' ''' www.thetimes.co.uk ''' @@ -21,6 +21,7 @@ class TimesOnline(BasicNewsRecipe): encoding = 'utf-8' delay = 1 needs_subscription = True + auto_cleanup = False publication_type = 'newspaper' masthead_url = 'http://www.thetimes.co.uk/tto/public/img/the_times_460.gif' INDEX = 'http://www.thetimes.co.uk' @@ -41,13 +42,14 @@ class TimesOnline(BasicNewsRecipe): def get_browser(self): br = BasicNewsRecipe.get_browser() - br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thetimes.co.uk/tto/news/?lightbox=false') + br.open('http://www.thetimes.co.uk/tto/news/') if self.username is not None and self.password is not None: - data = urllib.urlencode({ 'userName':self.username + data = urllib.urlencode({ + 'gotoUrl' :self.INDEX + ,'username':self.username ,'password':self.password - ,'keepMeLoggedIn':'false' }) - br.open('https://www.timesplus.co.uk/iam/app/authenticate',data) + br.open('https://acs.thetimes.co.uk/user/login',data) return br remove_tags = [ @@ -58,6 +60,7 @@ class TimesOnline(BasicNewsRecipe): keep_only_tags = [ dict(attrs={'class':'heading' }) ,dict(attrs={'class':'f-author'}) + ,dict(attrs={'class':['media','byline-timestamp']}) ,dict(attrs={'id':'bodycopy'}) ] @@ -79,11 +82,6 @@ class TimesOnline(BasicNewsRecipe): ,(u'Arts' , PREFIX + u'arts/?view=list' ) ] - def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - return self.adeify_images(soup) - def parse_index(self): totalfeeds = [] lfeeds = self.get_feeds() From d161a6b06d448858a5b5104764836c4b62b3c007 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 09:32:56 +0530 Subject: [PATCH 09/22] Fix #1039286 (Updated recipe for The Sunday Times UK) --- recipes/sunday_times.recipe | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/recipes/sunday_times.recipe b/recipes/sunday_times.recipe index 1f20f73cd9..973f1792c7 100644 --- a/recipes/sunday_times.recipe +++ b/recipes/sunday_times.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Darko Miletic ' +__copyright__ = '2010-2012, Darko Miletic ' ''' www.thesundaytimes.co.uk ''' @@ -43,13 +43,14 @@ class TimesOnline(BasicNewsRecipe): def get_browser(self): br = BasicNewsRecipe.get_browser() - br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thesundaytimes.co.uk/sto/') + br.open('http://www.thesundaytimes.co.uk/sto/') if self.username is not None and self.password is not None: - data = urllib.urlencode({ 'userName':self.username + data = urllib.urlencode({ + 'gotoUrl' :self.INDEX + ,'username':self.username ,'password':self.password - ,'keepMeLoggedIn':'false' }) - br.open('https://www.timesplus.co.uk/iam/app/authenticate',data) + br.open('https://acs.thetimes.co.uk/user/login',data) return br remove_tags = [ From 160d88ecccac516e08cd6687e5cadfa017c4727f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 09:38:53 +0530 Subject: [PATCH 10/22] Fix #1039289 (Updated recipe for Variety) --- recipes/variety.recipe | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/recipes/variety.recipe b/recipes/variety.recipe index 35418174e1..4980a55dee 100644 --- a/recipes/variety.recipe +++ b/recipes/variety.recipe @@ -19,7 +19,13 @@ class Variety(BasicNewsRecipe): category = 'Entertainment Industry News, Daily Variety, Movie Reviews, TV, Awards, Oscars, Cannes, Box Office, Hollywood' language = 'en' masthead_url = 'http://images1.variety.com/graphics/variety/Variety_logo_green_tm.gif' - extra_css = ' body{font-family: Georgia,"Times New Roman",Times,Courier,serif } img{margin-bottom: 1em} ' + extra_css = """ + body{font-family: Arial,Helvetica,sans-serif; font-size: 1.275em} + .date{font-size: small; border: 1px dotted rgb(204, 204, 204); font-style: italic; color: rgb(102, 102, 102); margin: 5px 0px; padding: 0.5em;} + .author{margin: 5px 0px 5px 20px; padding: 0.5em; background: none repeat scroll 0% 0% rgb(247, 247, 247);} + .art h2{color: rgb(153, 0, 0); font-size: 1.275em; font-weight: bold;} + img{margin-bottom: 1em} + """ conversion_options = { 'comments' : description @@ -29,7 +35,7 @@ class Variety(BasicNewsRecipe): } remove_tags = [dict(name=['object','link','map'])] - + remove_attributes=['lang','vspace','hspace','xmlns:ms','xmlns:dt'] keep_only_tags = [dict(name='div', attrs={'class':'art control'})] feeds = [(u'News & Articles', u'http://feeds.feedburner.com/variety/headlines' )] @@ -37,3 +43,29 @@ class Variety(BasicNewsRecipe): def print_version(self, url): rpt = url.rpartition('.html')[0] return rpt + '?printerfriendly=true' + + def preprocess_raw_html(self, raw, url): + return ''+raw[raw.find(''):] + + def get_article_url(self, article): + url = BasicNewsRecipe.get_article_url(self, article) + return url.rpartition('?')[0] + + def preprocess_html(self, soup): + for item in soup.findAll('a'): + limg = item.find('img') + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + if limg: + item.name = 'div' + item.attrs = [] + else: + str = self.tag_to_string(item) + item.replaceWith(str) + for item in soup.findAll('img'): + if not item.has_key('alt'): + item['alt'] = 'image' + return soup + \ No newline at end of file From 190e5b94156e8e3633482a0137b8385e778dc43d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 09:41:05 +0530 Subject: [PATCH 11/22] ... --- src/calibre/devices/android/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index ef2cffa3a0..45672fdbd1 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -197,7 +197,7 @@ class ANDROID(USBMS): 'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON', 'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP', 'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD', - 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ'] + 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', From 80e0f15f85b68a402875d536c63f3d10b80f7aac Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 09:43:43 +0530 Subject: [PATCH 12/22] ... --- recipes/le_monde_sub.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/le_monde_sub.recipe b/recipes/le_monde_sub.recipe index ab75b96ae1..97505c7263 100644 --- a/recipes/le_monde_sub.recipe +++ b/recipes/le_monde_sub.recipe @@ -70,7 +70,7 @@ class LeMondeAbonne(BasicNewsRecipe): second = time.time() second += self.decalage - ltime = time.localtime(second) + ltime = time.gmtime(second) url = time.strftime(self.zipurl_format, ltime) self.timefmt=strftime(" %A %d %B %Y", ltime) From 3a757cd54cc8e4bf9dcbf683ad3f3b61cb8ad6ac Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 09:49:32 +0530 Subject: [PATCH 13/22] ... --- recipes/le_monde_sub.recipe | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/recipes/le_monde_sub.recipe b/recipes/le_monde_sub.recipe index 97505c7263..6f5c16e3d1 100644 --- a/recipes/le_monde_sub.recipe +++ b/recipes/le_monde_sub.recipe @@ -59,10 +59,7 @@ class LeMondeAbonne(BasicNewsRecipe): decalage = 24 * 60 * 60 # today Monde has tomorow date def get_cover_url(self): - second = time.time() - second += self.decalage - ltime = time.localtime(second) - url = time.strftime(self.coverurl_format, ltime) + url = time.strftime(self.coverurl_format, self.ltime) return self.articles_path + url def parse_index(self): @@ -70,7 +67,7 @@ class LeMondeAbonne(BasicNewsRecipe): second = time.time() second += self.decalage - ltime = time.gmtime(second) + ltime = self.ltime = time.gmtime(second) url = time.strftime(self.zipurl_format, ltime) self.timefmt=strftime(" %A %d %B %Y", ltime) From 2fd79d209f3ca2c7bc34afc5eced2c91c813940a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 10:24:31 +0530 Subject: [PATCH 14/22] WPD: Implement non bulk filesystem metadata transfers for devices that do not support bulk transfers --- src/calibre/devices/mtp/filesystem_cache.py | 16 +++ .../mtp/windows/content_enumeration.cpp | 107 ++++++++++++++++-- src/calibre/devices/mtp/windows/global.h | 2 +- src/calibre/devices/mtp/windows/utils.cpp | 2 +- 4 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 src/calibre/devices/mtp/filesystem_cache.py diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py new file mode 100644 index 0000000000..a94172b6b0 --- /dev/null +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +class FilesystemCache(object): + + def __init__(self, storage_map): + self.tree = {} + for storage_id, id_map in storage_map.iteritems(): + self.tree[storage_id] = self.build_tree(id_map) + diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp index f9026c35f8..4fe20523cd 100644 --- a/src/calibre/devices/mtp/windows/content_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -87,8 +87,25 @@ static void set_content_type_property(PyObject *dict, IPortableDeviceValues *pro if (SUCCEEDED(properties->GetGuidValue(WPD_OBJECT_CONTENT_TYPE, &guid)) && IsEqualGUID(guid, WPD_CONTENT_TYPE_FOLDER)) is_folder = 1; PyDict_SetItemString(dict, "is_folder", (is_folder) ? Py_True : Py_False); } + +static void set_properties(PyObject *obj, IPortableDeviceValues *values) { + set_content_type_property(obj, values); + + set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", values); + set_string_property(obj, WPD_OBJECT_NAME, "name", values); + set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values); + set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", values); + + set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", values); + set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", values); + set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", values); + + set_size_property(obj, WPD_OBJECT_SIZE, "size", values); +} + // }}} +// Bulk get filesystem {{{ class GetBulkCallback : public IPortableDevicePropertiesBulkCallback { public: @@ -154,19 +171,8 @@ public: } Py_DECREF(temp); - set_content_type_property(obj, properties); + set_properties(obj, properties); - set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", properties); - set_string_property(obj, WPD_OBJECT_NAME, "name", properties); - set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", properties); - set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", properties); - - set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", properties); - set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", properties); - set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", properties); - - set_size_property(obj, WPD_OBJECT_SIZE, "size", properties); - properties->Release(); properties = NULL; } } // end for loop @@ -240,6 +246,9 @@ end: return folders; } +// }}} + +// find_all_objects_in() {{{ static BOOL find_all_objects_in(IPortableDeviceContent *content, IPortableDevicePropVariantCollection *object_ids, const wchar_t *parent_id) { /* * Find all children of the object identified by parent_id, recursively. @@ -286,8 +295,81 @@ end: if (children != NULL) children->Release(); PropVariantClear(&pv); return ok; +} // }}} + +// Single get filesystem {{{ + +static PyObject* get_object_properties(IPortableDeviceProperties *devprops, IPortableDeviceKeyCollection *properties, const wchar_t *object_id) { + IPortableDeviceValues *values = NULL; + HRESULT hr; + PyObject *ans = NULL, *temp = NULL; + + Py_BEGIN_ALLOW_THREADS; + hr = devprops->GetValues(object_id, properties, &values); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to get properties for object", hr); goto end; } + + temp = wchar_to_unicode(object_id); + if (temp == NULL) goto end; + + ans = PyDict_New(); + if (ans == NULL) { PyErr_NoMemory(); goto end; } + if (PyDict_SetItemString(ans, "id", temp) != 0) { Py_DECREF(ans); ans = NULL; PyErr_NoMemory(); goto end; } + + set_properties(ans, values); + +end: + Py_XDECREF(temp); + if (values != NULL) values->Release(); + return ans; } +static PyObject* single_get_filesystem(IPortableDeviceContent *content, const wchar_t *storage_id, IPortableDevicePropVariantCollection *object_ids) { + DWORD num, i; + PROPVARIANT pv; + HRESULT hr; + BOOL ok = 1; + PyObject *ans = NULL, *item = NULL; + IPortableDeviceProperties *devprops = NULL; + IPortableDeviceKeyCollection *properties = NULL; + + hr = content->Properties(&devprops); + if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; } + + properties = create_filesystem_properties_collection(); + if (properties == NULL) goto end; + + hr = object_ids->GetCount(&num); + if (FAILED(hr)) { hresult_set_exc("Failed to get object id count", hr); goto end; } + + ans = PyDict_New(); + if (ans == NULL) goto end; + + for (i = 0; i < num; i++) { + ok = 0; + PropVariantInit(&pv); + hr = object_ids->GetAt(i, &pv); + if (SUCCEEDED(hr) && pv.pwszVal != NULL) { + item = get_object_properties(devprops, properties, pv.pwszVal); + if (item != NULL) { + PyDict_SetItem(ans, PyDict_GetItemString(item, "id"), item); + Py_DECREF(item); item = NULL; + ok = 1; + } + } else hresult_set_exc("Failed to get item from IPortableDevicePropVariantCollection", hr); + + PropVariantClear(&pv); + if (!ok) { Py_DECREF(ans); ans = NULL; break; } + } + +end: + if (devprops != NULL) devprops->Release(); + if (properties != NULL) properties->Release(); + + return ans; +} +// }}} + PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { PyObject *folders = NULL; IPortableDevicePropVariantCollection *object_ids = NULL; @@ -310,6 +392,7 @@ PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id if (!ok) goto end; if (bulk_properties != NULL) folders = bulk_get_filesystem(device, bulk_properties, storage_id, object_ids); + else folders = single_get_filesystem(content, storage_id, object_ids); end: if (content != NULL) content->Release(); diff --git a/src/calibre/devices/mtp/windows/global.h b/src/calibre/devices/mtp/windows/global.h index cbf489d424..e58fafa1d4 100644 --- a/src/calibre/devices/mtp/windows/global.h +++ b/src/calibre/devices/mtp/windows/global.h @@ -50,7 +50,7 @@ extern PyTypeObject DeviceType; // Utility functions PyObject *hresult_set_exc(const char *msg, HRESULT hr); wchar_t *unicode_to_wchar(PyObject *o); -PyObject *wchar_to_unicode(wchar_t *o); +PyObject *wchar_to_unicode(const wchar_t *o); int pump_waiting_messages(); extern IPortableDeviceValues* get_client_information(); diff --git a/src/calibre/devices/mtp/windows/utils.cpp b/src/calibre/devices/mtp/windows/utils.cpp index 243bcc0f59..7592e0d4a9 100644 --- a/src/calibre/devices/mtp/windows/utils.cpp +++ b/src/calibre/devices/mtp/windows/utils.cpp @@ -43,7 +43,7 @@ wchar_t *wpd::unicode_to_wchar(PyObject *o) { return buf; } -PyObject *wpd::wchar_to_unicode(wchar_t *o) { +PyObject *wpd::wchar_to_unicode(const wchar_t *o) { PyObject *ans; if (o == NULL) return NULL; ans = PyUnicode_FromWideChar(o, wcslen(o)); From 674edcb013084300bf9de84d1c7ca48783c0c908 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 12:38:12 +0530 Subject: [PATCH 15/22] Get Books: Update B&N plugin to handle changes to the B&N website --- src/calibre/gui2/store/stores/bn_plugin.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/store/stores/bn_plugin.py b/src/calibre/gui2/store/stores/bn_plugin.py index 1f7eb2a91e..65a7eee194 100644 --- a/src/calibre/gui2/store/stores/bn_plugin.py +++ b/src/calibre/gui2/store/stores/bn_plugin.py @@ -34,26 +34,29 @@ class BNStore(BasicStoreConfig, StorePlugin): d.exec_() def search(self, query, max_results=10, timeout=60): - url = 'http://www.barnesandnoble.com/s/%s?keyword=%s&store=ebook' % (query.replace(' ', '-'), urllib.quote_plus(query)) + url = 'http://www.barnesandnoble.com/s/%s?keyword=%s&store=ebook&view=list' % (query.replace(' ', '-'), urllib.quote_plus(query)) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) + raw = f.read() + doc = html.fromstring(raw) for data in doc.xpath('//ul[contains(@class, "result-set")]/li[contains(@class, "result")]'): if counter <= 0: break - id = ''.join(data.xpath('.//div[contains(@class, "image-bounding-box")]/a/@href')) + id = ''.join(data.xpath('.//div[contains(@class, "image-block")]/a/@href')) if not id: continue cover_url = ''.join(data.xpath('.//img[contains(@class, "product-image")]/@src')) - title = ''.join(data.xpath('.//a[@class="title"]//text()')) - author = ', '.join(data.xpath('.//a[@class="contributor"]//text()')) - price = ''.join(data.xpath('.//div[@class="price-format"]//span[contains(@class, "price")]/text()')) + title = ''.join(data.xpath('descendant::p[@class="title"]//span[@class="name"]//text()')).strip() + if not title: continue + + author = ', '.join(data.xpath('.//ul[@class="contributors"]//a[@class="subtle"]//text()')).strip() + price = ''.join(data.xpath('.//a[contains(@class, "bn-price")]//text()')) counter -= 1 From 35eb01234ddfe2af803b18613d284111bd407346 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 17:27:12 +0530 Subject: [PATCH 16/22] WPD: Implement reading files from device --- .../mtp/windows/content_enumeration.cpp | 105 +++++++++++++++++- src/calibre/devices/mtp/windows/device.cpp | 21 +++- src/calibre/devices/mtp/windows/driver.py | 10 +- src/calibre/devices/mtp/windows/global.h | 3 +- src/calibre/devices/mtp/windows/remote.py | 9 +- src/calibre/devices/mtp/windows/wpd.cpp | 5 +- 6 files changed, 143 insertions(+), 10 deletions(-) diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp index 4fe20523cd..29d227d710 100644 --- a/src/calibre/devices/mtp/windows/content_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -14,7 +14,7 @@ namespace wpd { static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { // {{{ - IPortableDeviceKeyCollection *properties; + IPortableDeviceKeyCollection *properties = NULL; HRESULT hr; Py_BEGIN_ALLOW_THREADS; @@ -370,7 +370,7 @@ end: } // }}} -PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { +PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { // {{{ PyObject *folders = NULL; IPortableDevicePropVariantCollection *object_ids = NULL; IPortableDeviceContent *content = NULL; @@ -399,6 +399,105 @@ end: if (object_ids != NULL) object_ids->Release(); return folders; -} +} // }}} + +PyObject* wpd::get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback) { // {{{ + IPortableDeviceContent *content = NULL; + IPortableDeviceResources *resources = NULL; + IPortableDeviceProperties *devprops = NULL; + IPortableDeviceValues *values = NULL; + IPortableDeviceKeyCollection *properties = NULL; + IStream *stream = NULL; + HRESULT hr; + DWORD bufsize = 4096; + char *buf = NULL; + ULONG bytes_read = 0, total_read = 0; + BOOL ok = FALSE; + PyObject *res = NULL; + ULONGLONG filesize = 0; + + Py_BEGIN_ALLOW_THREADS; + hr = device->Content(&content); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = content->Properties(&devprops); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = CoCreateInstance(CLSID_PortableDeviceKeyCollection, NULL, + CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&properties)); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create filesystem properties collection", hr); goto end; } + hr = properties->Add(WPD_OBJECT_SIZE); + if (FAILED(hr)) { hresult_set_exc("Failed to add filesize property to properties collection", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = devprops->GetValues(object_id, properties, &values); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to get filesize for object", hr); goto end; } + hr = values->GetUnsignedLargeIntegerValue(WPD_OBJECT_SIZE, &filesize); + if (FAILED(hr)) { hresult_set_exc("Failed to get filesize from values collection", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = content->Transfer(&resources); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { hresult_set_exc("Failed to create resources interface", hr); goto end; } + + Py_BEGIN_ALLOW_THREADS; + hr = resources->GetStream(object_id, WPD_RESOURCE_DEFAULT, STGM_READ, &bufsize, &stream); + Py_END_ALLOW_THREADS; + if (FAILED(hr)) { + if (HRESULT_FROM_WIN32(ERROR_BUSY) == hr) { + PyErr_SetString(WPDFileBusy, "Object is in use"); + } else hresult_set_exc("Failed to create stream interface to read from object", hr); + goto end; + } + + buf = (char *)calloc(bufsize+10, 1); + if (buf == NULL) { PyErr_NoMemory(); goto end; } + + while (TRUE) { + bytes_read = 0; + Py_BEGIN_ALLOW_THREADS; + hr = stream->Read(buf, bufsize, &bytes_read); + Py_END_ALLOW_THREADS; + total_read = total_read + bytes_read; + if (hr == STG_E_ACCESSDENIED) { + PyErr_SetString(PyExc_IOError, "Read access is denied to this object"); break; + } else if (hr == S_OK || hr == S_FALSE) { + if (bytes_read > 0) { + res = PyObject_CallMethod(dest, "write", "s#", buf, bytes_read); + if (res == NULL) break; + Py_DECREF(res); res = NULL; + if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_read, filesize)); + } + } else { hresult_set_exc("Failed to read file from device", hr); break; } + + if (hr == S_FALSE || bytes_read < bufsize) { + ok = TRUE; + Py_XDECREF(PyObject_CallMethod(dest, "flush", NULL)); + break; + } + } + + if (ok && total_read != filesize) { + ok = FALSE; + PyErr_SetString(WPDError, "Failed to read all data from file"); + } + +end: + if (content != NULL) content->Release(); + if (devprops != NULL) devprops->Release(); + if (resources != NULL) resources->Release(); + if (stream != NULL) stream->Release(); + if (values != NULL) values->Release(); + if (properties != NULL) properties->Release(); + if (buf != NULL) free(buf); + if (!ok) return NULL; + Py_RETURN_NONE; +} // }}} } // namespace wpd diff --git a/src/calibre/devices/mtp/windows/device.cpp b/src/calibre/devices/mtp/windows/device.cpp index 0a03b9e735..d79db0a2d3 100644 --- a/src/calibre/devices/mtp/windows/device.cpp +++ b/src/calibre/devices/mtp/windows/device.cpp @@ -78,7 +78,7 @@ update_data(Device *self, PyObject *args, PyObject *kwargs) { // get_filesystem() {{{ static PyObject* py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) { - PyObject *storage_id, *ans = NULL; + PyObject *storage_id; wchar_t *storage; if (!PyArg_ParseTuple(args, "O", &storage_id)) return NULL; @@ -88,6 +88,21 @@ py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) { return wpd::get_filesystem(self->device, storage, self->bulk_properties); } // }}} +// get_file() {{{ +static PyObject* +py_get_file(Device *self, PyObject *args, PyObject *kwargs) { + PyObject *object_id, *stream, *callback = NULL; + wchar_t *object; + + if (!PyArg_ParseTuple(args, "OO|O", &object_id, &stream, &callback)) return NULL; + object = unicode_to_wchar(object_id); + if (object == NULL) return NULL; + + if (callback == NULL || !PyCallable_Check(callback)) callback = NULL; + + return wpd::get_file(self->device, object, stream, callback); +} // }}} + static PyMethodDef Device_methods[] = { {"update_data", (PyCFunction)update_data, METH_VARARGS, "update_data() -> Reread the basic device data from the device (total, space, free space, storage locations, etc.)" @@ -97,6 +112,10 @@ static PyMethodDef Device_methods[] = { "get_filesystem(storage_id) -> Get all files/folders on the storage identified by storage_id. Tries to use bulk operations when possible." }, + {"get_file", (PyCFunction)py_get_file, METH_VARARGS, + "get_file(object_id, stream, callback=None) -> Get the file identified by object_id from the device. The file is written to the stream object, which must be a file like object. If callback is not None, it must be a callable that accepts two arguments: (bytes_read, total_size). It will be called after each chunk is read from the device. Note that it can be called multiple times with the same values." + }, + {NULL} }; diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index fcfc415c90..4809ea5054 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -12,6 +12,7 @@ from threading import RLock from calibre import as_unicode, prints from calibre.constants import plugins, __appname__, numeric_version +from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed from calibre.devices.mtp.base import MTPDeviceBase, synchronous @@ -196,5 +197,12 @@ class MTP_DEVICE(MTPDeviceBase): ans[i] = s['free_space'] return tuple(ans) - + def get_file(self, object_id, callback=None): + stream = SpooledTemporaryFile(5*1024*1024) + try: + self.dev.get_file(object_id, stream, callback) + except self.wpd.WPDFileBusy: + time.sleep(2) + self.dev.get_file(object_id, stream, callback) + return stream diff --git a/src/calibre/devices/mtp/windows/global.h b/src/calibre/devices/mtp/windows/global.h index e58fafa1d4..47f0786249 100644 --- a/src/calibre/devices/mtp/windows/global.h +++ b/src/calibre/devices/mtp/windows/global.h @@ -20,7 +20,7 @@ namespace wpd { // Module exception types -extern PyObject *WPDError, *NoWPD; +extern PyObject *WPDError, *NoWPD, *WPDFileBusy; // The global device manager extern IPortableDeviceManager *portable_device_manager; @@ -57,6 +57,7 @@ extern IPortableDeviceValues* get_client_information(); extern IPortableDevice* open_device(const wchar_t *pnp_id, IPortableDeviceValues *client_information); extern PyObject* get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **bulk_properties); extern PyObject* get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties); +extern PyObject* get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback); } diff --git a/src/calibre/devices/mtp/windows/remote.py b/src/calibre/devices/mtp/windows/remote.py index a6502b991b..6f883f8baf 100644 --- a/src/calibre/devices/mtp/windows/remote.py +++ b/src/calibre/devices/mtp/windows/remote.py @@ -7,8 +7,8 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import subprocess, sys, os, pprint, signal, time, glob -pprint +import subprocess, sys, os, pprint, signal, time, glob, io +pprint, io def build(mod='wpd'): master = subprocess.Popen('ssh -MN getafix'.split()) @@ -70,7 +70,10 @@ def main(): print ('Connected to:', dev.get_gui_name()) print ('Total space', dev.total_space()) print ('Free space', dev.free_space()) - pprint.pprint(dev.dev.get_filesystem(dev._main_id)) + # pprint.pprint(dev.dev.get_filesystem(dev._main_id)) + print ('Fetching file: oFF (198214 bytes)') + stream = dev.get_file('oFF') + print ("Fetched size: ", stream.tell()) finally: dev.shutdown() diff --git a/src/calibre/devices/mtp/windows/wpd.cpp b/src/calibre/devices/mtp/windows/wpd.cpp index 561eeb1bbc..51a55b97ac 100644 --- a/src/calibre/devices/mtp/windows/wpd.cpp +++ b/src/calibre/devices/mtp/windows/wpd.cpp @@ -10,7 +10,7 @@ using namespace wpd; // Module exception types -PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL; +PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL, *wpd::WPDFileBusy = NULL; // The global device manager IPortableDeviceManager *wpd::portable_device_manager = NULL; @@ -199,6 +199,9 @@ initwpd(void) { NoWPD = PyErr_NewException("wpd.NoWPD", NULL, NULL); if (NoWPD == NULL) return; + WPDFileBusy = PyErr_NewException("wpd.WPDFileBusy", NULL, NULL); + if (WPDFileBusy == NULL) return; + Py_INCREF(&DeviceType); PyModule_AddObject(m, "Device", (PyObject *)&DeviceType); From d14d86766d4aebbaaaf351fc7046157036775a85 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 19:59:59 +0530 Subject: [PATCH 17/22] WPD driver: Ensure that all methods are used in a single thread --- src/calibre/devices/mtp/windows/driver.py | 40 ++++++++++++++--------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 4809ea5054..ac2933eae0 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -7,14 +7,24 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import time -from threading import RLock +import time, threading +from functools import wraps from calibre import as_unicode, prints from calibre.constants import plugins, __appname__, numeric_version from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed -from calibre.devices.mtp.base import MTPDeviceBase, synchronous +from calibre.devices.mtp.base import MTPDeviceBase + +def same_thread(func): + @wraps(func) + def check_thread(self, *args, **kwargs): + if self.start_thread is not threading.current_thread(): + raise Exception('You cannot use %s from a thread other than the ' + ' thread in which startup() was called'%self.__class__.__name__) + return func(self, *args, **kwargs) + return check_thread + class MTP_DEVICE(MTPDeviceBase): @@ -23,7 +33,6 @@ class MTP_DEVICE(MTPDeviceBase): def __init__(self, *args, **kwargs): MTPDeviceBase.__init__(self, *args, **kwargs) self.dev = None - self.lock = RLock() self.blacklisted_devices = set() self.ejected_devices = set() self.currently_connected_pnp_id = None @@ -32,9 +41,10 @@ class MTP_DEVICE(MTPDeviceBase): self.last_refresh_devices_time = time.time() self.wpd = self.wpd_error = None self._main_id = self._carda_id = self._cardb_id = None + self.start_thread = None - @synchronous def startup(self): + self.start_thread = threading.current_thread() self.wpd, self.wpd_error = plugins['wpd'] if self.wpd is not None: try: @@ -47,13 +57,13 @@ class MTP_DEVICE(MTPDeviceBase): except Exception as e: self.wpd_error = as_unicode(e) - @synchronous + @same_thread def shutdown(self): - self.dev = self.filesystem_cache = None + self.dev = self.filesystem_cache = self.start_thread = None if self.wpd is not None: self.wpd.uninit() - @synchronous + @same_thread def detect_managed_devices(self, devices_on_system): if self.wpd is None: return None @@ -120,13 +130,13 @@ class MTP_DEVICE(MTPDeviceBase): return True - @synchronous + @same_thread def post_yank_cleanup(self): self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None self.dev = self.filesystem_cache = None - @synchronous + @same_thread def eject(self): if self.currently_connected_pnp_id is None: return self.ejected_devices.add(self.currently_connected_pnp_id) @@ -134,7 +144,7 @@ class MTP_DEVICE(MTPDeviceBase): self._main_id = self._carda_id = self._cardb_id = None self.dev = self.filesystem_cache = None - @synchronous + @same_thread def open(self, connected_device, library_uuid): self.dev = self.filesystem_cache = None try: @@ -159,13 +169,13 @@ class MTP_DEVICE(MTPDeviceBase): self._cardb_id = storage[2]['id'] self.current_friendly_name = devdata.get('friendly_name', None) - @synchronous + @same_thread def get_device_information(self, end_session=True): d = self.dev.data dv = d.get('device_version', '') return (self.current_friendly_name, dv, dv, '') - @synchronous + @same_thread def card_prefix(self, end_session=True): ans = [None, None] if self._carda_id is not None: @@ -174,7 +184,7 @@ class MTP_DEVICE(MTPDeviceBase): ans[1] = 'mtp:::%s:::'%self._cardb_id return tuple(ans) - @synchronous + @same_thread def total_space(self, end_session=True): ans = [0, 0, 0] dd = self.dev.data @@ -185,7 +195,7 @@ class MTP_DEVICE(MTPDeviceBase): ans[i] = s['capacity'] return tuple(ans) - @synchronous + @same_thread def free_space(self, end_session=True): self.dev.update_data() ans = [0, 0, 0] From cf0c0d3fa96790ce9437632e36f1c4f44b98ea6f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 20:01:56 +0530 Subject: [PATCH 18/22] ... --- src/calibre/devices/mtp/windows/driver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index ac2933eae0..2bdcff442d 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -207,8 +207,9 @@ class MTP_DEVICE(MTPDeviceBase): ans[i] = s['free_space'] return tuple(ans) - def get_file(self, object_id, callback=None): - stream = SpooledTemporaryFile(5*1024*1024) + def get_file(self, object_id, stream=None, callback=None): + if stream is None: + stream = SpooledTemporaryFile(5*1024*1024, '_wpd_receive_file.dat') try: self.dev.get_file(object_id, stream, callback) except self.wpd.WPDFileBusy: From 2e0863a5fc4e45e331809b78c47a35401fdb6ef4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Aug 2012 20:02:32 +0530 Subject: [PATCH 19/22] ... --- src/calibre/devices/mtp/windows/driver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 2bdcff442d..9d0ac55e82 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -207,6 +207,7 @@ class MTP_DEVICE(MTPDeviceBase): ans[i] = s['free_space'] return tuple(ans) + @same_thread def get_file(self, object_id, stream=None, callback=None): if stream is None: stream = SpooledTemporaryFile(5*1024*1024, '_wpd_receive_file.dat') From d70d85bb53c5657158c0d158cdcc49e7eebdacaf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 22 Aug 2012 09:11:37 +0530 Subject: [PATCH 20/22] Driver for SONY PRS-T2 --- src/calibre/devices/prst1/driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index b51e55b829..8b76255532 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -50,10 +50,10 @@ class PRST1(USBMS): VENDOR_NAME = 'SONY' WINDOWS_MAIN_MEM = re.compile( - r'(PRS-T1&)' + r'(PRS-T(1|2)&)' ) WINDOWS_CARD_A_MEM = re.compile( - r'(PRS-T1__SD&)' + r'(PRS-T(1|2)__SD&)' ) MAIN_MEMORY_VOLUME_LABEL = 'SONY Reader Main Memory' STORAGE_CARD_VOLUME_LABEL = 'SONY Reader Storage Card' From 67ce036b439d683369956edafb3a558e3f059c31 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 22 Aug 2012 09:42:36 +0530 Subject: [PATCH 21/22] ... --- src/calibre/devices/mtp/unix/libmtp.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/mtp/unix/libmtp.c b/src/calibre/devices/mtp/unix/libmtp.c index 7ea987782a..856fd057bf 100644 --- a/src/calibre/devices/mtp/unix/libmtp.c +++ b/src/calibre/devices/mtp/unix/libmtp.c @@ -55,7 +55,7 @@ static int report_progress(uint64_t const sent, uint64_t const total, void const cb = (ProgressCallback *)data; if (cb->obj != NULL) { PyEval_RestoreThread(cb->state); - res = PyObject_CallMethod(cb->obj, "report_progress", "KK", sent, total); + res = PyObject_CallFunction(cb->obj, "KK", sent, total); Py_XDECREF(res); cb->state = PyEval_SaveThread(); } @@ -339,6 +339,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs if (!PyArg_ParseTuple(args, "|O", &callback)) return NULL; + if (callback == NULL || !PyCallable_Check(callback)) callback = NULL; cb.obj = callback; ans = PyList_New(0); @@ -377,7 +378,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs if (callback != NULL) { // Bug in libmtp where it does not call callback with 100% - fo = PyObject_CallMethod(callback, "report_progress", "KK", PyList_Size(ans), PyList_Size(ans)); + fo = PyObject_CallFunction(callback, "KK", PyList_Size(ans), PyList_Size(ans)); Py_XDECREF(fo); } @@ -454,6 +455,7 @@ libmtp_Device_get_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) { if (!PyArg_ParseTuple(args, "kO|O", &fileid, &stream, &callback)) return NULL; errs = PyList_New(0); if (errs == NULL) { PyErr_NoMemory(); return NULL; } + if (callback == NULL || !PyCallable_Check(callback)) callback = NULL; cb.obj = callback; cb.extra = stream; Py_XINCREF(callback); Py_INCREF(stream); @@ -486,6 +488,7 @@ libmtp_Device_put_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) { if (!PyArg_ParseTuple(args, "kksOK|O", &storage_id, &parent_id, &name, &stream, &filesize, &callback)) return NULL; errs = PyList_New(0); if (errs == NULL) { PyErr_NoMemory(); return NULL; } + if (callback == NULL || !PyCallable_Check(callback)) callback = NULL; cb.obj = callback; cb.extra = stream; f.parent_id = parent_id; f.storage_id = storage_id; f.item_id = 0; f.filename = name; f.filetype = LIBMTP_FILETYPE_UNKNOWN; f.filesize = filesize; @@ -599,7 +602,7 @@ static PyMethodDef libmtp_Device_methods[] = { }, {"get_filelist", (PyCFunction)libmtp_Device_get_filelist, METH_VARARGS, - "get_filelist(callback=None) -> Get the list of files on the device. callback must be an object that has a method named 'report_progress(current, total)'. Returns files, errors." + "get_filelist(callback=None) -> Get the list of files on the device. callback must be callable accepts arguments (current, total)'. Returns files, errors." }, {"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS, From 922e02e0d74fb8f3e264222a19fb7799040c4126 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 22 Aug 2012 14:09:21 +0530 Subject: [PATCH 22/22] MTP: Cross platform filesystem cache --- src/calibre/__init__.py | 4 +- src/calibre/devices/mtp/filesystem_cache.py | 83 +++++++++++- src/calibre/devices/mtp/unix/driver.py | 127 ++++++------------ src/calibre/devices/mtp/unix/libmtp.c | 10 +- .../mtp/windows/content_enumeration.cpp | 4 +- src/calibre/devices/mtp/windows/driver.py | 43 +++++- src/calibre/devices/mtp/windows/remote.py | 8 +- 7 files changed, 170 insertions(+), 109 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index ac8e681fed..58390a314a 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -674,7 +674,7 @@ def get_download_filename(url, cookie_file=None): return filename -def human_readable(size): +def human_readable(size, sep=' '): """ Convert a size in bytes into a human readable form """ divisor, suffix = 1, "B" for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')): @@ -686,7 +686,7 @@ def human_readable(size): size = size[:size.find(".")+2] if size.endswith('.0'): size = size[:-2] - return size + " " + suffix + return size + sep + suffix def remove_bracketed_text(src, brackets={u'(':u')', u'[':u']', u'{':u'}'}): diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index a94172b6b0..cc7d41e09b 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -7,10 +7,85 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import weakref, sys +from operator import attrgetter +from future_builtins import map + +from calibre import human_readable, prints, force_unicode +from calibre.utils.icu import sort_key + +class FileOrFolder(object): + + def __init__(self, entry, fs_cache, all_storage_ids): + self.object_id = entry['id'] + self.is_folder = entry['is_folder'] + self.name = force_unicode(entry.get('name', '___'), 'utf-8') + self.persistent_id = entry.get('persistent_id', self.object_id) + self.size = entry.get('size', 0) + # self.parent_id is None for storage objects + self.parent_id = entry.get('parent_id', None) + if self.parent_id == 0: + sid = entry['storage_id'] + if sid not in all_storage_ids: + sid = all_storage_ids[0] + self.parent_id = sid + self.is_hidden = entry.get('is_hidden', False) + self.is_system = entry.get('is_system', False) + self.can_delete = entry.get('can_delete', True) + + self.files = [] + self.folders = [] + fs_cache.id_map[self.object_id] = self + self.fs_cache = weakref.ref(fs_cache) + + @property + def id_map(self): + return self.fs_cache().id_map + + @property + def parent(self): + return None if self.parent_id is None else self.id_map[self.parent_id] + + def __iter__(self): + for e in self.folders: + yield e + for e in self.files: + yield e + + def dump(self, prefix='', out=sys.stdout): + c = '+' if self.is_folder else '-' + data = ('%s children'%(sum(map(len, (self.files, self.folders)))) + if self.is_folder else human_readable(self.size)) + line = '%s%s %s [id:%s %s]'%(prefix, c, self.name, self.object_id, data) + prints(line, file=out) + for c in (self.folders, self.files): + for e in sorted(c, key=lambda x:sort_key(x.name)): + e.dump(prefix=prefix+' ', out=out) + class FilesystemCache(object): - def __init__(self, storage_map): - self.tree = {} - for storage_id, id_map in storage_map.iteritems(): - self.tree[storage_id] = self.build_tree(id_map) + def __init__(self, all_storage, entries): + self.entries = [] + self.id_map = {} + + for storage in all_storage: + e = FileOrFolder(storage, self, []) + self.entries.append(e) + + self.entries.sort(key=attrgetter('object_id')) + all_storage_ids = [x.object_id for x in self.entries] + + for entry in entries: + FileOrFolder(entry, self, all_storage_ids) + + for item in self.id_map.itervalues(): + p = item.parent + if p is not None: + t = p.folders if item.is_folder else p.files + t.append(item) + + def dump(self, out=sys.stdout): + for e in self.entries: + e.dump(out=out) + diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index c94a2e2458..835f2245d0 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -9,77 +9,13 @@ __docformat__ = 'restructuredtext en' import time, operator from threading import RLock -from itertools import chain -from collections import deque, OrderedDict from io import BytesIO -from calibre import prints from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.mtp.base import MTPDeviceBase, synchronous +from calibre.devices.mtp.filesystem_cache import FilesystemCache from calibre.devices.mtp.unix.detect import MTPDetect -class FilesystemCache(object): - - def __init__(self, files, folders): - self.files = files - self.folders = folders - self.file_id_map = {f['id']:f for f in files} - self.folder_id_map = {f['id']:f for f in self.iterfolders(set_level=0)} - - # Set the parents of each file - self.files_in_root = OrderedDict() - for f in files: - parents = deque() - pid = f['parent_id'] - while pid is not None and pid > 0: - try: - parent = self.folder_id_map[pid] - except KeyError: - break - parents.appendleft(pid) - pid = parent['parent_id'] - f['parents'] = parents - if not parents: - self.files_in_root[f['id']] = f - - # Set the files in each folder - for f in self.iterfolders(): - f['files'] = [i for i in files if i['parent_id'] == - f['id']] - - # Decode the file and folder names - for f in chain(files, folders): - try: - name = f['name'].decode('utf-8') - except UnicodeDecodeError: - name = 'undecodable_%d'%f['id'] - f['name'] = name - - def iterfolders(self, folders=None, set_level=None): - clevel = None if set_level is None else set_level + 1 - if folders is None: - folders = self.folders - for f in folders: - if set_level is not None: - f['level'] = set_level - yield f - for c in f['children']: - for child in self.iterfolders([c], set_level=clevel): - yield child - - def dump_filesystem(self): - indent = 2 - for f in self.iterfolders(): - prefix = ' '*(indent*f['level']) - prints(prefix, '+', f['name'], 'id=%s'%f['id']) - for leaf in f['files']: - prints(prefix, ' '*indent, '-', leaf['name'], - 'id=%d'%leaf['id'], 'size=%d'%leaf['size'], - 'modtime=%d'%leaf['modtime']) - for leaf in self.files_in_root.itervalues(): - prints('-', leaf['name'], 'id=%d'%leaf['id'], - 'size=%d'%leaf['size'], 'modtime=%d'%leaf['modtime']) - class MTP_DEVICE(MTPDeviceBase): supported_platforms = ['linux'] @@ -87,7 +23,7 @@ class MTP_DEVICE(MTPDeviceBase): def __init__(self, *args, **kwargs): MTPDeviceBase.__init__(self, *args, **kwargs) self.dev = None - self.filesystem_cache = None + self._filesystem_cache = None self.lock = RLock() self.blacklisted_devices = set() @@ -129,7 +65,7 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def post_yank_cleanup(self): - self.dev = self.filesystem_cache = self.current_friendly_name = None + self.dev = self._filesystem_cache = self.current_friendly_name = None @synchronous def startup(self): @@ -140,7 +76,7 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def shutdown(self): - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None def format_errorstack(self, errs): return '\n'.join(['%d:%s'%(code, msg.decode('utf-8', 'replace')) for @@ -148,7 +84,7 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def open(self, connected_device, library_uuid): - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None def blacklist_device(): d = connected_device self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id, @@ -179,23 +115,41 @@ class MTP_DEVICE(MTPDeviceBase): self._carda_id = storage[1]['id'] if len(storage) > 2: self._cardb_id = storage[2]['id'] - self.current_friendly_name = self.dev.name + self.current_friendly_name = self.dev.friendly_name - @synchronous - def read_filesystem_cache(self): - try: - files, errs = self.dev.get_filelist(self) - if errs and not files: - raise DeviceError('Failed to read files from device. Underlying errors:\n' - +self.format_errorstack(errs)) - folders, errs = self.dev.get_folderlist() - if errs and not folders: - raise DeviceError('Failed to read folders from device. Underlying errors:\n' - +self.format_errorstack(errs)) - self.filesystem_cache = FilesystemCache(files, folders) - except: - self.dev = self._main_id = self._carda_id = self._cardb_id = None - raise + @property + def filesystem_cache(self): + if self._filesystem_cache is None: + with self.lock: + files, errs = self.dev.get_filelist(self) + if errs and not files: + raise DeviceError('Failed to read files from device. Underlying errors:\n' + +self.format_errorstack(errs)) + folders, errs = self.dev.get_folderlist() + if errs and not folders: + raise DeviceError('Failed to read folders from device. Underlying errors:\n' + +self.format_errorstack(errs)) + storage = [] + for sid, capacity in zip([self._main_id, self._carda_id, + self._cardb_id], self.total_space()): + if sid is not None: + name = _('Unknown') + for x in self.dev.storage_info: + if x['id'] == sid: + name = x['name'] + break + storage.append({'id':sid, 'size':capacity, + 'is_folder':True, 'name':name}) + all_folders = [] + def recurse(f): + all_folders.append(f) + for c in f['children']: + recurse(c) + + for f in folders: recurse(f) + self._filesystem_cache = FilesystemCache(storage, + all_folders+files) + return self._filesystem_cache @synchronous def get_device_information(self, end_session=True): @@ -246,7 +200,6 @@ if __name__ == '__main__': devs = linux_scanner() mtp_devs = dev.detect(devs) dev.open(list(mtp_devs)[0], 'xxx') - dev.read_filesystem_cache() d = dev.dev print ("Opened device:", dev.get_gui_name()) print ("Storage info:") @@ -257,7 +210,7 @@ if __name__ == '__main__': # fname = b'moose.txt' # src = BytesIO(raw) # print (d.put_file(dev._main_id, 0, fname, src, len(raw), PR())) - dev.filesystem_cache.dump_filesystem() + dev.filesystem_cache.dump() # with open('/tmp/flint.epub', 'wb') as f: # print(d.get_file(786, f, PR())) # print() diff --git a/src/calibre/devices/mtp/unix/libmtp.c b/src/calibre/devices/mtp/unix/libmtp.c index 856fd057bf..ffab2e8b30 100644 --- a/src/calibre/devices/mtp/unix/libmtp.c +++ b/src/calibre/devices/mtp/unix/libmtp.c @@ -315,7 +315,7 @@ libmtp_Device_storage_info(libmtp_Device *self, void *closure) { "capacity", storage->MaxCapacity, "freespace_bytes", storage->FreeSpaceInBytes, "freespace_objects", storage->FreeSpaceInObjects, - "storage_desc", storage->StorageDescription, + "name", storage->StorageDescription, "volume_id", storage->VolumeIdentifier ); @@ -358,13 +358,14 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs } for (f=tf; f != NULL; f=f->next) { - fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}", + fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k,s:O}", "id", f->item_id, "parent_id", f->parent_id, "storage_id", f->storage_id, "name", f->filename, "size", f->filesize, - "modtime", f->modificationdate + "modtime", f->modificationdate, + "is_folder", Py_False ); if (fo == NULL || PyList_Append(ans, fo) != 0) break; Py_DECREF(fo); @@ -393,11 +394,12 @@ int folderiter(LIBMTP_folder_t *f, PyObject *parent) { children = PyList_New(0); if (children == NULL) { PyErr_NoMemory(); return 1;} - folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}", + folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:O,s:N}", "id", f->folder_id, "parent_id", f->parent_id, "storage_id", f->storage_id, "name", f->name, + "is_folder", Py_True, "children", children); if (folder == NULL) return 1; PyList_Append(parent, folder); diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp index 29d227d710..70cead4893 100644 --- a/src/calibre/devices/mtp/windows/content_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -28,7 +28,7 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { ADDPROP(WPD_OBJECT_PARENT_ID); ADDPROP(WPD_OBJECT_PERSISTENT_UNIQUE_ID); ADDPROP(WPD_OBJECT_NAME); - ADDPROP(WPD_OBJECT_SYNC_ID); + // ADDPROP(WPD_OBJECT_SYNC_ID); ADDPROP(WPD_OBJECT_ISSYSTEM); ADDPROP(WPD_OBJECT_ISHIDDEN); ADDPROP(WPD_OBJECT_CAN_DELETE); @@ -93,7 +93,7 @@ static void set_properties(PyObject *obj, IPortableDeviceValues *values) { set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", values); set_string_property(obj, WPD_OBJECT_NAME, "name", values); - set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values); + // set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values); set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", values); set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", values); diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 9d0ac55e82..51f5bfd60d 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -9,19 +9,27 @@ __docformat__ = 'restructuredtext en' import time, threading from functools import wraps +from future_builtins import zip +from itertools import chain from calibre import as_unicode, prints from calibre.constants import plugins, __appname__, numeric_version from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed from calibre.devices.mtp.base import MTPDeviceBase +from calibre.devices.mtp.filesystem_cache import FilesystemCache + +class ThreadingViolation(Exception): + + def __init__(self): + Exception.__init__('You cannot use the MTP driver from a thread other than the ' + ' thread in which startup() was called') def same_thread(func): @wraps(func) def check_thread(self, *args, **kwargs): if self.start_thread is not threading.current_thread(): - raise Exception('You cannot use %s from a thread other than the ' - ' thread in which startup() was called'%self.__class__.__name__) + raise ThreadingViolation() return func(self, *args, **kwargs) return check_thread @@ -42,6 +50,7 @@ class MTP_DEVICE(MTPDeviceBase): self.wpd = self.wpd_error = None self._main_id = self._carda_id = self._cardb_id = None self.start_thread = None + self._filesystem_cache = None def startup(self): self.start_thread = threading.current_thread() @@ -59,7 +68,7 @@ class MTP_DEVICE(MTPDeviceBase): @same_thread def shutdown(self): - self.dev = self.filesystem_cache = self.start_thread = None + self.dev = self._filesystem_cache = self.start_thread = None if self.wpd is not None: self.wpd.uninit() @@ -130,11 +139,33 @@ class MTP_DEVICE(MTPDeviceBase): return True + @property + def filesystem_cache(self): + if self._filesystem_cache is None: + ts = self.total_space() + all_storage = [] + items = [] + for storage_id, capacity in zip([self._main_id, self._carda_id, + self._cardb_id], ts): + if storage_id is None: continue + name = _('Unknown') + for s in self.dev.data['storage']: + if s['id'] == storage_id: + name = s['name'] + break + storage = {'id':storage_id, 'size':capacity, 'name':name, + 'is_folder':True} + id_map = self.dev.get_filesystem(storage_id) + all_storage.append(storage) + items.append(id_map.itervalues()) + self._filesystem_cache = FilesystemCache(all_storage, chain(*items)) + return self._filesystem_cache + @same_thread def post_yank_cleanup(self): self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None @same_thread def eject(self): @@ -142,11 +173,11 @@ class MTP_DEVICE(MTPDeviceBase): self.ejected_devices.add(self.currently_connected_pnp_id) self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None @same_thread def open(self, connected_device, library_uuid): - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None try: self.dev = self.wpd.Device(connected_device) except self.wpd.WPDError: diff --git a/src/calibre/devices/mtp/windows/remote.py b/src/calibre/devices/mtp/windows/remote.py index 6f883f8baf..a3686ce88c 100644 --- a/src/calibre/devices/mtp/windows/remote.py +++ b/src/calibre/devices/mtp/windows/remote.py @@ -70,10 +70,10 @@ def main(): print ('Connected to:', dev.get_gui_name()) print ('Total space', dev.total_space()) print ('Free space', dev.free_space()) - # pprint.pprint(dev.dev.get_filesystem(dev._main_id)) - print ('Fetching file: oFF (198214 bytes)') - stream = dev.get_file('oFF') - print ("Fetched size: ", stream.tell()) + dev.filesystem_cache.dump() + # print ('Fetching file: oFF (198214 bytes)') + # stream = dev.get_file('oFF') + # print ("Fetched size: ", stream.tell()) finally: dev.shutdown()