From 6a40749075f612d347520dd97664380a838c4e4b Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 19 Apr 2011 07:24:50 -0400 Subject: [PATCH 01/23] Store: Clean calibre query logic from searches better. --- src/calibre/gui2/store/search.py | 35 ++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index ce74d52547..43fde1e4ed 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -33,6 +33,13 @@ TIMEOUT = 75 # seconds SEARCH_THREAD_TOTAL = 4 COVER_DOWNLOAD_THREAD_TOTAL = 2 +def comparable_price(text): + if len(text) < 3 or text[-3] not in ('.', ','): + text += '00' + text = re.sub(r'\D', '', text) + text = text.rjust(6, '0') + + class SearchDialog(QDialog, Ui_Dialog): def __init__(self, istores, *args): @@ -294,9 +301,7 @@ class SearchThread(Thread): while self._run and not self.tasks.empty(): try: query, store_name, store_plugin, timeout = self.tasks.get() - squery = query - for loc in SearchFilter.USABLE_LOCATIONS: - squery = re.sub(r'%s:"?(?P[^\s"]+)"?' % loc, '\g', squery) + squery = self._clean_query(query) for res in store_plugin.search(squery, timeout=timeout): if not self._run: return @@ -306,6 +311,21 @@ class SearchThread(Thread): self.tasks.task_done() except: pass + + def _clean_query(self, query): + query = query.lower() + for loc in ( 'all', 'author', 'authors', 'title'): + query = re.sub(r'%s:"?(?P[^\s"]+)"?' % loc, '\g', query) + for loc in ('cover', 'drm', 'format', 'formats', 'price', 'store'): + query = re.sub(r'%s:"[^"]"' % loc, '', query) + query = re.sub(r'%s:[^\s]*' % loc, '', query) + query = re.sub(r'(^|\s)(and|not|or)(\s|$)', ' ', query) + query = query.replace('\\', '') + query = query.replace('!', '') + query = query.replace('=', '') + query = query.replace('~', '') + query = re.sub(r'\s{2,}', ' ', query) + return query class CoverThreadPool(GenericDownloadThreadPool): @@ -439,11 +459,7 @@ class Matches(QAbstractItemModel): elif col == 2: text = result.author elif col == 3: - text = result.price - if len(text) < 3 or text[-3] not in ('.', ','): - text += '00' - text = re.sub(r'\D', '', text) - text = text.rjust(6, '0') + text = comparable_price(result.price) elif col == 4: text = result.store_name return text @@ -504,8 +520,9 @@ class SearchFilter(SearchQueryParser): q = { 'author': self.search_result.author.lower(), 'cover': self.search_result.cover_url, + 'drm': '', 'format': '', - 'price': self.search_result.price, + 'price': comparable_price(self.search_result.price), 'store': self.search_result.store_name.lower(), 'title': self.search_result.title.lower(), } From 9c09756b1cf011643ed24c18fe4d9b3b30fdd3c8 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 19 Apr 2011 07:25:54 -0400 Subject: [PATCH 02/23] Store: Clean calibre query logic from searches better. --- src/calibre/gui2/store/search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 43fde1e4ed..4514f6441a 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -324,6 +324,8 @@ class SearchThread(Thread): query = query.replace('!', '') query = query.replace('=', '') query = query.replace('~', '') + query = query.replace('>', '') + query = query.replace('<', '') query = re.sub(r'\s{2,}', ' ', query) return query From 094d09c61af97d14836a916ae9778cc4bc718de3 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 19 Apr 2011 07:27:00 -0400 Subject: [PATCH 03/23] Store: comments. --- src/calibre/gui2/store/search.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 4514f6441a..ce8296ba7e 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -314,11 +314,14 @@ class SearchThread(Thread): def _clean_query(self, query): query = query.lower() + # Remove the prefix. for loc in ( 'all', 'author', 'authors', 'title'): query = re.sub(r'%s:"?(?P[^\s"]+)"?' % loc, '\g', query) + # Remove the prefix and search text. for loc in ('cover', 'drm', 'format', 'formats', 'price', 'store'): query = re.sub(r'%s:"[^"]"' % loc, '', query) query = re.sub(r'%s:[^\s]*' % loc, '', query) + # Remove control modifiers. query = re.sub(r'(^|\s)(and|not|or)(\s|$)', ' ', query) query = query.replace('\\', '') query = query.replace('!', '') @@ -326,6 +329,7 @@ class SearchThread(Thread): query = query.replace('~', '') query = query.replace('>', '') query = query.replace('<', '') + # Remove excess whitespace. query = re.sub(r'\s{2,}', ' ', query) return query From 0b64888105f7c64db93fa81796b921c9e2dd892b Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 19 Apr 2011 18:42:23 -0400 Subject: [PATCH 04/23] Store: Remove control modifier sooner when cleaning a query. --- src/calibre/gui2/store/gutenberg_plugin.py | 1 + src/calibre/gui2/store/mobileread_plugin.py | 2 -- src/calibre/gui2/store/search.py | 16 +++++++++------- src/calibre/gui2/store/search_result.py | 5 +++++ 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 8d04b6236d..8166185ff5 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -79,5 +79,6 @@ class GutenbergStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/ebooks/' + id.strip() + s.drm = False yield s diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index 49c265d7fe..d520b42fae 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -129,8 +129,6 @@ class BookRef(SearchResult): def __init__(self): SearchResult.__init__(self) - - self.format = '' class MobeReadStoreDialog(QDialog, Ui_Dialog): diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index ce8296ba7e..c94c43537f 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -38,6 +38,7 @@ def comparable_price(text): text += '00' text = re.sub(r'\D', '', text) text = text.rjust(6, '0') + return text class SearchDialog(QDialog, Ui_Dialog): @@ -314,6 +315,13 @@ class SearchThread(Thread): def _clean_query(self, query): query = query.lower() + # Remove control modifiers. + query = query.replace('\\', '') + query = query.replace('!', '') + query = query.replace('=', '') + query = query.replace('~', '') + query = query.replace('>', '') + query = query.replace('<', '') # Remove the prefix. for loc in ( 'all', 'author', 'authors', 'title'): query = re.sub(r'%s:"?(?P[^\s"]+)"?' % loc, '\g', query) @@ -321,14 +329,8 @@ class SearchThread(Thread): for loc in ('cover', 'drm', 'format', 'formats', 'price', 'store'): query = re.sub(r'%s:"[^"]"' % loc, '', query) query = re.sub(r'%s:[^\s]*' % loc, '', query) - # Remove control modifiers. + # Remove logic. query = re.sub(r'(^|\s)(and|not|or)(\s|$)', ' ', query) - query = query.replace('\\', '') - query = query.replace('!', '') - query = query.replace('=', '') - query = query.replace('~', '') - query = query.replace('>', '') - query = query.replace('<', '') # Remove excess whitespace. query = re.sub(r'\s{2,}', ' ', query) return query diff --git a/src/calibre/gui2/store/search_result.py b/src/calibre/gui2/store/search_result.py index 6e0ed0b572..25ff32d0ec 100644 --- a/src/calibre/gui2/store/search_result.py +++ b/src/calibre/gui2/store/search_result.py @@ -16,3 +16,8 @@ class SearchResult(object): self.author = '' self.price = '' self.detail_item = '' + # None = Unknown. + # True = Has DRM. + # False = Does not have DRM. + self.drm = None + self.formats = '' From ebf4ee8fedcb571c6d82b901304c9d8822280b80 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 19 Apr 2011 20:32:24 -0400 Subject: [PATCH 05/23] Store: Add drm status indicators. Add DRM status results for Manybooks, amazon, baen and bwrite. --- resources/images/drm-locked.png | Bin 0 -> 1612 bytes resources/images/drm-unlocked.png | Bin 0 -> 1675 bytes src/calibre/gui2/store/amazon_plugin.py | 11 ++++ .../gui2/store/baen_webscription_plugin.py | 1 + src/calibre/gui2/store/bewrite_plugin.py | 1 + src/calibre/gui2/store/manybooks_plugin.py | 1 + src/calibre/gui2/store/search.py | 52 +++++++++++++++--- 7 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 resources/images/drm-locked.png create mode 100644 resources/images/drm-unlocked.png diff --git a/resources/images/drm-locked.png b/resources/images/drm-locked.png new file mode 100644 index 0000000000000000000000000000000000000000..84c7a5e7b1db1d4f92b896cce2a4a82e07e4de34 GIT binary patch literal 1612 zcmV-S2DABzP)8WRRCX%pC=)6|cSu-rmy|BA zOgdBH0HK+x9`21Z=}(A_szYZ@B4OU_ik)AZQI^ps~3Uy>Mfm>3j!2C z&CW?766F959Yo`-?;{Z3%vIy z^h1c@>>7STzI7u70ODsr`v>&xf~dcNukRp&!-78DU2T*Ay0!ghpeh2eYOmq=0fca3 z6<|!i8}k_0UG=X3o+bIhjVvdx5bv=ZWN#ZGfR2pcLB@0>LMDiFfVc-E#WM?&ehB~x z-vXU|GlX=>gRW{r4ze7`jbvYwrROGckR>5>OS1$3c)9vD@`ZGHCO09!WAYJR zWm)KJQy3rj4Txe(BLD!lKLE6EK>lLDxGN9ouIdK^96T8?#(hW%0pRMFD{7{oH3vC| z%9tW%#CZ)zKPt|)koTT4#@(Dk06^&>y4+#+wXgb&2DYd0d8YXNQn9<6vk2e|1*zli zs#J!RGb5hd=xMXS46}eH+;T}>!BcnvbkqMWr8c=5J=!xRsE|KX#<-fp3t#|%rx^3n z14YaN3&fo(v2QDOS91mdXaLGLaoSpJ3GyF`^E-;&)to^9I|@R7Dt2r03njo#1G^X-M0i>dFu=qjHq1yCN- zR30gSRCJY3mI5e`X)2EtKq|V*Crbg8$265k3Lq6-<&&iV%43?!BL$F(uJXxJ0Oc`F z#0OE6Gnjr;X3U7Sp6hM5AOf#eaOyQ01oC1i?k!gk$ zfGNE3ol^kuIWo~H99LGQ3v6{&l=0{f7reTfBk8p9#nzi#QZ7~$iR>_YFXS(A|zJ4@6&g5J_ zRAkI`?sAHADeYLHW)mvT4kyw?agJ*FF`Vx1zx(I@c)#C|_xt^PydLkz`;YfKm+AM3 z9@-oY0MPUDr0-vi@Owk8S@kUxt^$BH`@QJ91CpSrF#G&LI#y`V=DkI9SmE=N^plme0AlY(>u$B5KM1 zjoY|mVtB~CYF&+SaJ_Bd;9qiQ22$D z?V_Vn6&p^+#Kd9nXO1hFoYGD6#ICM!dv~k$^9i{5z&YO-KDsJ<+N{QGR+Stk3;tE# zSUs%KrKjdB`+u}BwXdVDztV*`=FGeYs^o05pSCMDbEhcHHaE@uWy1C{iE+Fxc&rkr z=1i3Za`ofd(D^d>xuWQoa~?hTf%HMog-geGH83-e>$o|^LY7J(30)%0o=!}*n$FTx zRCdj58OXT)aD+dpi<&3FWZDC^f7`5+ESz1v)jaRo0=Km=oZi|aU;a{|n+1xLL{l;(c19f0X|rruzPP8KD%68GH~qa8>^3g0WA7VA@W@#)=x74^>YUgal?gu(z>xb~rvSTsk^ z2PO5%Y6{JMmD)nSLAx1&AAJBdKiLaab9Eq6TCVO?+p92%RIi*`HI(o~Xp2hV>gS{U`1nTa?o7?JS!< zd<+ZgZ#)y%HEPsH9_~xXeyKcm5ETR;YaN4z1EJ>gZUGOfqX}5*hC}7QxWt})Q`L;{ z;A8#_8pKMSh{Ee7=pt7^AqLl#iMVCucV(KlDLZMujDO0q3^N#0iiHh%flrtq@FMGn zuMkW`e~dX=N_O-<$pYTgki8yIa0KJ~ZA~UM+V#CVLZNC0aeygO^gd}{@#Cl?$Ce`G zFI(?a+ccMk%=OyEB~((cn(@lgguzq?}ykLE}Q84vRjIT_KC-B@YTZQ-T$ zNDT5xQYHzWNx(TdqVedfGiz92ROK9V6idbWj_Or_`-&4i9VD1kd>==L@-GZjU1R~4 zozSJ~ZP2~W&Q7bhr=%kzBb;URbdh}!FeMw+v4D}qfK%`Dt?>0MHdkfCW&1|i=Ya+w zZjCy(2Z|z`*rAW)oHhHgjs&(=3dhPpzM+_Hf)t&ufAldHW*w@RSAhCL=S$fLJMa9! z16bJPG2=B8FmfM}ze04=`lCKXOG)+zBUs>|C+^`l8VWV}Q@&+&e$Le48gO%dLgI=R z@p4XBdk!Jyyo1kinhXQmvP{INc-{Y_=K+~nGOpR8sk+hPWYYf{x_w&9tkhJIN5?NU z3l()|3SY&&d1t{ZpxAc3b{9CbAAvX(D-};|DcKHjzaGGM3&8T`OD%gudXNk)b!KXl z;B>>jz~8Xc5YG=1OPsZ(DR6qx_J{@o7}8<_Ee7Y=x9_iRk-_jTszcV2p43?6KDqbE zRp8s*9d3Z7rYSGW1_dBVpOYc~Juk`wXf|?OCij-KnbQ6yR+sZ1-WMtsl{Z_E;N=@G bU(cr<`1(Q8GaR;m^~nIQJ%02W8Ykl~+{W{F literal 0 HcmV?d00001 diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 51986ee4df..067e9e899e 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -159,6 +159,16 @@ class AmazonKindleStore(StorePlugin): author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) author = author.split('by')[-1] price = ''.join(data.xpath('div[@class="newPrice"]/span/text()')) + + with closing(br.open(asin_href, timeout=timeout/4)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "Simultaneous Device Usage")])'): + if idata.xpath('boolean(//div[@class="content"]//li[contains(., "Unlimited") and contains(b, "Simultaneous Device Usage")])'): + drm = False + else: + drm = None + else: + drm = True counter -= 1 @@ -168,5 +178,6 @@ class AmazonKindleStore(StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = asin.strip() + s.drm = drm yield s diff --git a/src/calibre/gui2/store/baen_webscription_plugin.py b/src/calibre/gui2/store/baen_webscription_plugin.py index 46a7c1ec7c..79b8751796 100644 --- a/src/calibre/gui2/store/baen_webscription_plugin.py +++ b/src/calibre/gui2/store/baen_webscription_plugin.py @@ -85,5 +85,6 @@ class BaenWebScriptionStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price s.detail_item = id.strip() + s.drm = False yield s diff --git a/src/calibre/gui2/store/bewrite_plugin.py b/src/calibre/gui2/store/bewrite_plugin.py index 37bd9cf9a5..887390a3af 100644 --- a/src/calibre/gui2/store/bewrite_plugin.py +++ b/src/calibre/gui2/store/bewrite_plugin.py @@ -76,5 +76,6 @@ class BeWriteStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = id.strip() + s.drm = False yield s diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 72fe54c427..697a31eb85 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -89,5 +89,6 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/titles/' + id + s.drm = False yield s diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index c94c43537f..61e278afee 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -95,9 +95,11 @@ class SearchDialog(QDialog, Ui_Dialog): # Author self.results_view.setColumnWidth(2,int(total*.35)) # Price - self.results_view.setColumnWidth(3, int(total*.10)) + self.results_view.setColumnWidth(3, int(total*.5)) + # DRM + self.results_view.setColumnWidth(4, int(total*.5)) # Store - self.results_view.setColumnWidth(4, int(total*.20)) + self.results_view.setColumnWidth(5, int(total*.20)) def do_search(self, checked=False): # Stop all running threads. @@ -311,6 +313,7 @@ class SearchThread(Thread): self.results.put(res) self.tasks.task_done() except: + traceback.print_exc() pass def _clean_query(self, query): @@ -379,10 +382,15 @@ class CoverThread(Thread): class Matches(QAbstractItemModel): - HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('Store')] + HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('DRM'), _('Store')] def __init__(self): QAbstractItemModel.__init__(self) + + self.DRM_LOCKED_ICON = QPixmap(I('drm-locked.png')).scaledToHeight(64) + self.DRM_UNLOCKED_ICON = QPixmap(I('drm-unlocked.png')).scaledToHeight(64) + self.DRM_UNKNOWN_ICON = QPixmap(I('dialog_warning.png')).scaledToHeight(64) + self.matches = [] self.cover_pool = CoverThreadPool(CoverThread, 2) self.cover_pool.start_threads() @@ -448,7 +456,7 @@ class Matches(QAbstractItemModel): return QVariant(result.author) elif col == 3: return QVariant(result.price) - elif col == 4: + elif col == 5: return QVariant(result.store_name) return NONE elif role == Qt.DecorationRole: @@ -456,6 +464,13 @@ class Matches(QAbstractItemModel): p = QPixmap() p.loadFromData(result.cover_data) return QVariant(p) + if col == 4: + if result.drm: + return QVariant(self.DRM_LOCKED_ICON) + if result.drm == False: + return QVariant(self.DRM_UNLOCKED_ICON) + else: + return QVariant(self.DRM_UNKNOWN_ICON) elif role == Qt.SizeHintRole: return QSize(64, 64) return NONE @@ -469,6 +484,13 @@ class Matches(QAbstractItemModel): elif col == 3: text = comparable_price(result.price) elif col == 4: + if result.drm: + text = 'a' + elif result.drm == False: + text = 'b' + else: + text = 'c' + elif col == 5: text = result.store_name return text @@ -490,6 +512,7 @@ class SearchFilter(SearchQueryParser): 'author', 'authors', 'cover', + 'drm', 'price', 'title', 'store', @@ -528,7 +551,7 @@ class SearchFilter(SearchQueryParser): q = { 'author': self.search_result.author.lower(), 'cover': self.search_result.cover_url, - 'drm': '', + 'drm': self.search_result.drm, 'format': '', 'price': comparable_price(self.search_result.price), 'store': self.search_result.store_name.lower(), @@ -539,12 +562,23 @@ class SearchFilter(SearchQueryParser): for locvalue in locations: ac_val = q[locvalue] if query == 'true': - if ac_val is not None: - matches.add(self.search_result) + if locvalue == 'drm': + if ac_val == True: + matches.add(self.search_result) + else: + if ac_val is not None: + matches.add(self.search_result) continue if query == 'false': - if ac_val is None: - matches.add(self.search_result) + if locvalue == 'drm': + if ac_val == False: + matches.add(self.search_result) + else: + if ac_val is None: + matches.add(self.search_result) + continue + # this is bool, so can't match below + if locvalue == 'drm': continue try: ### Can't separate authors because comma is used for name sep and author sep From 7680faa48f2a8ede3f31c0a0f0d91c5dff452406 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 19 Apr 2011 20:40:11 -0400 Subject: [PATCH 06/23] Store: Diesel drm status. --- src/calibre/gui2/store/diesel_ebooks_plugin.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py index 66c22f847f..e44896e1f8 100644 --- a/src/calibre/gui2/store/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -74,6 +74,13 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): price_elem = data.xpath('//td[@class="price"]/text()') if price_elem: price = price_elem[0] + + with closing(br.open('http://www.diesel-ebooks.com/item/' + id.strip(), timeout=timeout/4)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//table[@class="format-info"]//tr[contains(th, "DRM") and contains(td, "No")])'): + drm = False + else: + drm = True counter -= 1 @@ -83,5 +90,6 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/item/' + id.strip() + s.drm = drm yield s From 8c6d4e66d334b05e359a7874b2806518e9d81dc8 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 19 Apr 2011 21:16:46 -0400 Subject: [PATCH 07/23] Store: ebooks.com drm status. --- src/calibre/gui2/store/ebooks_com_plugin.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py index 259e996ebe..b1fc7e9c11 100644 --- a/src/calibre/gui2/store/ebooks_com_plugin.py +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -70,6 +70,11 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()') if len(pdata) >= 2: price = pdata[1] + drm = False + for sec in ('Printing', 'Copying', 'Lending'): + if pdoc.xpath('boolean(//div[@class="formatTableInner"]//table//tr[contains(th, "%s") and contains(td, "Off")])' % sec): + drm = True + break if not price: continue @@ -91,5 +96,6 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '?url=http://www.ebooks.com/cj.asp?IID=' + id.strip() + '&cjsku=' + id.strip() + s.drm = drm yield s From 2b01f9ab68480a1583bf87e3c509ac1cb96cad7e Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 19 Apr 2011 21:38:15 -0400 Subject: [PATCH 08/23] Store: eharlequin report DRM status. --- src/calibre/gui2/store/eharlequin_plugin.py | 9 ++++++++- src/calibre/gui2/store/search.py | 1 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/eharlequin_plugin.py b/src/calibre/gui2/store/eharlequin_plugin.py index 1886671b0a..4624732579 100644 --- a/src/calibre/gui2/store/eharlequin_plugin.py +++ b/src/calibre/gui2/store/eharlequin_plugin.py @@ -68,13 +68,20 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin): price = ''.join(data.xpath('.//div[@class="ourprice"]/font/text()')) cover_url = ''.join(data.xpath('.//a[@href="%s"]/img/@src' % id)) + with closing(br.open('http://ebooks.eharlequin.com/' + id.strip(), timeout=timeout/4)) as nf: + idata = html.fromstring(nf.read()) + drm = None + if idata.xpath('boolean(//div[@class="drm_head"])'): + drm = idata.xpath('boolean(//td[contains(., "Copy") and contains(., "not")])') + counter -= 1 - + s = SearchResult() s.cover_url = cover_url s.title = title.strip() s.author = author.strip() s.price = price.strip() s.detail_item = '?url=http://ebooks.eharlequin.com/' + id.strip() + s.drm = drm yield s diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 61e278afee..e536013e4b 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -314,7 +314,6 @@ class SearchThread(Thread): self.tasks.task_done() except: traceback.print_exc() - pass def _clean_query(self, query): query = query.lower() From f9a931dbfb47d6c49826c7b4244688f97a5e4d99 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 19 Apr 2011 21:39:50 -0400 Subject: [PATCH 09/23] Store: mobileread report DRM status. --- src/calibre/gui2/store/mobileread_plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index d520b42fae..54f19cf814 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -76,6 +76,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): matches = heapq.nlargest(max_results, matches) for score, book in matches: book.price = '$0.00' + book.drm = False yield book def update_book_list(self, timeout=10): From b415c44cb6f2487fb1a206c885a3b5b0479eb1f8 Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 20 Apr 2011 07:46:27 -0400 Subject: [PATCH 10/23] Store: Filter in Matches model. --- src/calibre/gui2/store/search.py | 142 +++++++++++++++++++------------ 1 file changed, 88 insertions(+), 54 deletions(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index e536013e4b..322680178b 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -112,6 +112,9 @@ class SearchDialog(QDialog, Ui_Dialog): query = unicode(self.search_edit.text()) if not query.strip(): return + # Give the query to the results model so it can do + # futher filtering. + self.results_view.model().set_query(query) # Plugins are in alphebetic order. Randomize the # order of plugin names. This way plugins closer @@ -304,13 +307,12 @@ class SearchThread(Thread): while self._run and not self.tasks.empty(): try: query, store_name, store_plugin, timeout = self.tasks.get() - squery = self._clean_query(query) - for res in store_plugin.search(squery, timeout=timeout): + query = self._clean_query(query) + for res in store_plugin.search(query, timeout=timeout): if not self._run: return res.store_name = store_name - if SearchFilter(res).parse(query): - self.results.put(res) + self.results.put(res) self.tasks.task_done() except: traceback.print_exc() @@ -390,7 +392,14 @@ class Matches(QAbstractItemModel): self.DRM_UNLOCKED_ICON = QPixmap(I('drm-unlocked.png')).scaledToHeight(64) self.DRM_UNKNOWN_ICON = QPixmap(I('dialog_warning.png')).scaledToHeight(64) + # All matches. Used to determine the order to display + # self.matches because the SearchFilter returns + # matches unordered. + self.all_matches = [] + # Only the showing matches. self.matches = [] + self.query = '' + self.search_filter = SearchFilter() self.cover_pool = CoverThreadPool(CoverThread, 2) self.cover_pool.start_threads() @@ -398,15 +407,21 @@ class Matches(QAbstractItemModel): self.cover_pool.abort() def clear_results(self): + self.all_matches = [] self.matches = [] + self.all_matches = [] + self.search_filter.clear_search_results() + self.query = '' self.cover_pool.abort() self.cover_pool.start_threads() self.reset() def add_result(self, result): self.layoutAboutToBeChanged.emit() - self.matches.append(result) - self.cover_pool.add_task(result, self.update_result) + self.all_matches.append(result) + self.search_filter.add_search_result(result) + self.cover_pool.add_task(result, self.filter_results) + self.filter_results() self.layoutChanged.emit() def get_result(self, index): @@ -415,10 +430,18 @@ class Matches(QAbstractItemModel): return self.matches[row] else: return None - - def update_result(self): + + def filter_results(self): self.layoutAboutToBeChanged.emit() - self.layoutChanged.emit() + if self.query: + self.matches = list(self.search_filter.parse(self.query)) + else: + self.matches = list(self.search_filter.universal_set()) + self.reorder_matches() + self.layoutAboutToBeChanged.emit() + + def set_query(self, query): + self.query = query def index(self, row, column, parent=QModelIndex()): return self.createIndex(row, column) @@ -497,11 +520,15 @@ class Matches(QAbstractItemModel): if not self.matches: return descending = order == Qt.DescendingOrder - self.matches.sort(None, + self.all_matches.sort(None, lambda x: sort_key(unicode(self.data_as_text(x, col))), descending) + self.reorder_matches() if reset: self.reset() + + def reorder_matches(self): + self.matches = sorted(self.matches, key=lambda x: self.all_matches.index(x)) class SearchFilter(SearchQueryParser): @@ -517,12 +544,18 @@ class SearchFilter(SearchQueryParser): 'store', ] - def __init__(self, search_result): + def __init__(self): SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS) - self.search_result = search_result + self.srs = set([]) + + def add_search_result(self, search_result): + self.srs.add(search_result) + + def clear_search_results(self): + self.srs = set([]) def universal_set(self): - return set([self.search_result]) + return self.srs def get_matches(self, location, query): location = location.lower().strip() @@ -548,50 +581,51 @@ class SearchFilter(SearchQueryParser): all_locs = set(self.USABLE_LOCATIONS) - set(['all']) locations = all_locs if location == 'all' else [location] q = { - 'author': self.search_result.author.lower(), - 'cover': self.search_result.cover_url, - 'drm': self.search_result.drm, + 'author': lambda x: x.author.lower(), + 'cover': lambda x: x.cover_url, + 'drm': lambda x: x.drm, 'format': '', - 'price': comparable_price(self.search_result.price), - 'store': self.search_result.store_name.lower(), - 'title': self.search_result.title.lower(), + 'price': lambda x: comparable_price(x.price), + 'store': lambda x: x.store_name.lower(), + 'title': lambda x: x.title.lower(), } for x in ('author', 'format'): q[x+'s'] = q[x] - for locvalue in locations: - ac_val = q[locvalue] - if query == 'true': + for sr in self.srs: + for locvalue in locations: + accessor = q[locvalue] + if query == 'true': + if locvalue == 'drm': + if accessor(sr) == True: + matches.add(sr) + else: + if accessor(sr) is not None: + matches.add(sr) + continue + if query == 'false': + if locvalue == 'drm': + if accessor(sr) == False: + matches.add(sr) + else: + if accessor(sr) is None: + matches.add(sr) + continue + # this is bool, so can't match below if locvalue == 'drm': - if ac_val == True: - matches.add(self.search_result) - else: - if ac_val is not None: - matches.add(self.search_result) - continue - if query == 'false': - if locvalue == 'drm': - if ac_val == False: - matches.add(self.search_result) - else: - if ac_val is None: - matches.add(self.search_result) - continue - # this is bool, so can't match below - if locvalue == 'drm': - continue - try: - ### Can't separate authors because comma is used for name sep and author sep - ### Exact match might not get what you want. For that reason, turn author - ### exactmatch searches into contains searches. - if locvalue == 'author' and matchkind == EQUALS_MATCH: - m = CONTAINS_MATCH - else: - m = matchkind - - vals = [ac_val] - if _match(query, vals, m): - matches.add(self.search_result) - break - except ValueError: # Unicode errors - traceback.print_exc() + continue + try: + ### Can't separate authors because comma is used for name sep and author sep + ### Exact match might not get what you want. For that reason, turn author + ### exactmatch searches into contains searches. + if locvalue == 'author' and matchkind == EQUALS_MATCH: + m = CONTAINS_MATCH + else: + m = matchkind + + vals = [accessor(sr)] + if _match(query, vals, m): + matches.add(sr) + break + except ValueError: # Unicode errors + traceback.print_exc() return matches From 08b8ef818cafb806357bc279d614d17586ea1b19 Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 20 Apr 2011 11:48:15 -0400 Subject: [PATCH 11/23] Store: SearchResult uses a different representation for DRM. Add Details download thread/pool. Change Amazon plugin to use details function instead of reading data in the search function. --- src/calibre/gui2/store/__init__.py | 13 ++- src/calibre/gui2/store/amazon_plugin.py | 27 +++--- src/calibre/gui2/store/search.py | 111 +++++++++++++++++------- src/calibre/gui2/store/search_result.py | 7 +- 4 files changed, 109 insertions(+), 49 deletions(-) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index 26bafd2c95..d0ec0cc479 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -76,10 +76,16 @@ class StorePlugin(object): # {{{ return items as a generator. Don't be lazy with the search! Load as much data as possible in the - :class:`calibre.gui2.store.search_result.SearchResult` object. If you have to parse - multiple pages to get all of the data then do so. However, if data (such as cover_url) + :class:`calibre.gui2.store.search_result.SearchResult` object. + However, if data (such as cover_url) isn't available because the store does not display cover images then it's okay to ignore it. + + At the very least a :class:`calibre.gui2.store.search_result.SearchResult` + returned by this function must have the title, author and id. + + If you have to parse multiple pages to get all of the data then implement + :meth:`get_deatils` for retrieving additional information. Also, by default search results can only include ebooks. A plugin can offer users an option to include physical books in the search results but this must be @@ -96,6 +102,9 @@ class StorePlugin(object): # {{{ item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. ''' raise NotImplementedError() + + def get_details(self, search_result, timeout=60): + raise NotImplementedError() def get_settings(self): ''' diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 067e9e899e..2fc5f8ba1e 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -159,16 +159,6 @@ class AmazonKindleStore(StorePlugin): author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) author = author.split('by')[-1] price = ''.join(data.xpath('div[@class="newPrice"]/span/text()')) - - with closing(br.open(asin_href, timeout=timeout/4)) as nf: - idata = html.fromstring(nf.read()) - if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "Simultaneous Device Usage")])'): - if idata.xpath('boolean(//div[@class="content"]//li[contains(., "Unlimited") and contains(b, "Simultaneous Device Usage")])'): - drm = False - else: - drm = None - else: - drm = True counter -= 1 @@ -178,6 +168,21 @@ class AmazonKindleStore(StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = asin.strip() - s.drm = drm yield s + + def get_details(self, search_result, timeout): + url = 'http://amazon.com/dp/' + + br = browser() + with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "Simultaneous Device Usage")])'): + if idata.xpath('boolean(//div[@class="content"]//li[contains(., "Unlimited") and contains(b, "Simultaneous Device Usage")])'): + search_result.drm = SearchResult.DRM_UNLOCKED + else: + search_result.drm = SearchResult.DRM_UNKNOWN + else: + search_result.drm = SearchResult.DRM_LOCKED + + \ No newline at end of file diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 322680178b..52b0761a7d 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -21,6 +21,7 @@ from calibre import browser from calibre.gui2 import NONE from calibre.gui2.progress_indicator import ProgressIndicator from calibre.gui2.store.search_ui import Ui_Dialog +from calibre.gui2.store.search_result import SearchResult from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ REGEXP_MATCH from calibre.utils.config import DynamicConfig @@ -123,6 +124,8 @@ class SearchDialog(QDialog, Ui_Dialog): store_names = self.store_plugins.keys() if not store_names: return + # Remove all of our internal filtering logic from the query. + query = self.clean_query(query) shuffle(store_names) # Add plugins that the user has checked to the search pool's work queue. for n in store_names: @@ -134,6 +137,29 @@ class SearchDialog(QDialog, Ui_Dialog): self.search_pool.start_threads() self.pi.startAnimation() + def clean_query(self, query): + query = query.lower() + # Remove control modifiers. + query = query.replace('\\', '') + query = query.replace('!', '') + query = query.replace('=', '') + query = query.replace('~', '') + query = query.replace('>', '') + query = query.replace('<', '') + # Remove the prefix. + for loc in ( 'all', 'author', 'authors', 'title'): + query = re.sub(r'%s:"?(?P[^\s"]+)"?' % loc, '\g', query) + # Remove the prefix and search text. + for loc in ('cover', 'drm', 'format', 'formats', 'price', 'store'): + query = re.sub(r'%s:"[^"]"' % loc, '', query) + query = re.sub(r'%s:[^\s]*' % loc, '', query) + # Remove logic. + query = re.sub(r'(^|\s)(and|not|or)(\s|$)', ' ', query) + # Remove excess whitespace. + query = re.sub(r'\s{2,}', ' ', query) + query = query.strip() + return query + def save_state(self): self.config['store_search_geometry'] = self.saveGeometry() self.config['store_search_store_splitter_state'] = self.store_splitter.saveState() @@ -183,9 +209,9 @@ class SearchDialog(QDialog, Ui_Dialog): self.pi.stopAnimation() while self.search_pool.has_results(): - res = self.search_pool.get_result() + res, store_plugin = self.search_pool.get_result() if res: - self.results_view.model().add_result(res) + self.results_view.model().add_result(res, store_plugin) def open_store(self, index): result = self.results_view.model().get_result(index) @@ -307,37 +333,14 @@ class SearchThread(Thread): while self._run and not self.tasks.empty(): try: query, store_name, store_plugin, timeout = self.tasks.get() - query = self._clean_query(query) for res in store_plugin.search(query, timeout=timeout): if not self._run: return res.store_name = store_name - self.results.put(res) + self.results.put((res, store_plugin)) self.tasks.task_done() except: traceback.print_exc() - - def _clean_query(self, query): - query = query.lower() - # Remove control modifiers. - query = query.replace('\\', '') - query = query.replace('!', '') - query = query.replace('=', '') - query = query.replace('~', '') - query = query.replace('>', '') - query = query.replace('<', '') - # Remove the prefix. - for loc in ( 'all', 'author', 'authors', 'title'): - query = re.sub(r'%s:"?(?P[^\s"]+)"?' % loc, '\g', query) - # Remove the prefix and search text. - for loc in ('cover', 'drm', 'format', 'formats', 'price', 'store'): - query = re.sub(r'%s:"[^"]"' % loc, '', query) - query = re.sub(r'%s:[^\s]*' % loc, '', query) - # Remove logic. - query = re.sub(r'(^|\s)(and|not|or)(\s|$)', ' ', query) - # Remove excess whitespace. - query = re.sub(r'\s{2,}', ' ', query) - return query class CoverThreadPool(GenericDownloadThreadPool): @@ -381,6 +384,42 @@ class CoverThread(Thread): continue +class DetailsThreadPool(GenericDownloadThreadPool): + ''' + Once started all threads run until abort is called. + ''' + + def add_task(self, search_result, store_plugin, update_callback, timeout=10): + self.tasks.put((search_result, store_plugin, update_callback, timeout)) + + +class DetailsThread(Thread): + + def __init__(self, tasks, results): + Thread.__init__(self) + self.daemon = True + self.tasks = tasks + self.results = results + self._run = True + + def abort(self): + self._run = False + + def run(self): + while self._run: + try: + time.sleep(.1) + while not self.tasks.empty(): + if not self._run: + break + result, store_plugin, callback, timeout = self.tasks.get() + if result: + store_plugin.get_details(result, timeout) + callback() + self.tasks.task_done() + except: + continue + class Matches(QAbstractItemModel): HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('DRM'), _('Store')] @@ -402,9 +441,12 @@ class Matches(QAbstractItemModel): self.search_filter = SearchFilter() self.cover_pool = CoverThreadPool(CoverThread, 2) self.cover_pool.start_threads() + self.details_pool = DetailsThreadPool(DetailsThread, 4) + self.details_pool.start_threads() def closing(self): self.cover_pool.abort() + self.details_pool.abort() def clear_results(self): self.all_matches = [] @@ -414,13 +456,16 @@ class Matches(QAbstractItemModel): self.query = '' self.cover_pool.abort() self.cover_pool.start_threads() + self.details_pool.abort() + self.details_pool.start_threads() self.reset() - def add_result(self, result): + def add_result(self, result, store_plugin): self.layoutAboutToBeChanged.emit() self.all_matches.append(result) self.search_filter.add_search_result(result) self.cover_pool.add_task(result, self.filter_results) + self.details_pool.add_task(result, store_plugin, self.filter_results) self.filter_results() self.layoutChanged.emit() @@ -438,7 +483,7 @@ class Matches(QAbstractItemModel): else: self.matches = list(self.search_filter.universal_set()) self.reorder_matches() - self.layoutAboutToBeChanged.emit() + self.layoutChanged.emit() def set_query(self, query): self.query = query @@ -487,11 +532,11 @@ class Matches(QAbstractItemModel): p.loadFromData(result.cover_data) return QVariant(p) if col == 4: - if result.drm: + if result.drm == SearchResult.DRM_LOCKED: return QVariant(self.DRM_LOCKED_ICON) - if result.drm == False: + if result.drm == SearchResult.DRM_UNLOCKED: return QVariant(self.DRM_UNLOCKED_ICON) - else: + elif result.drm == SearchResult.DRM_UNKNOWN: return QVariant(self.DRM_UNKNOWN_ICON) elif role == Qt.SizeHintRole: return QSize(64, 64) @@ -596,7 +641,7 @@ class SearchFilter(SearchQueryParser): accessor = q[locvalue] if query == 'true': if locvalue == 'drm': - if accessor(sr) == True: + if accessor(sr) == SearchResult.DRM_LOCKED: matches.add(sr) else: if accessor(sr) is not None: @@ -604,7 +649,7 @@ class SearchFilter(SearchQueryParser): continue if query == 'false': if locvalue == 'drm': - if accessor(sr) == False: + if accessor(sr) == SearchResult.DRM_UNKNOWN: matches.add(sr) else: if accessor(sr) is None: diff --git a/src/calibre/gui2/store/search_result.py b/src/calibre/gui2/store/search_result.py index 25ff32d0ec..b817b68a77 100644 --- a/src/calibre/gui2/store/search_result.py +++ b/src/calibre/gui2/store/search_result.py @@ -8,6 +8,10 @@ __docformat__ = 'restructuredtext en' class SearchResult(object): + DRM_LOCKED = 1 + DRM_UNLOCKED = 2 + DRM_UNKNOWN = 3 + def __init__(self): self.store_name = '' self.cover_url = '' @@ -16,8 +20,5 @@ class SearchResult(object): self.author = '' self.price = '' self.detail_item = '' - # None = Unknown. - # True = Has DRM. - # False = Does not have DRM. self.drm = None self.formats = '' From 2390a90e7b4d31c08b347287da27946cdd6c3fe0 Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 20 Apr 2011 21:09:01 -0400 Subject: [PATCH 12/23] Store: Get DRM status for all stores. Load extra details in a separate thread. delay loading covers where the cover url is gotten via extra details. --- src/calibre/gui2/store/__init__.py | 2 +- .../gui2/store/baen_webscription_plugin.py | 2 +- src/calibre/gui2/store/bewrite_plugin.py | 23 ++++++----- src/calibre/gui2/store/bn_plugin.py | 1 + .../gui2/store/diesel_ebooks_plugin.py | 19 +++++---- src/calibre/gui2/store/ebooks_com_plugin.py | 40 +++++++++++-------- src/calibre/gui2/store/eharlequin_plugin.py | 29 ++++++++++---- src/calibre/gui2/store/feedbooks_plugin.py | 11 +++++ src/calibre/gui2/store/gutenberg_plugin.py | 2 +- src/calibre/gui2/store/kobo_plugin.py | 4 +- src/calibre/gui2/store/manybooks_plugin.py | 2 +- src/calibre/gui2/store/mobileread_plugin.py | 2 +- src/calibre/gui2/store/open_library_plugin.py | 1 + src/calibre/gui2/store/search.py | 20 ++++++++-- src/calibre/gui2/store/smashwords_plugin.py | 1 + 15 files changed, 110 insertions(+), 49 deletions(-) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index d0ec0cc479..2fc752ed55 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -104,7 +104,7 @@ class StorePlugin(object): # {{{ raise NotImplementedError() def get_details(self, search_result, timeout=60): - raise NotImplementedError() + pass def get_settings(self): ''' diff --git a/src/calibre/gui2/store/baen_webscription_plugin.py b/src/calibre/gui2/store/baen_webscription_plugin.py index 79b8751796..34ccc8f917 100644 --- a/src/calibre/gui2/store/baen_webscription_plugin.py +++ b/src/calibre/gui2/store/baen_webscription_plugin.py @@ -85,6 +85,6 @@ class BaenWebScriptionStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price s.detail_item = id.strip() - s.drm = False + s.drm = SearchResult.DRM_UNLOCKED yield s diff --git a/src/calibre/gui2/store/bewrite_plugin.py b/src/calibre/gui2/store/bewrite_plugin.py index 887390a3af..1f1cc59224 100644 --- a/src/calibre/gui2/store/bewrite_plugin.py +++ b/src/calibre/gui2/store/bewrite_plugin.py @@ -60,14 +60,6 @@ class BeWriteStore(BasicStoreConfig, StorePlugin): cover_url = '' price = '' - with closing(br.open(id.strip(), timeout=timeout/4)) as nf: - idata = html.fromstring(nf.read()) - price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()')) - price = '$' + price.split('$')[-1] - cover_img = idata.xpath('//div[@id="content"]//img[1]/@src') - if cover_img: - cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0] - counter -= 1 s = SearchResult() @@ -76,6 +68,19 @@ class BeWriteStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = id.strip() - s.drm = False + s.drm = SearchResult.DRM_UNLOCKED yield s + + def get_details(self, search_result, timeout): + br = browser() + + with closing(br.open(search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()')) + price = '$' + price.split('$')[-1] + search_result.price = price.strip() + cover_img = idata.xpath('//div[@id="content"]//img[1]/@src') + if cover_img: + cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0] + search_result.cover_url = cover_url.strip() diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index 4da551fd92..aa74eebf54 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -78,5 +78,6 @@ class BNStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price s.detail_item = id.strip() + s.drm = SearchResult.DRM_UNKNOWN yield s diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py index e44896e1f8..93edfce272 100644 --- a/src/calibre/gui2/store/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -74,13 +74,6 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): price_elem = data.xpath('//td[@class="price"]/text()') if price_elem: price = price_elem[0] - - with closing(br.open('http://www.diesel-ebooks.com/item/' + id.strip(), timeout=timeout/4)) as nf: - idata = html.fromstring(nf.read()) - if idata.xpath('boolean(//table[@class="format-info"]//tr[contains(th, "DRM") and contains(td, "No")])'): - drm = False - else: - drm = True counter -= 1 @@ -90,6 +83,16 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/item/' + id.strip() - s.drm = drm yield s + + def get_details(self, search_result, timeout): + url = 'http://www.diesel-ebooks.com/item/' + + br = browser() + with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//table[@class="format-info"]//tr[contains(th, "DRM") and contains(td, "No")])'): + search_result.drm = SearchResult.DRM_UNLOCKED + else: + search_result.drm = SearchResult.DRM_LOCKED diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py index b1fc7e9c11..1405b2f33e 100644 --- a/src/calibre/gui2/store/ebooks_com_plugin.py +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -7,6 +7,7 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' import random +import re import urllib2 from contextlib import closing @@ -63,20 +64,6 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): id = id.split('=')[-1] if not id: continue - - price = '' - with closing(br.open('http://www.ebooks.com/ebooks/book_display.asp?IID=' + id.strip(), timeout=timeout)) as fp: - pdoc = html.fromstring(fp.read()) - pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()') - if len(pdata) >= 2: - price = pdata[1] - drm = False - for sec in ('Printing', 'Copying', 'Lending'): - if pdoc.xpath('boolean(//div[@class="formatTableInner"]//table//tr[contains(th, "%s") and contains(td, "Off")])' % sec): - drm = True - break - if not price: - continue cover_url = ''.join(data.xpath('.//img[1]/@src')) @@ -94,8 +81,29 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): s.cover_url = cover_url s.title = title.strip() s.author = author.strip() - s.price = price.strip() s.detail_item = '?url=http://www.ebooks.com/cj.asp?IID=' + id.strip() + '&cjsku=' + id.strip() - s.drm = drm yield s + + def get_details(self, search_result, timeout): + url = 'http://www.ebooks.com/ebooks/book_display.asp?IID=' + + mo = re.search(r'\?IID=(?P\d+)', search_result.detail_item) + if mo: + id = mo.group('id') + if not id: + return + + price = _('Not Available') + br = browser() + with closing(br.open(url + id, timeout=timeout)) as nf: + pdoc = html.fromstring(nf.read()) + pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()') + if len(pdata) >= 2: + price = pdata[1] + search_result.drm = SearchResult.DRM_UNLOCKED + for sec in ('Printing', 'Copying', 'Lending'): + if pdoc.xpath('boolean(//div[@class="formatTableInner"]//table//tr[contains(th, "%s") and contains(td, "Off")])' % sec): + search_result.drm = SearchResult.DRM_LOCKED + break + search_result.price = price.strip() diff --git a/src/calibre/gui2/store/eharlequin_plugin.py b/src/calibre/gui2/store/eharlequin_plugin.py index 4624732579..e77e6f11ce 100644 --- a/src/calibre/gui2/store/eharlequin_plugin.py +++ b/src/calibre/gui2/store/eharlequin_plugin.py @@ -7,6 +7,7 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' import random +import re import urllib2 from contextlib import closing @@ -68,12 +69,6 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin): price = ''.join(data.xpath('.//div[@class="ourprice"]/font/text()')) cover_url = ''.join(data.xpath('.//a[@href="%s"]/img/@src' % id)) - with closing(br.open('http://ebooks.eharlequin.com/' + id.strip(), timeout=timeout/4)) as nf: - idata = html.fromstring(nf.read()) - drm = None - if idata.xpath('boolean(//div[@class="drm_head"])'): - drm = idata.xpath('boolean(//td[contains(., "Copy") and contains(., "not")])') - counter -= 1 s = SearchResult() @@ -82,6 +77,26 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '?url=http://ebooks.eharlequin.com/' + id.strip() - s.drm = drm yield s + + def get_details(self, search_result, timeout): + url = 'http://ebooks.eharlequin.com/en/ContentDetails.htm?ID=' + + mo = re.search(r'\?ID=(?P.+)', search_result.detail_item) + if mo: + id = mo.group('id') + if not id: + return + + + br = browser() + with closing(br.open(url + id, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + drm = SearchResult.DRM_UNKNOWN + if idata.xpath('boolean(//div[@class="drm_head"])'): + if idata.xpath('boolean(//td[contains(., "Copy") and contains(., "not")])'): + drm = SearchResult.DRM_LOCKED + else: + drm = SearchResult.DRM_UNLOCKED + search_result.drm = drm diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 12873f8bc9..67de97126e 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -90,3 +90,14 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin): s.detail_item = id.strip() yield s + + def get_details(self, search_result, timeout): + url = 'http://m.feedbooks.com/' + + br = browser() + with closing(br.open(url_slash_cleaner(url + search_result.detail_item), timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//div[contains(@class, "m-description-long")]//p[contains(., "DRM") or contains(b, "Protection")])'): + search_result.drm = SearchResult.DRM_LOCKED + else: + search_result.drm = SearchResult.DRM_UNLOCKED diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 8166185ff5..0551c1b40b 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -79,6 +79,6 @@ class GutenbergStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/ebooks/' + id.strip() - s.drm = False + s.drm = SearchResult.DRM_UNLOCKED yield s diff --git a/src/calibre/gui2/store/kobo_plugin.py b/src/calibre/gui2/store/kobo_plugin.py index d37e806c3f..e08094adf0 100644 --- a/src/calibre/gui2/store/kobo_plugin.py +++ b/src/calibre/gui2/store/kobo_plugin.py @@ -63,7 +63,7 @@ class KoboStore(BasicStoreConfig, StorePlugin): if not id: continue - price = ''.join(data.xpath('.//span[@class="SCOurPrice"]/strong/text()')) + price = ''.join(data.xpath('.//li[@class="OurPrice"]/strong/text()')) if not price: price = '$0.00' @@ -71,6 +71,7 @@ class KoboStore(BasicStoreConfig, StorePlugin): title = ''.join(data.xpath('.//div[@class="SCItemHeader"]/h1/a[1]/text()')) author = ''.join(data.xpath('.//div[@class="SCItemSummary"]/span/a[1]/text()')) + drm = data.xpath('boolean(.//span[@class="SCAvailibilityFormatsText" and contains(text(), "DRM")])') counter -= 1 @@ -80,5 +81,6 @@ class KoboStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '?url=http://www.kobobooks.com/' + id.strip() + s.drm = SearchResult.DRM_LOCKED if drm else SearchResult.DRM_UNLOCKED yield s diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 697a31eb85..aa5c45300a 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -89,6 +89,6 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/titles/' + id - s.drm = False + s.drm = SearchResult.DRM_UNLOCKED yield s diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index 54f19cf814..3ac035d378 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -76,7 +76,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): matches = heapq.nlargest(max_results, matches) for score, book in matches: book.price = '$0.00' - book.drm = False + book.drm = SearchResult.DRM_UNLOCKED yield book def update_book_list(self, timeout=10): diff --git a/src/calibre/gui2/store/open_library_plugin.py b/src/calibre/gui2/store/open_library_plugin.py index 15b674f262..2bd38aae4f 100644 --- a/src/calibre/gui2/store/open_library_plugin.py +++ b/src/calibre/gui2/store/open_library_plugin.py @@ -68,5 +68,6 @@ class OpenLibraryStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price s.detail_item = id.strip() + s.drm = SearchResult.DRM_UNKNOWN yield s diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 52b0761a7d..3c5d54afcf 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -415,9 +415,10 @@ class DetailsThread(Thread): result, store_plugin, callback, timeout = self.tasks.get() if result: store_plugin.get_details(result, timeout) - callback() + callback(result) self.tasks.task_done() except: + traceback.print_exc() continue class Matches(QAbstractItemModel): @@ -464,8 +465,12 @@ class Matches(QAbstractItemModel): self.layoutAboutToBeChanged.emit() self.all_matches.append(result) self.search_filter.add_search_result(result) - self.cover_pool.add_task(result, self.filter_results) - self.details_pool.add_task(result, store_plugin, self.filter_results) + if result.cover_url: + result.cover_queued = True + self.cover_pool.add_task(result, self.filter_results) + else: + result.cover_queued = False + self.details_pool.add_task(result, store_plugin, self.got_result_details) self.filter_results() self.layoutChanged.emit() @@ -485,6 +490,15 @@ class Matches(QAbstractItemModel): self.reorder_matches() self.layoutChanged.emit() + def got_result_details(self, result): + if not result.cover_queued and result.cover_url: + result.cover_queued = True + self.cover_pool.add_task(result, self.filter_results) + if result in self.matches: + row = self.matches.index(result) + self.dataChanged.emit(self.index(row, 0), self.index(row, self.columnCount() - 1)) + self.filter_results() + def set_query(self, query): self.query = query diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index 1806e9f4e1..43efe549cc 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -90,5 +90,6 @@ class SmashwordsStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/books/view/' + id.strip() + s.drm = SearchResult.DRM_UNLOCKED yield s From 7a466b28257f2d5080510308d939379a957724ac Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 20 Apr 2011 21:51:49 -0400 Subject: [PATCH 13/23] Store: Format support. --- src/calibre/gui2/store/amazon_plugin.py | 1 + .../gui2/store/baen_webscription_plugin.py | 1 + src/calibre/gui2/store/bewrite_plugin.py | 15 +++++++++++++++ src/calibre/gui2/store/bn_plugin.py | 1 + src/calibre/gui2/store/diesel_ebooks_plugin.py | 3 +++ src/calibre/gui2/store/ebooks_com_plugin.py | 9 +++++++++ src/calibre/gui2/store/eharlequin_plugin.py | 1 + src/calibre/gui2/store/search.py | 16 ++++++++++++---- 8 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 2fc5f8ba1e..fe9d9c81c6 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -168,6 +168,7 @@ class AmazonKindleStore(StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = asin.strip() + s.formats = 'Kindle' yield s diff --git a/src/calibre/gui2/store/baen_webscription_plugin.py b/src/calibre/gui2/store/baen_webscription_plugin.py index 34ccc8f917..d4f7924851 100644 --- a/src/calibre/gui2/store/baen_webscription_plugin.py +++ b/src/calibre/gui2/store/baen_webscription_plugin.py @@ -86,5 +86,6 @@ class BaenWebScriptionStore(BasicStoreConfig, StorePlugin): s.price = price s.detail_item = id.strip() s.drm = SearchResult.DRM_UNLOCKED + s.formats = 'RB, MOBI, EPUB, LIT, LRF, RTF, HTML' yield s diff --git a/src/calibre/gui2/store/bewrite_plugin.py b/src/calibre/gui2/store/bewrite_plugin.py index 1f1cc59224..716e147bbc 100644 --- a/src/calibre/gui2/store/bewrite_plugin.py +++ b/src/calibre/gui2/store/bewrite_plugin.py @@ -77,10 +77,25 @@ class BeWriteStore(BasicStoreConfig, StorePlugin): with closing(br.open(search_result.detail_item, timeout=timeout)) as nf: idata = html.fromstring(nf.read()) + price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()')) + if not price: + price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "MOBI")]/text()')) + if not price: + price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "PDF")]/text()')) price = '$' + price.split('$')[-1] search_result.price = price.strip() + cover_img = idata.xpath('//div[@id="content"]//img[1]/@src') if cover_img: cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0] search_result.cover_url = cover_url.strip() + + formats = set([]) + if idata.xpath('boolean(//div[@id="content"]//td[contains(text(), "ePub")])'): + formats.add('EPUB') + if idata.xpath('boolean(//div[@id="content"]//td[contains(text(), "PDF")])'): + formats.add('PDF') + if idata.xpath('boolean(//div[@id="content"]//td[contains(text(), "MOBI")])'): + formats.add('MOBI') + search_result.formats = ', '.join(list(formats)) diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index aa74eebf54..8b1cfa03f0 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -79,5 +79,6 @@ class BNStore(BasicStoreConfig, StorePlugin): s.price = price s.detail_item = id.strip() s.drm = SearchResult.DRM_UNKNOWN + s.formats = 'Nook' yield s diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py index 93edfce272..9cea376341 100644 --- a/src/calibre/gui2/store/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -75,6 +75,8 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): if price_elem: price = price_elem[0] + formats = ', '.join(data.xpath('.//td[@class="format"]/text()')) + counter -= 1 s = SearchResult() @@ -83,6 +85,7 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/item/' + id.strip() + s.formats = formats yield s diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py index 1405b2f33e..2c3b3311c4 100644 --- a/src/calibre/gui2/store/ebooks_com_plugin.py +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -98,12 +98,21 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): br = browser() with closing(br.open(url + id, timeout=timeout)) as nf: pdoc = html.fromstring(nf.read()) + pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()') if len(pdata) >= 2: price = pdata[1] + search_result.drm = SearchResult.DRM_UNLOCKED for sec in ('Printing', 'Copying', 'Lending'): if pdoc.xpath('boolean(//div[@class="formatTableInner"]//table//tr[contains(th, "%s") and contains(td, "Off")])' % sec): search_result.drm = SearchResult.DRM_LOCKED break + + fdata = ', '.join(pdoc.xpath('//table[@class="price"]//tr//td[1]/text()')) + fdata = fdata.replace(':', '') + fdata = re.sub(r'\s{2,}', ' ', fdata) + fdata = fdata.strip() + search_result.formats = fdata + search_result.price = price.strip() diff --git a/src/calibre/gui2/store/eharlequin_plugin.py b/src/calibre/gui2/store/eharlequin_plugin.py index e77e6f11ce..325cb2f237 100644 --- a/src/calibre/gui2/store/eharlequin_plugin.py +++ b/src/calibre/gui2/store/eharlequin_plugin.py @@ -77,6 +77,7 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '?url=http://ebooks.eharlequin.com/' + id.strip() + s.formats = 'EPUB' yield s diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 3c5d54afcf..506e3d4d11 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -418,12 +418,11 @@ class DetailsThread(Thread): callback(result) self.tasks.task_done() except: - traceback.print_exc() continue class Matches(QAbstractItemModel): - HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('DRM'), _('Store')] + HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('DRM'), _('Store'), _('Formats')] def __init__(self): QAbstractItemModel.__init__(self) @@ -539,6 +538,8 @@ class Matches(QAbstractItemModel): return QVariant(result.price) elif col == 5: return QVariant(result.store_name) + elif col == 6: + return QVariant(result.formats) return NONE elif role == Qt.DecorationRole: if col == 0 and result.cover_data: @@ -573,6 +574,8 @@ class Matches(QAbstractItemModel): text = 'c' elif col == 5: text = result.store_name + elif col == 6: + text = ', '.join(sorted(result.formats.split(','))) return text def sort(self, col, order, reset=True): @@ -598,6 +601,8 @@ class SearchFilter(SearchQueryParser): 'authors', 'cover', 'drm', + 'format', + 'formats', 'price', 'title', 'store', @@ -643,7 +648,7 @@ class SearchFilter(SearchQueryParser): 'author': lambda x: x.author.lower(), 'cover': lambda x: x.cover_url, 'drm': lambda x: x.drm, - 'format': '', + 'format': lambda x: x.formats, 'price': lambda x: comparable_price(x.price), 'store': lambda x: x.store_name.lower(), 'title': lambda x: x.title.lower(), @@ -681,7 +686,10 @@ class SearchFilter(SearchQueryParser): else: m = matchkind - vals = [accessor(sr)] + if locvalue == 'format': + vals = accessor(sr).split(',') + else: + vals = [accessor(sr)] if _match(query, vals, m): matches.add(sr) break From f8df01905957c716b4f0aa597f47f2e0a42ff3df Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 21 Apr 2011 06:39:19 -0400 Subject: [PATCH 14/23] Store: Documentation, Tooltips, Ensure DRM status is set to unknown after extra details are donwloaded if DRM status is not set. --- src/calibre/gui2/store/__init__.py | 13 +++++++++++++ src/calibre/gui2/store/search.py | 26 ++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index 2fc752ed55..ddce2f5ec4 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -104,6 +104,19 @@ class StorePlugin(object): # {{{ raise NotImplementedError() def get_details(self, search_result, timeout=60): + ''' + Delayed search for information about specific search items. + + Typically, this will be used when certain information such as + formats, drm status, cover url are not part of the main search + results and the information is on another web page. + + Using this function allows for the main information (title, author) + to be displayed in the search results while other information can + take extra time to load. Splitting retrieving data that takes longer + to load into a separate function will give the illusion of the search + being faster. + ''' pass def get_settings(self): diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 506e3d4d11..1c9564a386 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -100,7 +100,9 @@ class SearchDialog(QDialog, Ui_Dialog): # DRM self.results_view.setColumnWidth(4, int(total*.5)) # Store - self.results_view.setColumnWidth(5, int(total*.20)) + self.results_view.setColumnWidth(5, int(total*.15)) + # Formats + self.results_view.setColumnWidth(6, int(total*.5)) def do_search(self, checked=False): # Stop all running threads. @@ -496,6 +498,8 @@ class Matches(QAbstractItemModel): if result in self.matches: row = self.matches.index(result) self.dataChanged.emit(self.index(row, 0), self.index(row, self.columnCount() - 1)) + if result.drm not in (SearchResult.DRM_LOCKED, SearchResult.DRM_UNLOCKED, SearchResult.DRM_UNKNOWN): + result.drm = SearchResult.DRM_UNKNOWN self.filter_results() def set_query(self, query): @@ -549,10 +553,28 @@ class Matches(QAbstractItemModel): if col == 4: if result.drm == SearchResult.DRM_LOCKED: return QVariant(self.DRM_LOCKED_ICON) - if result.drm == SearchResult.DRM_UNLOCKED: + elif result.drm == SearchResult.DRM_UNLOCKED: return QVariant(self.DRM_UNLOCKED_ICON) elif result.drm == SearchResult.DRM_UNKNOWN: return QVariant(self.DRM_UNKNOWN_ICON) + elif role == Qt.ToolTipRole: + if col == 1: + return QVariant(result.title) + elif col == 2: + return QVariant(result.author) + elif col == 3: + return QVariant(_('Detected price as: %s. Check with the store before making a purchase to verify this price information is correct.') % result.price) + elif col == 4: + if result.drm == SearchResult.DRM_LOCKED: + return QVariant(_('This book as been detected as having DRM restrictions. This book may not work with your reader and you will have limitations placed upon you as to what you can do with this book. Check with the store before making any purchases to ensure you can actually read this book.')) + elif result.drm == SearchResult.DRM_UNLOCKED: + return QVariant(_('This book has been detected as being DRM Free. You should be able to use this book on any device provided it is in a format calibre supports for conversion. However, before making a purchase double check the DRM status with the store. The store may not be disclosing the use of DRM.')) + else: + return QVariant(_('The DRM status of this book could not be determined. There is a very high likelihood that this book is actually DRM restricted.')) + elif col == 5: + return QVariant(result.store_name) + elif col == 6: + return QVariant(result.formats) elif role == Qt.SizeHintRole: return QSize(64, 64) return NONE From d6c7447e0aa9154208ee67e9632349aba6a9389e Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 21 Apr 2011 06:41:37 -0400 Subject: [PATCH 15/23] Store: Update drm check. --- src/calibre/gui2/store/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 1c9564a386..ba84d4583c 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -588,9 +588,9 @@ class Matches(QAbstractItemModel): elif col == 3: text = comparable_price(result.price) elif col == 4: - if result.drm: + if result.drm == SearchResult.DRM_UNLOCKED: text = 'a' - elif result.drm == False: + elif result.drm == SearchResult.DRM_LOCKED: text = 'b' else: text = 'c' From 94e4a48bbafaa64dd81917eb81ce5f489052784d Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 21 Apr 2011 06:43:09 -0400 Subject: [PATCH 16/23] Store: Fix spacing on ebooks.com formats. --- src/calibre/gui2/store/ebooks_com_plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py index 2c3b3311c4..3131cd0621 100644 --- a/src/calibre/gui2/store/ebooks_com_plugin.py +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -112,6 +112,7 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): fdata = ', '.join(pdoc.xpath('//table[@class="price"]//tr//td[1]/text()')) fdata = fdata.replace(':', '') fdata = re.sub(r'\s{2,}', ' ', fdata) + fdata = fdata.replace(' ,', ',') fdata = fdata.strip() search_result.formats = fdata From 4695840b2cc9eab54f0e6432289a719171fb6276 Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 21 Apr 2011 06:43:52 -0400 Subject: [PATCH 17/23] Store: ... --- src/calibre/gui2/store/search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index ba84d4583c..193ebbc420 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -647,6 +647,8 @@ class SearchFilter(SearchQueryParser): location = location.lower().strip() if location == 'authors': location = 'author' + elif location == 'formats': + location = 'format' matchkind = CONTAINS_MATCH if len(query) > 1: From b17ecef76796810d6074f3e1fa33e2c7f286e003 Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 21 Apr 2011 07:43:44 -0400 Subject: [PATCH 18/23] Store: Documentation. Return value for get_details. Formats for more stores. --- src/calibre/gui2/store/__init__.py | 9 +++++++-- src/calibre/gui2/store/amazon_plugin.py | 1 + src/calibre/gui2/store/bewrite_plugin.py | 4 +++- src/calibre/gui2/store/diesel_ebooks_plugin.py | 1 + src/calibre/gui2/store/ebooks_com_plugin.py | 1 + src/calibre/gui2/store/eharlequin_plugin.py | 1 + src/calibre/gui2/store/feedbooks_plugin.py | 4 ++++ src/calibre/gui2/store/gutenberg_plugin.py | 9 +++++++++ src/calibre/gui2/store/kobo_plugin.py | 1 + 9 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index ddce2f5ec4..43909e9d8b 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -96,7 +96,7 @@ class StorePlugin(object): # {{{ :param query: The string query search with. :param max_results: The maximum number of results to return. - :param timeout: The maximum amount of time in seconds to spend download the search results. + :param timeout: The maximum amount of time in seconds to spend downloading data for search results. :return: :class:`calibre.gui2.store.search_result.SearchResult` objects item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. @@ -116,8 +116,13 @@ class StorePlugin(object): # {{{ take extra time to load. Splitting retrieving data that takes longer to load into a separate function will give the illusion of the search being faster. + + :param search_result: A search result that need details set. + :param timeout: The maximum amount of time in seconds to spend downloading details. + + :return: True if the search_result was modified otherwise False ''' - pass + return False def get_settings(self): ''' diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index fe9d9c81c6..d5d8b54600 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -185,5 +185,6 @@ class AmazonKindleStore(StorePlugin): search_result.drm = SearchResult.DRM_UNKNOWN else: search_result.drm = SearchResult.DRM_LOCKED + return True \ No newline at end of file diff --git a/src/calibre/gui2/store/bewrite_plugin.py b/src/calibre/gui2/store/bewrite_plugin.py index 716e147bbc..8cc4a2745f 100644 --- a/src/calibre/gui2/store/bewrite_plugin.py +++ b/src/calibre/gui2/store/bewrite_plugin.py @@ -98,4 +98,6 @@ class BeWriteStore(BasicStoreConfig, StorePlugin): formats.add('PDF') if idata.xpath('boolean(//div[@id="content"]//td[contains(text(), "MOBI")])'): formats.add('MOBI') - search_result.formats = ', '.join(list(formats)) + search_result.formats = ', '.join(list(formats)) + + return True diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py index 9cea376341..b33bad15f9 100644 --- a/src/calibre/gui2/store/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -99,3 +99,4 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): search_result.drm = SearchResult.DRM_UNLOCKED else: search_result.drm = SearchResult.DRM_LOCKED + return True diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py index 3131cd0621..1597cc89ca 100644 --- a/src/calibre/gui2/store/ebooks_com_plugin.py +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -117,3 +117,4 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): search_result.formats = fdata search_result.price = price.strip() + return True diff --git a/src/calibre/gui2/store/eharlequin_plugin.py b/src/calibre/gui2/store/eharlequin_plugin.py index 325cb2f237..4f54508c80 100644 --- a/src/calibre/gui2/store/eharlequin_plugin.py +++ b/src/calibre/gui2/store/eharlequin_plugin.py @@ -101,3 +101,4 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin): else: drm = SearchResult.DRM_UNLOCKED search_result.drm = drm + return True diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 67de97126e..e56964b339 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -72,8 +72,10 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin): title = ''.join(data.xpath('//h5//a/text()')) author = ''.join(data.xpath('//h6//a/text()')) price = ''.join(data.xpath('//a[@class="buy"]/text()')) + formats = 'EPUB' if not price: price = '$0.00' + formats = 'EPUB, MOBI, PDF' cover_url = '' cover_url_img = data.xpath('//img') if cover_url_img: @@ -88,6 +90,7 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.replace(' ', '').strip() s.detail_item = id.strip() + s.formats = formats yield s @@ -101,3 +104,4 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin): search_result.drm = SearchResult.DRM_LOCKED else: search_result.drm = SearchResult.DRM_UNLOCKED + return True diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 0551c1b40b..04fe4da0fb 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -82,3 +82,12 @@ class GutenbergStore(BasicStoreConfig, StorePlugin): s.drm = SearchResult.DRM_UNLOCKED yield s + + def get_details(self, search_result, timeout): + url = 'http://m.gutenberg.org/' + + br = browser() + with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + search_result.formats = ', '.join(idata.xpath('//a[@type!="application/atom+xml"]//span[@class="title"]/text()')) + return True \ No newline at end of file diff --git a/src/calibre/gui2/store/kobo_plugin.py b/src/calibre/gui2/store/kobo_plugin.py index e08094adf0..421348d210 100644 --- a/src/calibre/gui2/store/kobo_plugin.py +++ b/src/calibre/gui2/store/kobo_plugin.py @@ -82,5 +82,6 @@ class KoboStore(BasicStoreConfig, StorePlugin): s.price = price.strip() s.detail_item = '?url=http://www.kobobooks.com/' + id.strip() s.drm = SearchResult.DRM_LOCKED if drm else SearchResult.DRM_UNLOCKED + s.formats = 'EPUB' yield s From 364729d28fe54ceb34f0162add62826fd08ca6d4 Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 21 Apr 2011 11:43:01 -0400 Subject: [PATCH 19/23] Store: more formats for searches. Wordwrap tooltips. --- src/calibre/gui2/store/manybooks_plugin.py | 1 + src/calibre/gui2/store/mobileread_plugin.py | 10 +++++----- src/calibre/gui2/store/open_library_plugin.py | 9 +++++++++ src/calibre/gui2/store/search.py | 16 ++++++++-------- src/calibre/gui2/store/smashwords_plugin.py | 8 ++++++++ 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index aa5c45300a..57eb42c13e 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -90,5 +90,6 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin): s.price = price.strip() s.detail_item = '/titles/' + id s.drm = SearchResult.DRM_UNLOCKED + s.formts = 'EPUB, PDB (eReader, PalmDoc, zTXT, Plucker, iSilo), FB2, ZIP, AZW, MOBI, PRC, LIT, PKG, PDF, TXT, RB, RTF, LRF, TCR, JAR' yield s diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index 3ac035d378..56fcb207bf 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -104,8 +104,8 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): for book_data in data.xpath('//ul/li'): book = BookRef() book.detail_item = ''.join(book_data.xpath('.//a/@href')) - book.format = ''.join(book_data.xpath('.//i/text()')) - book.format = book.format.strip() + book.formats = ''.join(book_data.xpath('.//i/text()')) + book.formats = book.formats.strip() text = ''.join(book_data.xpath('.//a/text()')) if ':' in text: @@ -222,7 +222,7 @@ class BooksModel(QAbstractItemModel): self.books = [] if self.filter: for b in self.all_books: - test = '%s %s %s' % (b.title, b.author, b.format) + test = '%s %s %s' % (b.title, b.author, b.formats) test = test.lower() include = True for item in self.filter.split(' '): @@ -275,7 +275,7 @@ class BooksModel(QAbstractItemModel): elif col == 1: return QVariant(result.author) elif col == 2: - return QVariant(result.format) + return QVariant(result.formats) return NONE def data_as_text(self, result, col): @@ -285,7 +285,7 @@ class BooksModel(QAbstractItemModel): elif col == 1: text = result.author elif col == 2: - text = result.format + text = result.formats return text def sort(self, col, order, reset=True): diff --git a/src/calibre/gui2/store/open_library_plugin.py b/src/calibre/gui2/store/open_library_plugin.py index 2bd38aae4f..0e2fa4b14f 100644 --- a/src/calibre/gui2/store/open_library_plugin.py +++ b/src/calibre/gui2/store/open_library_plugin.py @@ -71,3 +71,12 @@ class OpenLibraryStore(BasicStoreConfig, StorePlugin): s.drm = SearchResult.DRM_UNKNOWN yield s + + def get_details(self, search_result, timeout): + url = 'http://openlibrary.org/' + + br = browser() + with closing(br.open(url_slash_cleaner(url + search_result.detail_item), timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + search_result.formats = ', '.join(list(set(idata.xpath('//a[contains(@title, "Download")]/text()')))) + return True diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 193ebbc420..d5fbe19e93 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -559,22 +559,22 @@ class Matches(QAbstractItemModel): return QVariant(self.DRM_UNKNOWN_ICON) elif role == Qt.ToolTipRole: if col == 1: - return QVariant(result.title) + return QVariant('

%s

' % result.title) elif col == 2: - return QVariant(result.author) + return QVariant('

%s

' % result.author) elif col == 3: - return QVariant(_('Detected price as: %s. Check with the store before making a purchase to verify this price information is correct.') % result.price) + return QVariant('

' + _('Detected price as: %s. Check with the store before making a purchase to verify this price is correct. This price often does not include promotions the store may be running.') % result.price + '

') elif col == 4: if result.drm == SearchResult.DRM_LOCKED: - return QVariant(_('This book as been detected as having DRM restrictions. This book may not work with your reader and you will have limitations placed upon you as to what you can do with this book. Check with the store before making any purchases to ensure you can actually read this book.')) + return QVariant('

' + _('This book as been detected as having DRM restrictions. This book may not work with your reader and you will have limitations placed upon you as to what you can do with this book. Check with the store before making any purchases to ensure you can actually read this book.') + '

') elif result.drm == SearchResult.DRM_UNLOCKED: - return QVariant(_('This book has been detected as being DRM Free. You should be able to use this book on any device provided it is in a format calibre supports for conversion. However, before making a purchase double check the DRM status with the store. The store may not be disclosing the use of DRM.')) + return QVariant('

' + _('This book has been detected as being DRM Free. You should be able to use this book on any device provided it is in a format calibre supports for conversion. However, before making a purchase double check the DRM status with the store. The store may not be disclosing the use of DRM.') + '

') else: - return QVariant(_('The DRM status of this book could not be determined. There is a very high likelihood that this book is actually DRM restricted.')) + return QVariant('

' + _('The DRM status of this book could not be determined. There is a very high likelihood that this book is actually DRM restricted.') + '

') elif col == 5: - return QVariant(result.store_name) + return QVariant('

%s

' % result.store_name) elif col == 6: - return QVariant(result.formats) + return QVariant('

%s

' % result.formats) elif role == Qt.SizeHintRole: return QSize(64, 64) return NONE diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index 43efe549cc..f91d5d09f3 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -93,3 +93,11 @@ class SmashwordsStore(BasicStoreConfig, StorePlugin): s.drm = SearchResult.DRM_UNLOCKED yield s + def get_details(self, search_result, timeout): + url = 'http://www.smashwords.com/' + + br = browser() + with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + search_result.formats = ', '.join(list(set(idata.xpath('//td//b//text()')))) + return True From cafd67924eccd0fc0423868e76ede59e4d782551 Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 21 Apr 2011 18:13:37 -0400 Subject: [PATCH 20/23] Store: Add first run disclaimer. --- src/calibre/gui2/actions/store.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index 4e96960243..31c383c79d 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -10,6 +10,7 @@ from functools import partial from PyQt4.Qt import QMenu +from calibre.gui2 import JSONConfig from calibre.gui2.actions import InterfaceAction class StoreAction(InterfaceAction): @@ -18,6 +19,8 @@ class StoreAction(InterfaceAction): action_spec = (_('Get books'), 'store.png', None, None) def genesis(self): + self.config = JSONConfig('store_action') + self.qaction.triggered.connect(self.search) self.store_menu = QMenu() self.load_menu() @@ -31,9 +34,30 @@ class StoreAction(InterfaceAction): self.qaction.setMenu(self.store_menu) def search(self): + self.first_run_check() from calibre.gui2.store.search import SearchDialog sd = SearchDialog(self.gui.istores, self.gui) sd.exec_() def open_store(self, store_plugin): + self.first_run_check() store_plugin.open(self.gui) + + def first_run_check(self): + if self.config.get('first_run', True): + self.config['first_run'] = False + from calibre.gui2 import info_dialog + info_dialog(self.gui, _('Get Books Disclaimer'), + _('

Calibre helps you find books to read by connecting you with outside stores. ' + 'The stores are a variety of big, independent, free, and public domain sources.

' + '

Using the integrated search you can easily find what store has the book you\'re ' + 'looking for. It will also give you a price, DRM status as well as a lot of ' + 'other useful information.

' + '

All transaction (paid or otherwise) are handled between you and the store. ' + 'Calibre is not part of this process and any issues related to a purchase need to ' + 'be directed to the actual store. Be sure to double check that any books you get ' + 'will work with you device. Double check for format and ' + 'DRM ' + 'restrictions.

'), + show=True, show_copy_button=False) + \ No newline at end of file From 672fd2d6ec9ecaa27de930d65042522a1823da3c Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 21 Apr 2011 18:23:13 -0400 Subject: [PATCH 21/23] ... --- src/calibre/gui2/actions/store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index 31c383c79d..5f8a3efc26 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -60,4 +60,3 @@ class StoreAction(InterfaceAction): 'DRM ' 'restrictions.

'), show=True, show_copy_button=False) - \ No newline at end of file From 88de42894e0c6c8baa0a9825a7dbb2f9226bf022 Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 21 Apr 2011 18:31:25 -0400 Subject: [PATCH 22/23] ... --- src/calibre/gui2/store/smashwords_plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index f91d5d09f3..629a85dca5 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -93,6 +93,7 @@ class SmashwordsStore(BasicStoreConfig, StorePlugin): s.drm = SearchResult.DRM_UNLOCKED yield s + def get_details(self, search_result, timeout): url = 'http://www.smashwords.com/' From 4c77213c9b71afa0a71ab7bc762b991f15c67e15 Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 21 Apr 2011 18:51:42 -0400 Subject: [PATCH 23/23] APNX: Modify the accurate parser so it includes div tags a paragraph tags. This will throw off the output a little bit when dealing with things like del tags. --- src/calibre/devices/kindle/apnx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/kindle/apnx.py b/src/calibre/devices/kindle/apnx.py index 178c1091f3..ee519750e0 100644 --- a/src/calibre/devices/kindle/apnx.py +++ b/src/calibre/devices/kindle/apnx.py @@ -164,7 +164,7 @@ class APNXBuilder(object): if c == '/': closing = True continue - elif c == 'p': + elif c in ('d', 'p'): if closing: in_p = False else: