From ab44da6f50cc3490bd9e86a9d00005ddc8acce49 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Jun 2011 15:56:38 -0600 Subject: [PATCH 1/5] El club del ebook by DM. Fixes #800406 (New recipe for ebook related blog in spanish - El club del ebook) --- recipes/elclubdelebook.recipe | 61 +++++++++++++++++++++++++++++++ recipes/icons/elclubdelebook.png | Bin 0 -> 5423 bytes 2 files changed, 61 insertions(+) create mode 100644 recipes/elclubdelebook.recipe create mode 100644 recipes/icons/elclubdelebook.png diff --git a/recipes/elclubdelebook.recipe b/recipes/elclubdelebook.recipe new file mode 100644 index 0000000000..e05b176cc5 --- /dev/null +++ b/recipes/elclubdelebook.recipe @@ -0,0 +1,61 @@ + +__license__ = 'GPL v3' +__copyright__ = '2011, Darko Miletic ' +''' +www.clubdelebook.com +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class ElClubDelEbook(BasicNewsRecipe): + title = 'El club del ebook' + __author__ = 'Darko Miletic' + description = 'El Club del eBook, es la primera fuente de informacion sobre ebooks de Argentina. Aca vas a encontrar noticias, tips, tutoriales, recursos y opiniones sobre el mundo de los libros electronicos.' + tags = 'ebook, libro electronico, e-book, ebooks, libros electronicos, e-books' + oldest_article = 7 + max_articles_per_feed = 100 + language = 'es_AR' + encoding = 'utf-8' + no_stylesheets = True + use_embedded_content = True + publication_type = 'blog' + masthead_url = 'http://dl.dropbox.com/u/2845131/elclubdelebook.png' + extra_css = """ + body{font-family: Arial,Helvetica,sans-serif} + img{ margin-bottom: 0.8em; + border: 1px solid #333333; + padding: 4px; display: block + } + """ + + conversion_options = { + 'comment' : description + , 'tags' : tags + , 'publisher': title + , 'language' : language + } + + remove_tags = [dict(attrs={'id':'crp_related'})] + remove_tags_after = dict(attrs={'id':'crp_related'}) + + feeds = [(u'Articulos', u'http://feeds.feedburner.com/ElClubDelEbook')] + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll('a'): + limg = item.find('img') + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + if limg: + item.name = 'div' + item.attrs = [] + else: + str = self.tag_to_string(item) + item.replaceWith(str) + for item in soup.findAll('img'): + if not item.has_key('alt'): + item['alt'] = 'image' + return soup diff --git a/recipes/icons/elclubdelebook.png b/recipes/icons/elclubdelebook.png new file mode 100644 index 0000000000000000000000000000000000000000..c43f0454848f640c5d035596b9dc2705d15e38ba GIT binary patch literal 5423 zcmV+~70~L5P)7L&AwYL1XdSuDV_r9dQ;!<5*bL+qVIp;s;+-4GLf)J7< z7KRXo5Rs65dpp;zU2D->$yY~~(__qNqrm7AiG;x@Fbu;Bax4}ruP*g^y%viJ->M8t z|3DSAkzSkfI*bgFv{9t@*&O{+O0q25tEP1D;KBD#x=m7uBoeEhxQR@kjH_{9)aP{c z!-utRfAp8X{6){Z4F-dlmi{FyC9j%Hree3#9SSi$J>F_6q0ddfvhKC-yzsyYk znn|mwqLMYQZ`rqR-?v7Mn>=}(QBQc5sPeU$a_lu&))X-sNh(du=ALgwg>Wo^6|N$> z0oJwE-2hAY>O)Q>sW_QG|H(umcJICSJ~(kCurjfv`uXR+msUMLdi3b2lgC&rB$H0- z^?F_?7?Gk=2&;`8Mm`}EgCXYk*c}d{V-lf|EK82EGR-ru9^7@8q6V~pPPjUz(`qi4 zvzQG$PyaHNA~PPDkM~Zz|IRUEMzQHM*|~EkKHqZ>#Ux2Yr_=E~M^qK3O8)r-rbHkB z=bQGx1K%1s(&};nQ{3+-it;bd8$PJosr_d(G7g6UZ_i{%GD*2#x^!tM6#C$U4?G?} zv+2(r8#iv8IsJQM$2Rh!BFl0v-5HO^O;*EIhKS13Wh->WyshTF_ugOr!t-@?bskSA zTz=oj@5kf0OeRImbiMtw!$ACW9;RSi%9El6L?W3-L=SHKS|*c$*K6-0D^|Sp!O@oa z^XD@QpML+;sZ-nET7>0T=-Wv??bQK1BVT6@?F$XoPiFq&^5x4myCs=S4jee(^Upsw z>UFz!?@%>B{}&cZ!q>S~AwuaWNh~i2Af}?CG@mE3Omqgacklnc@W%ED6DBa@=B)4T z?%uxsXErm@XH(E2o+M$tw9LV>l4=Qu!*$ht^?Gt--`}5m?z!q=Lr$DH;Vv$I_St7~ zv3>jYQ^!w?9Xs}!Ws8ErG!G?^B#vjncag)6d8AYV3J10@sv2sQq23`W5_C&pRooZ! zIuOS5JPbT(;ak+@2aJ8HzP^6rifQQ_Dbq=KC=uzfI-T*9Xf<0ULbXaxk(Xayy?@sS z4Gj&uj~$&lb?OhMPQ?@z<+e9AZr!u%gVU!^H$U3c+&o>C2*(joB7#5^i3Ee;GN+wt z3Cn1JuEAnCRtqRalo84*oB1PtE!f7r$c2c=rEk+fS#A9X-6hu2{eKe_lWK z;ch|6Y+kj}clvK^%rAOd%A8DhNbC;gyZu>u22wbEfv7(iyWH8;;qCVNypc#Gn?t4% z1y(WXIo4LDn%&Gz)1LvlJ!_r-c5^mjFc^udB%@KivjR&y+m?i}rrkIB-FM$@J8^W) znl%kWO3xfWZZ(_FoHIO+fN?zWLFo01F&p18%MZwIxX;~Xf$dy zTKo0umyX9EJZDZFL+b6y?et2o_Xe(5bwbHj(mAx_3G7w`&v6XI;7xPJbx+e z!ZJ`tz=(qyZbI!KXGJ_&TUiCrf7-sKrKP3R?YQf%yTYMvm{Qf{K^4rSDJzQ^Si)h< zrYKt+;W2=F1EgF7I%4K38W33el`B_3QAsfeOu0Z{*|KGj3BTVDC_8p+EiH9_@x>Pi zcpA@&f;GTN@c?|t=ks9-%z*L2g$uoU^%^y56jltI;^A@Q#^Eui-RW|=O57!Gx4Wc- zj`***q@=1>ukz~ZlFABKaWTcj^`tsDE7A;`jY|$wG19h@AfA>}caHi-2tjMJG&c5m z=biP=SZn^vgAO@w&ZYuNAH{i#Rg^7xNr?umYTl^C4RTCOb=eHrT*lkgacbaz-amch zf%3A-_3Jk5-hH5=q7Ref68@y;l0VoPjf7G-&tJHhNM$c}w6}XbT|sXb;xZG;s98lP zD@HkuGy@R^hgdhF0fO1y-VSdlE-r@OJoeaQNDk0a)K$>8R4P?gRt8VXLy`&2n^+d~ z7Cr$P1UHqHmC*gp&d%?C_q$*VAHnf!udRc(+gz@SnwrweN>{N9u3cYWhnM!Mt*xr8 zEH5j^>ns)v=0U=hd+HX#;x%}spk;+2Lx#+rJ^Or1OHFO<*|TQ>7r7J6!!G^%_lHgK z$N79#Rb^h_a57XF5o2&1(P%Uz67k~VqN7KTJlfpkaN6+krI%J6ICK~u4j+Ioc6NEg z(O7qPARbS4bar~XyS)KlI2KLkax%@Bm{90}1>mZtD%q@RC@C5B{qMv2U~A>dl}C;o z89H<*)(2v`y1KwkHk+lMOCyVxC3p-YKqJB90YFFaLNXABSRQ;1XdZw3ai6yv7lQ^5 zs;VkS9a~oBE-fvt#s^%v-H;wi71jscQhG{aR#p|HBvB?NRP(4X2tnw?YG~K3&=W_4m|^5x4R8AT3fZFLR2 z{*_l==|7;sVYSH;QUp&s2NpdK3wjE{3{gs@5?E|ekpsq}9TCrK?%&?fP(ShhDMpJe znacjOX^!1)3WwuLUcUHcTl?k9p3Y96*AtIK@|i57AcHeJ%lE*79wHV1RV&3vbwdO6 z2EKxMZ@u+aSQ8N6+qxC%f@@8YX}ZArKoT%%4h3P!2^sZ_7=7<>-XqB0vlemry^ zHRI8vM`z8N?{ZOx>)Wp{L=I`Kyu2q8d5*uKN;BE4u~ZU?GpF&VCKAJj4O_TyAtr)R zkcP`kN;kjx$G;sqe9O?G5;`8j6cxGJS}*0Y6e8%9WwpM(!RHGg=>d}lT5q>I&1UO> z0RuO0e!IKdFXr=g{rhiN_u8Mg?Xa8lawe5eMB6W%kNCXtpg$V)vqVOnPOceIrz_+m zL4(V%G(X_jvSkax3Kx(oxZ2AvzZ{7~d-v{*=OIYLhYtrIkPL`^pFVw%r12z3f}p^E z@l0)PEv8+tV8O0kyP!3o4Jl#d$dR*V%|a}5n$&@Qr`wGa&m%R$w>Y+^H(^tmG)~$T zWoTN@WHW_h@#4kEC%bp=zT=KNV0fp^{>xu1cG?{RZ?IY%UXKqkRa4z3oyvc4?37M0 zNusLAj6rYF>&!#HHp0{8U-ZP!eBD8Rw{Ob*lN;*lW;Zvx?UtG{cdv3c3Ugl18i|;X z`lFu9Vk*wbc~%w~^((euSHyx&ASU>hAQ+*c4!g~6!)pHUhZR$%O!@TFPtnZ^wC?u2 z_~MK32Bd1R0$anYU_{Ufm%>FU2cLs)fguzyU=~^ctjK_fEPVU)(@*c%v7?|>U=D17 z9uS7g5&kgYd*h*e?`__U7{s?wIIxaB;MlQattU=ko}MZeu#)K%d<#{620-*PLmRFRPR;*pS{=%0RZ~yu?Sq8331`oaoq9Nwhg$thi z;)|12Rkb{4a2J=G%@&U1kzz_*MOWHdhuwVBupy17PaHdY{OHElUfK8Iu52RG(7(^X zy1oLVc6YWX!hx&ZIL(PXtpSJu8pQMFuY9YjsuYfoK!*{3IAaFH20Wqhf-T@un>K9% zbkI}y7zhOB&Yg>t4+7vK)LAuZ;Y|1zJWZN336Y3G7hM*l2Qi4?-M@c7s0ERDCweOA z5%NHfZ;hDwItc%0^+O0ow-}BGgDj!dW=!Y8i&MNEZ*uS|W>0stz)6pt_o3Y|o9!^o z{CNuy3Lk&G4>_ukh(e(N^n&FWEGeCiqb@4p!nD-MXO15~USzSrd?pi;HiNX-ia|z+hSrYbi~=D0;V~R@LyxtH`r9;sebY5myoXpVa ztH9RB>mCI%o1CU>aa#GMnqZe4OS>mtq#Ha z$1|F02Mj!arNa}6>FrKOS%tZ%h%p$VnKXJem4FDAxnm5oNkrpx0`LWy_Xt-MY1|{{XNB`=Z^QJmtRM{qEU(UO{InN?BCW z+CBsNLruXm2|5wap&^u_5mdhjLpTePI{Yl1&di)Ucj%&>)aCDL`aR-q>#K87K{zE5 z53wXeJ2it1!JpSQeFUU;1e!wk^>Pe)PT6Lo5gLsoX3m^>`BFO~;nbN^-Y#!hxht8> z01#X_pU)u^=*$)b0Y?<1AYP^OSadArH`;9lE$C`({ph2;$mQuw?)eofhR^vUh4uAm z&mz_A+^`5A>@=?%lip+jrkRy!j0}!MA_B z1hBSkTtcTy$p~Xk&47=>#%y8FlhLkO%?>h~r*$`@qpr$mO3czK1qqJsT1hlXpckSw zFhfU)r*$@`?ZI+dZeg`DK^7&bZay!;{Vi69!|C+-{K!amHvb+AX2v)D7BjxJ{^xkV zMO$hvmC|bV0xX7H-O4K@cvk$1L2aw*UmgrC%^7V!F{kYS7LrPVv%(%dIu%8hSzf0| zQZ^?+mIa*-?lu1LwaBnc^RLz+>HcBmb4aXWLecA4DHBIRwCGuFWy)2~3L_`bm#GSE z<5Wd^ze-2YrZslg_GG94F-%s{<|zOwY-DQJg3f?an>WyIPiJ&MhYd!uVdJ)~yFVR0 zdNlL#$?(#pOD9hn|HCPFatdw9iftq>s-}XMs67<*xC|ZsEw;4(B<-JYVq~7~yGsef=&xqtHE4(zBg(b9%IlVo;x2~vh-Kb=y~&+9S+vUlXOOL8VehEg%{{FYBWE)eh0chiuzCQ?HN_g z@mZGtr>_f-=VgeOnj?(KKz|=;yOjL(@)r=NPc}^*Jg6TN%BcwX-~RrcZQHgDx~T!B z%jnyO+O2|4qTl+7DF< zLxE;QVldEOll(z>&;Gxo6G@BmJ@?%6=(r*1oR}mQCDX+pJKnu!-D*hRtvB}@HEPuD zL+iDfG*8lCDU8ODLLIsA>zB4Q(nC;BUw*@{O==I`Yj@bItOHgwn%aHfZwC$>2&Yu2 z!L*;uxZ{q#mB5qF(;&hv^wp>@PG0B$>I0{{=%zsF!1L$NyQ=G;VQQXMXD+jVqD$s= zDGw%Z!YEoee-ZQv1!@Ud(zYt(*T Z{Rh?U!$@|sqe}n)002ovPDHLkV1mm`cL)Fg literal 0 HcmV?d00001 From 767f0d15841319efb5b9a888bdeab977dd3126b1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Jun 2011 16:07:34 -0600 Subject: [PATCH 2/5] Daytona Beach Journal by BRGriff --- recipes/daytona_beach.recipe | 78 ++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 recipes/daytona_beach.recipe diff --git a/recipes/daytona_beach.recipe b/recipes/daytona_beach.recipe new file mode 100644 index 0000000000..1230c1d8ed --- /dev/null +++ b/recipes/daytona_beach.recipe @@ -0,0 +1,78 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class DaytonBeachNewsJournal(BasicNewsRecipe): + title ='Daytona Beach News Journal' + __author__ = 'BRGriff' + pubisher = 'News-JournalOnline.com' + description = 'Daytona Beach, Florida, Newspaper' + category = 'News, Daytona Beach, Florida' + oldest_article = 1 + max_articles_per_feed = 100 + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + language = 'en' + filterDuplicates = True + remove_attributes = ['style'] + + keep_only_tags = [dict(name='div', attrs={'class':'page-header'}), + dict(name='div', attrs={'class':'asset-body'}) + ] + remove_tags = [dict(name='div', attrs={'class':['byline-section', 'asset-meta']}) + ] + + feeds = [ + #####NEWS##### + (u"News", u"http://www.news-journalonline.com/rss.xml"), + (u"Breaking News", u"http://www.news-journalonline.com/breakingnews/rss.xml"), + (u"Local - East Volusia", u"http://www.news-journalonline.com/news/local/east-volusia/rss.xml"), + (u"Local - West Volusia", u"http://www.news-journalonline.com/news/local/west-volusia/rss.xml"), + (u"Local - Southeast", u"http://www.news-journalonline.com/news/local/southeast-volusia/rss.xml"), + (u"Local - Flagler", u"http://www.news-journalonline.com/news/local/flagler/rss.xml"), + (u"Florida", u"http://www.news-journalonline.com/news/florida/rss.xml"), + (u"National/World", u"http://www.news-journalonline.com/news/nationworld/rss.xml"), + (u"Politics", u"http://www.news-journalonline.com/news/politics/rss.xml"), + (u"News of Record", u"http://www.news-journalonline.com/news/news-of-record/rss.xml"), + ####BUSINESS#### + (u"Business", u"http://www.news-journalonline.com/business/rss.xml"), + #(u"Jobs", u"http://www.news-journalonline.com/business/jobs/rss.xml"), + #(u"Markets", u"http://www.news-journalonline.com/business/markets/rss.xml"), + #(u"Real Estate", u"http://www.news-journalonline.com/business/real-estate/rss.xml"), + #(u"Technology", u"http://www.news-journalonline.com/business/technology/rss.xml"), + ####SPORTS#### + (u"Sports", u"http://www.news-journalonline.com/sports/rss.xml"), + (u"Racing", u"http://www.news-journalonline.com/racing/rss.xml"), + (u"Highschool", u"http://www.news-journalonline.com/sports/highschool/rss.xml"), + (u"College", u"http://www.news-journalonline.com/sports/college/rss.xml"), + (u"Basketball", u"http://www.news-journalonline.com/sports/basketball/rss.xml"), + (u"Football", u"http://www.news-journalonline.com/sports/football/rss.xml"), + (u"Golf", u"http://www.news-journalonline.com/sports/golf/rss.xml"), + (u"Other Sports", u"http://www.news-journalonline.com/sports/other/rss.xml"), + ####LIFESTYLE#### + (u"Lifestyle", u"http://www.news-journalonline.com/lifestyle/rss.xml"), + #(u"Fashion", u"http://www.news-journalonline.com/lifestyle/fashion/rss.xml"), + (u"Food", u"http://www.news-journalonline.com/lifestyle/food/rss.xml"), + #(u"Health", u"http://www.news-journalonline.com/lifestyle/health/rss.xml"), + (u"Home and Garden", u"http://www.news-journalonline.com/lifestyle/home-and-garden/rss.xml"), + (u"Living", u"http://www.news-journalonline.com/lifestyle/living/rss.xml"), + (u"Religion", u"http://www.news-journalonline.com/lifestyle/religion/rss.xml"), + #(u"Travel", u"http://www.news-journalonline.com/lifestyle/travel/rss.xml"), + ####OPINION#### + #(u"Opinion", u"http://www.news-journalonline.com/opinion/rss.xml"), + #(u"Letters to Editor", u"http://www.news-journalonline.com/opinion/letters-to-the-editor/rss.xml"), + #(u"Columns", u"http://www.news-journalonline.com/columns/rss.xml"), + #(u"Podcasts", u"http://www.news-journalonline.com/podcasts/rss.xml"), + ####ENTERTAINMENT#### ##Weekly Feature## + (u"Entertainment", u"http://www.go386.com/rss.xml"), + (u"Go Out", u"http://www.go386.com/go/rss.xml"), + (u"Music", u"http://www.go386.com/music/rss.xml"), + (u"Movies", u"http://www.go386.com/movies/rss.xml"), + #(u"Culture", u"http://www.go386.com/culture/rss.xml"), + + ] + + extra_css = ''' + .page-header{font-family:Arial,Helvetica,sans-serif; font-style:bold;font-size:22pt;} + .asset-body{font-family:Helvetica,Arial,sans-serif; font-size:16pt;} + + ''' From cdd6637598fec9af389c13d6580e279ef6651e35 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Tue, 21 Jun 2011 23:05:57 -0300 Subject: [PATCH 3/5] Add support to display Favoutite as a collection for KTouch --- src/calibre/devices/kobo/driver.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index b523b7f296..4ccff13dd6 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -100,7 +100,7 @@ class KOBO(USBMS): for idx,b in enumerate(bl): bl_cache[b.lpath] = idx - def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType, expired): + def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType, expired, favouritesindex): changed = False try: lpath = path.partition(self.normalize_path(prefix))[2] @@ -111,17 +111,23 @@ class KOBO(USBMS): playlist_map = {} + if lpath not in playlist_map: + playlist_map[lpath] = [] + if readstatus == 1: - playlist_map[lpath]= "Im_Reading" + playlist_map[lpath].append('Im_Reading') elif readstatus == 2: - playlist_map[lpath]= "Read" + playlist_map[lpath].append('Read') elif readstatus == 3: - playlist_map[lpath]= "Closed" + playlist_map[lpath].append('Closed') # Related to a bug in the Kobo firmware that leaves an expired row for deleted books # this shows an expired Collection so the user can decide to delete the book if expired == 3: - playlist_map[lpath] = "Expired" + playlist_map[lpath].append('Expired') + # Favourites are supported on the touch but the data field is there on most earlier models + if favouritesindex == 1: + playlist_map[lpath].append('Favourite') path = self.normalize_path(path) # print "Normalized FileName: " + path @@ -149,7 +155,7 @@ class KOBO(USBMS): debug_print(" Strange: The file: ", prefix, lpath, " does mot exist!") if lpath in playlist_map and \ playlist_map[lpath] not in bl[idx].device_collections: - bl[idx].device_collections.append(playlist_map[lpath]) + bl[idx].device_collections = playlist_map.get(lpath,[]) else: if ContentType == '6' and MimeType == 'Shortcover': book = Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=1048576) @@ -168,7 +174,7 @@ class KOBO(USBMS): raise # print 'Update booklist' - book.device_collections = [playlist_map[lpath]] if lpath in playlist_map else [] + book.device_collections = playlist_map.get(lpath,[])# if lpath in playlist_map else [] if bl.add_book(book, replace_metadata=False): changed = True @@ -198,7 +204,7 @@ class KOBO(USBMS): self.dbversion = result[0] query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ - 'ImageID, ReadStatus, ___ExpirationStatus from content where BookID is Null' + 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex from content where BookID is Null' cursor.execute (query) @@ -213,10 +219,10 @@ class KOBO(USBMS): # debug_print("mime:", mime) if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"): - changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8]) + changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8], row[9]) # print "shortbook: " + path elif oncard == 'carda' and row[3].startswith("file:///mnt/sd/"): - changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8]) + changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8], row[9]) if changed: need_sync = True From a8a2db1313ee3b90b366f3a4fec2585af8feb3e3 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Tue, 21 Jun 2011 23:20:52 -0300 Subject: [PATCH 4/5] Query for FavouritesIndex only for KTouch and database versions that support it --- src/calibre/devices/kobo/driver.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 4ccff13dd6..650e965941 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -203,8 +203,12 @@ class KOBO(USBMS): result = cursor.fetchone() self.dbversion = result[0] - query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ + if self.dbversion >= 14: + query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex from content where BookID is Null' + else: + query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ + 'ImageID, ReadStatus, ___ExpirationStatus, "-1" as FavouritesIndex from content where BookID is Null' cursor.execute (query) From f1a4d06e512a1ab63a727a148005fdacf4a52e36 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Jun 2011 20:43:48 -0600 Subject: [PATCH 5/5] Try to ensure that all file I/O on library files happes in the LibraryDatabase2 class. This is in preparation for the new multi-threaded db backend. Also: Content server now sends the Content-Disposition header when sending ebook files. After uploading books to the device, delete the associated temp files. If the user tries to drag and drop more than 25 books out of calibre, only the first 25 will be sent. --- src/calibre/db/backend.py | 4 +- src/calibre/db/errors.py | 13 ++ src/calibre/devices/apple/driver.py | 1 + src/calibre/devices/interface.py | 7 +- src/calibre/ebooks/metadata/worker.py | 30 ++-- src/calibre/gui2/actions/copy_to_library.py | 16 +- src/calibre/gui2/actions/edit_metadata.py | 46 ++++-- src/calibre/gui2/actions/tweak_epub.py | 7 +- src/calibre/gui2/add.py | 1 + src/calibre/gui2/convert/regex_builder.py | 9 +- src/calibre/gui2/convert/single.py | 6 +- src/calibre/gui2/device.py | 28 +++- src/calibre/gui2/dialogs/metadata_bulk.py | 13 +- src/calibre/gui2/email.py | 3 +- src/calibre/gui2/library/models.py | 45 ++---- src/calibre/gui2/library/views.py | 12 +- src/calibre/gui2/metadata/basic_widgets.py | 3 +- src/calibre/gui2/tools.py | 18 ++- src/calibre/library/__init__.py | 18 --- src/calibre/library/catalog.py | 6 + src/calibre/library/database2.py | 155 ++++++++++++++------ src/calibre/library/save_to_disk.py | 30 ++-- src/calibre/library/server/content.py | 35 ++--- 23 files changed, 319 insertions(+), 187 deletions(-) create mode 100644 src/calibre/db/errors.py diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 159612e52d..ba683dde50 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -222,7 +222,9 @@ class DB(object, SchemaUpgrade): if self.user_version == 0: self.initialize_database() - SchemaUpgrade.__init__(self) + with self.conn: + SchemaUpgrade.__init__(self) + # Guarantee that the library_id is set self.library_id diff --git a/src/calibre/db/errors.py b/src/calibre/db/errors.py new file mode 100644 index 0000000000..d2657f9904 --- /dev/null +++ b/src/calibre/db/errors.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +class NoSuchFormat(ValueError): + pass + diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index dea5844028..cc531b7476 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -107,6 +107,7 @@ class DriverBase(DeviceConfig, DevicePlugin): # Needed for config_widget to work FORMATS = ['epub', 'pdf'] USER_CAN_ADD_NEW_FORMATS = False + KEEP_TEMP_FILES_AFTER_UPLOAD = True # Hide the standard customization widgets SUPPORTS_SUB_DIRS = False diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index b265331ace..925f9eafd0 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -327,12 +327,7 @@ class DevicePlugin(Plugin): free space on the device. The text of the FreeSpaceError must contain the word "card" if ``on_card`` is not None otherwise it must contain the word "memory". - :param files: A list of paths and/or file-like objects. If they are paths and - the paths point to temporary files, they may have an additional - attribute, original_file_path pointing to the originals. They may have - another optional attribute, deleted_after_upload which if True means - that the file pointed to by original_file_path will be deleted after - being uploaded to the device. + :param files: A list of paths :param names: A list of file names that the books should have once uploaded to the device. len(names) == len(files) :param metadata: If not None, it is a list of :class:`Metadata` objects. diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index c335cc4c13..ca8707258b 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -15,7 +15,7 @@ from calibre.utils.ipc.server import Server from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory from calibre import prints, isbytestring from calibre.constants import filesystem_encoding - +from calibre.db.errors import NoSuchFormat def debug(*args): prints(*args) @@ -201,27 +201,35 @@ class SaveWorker(Thread): self.spare_server = spare_server self.start() - def collect_data(self, ids): + def collect_data(self, ids, tdir): from calibre.ebooks.metadata.opf2 import metadata_to_opf data = {} for i in set(ids): - mi = self.db.get_metadata(i, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(i, index_is_id=True, get_cover=True, + cover_as_data=True) opf = metadata_to_opf(mi) if isbytestring(opf): opf = opf.decode('utf-8') cpath = None - if mi.cover: - cpath = mi.cover + if mi.cover_data and mi.cover_data[1]: + cpath = os.path.join(tdir, 'cover_%s.jpg'%i) + with lopen(cpath, 'wb') as f: + f.write(mi.cover_data[1]) if isbytestring(cpath): cpath = cpath.decode(filesystem_encoding) formats = {} if mi.formats: for fmt in mi.formats: - fpath = self.db.format_abspath(i, fmt, index_is_id=True) - if fpath is not None: - if isbytestring(fpath): - fpath = fpath.decode(filesystem_encoding) - formats[fmt.lower()] = fpath + fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower())) + with lopen(fpath, 'wb') as f: + try: + self.db.copy_format_to(i, fmt, f, index_is_id=True) + except NoSuchFormat: + continue + else: + if isbytestring(fpath): + fpath = fpath.decode(filesystem_encoding) + formats[fmt.lower()] = fpath data[i] = [opf, cpath, formats, mi.last_modified.isoformat()] return data @@ -244,7 +252,7 @@ class SaveWorker(Thread): for i, task in enumerate(tasks): tids = [x[-1] for x in task] - data = self.collect_data(tids) + data = self.collect_data(tids, tdir) dpath = os.path.join(tdir, '%d.json'%i) with open(dpath, 'wb') as f: f.write(json.dumps(data, ensure_ascii=False).encode('utf-8')) diff --git a/src/calibre/gui2/actions/copy_to_library.py b/src/calibre/gui2/actions/copy_to_library.py index 7190d1486f..97880faaa1 100644 --- a/src/calibre/gui2/actions/copy_to_library.py +++ b/src/calibre/gui2/actions/copy_to_library.py @@ -53,13 +53,18 @@ class Worker(Thread): # {{{ from calibre.library.database2 import LibraryDatabase2 newdb = LibraryDatabase2(self.loc) for i, x in enumerate(self.ids): - mi = self.db.get_metadata(x, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(x, index_is_id=True, get_cover=True, + cover_as_data=True) self.progress(i, mi.title) fmts = self.db.formats(x, index_is_id=True) if not fmts: fmts = [] else: fmts = fmts.split(',') - paths = [self.db.format_abspath(x, fmt, index_is_id=True) for fmt in - fmts] + paths = [] + for fmt in fmts: + p = self.db.format(x, fmt, index_is_id=True, + as_path=True) + if p: + paths.append(p) added = False if prefs['add_formats_to_existing']: identical_book_list = newdb.find_identical_books(mi) @@ -75,6 +80,11 @@ class Worker(Thread): # {{{ if co is not None: newdb.set_conversion_options(x, 'PIPE', co) self.processed.add(x) + for path in paths: + try: + os.remove(path) + except: + pass # }}} class CopyToLibraryAction(InterfaceAction): diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 650947100e..718ece46d2 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -17,6 +17,7 @@ from calibre.gui2.dialogs.tag_list_editor import TagListEditor from calibre.gui2.actions import InterfaceAction from calibre.ebooks.metadata import authors_to_string from calibre.utils.icu import sort_key +from calibre.db.errors import NoSuchFormat class EditMetadataAction(InterfaceAction): @@ -265,7 +266,7 @@ class EditMetadataAction(InterfaceAction): +'

', 'merge_too_many_books', self.gui): return - dest_id, src_books, src_ids = self.books_to_merge(rows) + dest_id, src_ids = self.books_to_merge(rows) title = self.gui.library_view.model().db.title(dest_id, index_is_id=True) if safe_merge: if not confirm('

'+_( @@ -277,7 +278,7 @@ class EditMetadataAction(InterfaceAction): 'Please confirm you want to proceed.')%title +'

', 'merge_books_safe', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.merge_metadata(dest_id, src_ids) elif merge_only_formats: if not confirm('

'+_( @@ -293,7 +294,7 @@ class EditMetadataAction(InterfaceAction): 'Are you sure you want to proceed?')%title +'

', 'merge_only_formats', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.delete_books_after_merge(src_ids) else: if not confirm('

'+_( @@ -308,7 +309,7 @@ class EditMetadataAction(InterfaceAction): 'Are you sure you want to proceed?')%title +'

', 'merge_books', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.merge_metadata(dest_id, src_ids) self.delete_books_after_merge(src_ids) # leave the selection highlight on first selected book @@ -329,8 +330,22 @@ class EditMetadataAction(InterfaceAction): self.gui.library_view.model().db.add_format(dest_id, fmt, f, index_is_id=True, notify=False, replace=replace) + def formats_for_books(self, rows): + m = self.gui.library_view.model() + ans = [] + for id_ in map(m.id, rows): + dbfmts = m.db.formats(id_, index_is_id=True) + if dbfmts: + for fmt in dbfmts.split(','): + try: + path = m.db.format(id_, fmt, index_is_id=True, + as_path=True) + ans.append(path) + except NoSuchFormat: + continue + return ans + def books_to_merge(self, rows): - src_books = [] src_ids = [] m = self.gui.library_view.model() for i, row in enumerate(rows): @@ -339,22 +354,19 @@ class EditMetadataAction(InterfaceAction): dest_id = id_ else: src_ids.append(id_) - dbfmts = m.db.formats(id_, index_is_id=True) - if dbfmts: - for fmt in dbfmts.split(','): - src_books.append(m.db.format_abspath(id_, fmt, - index_is_id=True)) - return [dest_id, src_books, src_ids] + return [dest_id, src_ids] def delete_books_after_merge(self, ids_to_delete): self.gui.library_view.model().delete_books_by_id(ids_to_delete) def merge_metadata(self, dest_id, src_ids): db = self.gui.library_view.model().db - dest_mi = db.get_metadata(dest_id, index_is_id=True, get_cover=True) + dest_mi = db.get_metadata(dest_id, index_is_id=True) orig_dest_comments = dest_mi.comments + dest_cover = db.cover(dest_id, index_is_id=True) + had_orig_cover = bool(dest_cover) for src_id in src_ids: - src_mi = db.get_metadata(src_id, index_is_id=True, get_cover=True) + src_mi = db.get_metadata(src_id, index_is_id=True) if src_mi.comments and orig_dest_comments != src_mi.comments: if not dest_mi.comments: dest_mi.comments = src_mi.comments @@ -372,8 +384,10 @@ class EditMetadataAction(InterfaceAction): dest_mi.tags = src_mi.tags else: dest_mi.tags.extend(src_mi.tags) - if src_mi.cover and not dest_mi.cover: - dest_mi.cover = src_mi.cover + if not dest_cover: + src_cover = db.cover(src_id, index_is_id=True) + if src_cover: + dest_cover = src_cover if not dest_mi.publisher: dest_mi.publisher = src_mi.publisher if not dest_mi.rating: @@ -382,6 +396,8 @@ class EditMetadataAction(InterfaceAction): dest_mi.series = src_mi.series dest_mi.series_index = src_mi.series_index db.set_metadata(dest_id, dest_mi, ignore_errors=False) + if not had_orig_cover and dest_cover: + db.set_cover(dest_id, dest_cover) for key in db.field_metadata: #loop thru all defined fields if db.field_metadata[key]['is_custom']: diff --git a/src/calibre/gui2/actions/tweak_epub.py b/src/calibre/gui2/actions/tweak_epub.py index c9f2d7a8c6..d3924e7cd3 100755 --- a/src/calibre/gui2/actions/tweak_epub.py +++ b/src/calibre/gui2/actions/tweak_epub.py @@ -5,6 +5,8 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os + from calibre.gui2 import error_dialog from calibre.gui2.actions import InterfaceAction from calibre.gui2.dialogs.tweak_epub import TweakEpub @@ -30,8 +32,8 @@ class TweakEpubAction(InterfaceAction): # Confirm 'EPUB' in formats book_id = self.gui.library_view.model().id(row) try: - path_to_epub = self.gui.library_view.model().db.format_abspath( - book_id, 'EPUB', index_is_id=True) + path_to_epub = self.gui.library_view.model().db.format( + book_id, 'EPUB', index_is_id=True, as_path=True) except: path_to_epub = None @@ -45,6 +47,7 @@ class TweakEpubAction(InterfaceAction): if dlg.exec_() == dlg.Accepted: self.update_db(book_id, dlg._output) dlg.cleanup() + os.remove(path_to_epub) def update_db(self, book_id, rebuilt): ''' diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 44b5bb446b..3dbf4b94df 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -445,6 +445,7 @@ class Saver(QObject): # {{{ self.pd.setModal(True) self.pd.show() self.pd.set_min(0) + self.pd.set_msg(_('Collecting data, please wait...')) self._parent = parent self.callback = callback self.callback_called = False diff --git a/src/calibre/gui2/convert/regex_builder.py b/src/calibre/gui2/convert/regex_builder.py index bf32bf472a..f79e6df3fe 100644 --- a/src/calibre/gui2/convert/regex_builder.py +++ b/src/calibre/gui2/convert/regex_builder.py @@ -4,7 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' -import re +import re, os from PyQt4.QtCore import SIGNAL, Qt, pyqtSignal from PyQt4.QtGui import QDialog, QWidget, QDialogButtonBox, \ @@ -134,7 +134,12 @@ class RegexBuilder(QDialog, Ui_RegexBuilder): _('Cannot build regex using the GUI builder without a book.'), show=True) return False - self.open_book(db.format_abspath(book_id, format, index_is_id=True)) + fpath = db.format(book_id, format, index_is_id=True, + as_path=True) + try: + self.open_book(fpath) + finally: + os.remove(fpath) return True def open_book(self, pathtoebook): diff --git a/src/calibre/gui2/convert/single.py b/src/calibre/gui2/convert/single.py index 3575fb5ffb..15e670ee32 100644 --- a/src/calibre/gui2/convert/single.py +++ b/src/calibre/gui2/convert/single.py @@ -106,7 +106,6 @@ class Config(ResizableDialog, Ui_Dialog): Configuration dialog for single book conversion. If accepted, has the following important attributes - input_path - Path to input file output_format - Output format (without a leading .) input_format - Input format (without a leading .) opf_path - Path to OPF file with user specified metadata @@ -156,13 +155,10 @@ class Config(ResizableDialog, Ui_Dialog): oidx = self.groups.currentIndex().row() input_format = self.input_format output_format = self.output_format - input_path = self.db.format_abspath(self.book_id, input_format, - index_is_id=True) - self.input_path = input_path output_path = 'dummy.'+output_format log = Log() log.outputs = [] - self.plumber = Plumber(input_path, output_path, log) + self.plumber = Plumber('dummy.'+input_format, output_path, log) def widget_factory(cls): return cls(self.stack, self.plumber.get_option_by_name, diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index a527cc2e27..d5805a8e09 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -396,8 +396,17 @@ class DeviceManager(Thread): # {{{ if DEBUG: prints(traceback.format_exc(), file=sys.__stdout__) - return self.device.upload_books(files, names, on_card, - metadata=metadata, end_session=False) + try: + return self.device.upload_books(files, names, on_card, + metadata=metadata, end_session=False) + finally: + if metadata: + for mi in metadata: + try: + if mi.cover: + os.remove(mi.cover) + except: + pass def upload_books(self, done, files, names, on_card=None, titles=None, metadata=None, plugboards=None, add_as_step_to_job=None): @@ -1072,8 +1081,6 @@ class DeviceMixin(object): # {{{ 'the device?'), autos): self.iactions['Convert Books'].auto_convert_news(auto, format) files = [f for f in files if f is not None] - for f in files: - f.deleted_after_upload = del_on_upload if not files: self.news_to_be_synced = set([]) return @@ -1315,8 +1322,17 @@ class DeviceMixin(object): # {{{ self.card_b_view if on_card == 'cardb' else self.memory_view view.model().resort(reset=False) view.model().research() - for f in files: - getattr(f, 'close', lambda : True)() + if files: + for f in files: + # Remove temporary files + try: + rem = not getattr( + self.device_manager.device, + 'KEEP_TEMP_FILES_AFTER_UPLOAD', False) + if rem and 'caltmpfmt.' in f: + os.remove(f) + except: + pass def book_on_device(self, id, reset=False): ''' diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index ce21eba00e..7c7c78629c 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -24,7 +24,7 @@ from calibre.utils.config import prefs, tweaks from calibre.utils.magick.draw import identify_data from calibre.utils.date import qt_to_dt -def get_cover_data(path): # {{{ +def get_cover_data(stream, ext): # {{{ from calibre.ebooks.metadata.meta import get_metadata old = prefs['read_file_metadata'] if not old: @@ -32,8 +32,8 @@ def get_cover_data(path): # {{{ cdata = area = None try: - mi = get_metadata(open(path, 'rb'), - os.path.splitext(path)[1][1:].lower()) + with stream: + mi = get_metadata(stream, ext) if mi.cover and os.access(mi.cover, os.R_OK): cdata = open(mi.cover).read() elif mi.cover_data[1] is not None: @@ -186,9 +186,10 @@ class MyBlockingBusy(QDialog): # {{{ if fmts: covers = [] for fmt in fmts.split(','): - fmt = self.db.format_abspath(id, fmt, index_is_id=True) - if not fmt: continue - cdata, area = get_cover_data(fmt) + fmtf = self.db.format(id, fmt, index_is_id=True, + as_file=True) + if fmtf is None: continue + cdata, area = get_cover_data(fmtf, fmt) if cdata: covers.append((cdata, area)) covers.sort(key=lambda x: x[1]) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index b82f421e1e..b9c760abff 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -174,7 +174,8 @@ class EmailMixin(object): # {{{ else: _auto_ids = [] - full_metadata = self.library_view.model().metadata_for(ids) + full_metadata = self.library_view.model().metadata_for(ids, + get_cover=False) bad, remove_ids, jobnames = [], [], [] texts, subjects, attachments, attachment_names = [], [], [], [] diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index f49c6db59a..40d6e2b6cf 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -5,8 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import shutil, functools, re, os, traceback -from contextlib import closing +import functools, re, os, traceback from collections import defaultdict from PyQt4.Qt import (QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage, @@ -36,14 +35,6 @@ TIME_FMT = '%d %b %Y' ALIGNMENT_MAP = {'left': Qt.AlignLeft, 'right': Qt.AlignRight, 'center': Qt.AlignHCenter} -class FormatPath(unicode): - - def __new__(cls, path, orig_file_path): - ans = unicode.__new__(cls, path) - ans.orig_file_path = orig_file_path - ans.deleted_after_upload = False - return ans - _default_image = None def default_image(): @@ -391,10 +382,14 @@ class BooksModel(QAbstractTableModel): # {{{ data = self.current_changed(index, None, False) return data - def metadata_for(self, ids): + def metadata_for(self, ids, get_cover=True): + ''' + WARNING: if get_cover=True temp files are created for mi.cover. + Remember to delete them once you are done with them. + ''' ans = [] for id in ids: - mi = self.db.get_metadata(id, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(id, index_is_id=True, get_cover=get_cover) ans.append(mi) return ans @@ -449,18 +444,14 @@ class BooksModel(QAbstractTableModel): # {{{ format = f break if format is not None: - pt = PersistentTemporaryFile(suffix='.'+format) - with closing(self.db.format(id, format, index_is_id=True, - as_file=True)) as src: - shutil.copyfileobj(src, pt) - pt.flush() - if getattr(src, 'name', None): - pt.orig_file_path = os.path.abspath(src.name) + pt = PersistentTemporaryFile(suffix='caltmpfmt.'+format) + self.db.copy_format_to(id, format, pt, index_is_id=True) pt.seek(0) if set_metadata: try: - _set_metadata(pt, self.db.get_metadata(id, get_cover=True, index_is_id=True), - format) + _set_metadata(pt, self.db.get_metadata( + id, get_cover=True, index_is_id=True, + cover_as_data=True), format) except: traceback.print_exc() pt.close() @@ -468,9 +459,7 @@ class BooksModel(QAbstractTableModel): # {{{ if isbytestring(x): x = x.decode(filesystem_encoding) return x - name, op = map(to_uni, map(os.path.abspath, (pt.name, - pt.orig_file_path))) - ans.append(FormatPath(name, op)) + ans.append(to_uni(os.path.abspath(pt.name))) else: need_auto.append(id) if not exclude_auto: @@ -499,13 +488,11 @@ class BooksModel(QAbstractTableModel): # {{{ break if format is not None: pt = PersistentTemporaryFile(suffix='.'+format) - with closing(self.db.format(row, format, as_file=True)) as src: - shutil.copyfileobj(src, pt) - pt.flush() + self.db.copy_format_to(id, format, pt, index_is_id=True) pt.seek(0) if set_metadata: - _set_metadata(pt, self.db.get_metadata(row, get_cover=True), - format) + _set_metadata(pt, self.db.get_metadata(row, get_cover=True, + cover_as_data=True), format) pt.close() if paths else pt.seek(0) ans.append(pt) else: diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 3ca898d15a..9aa926f5c5 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -584,14 +584,15 @@ class BooksView(QTableView): # {{{ m = self.model() db = m.db rows = self.selectionModel().selectedRows() - selected = map(m.id, rows) + selected = list(map(m.id, rows)) ids = ' '.join(map(str, selected)) md = QMimeData() md.setData('application/calibre+from_library', ids) fmt = prefs['output_format'] def url_for_id(i): - ans = db.format_abspath(i, fmt, index_is_id=True) + ans = db.format(i, fmt, index_is_id=True, as_path=True, + preserve_filename=True) if ans is None: fmts = db.formats(i, index_is_id=True) if fmts: @@ -599,14 +600,13 @@ class BooksView(QTableView): # {{{ else: fmts = [] for f in fmts: - ans = db.format_abspath(i, f, index_is_id=True) - if ans is not None: - break + ans = db.format(i, f, index_is_id=True, as_path=True, + preserve_filename=True) if ans is None: ans = db.abspath(i, index_is_id=True) return QUrl.fromLocalFile(ans) - md.setUrls([url_for_id(i) for i in selected]) + md.setUrls([url_for_id(i) for i in selected[:25]]) drag = QDrag(self) col = self.selectionModel().currentIndex().column() md.column_name = self.column_map[col] diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index f865a5c62c..303cc51c74 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -688,7 +688,8 @@ class FormatsManager(QWidget): # {{{ else: stream = open(fmt.path, 'r+b') try: - mi = get_metadata(stream, ext) + with stream: + mi = get_metadata(stream, ext) return mi, ext except: error_dialog(self, _('Could not read metadata'), diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 39224c8b35..03b32ca2fb 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -51,12 +51,15 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ # continue mi = db.get_metadata(book_id, True) - in_file = db.format_abspath(book_id, d.input_format, True) + in_file = PersistentTemporaryFile('.'+d.input_format) + with in_file: + db.copy_format_to(book_id, d.input_format, in_file, + index_is_id=True) out_file = PersistentTemporaryFile('.' + d.output_format) out_file.write(d.output_format) out_file.close() - temp_files = [] + temp_files = [in_file] try: dtitle = unicode(mi.title) @@ -74,7 +77,7 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ recs.append(('cover', d.cover_file.name, OptionRecommendation.HIGH)) temp_files.append(d.cover_file) - args = [in_file, out_file.name, recs] + args = [in_file.name, out_file.name, recs] temp_files.append(out_file) jobs.append(('gui_convert_override', args, desc, d.output_format.upper(), book_id, temp_files)) @@ -142,12 +145,15 @@ class QueueBulk(QProgressDialog): try: input_format = get_input_format_for_book(self.db, book_id, None)[0] mi, opf_file = create_opf_file(self.db, book_id) - in_file = self.db.format_abspath(book_id, input_format, True) + in_file = PersistentTemporaryFile('.'+input_format) + with in_file: + self.db.copy_format_to(book_id, input_format, in_file, + index_is_id=True) out_file = PersistentTemporaryFile('.' + self.output_format) out_file.write(self.output_format) out_file.close() - temp_files = [] + temp_files = [in_file] combined_recs = GuiRecommendations() default_recs = bulk_defaults_for_input_format(input_format) @@ -183,7 +189,7 @@ class QueueBulk(QProgressDialog): self.setLabelText(_('Queueing ')+dtitle) desc = _('Convert book %d of %d (%s)') % (self.i, len(self.book_ids), dtitle) - args = [in_file, out_file.name, lrecs] + args = [in_file.name, out_file.name, lrecs] temp_files.append(out_file) self.jobs.append(('gui_convert_override', args, desc, self.output_format.upper(), book_id, temp_files)) diff --git a/src/calibre/library/__init__.py b/src/calibre/library/__init__.py index 2e00db32c4..84a7acbc73 100644 --- a/src/calibre/library/__init__.py +++ b/src/calibre/library/__init__.py @@ -61,22 +61,4 @@ def generate_test_db(library_path, # {{{ print 'Time per record:', t/float(num_of_records) # }}} -def cover_load_timing(path=None): - from PyQt4.Qt import QApplication, QImage - import os, time - app = QApplication([]) - app - d = db(path) - paths = [d.cover(i, index_is_id=True, as_path=True) for i in - d.data.iterallids()] - paths = [p for p in paths if (p and os.path.exists(p) and os.path.isfile(p))] - - start = time.time() - - for p in paths: - with open(p, 'rb') as f: - img = QImage() - img.loadFromData(f.read()) - - print 'Average load time:', (time.time() - start)/len(paths), 'seconds' diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 8508fb266f..1b8bb365ab 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -5137,6 +5137,7 @@ Author '{0}': OptionRecommendation.HIGH)) # If cover exists, use it + cpath = None try: search_text = 'title:"%s" author:%s' % ( opts.catalog_title.replace('"', '\\"'), 'calibre') @@ -5157,5 +5158,10 @@ Author '{0}': plumber.merge_ui_recommendations(recommendations) plumber.run() + try: + os.remove(cpath) + except: + pass + # returns to gui2.actions.catalog:catalog_generated() return catalog.error diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index efb130676a..4c61438e35 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -12,8 +12,6 @@ import threading, random from itertools import repeat from math import ceil -from PyQt4.QtGui import QImage - from calibre import prints from calibre.ebooks.metadata import (title_sort, author_to_author_sort, string_to_authors, authors_to_string) @@ -27,7 +25,7 @@ from calibre.library.sqlite import connect, IntegrityError from calibre.library.prefs import DBPrefs from calibre.ebooks.metadata.book.base import Metadata from calibre.constants import preferred_encoding, iswindows, filesystem_encoding -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, base_dir from calibre.customize.ui import run_plugins_on_import from calibre import isbytestring from calibre.utils.filenames import ascii_filename @@ -39,8 +37,10 @@ from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.formatter_functions import load_user_template_functions +from calibre.db.errors import NoSuchFormat copyfile = os.link if hasattr(os, 'link') else shutil.copyfile +SPOOL_SIZE = 30*1024*1024 class Tag(object): @@ -601,14 +601,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f: f.write(cdata) for format in formats: - # Get data as string (can't use file as source and target files may be the same) - f = self.format(id, format, index_is_id=True, as_file=True) - if f is None: - continue - with tempfile.SpooledTemporaryFile(max_size=30*(1024**2)) as stream: - with f: - shutil.copyfileobj(f, stream) - stream.seek(0) + with tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) as stream: + try: + self.copy_format_to(id, format, stream, index_is_id=True) + stream.seek(0) + except NoSuchFormat: + continue self.add_format(id, format, stream, index_is_id=True, path=tpath, notify=False) self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id)) @@ -661,32 +659,53 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): continue def cover(self, index, index_is_id=False, as_file=False, as_image=False, - as_path=False): + as_path=False): ''' Return the cover image as a bytestring (in JPEG format) or None. - `as_file` : If True return the image as an open file object - `as_image`: If True return the image as a QImage object + WARNING: Using as_path will copy the cover to a temp file and return + the path to the temp file. You should delete the temp file when you are + done with it. + + :param as_file: If True return the image as an open file object (a SpooledTemporaryFile) + :param as_image: If True return the image as a QImage object ''' - id = index if index_is_id else self.id(index) + id = index if index_is_id else self.id(index) path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') if os.access(path, os.R_OK): - if as_path: - return path try: f = lopen(path, 'rb') except (IOError, OSError): time.sleep(0.2) f = lopen(path, 'rb') - if as_image: - img = QImage() - img.loadFromData(f.read()) - f.close() - return img - ans = f if as_file else f.read() - if ans is not f: - f.close() - return ans + with f: + if as_path: + pt = PersistentTemporaryFile('_dbcover.jpg') + with pt: + shutil.copyfileobj(f, pt) + return pt.name + if as_file: + ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE) + shutil.copyfileobj(f, ret) + ret.seek(0) + else: + ret = f.read() + if as_image: + from PyQt4.Qt import QImage + i = QImage() + i.loadFromData(ret) + ret = i + return ret + + def cover_last_modified(self, index, index_is_id=False): + id = index if index_is_id else self.id(index) + path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') + try: + return utcfromtimestamp(os.stat(path).st_mtime) + except: + # Cover doesn't exist + pass + return self.last_modified() ### The field-style interface. These use field keys. @@ -859,7 +878,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return (path, mi, sequence) def get_metadata(self, idx, index_is_id=False, get_cover=False, - get_user_categories=True): + get_user_categories=True, cover_as_data=False): ''' Convenience method to return metadata as a :class:`Metadata` object. Note that the list of formats is not verified. @@ -934,7 +953,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.user_categories = user_cat_vals if get_cover: - mi.cover = self.cover(id, index_is_id=True, as_path=True) + if cover_as_data: + cdata = self.cover(id, index_is_id=True) + if cdata: + mi.cover_data = ('jpeg', cdata) + else: + mi.cover = self.cover(id, index_is_id=True, as_path=True) return mi def has_book(self, mi): @@ -1099,7 +1123,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return utcfromtimestamp(os.stat(path).st_mtime) def format_abspath(self, index, format, index_is_id=False): - 'Return absolute path to the ebook file of format `format`' + ''' + Return absolute path to the ebook file of format `format` + + WARNING: This method will return a dummy path for a network backend DB, + so do not rely on it, use format(..., as_path=True) instead. + + Currently used only in calibredb list, the viewer and the catalogs (via + get_data_as_dict()). + + Apart from the viewer, I don't believe any of the others do any file + I/O with the results of this call. + ''' id = index if index_is_id else self.id(index) try: name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False) @@ -1119,25 +1154,63 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): shutil.copyfile(candidates[0], fmt_path) return fmt_path - def format(self, index, format, index_is_id=False, as_file=False, mode='r+b'): + def copy_format_to(self, index, fmt, dest, index_is_id=False): + ''' + Copy the format ``fmt`` to the file like object ``dest``. If the + specified format does not exist, raises :class:`NoSuchFormat` error. + ''' + path = self.format_abspath(index, fmt, index_is_id=index_is_id) + if path is None: + id_ = index if index_is_id else self.id(index) + raise NoSuchFormat('Record %d has no %s file'%(id_, fmt)) + with lopen(path, 'rb') as f: + shutil.copyfileobj(f, dest) + if hasattr(dest, 'flush'): + dest.flush() + + def format(self, index, format, index_is_id=False, as_file=False, + mode='r+b', as_path=False, preserve_filename=False): ''' Return the ebook format as a bytestring or `None` if the format doesn't exist, or we don't have permission to write to the ebook file. - `as_file`: If True the ebook format is returned as a file object opened in `mode` + :param as_file: If True the ebook format is returned as a file object. Note + that the file object is a SpooledTemporaryFile, so if what you want to + do is copy the format to another file, use :method:`copy_format_to` + instead for performance. + :param as_path: Copies the format file to a temp file and returns the + path to the temp file + :param preserve_filename: If True and returning a path the filename is + the same as that used in the library. Note that using + this means that repeated calls yield the same + temp file (which is re-created each time) + :param mode: This is ignored (present for legacy compatibility) ''' path = self.format_abspath(index, format, index_is_id=index_is_id) if path is not None: - f = lopen(path, mode) - try: - ret = f if as_file else f.read() - except IOError: - f.seek(0) - out = cStringIO.StringIO() - shutil.copyfileobj(f, out) - ret = out.getvalue() - if not as_file: - f.close() + with lopen(path, mode) as f: + if as_path: + if preserve_filename: + bd = base_dir() + d = os.path.join(bd, 'format_abspath') + try: + os.makedirs(d) + except: + pass + fname = os.path.basename(path) + ret = os.path.join(d, fname) + with lopen(ret, 'wb') as f2: + shutil.copyfileobj(f, f2) + else: + with PersistentTemporaryFile('.'+format.lower()) as pt: + shutil.copyfileobj(f, pt) + ret = pt.name + elif as_file: + ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) + shutil.copyfileobj(f, ret) + ret.seek(0) + else: + ret = f.read() return ret def add_format_with_hooks(self, index, format, fpath, index_is_id=False, diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 5f49833564..b5c4e2faf3 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback, cStringIO, re, shutil +import os, traceback, cStringIO, re from calibre.constants import DEBUG from calibre.utils.config import Config, StringConfig, tweaks @@ -238,7 +238,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, def save_book_to_disk(id_, db, root, opts, length): mi = db.get_metadata(id_, index_is_id=True) - cover = db.cover(id_, index_is_id=True, as_path=True) + cover = db.cover(id_, index_is_id=True) plugboards = db.prefs.get('plugboards', {}) available_formats = db.formats(id_, index_is_id=True) @@ -252,12 +252,20 @@ def save_book_to_disk(id_, db, root, opts, length): if fmts: fmts = fmts.split(',') for fmt in fmts: - fpath = db.format_abspath(id_, fmt, index_is_id=True) + fpath = db.format(id_, fmt, index_is_id=True, as_path=True) if fpath is not None: formats[fmt.lower()] = fpath - return do_save_book_to_disk(id_, mi, cover, plugboards, + try: + return do_save_book_to_disk(id_, mi, cover, plugboards, formats, root, opts, length) + finally: + for temp in formats.itervalues(): + try: + os.remove(temp) + except: + pass + def do_save_book_to_disk(id_, mi, cover, plugboards, @@ -289,10 +297,9 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, raise ocover = mi.cover - if opts.save_cover and cover and os.access(cover, os.R_OK): + if opts.save_cover and cover: with open(base_path+'.jpg', 'wb') as f: - with open(cover, 'rb') as s: - shutil.copyfileobj(s, f) + f.write(cover) mi.cover = base_name+'.jpg' else: mi.cover = None @@ -395,8 +402,13 @@ def save_serialized_to_disk(ids, data, plugboards, root, opts, callback): pass tb = '' try: - failed, id, title = do_save_book_to_disk(x, mi, cover, plugboards, - format_map, root, opts, length) + with open(cover, 'rb') as f: + cover = f.read() + except: + cover = None + try: + failed, id, title = do_save_book_to_disk(x, mi, cover, + plugboards, format_map, root, opts, length) tb = _('Requested formats not available') except: failed, id, title = True, x, mi.title diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index 08de4faecd..853ec7829c 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -10,12 +10,13 @@ import re, os, posixpath import cherrypy from calibre import fit_image, guess_type -from calibre.utils.date import fromtimestamp +from calibre.utils.date import fromtimestamp, utcnow from calibre.library.caches import SortKeyGenerator from calibre.library.save_to_disk import find_plugboard - -from calibre.utils.magick.draw import save_cover_data_to, Image, \ - thumbnail as generate_thumbnail +from calibre.ebooks.metadata import authors_to_string +from calibre.utils.magick.draw import (save_cover_data_to, Image, + thumbnail as generate_thumbnail) +from calibre.utils.filenames import ascii_filename plugboard_content_server_value = 'content_server' plugboard_content_server_formats = ['epub'] @@ -46,7 +47,7 @@ class ContentServer(object): # Utility methods {{{ def last_modified(self, updated): ''' - Generates a local independent, english timestamp from a datetime + Generates a locale independent, english timestamp from a datetime object ''' lm = updated.strftime('day, %d month %Y %H:%M:%S GMT') @@ -151,14 +152,12 @@ class ContentServer(object): try: cherrypy.response.headers['Content-Type'] = 'image/jpeg' cherrypy.response.timeout = 3600 - cover = self.db.cover(id, index_is_id=True, as_file=True) + cover = self.db.cover(id, index_is_id=True) if cover is None: cover = self.default_cover updated = self.build_time else: - with cover as f: - updated = fromtimestamp(os.fstat(f.fileno()).st_mtime) - cover = f.read() + updated = self.db.cover_last_modified(id, index_is_id=True) cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) if thumbnail: @@ -187,9 +186,9 @@ class ContentServer(object): mode='rb') if fmt is None: raise cherrypy.HTTPError(404, 'book: %d does not have format: %s'%(id, format)) + mi = self.db.get_metadata(id, index_is_id=True) if format == 'EPUB': # Get the original metadata - mi = self.db.get_metadata(id, index_is_id=True) # Get any EPUB plugboards for the content server plugboards = self.db.prefs.get('plugboards', {}) @@ -203,24 +202,22 @@ class ContentServer(object): newmi = mi # Write the updated file - from tempfile import TemporaryFile from calibre.ebooks.metadata.meta import set_metadata - raw = fmt.read() - fmt = TemporaryFile() - fmt.write(raw) - fmt.seek(0) set_metadata(fmt, newmi, 'epub') fmt.seek(0) mt = guess_type('dummy.'+format.lower())[0] if mt is None: mt = 'application/octet-stream' + au = authors_to_string(mi.authors if mi.authors else [_('Unknown')]) + title = mi.title if mi.title else _('Unknown') + fname = u'%s - %s_%s.%s'%(title[:30], au[:30], id, format.lower()) + fname = ascii_filename(fname).replace('"', '_') cherrypy.response.headers['Content-Type'] = mt + cherrypy.response.headers['Content-Disposition'] = \ + b'attachment; filename="%s"'%fname cherrypy.response.timeout = 3600 - path = getattr(fmt, 'name', None) - if path and os.path.exists(path): - updated = fromtimestamp(os.stat(path).st_mtime) - cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) + cherrypy.response.headers['Last-Modified'] = self.last_modified(utcnow()) return fmt # }}}