From b540537f30019ed965d4f7e74e0850f5054de315 Mon Sep 17 00:00:00 2001 From: Spedinfargo Date: Fri, 25 Feb 2011 16:05:02 -0600 Subject: [PATCH 01/70] Suggested patch for max issues per recipe --- src/calibre/gui2/actions/fetch_news.py | 10 +++++++ src/calibre/gui2/dialogs/scheduler.py | 14 +++++++--- src/calibre/gui2/dialogs/scheduler.ui | 30 +++++++++++++++++++++ src/calibre/library/database2.py | 12 +++++++++ src/calibre/web/feeds/recipes/collection.py | 9 +++++-- src/calibre/web/feeds/recipes/model.py | 5 ++-- 6 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/actions/fetch_news.py b/src/calibre/gui2/actions/fetch_news.py index 5c2a5e9663..009bf8b00b 100644 --- a/src/calibre/gui2/actions/fetch_news.py +++ b/src/calibre/gui2/actions/fetch_news.py @@ -58,6 +58,16 @@ class FetchNewsAction(InterfaceAction): self.scheduler.recipe_download_failed(arg) return self.gui.job_exception(job) id = self.gui.library_view.model().add_news(pt.name, arg) + + # Arg may contain a "keep_issues" variable. if it is non-zer, delete all but newest x issues. + try: + ikeep_issues = int(arg['keep_issues']) + except: + ikeep_issues = 0 + if ikeep_issues > 0: + ids2delete = self.gui.library_view.model().db.get_most_recent_by_tag(arg['keep_issues'], arg['title']) + self.gui.library_view.model().delete_books_by_id(ids2delete) + self.gui.library_view.model().reset() sync = self.gui.news_to_be_synced sync.add(id) diff --git a/src/calibre/gui2/dialogs/scheduler.py b/src/calibre/gui2/dialogs/scheduler.py index b6a3bed3eb..be2997a314 100644 --- a/src/calibre/gui2/dialogs/scheduler.py +++ b/src/calibre/gui2/dialogs/scheduler.py @@ -153,9 +153,10 @@ class SchedulerDialog(QDialog, Ui_Dialog): self.recipe_model.un_schedule_recipe(urn) add_title_tag = self.add_title_tag.isChecked() + keep_issues = unicode(self.keep_issues.text()) custom_tags = unicode(self.custom_tags.text()).strip() custom_tags = [x.strip() for x in custom_tags.split(',')] - self.recipe_model.customize_recipe(urn, add_title_tag, custom_tags) + self.recipe_model.customize_recipe(urn, add_title_tag, custom_tags, keep_issues) return True def initialize_detail_box(self, urn): @@ -215,9 +216,15 @@ class SchedulerDialog(QDialog, Ui_Dialog): if d < timedelta(days=366): self.last_downloaded.setText(_('Last downloaded')+': '+tm) - add_title_tag, custom_tags = customize_info + add_title_tag, custom_tags, keep_issues = customize_info self.add_title_tag.setChecked(add_title_tag) self.custom_tags.setText(u', '.join(custom_tags)) + try: + ikeep_issues = int(keep_issues) + except: + ikeep_issues = 0 + self.keep_issues.setValue(ikeep_issues) + class Scheduler(QObject): @@ -299,7 +306,7 @@ class Scheduler(QObject): un = pw = None if account_info is not None: un, pw = account_info - add_title_tag, custom_tags = customize_info + add_title_tag, custom_tags, keep_issues = customize_info script = self.recipe_model.get_recipe(urn) pt = PersistentTemporaryFile('_builtin.recipe') pt.write(script) @@ -312,6 +319,7 @@ class Scheduler(QObject): 'recipe':pt.name, 'title':recipe.get('title',''), 'urn':urn, + 'keep_issues':keep_issues } self.download_queue.add(urn) self.start_recipe_fetch.emit(arg) diff --git a/src/calibre/gui2/dialogs/scheduler.ui b/src/calibre/gui2/dialogs/scheduler.ui index 8e6ab37162..079a17dbb3 100644 --- a/src/calibre/gui2/dialogs/scheduler.ui +++ b/src/calibre/gui2/dialogs/scheduler.ui @@ -245,6 +245,36 @@ + + + + + + Maximum number of copies (issues) of this recipe to keep. Set to 0 to keep all (disable). + + + Maximum copies (0 to keep all): + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index dce0b34aef..234d1fcfc4 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1476,6 +1476,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ############# End get_categories + def get_most_recent_by_tag(self, number_to_keep, tag): + #Based on tag and number passed in, create a list of books matching that tag, keeping only the newest X versions + tag = tag.lower().strip() + idlist = [] + mycount = 0 + for myid in (self.conn.get('select a.book id from books_tags_link a inner join books b on a.book = b.id where a.tag in (select tags.id from tags where tags.name = ?) order by b.timestamp desc', [tag])): + myid = myid[0] + mycount = mycount + 1 + if mycount > int(number_to_keep): + idlist.append(myid) + return idlist + def tags_older_than(self, tag, delta): tag = tag.lower().strip() now = nowf() diff --git a/src/calibre/web/feeds/recipes/collection.py b/src/calibre/web/feeds/recipes/collection.py index 5dd360213b..6697a1f39f 100644 --- a/src/calibre/web/feeds/recipes/collection.py +++ b/src/calibre/web/feeds/recipes/collection.py @@ -201,12 +201,14 @@ class SchedulerConfig(object): self.root.append(sr) self.write_scheduler_file() - def customize_recipe(self, urn, add_title_tag, custom_tags): + # 'keep_issues' argument for recipe-specific number of copies to keep + def customize_recipe(self, urn, add_title_tag, custom_tags, keep_issues): with self.lock: for x in list(self.iter_customization()): if x.get('id') == urn: self.root.remove(x) cs = E.recipe_customization({ + 'keep_issues' : keep_issues, 'id' : urn, 'add_title_tag' : 'yes' if add_title_tag else 'no', 'custom_tags' : ','.join(custom_tags), @@ -316,17 +318,20 @@ class SchedulerConfig(object): if x.get('id', False) == urn: return x.get('username', ''), x.get('password', '') + # 'keep_issues' element for recipe-specific number of copies to keep (default 0 == all) def get_customize_info(self, urn): + keep_issues = 0 add_title_tag = True custom_tags = [] with self.lock: for x in self.iter_customization(): if x.get('id', False) == urn: + keep_issues = x.get('keep_issues',0) add_title_tag = x.get('add_title_tag', 'yes') == 'yes' custom_tags = [i.strip() for i in x.get('custom_tags', '').split(',')] break - return add_title_tag, custom_tags + return add_title_tag, custom_tags, keep_issues def get_schedule_info(self, urn): with self.lock: diff --git a/src/calibre/web/feeds/recipes/model.py b/src/calibre/web/feeds/recipes/model.py index 559a5c08dd..203f96b03d 100644 --- a/src/calibre/web/feeds/recipes/model.py +++ b/src/calibre/web/feeds/recipes/model.py @@ -354,9 +354,10 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser): self.scheduler_config.schedule_recipe(self.recipe_from_urn(urn), sched_type, schedule) - def customize_recipe(self, urn, add_title_tag, custom_tags): + # 'keep_issues' argument for recipe-specific number of copies to keep + def customize_recipe(self, urn, add_title_tag, custom_tags, keep_issues): self.scheduler_config.customize_recipe(urn, add_title_tag, - custom_tags) + custom_tags, keep_issues) def get_to_be_downloaded_recipes(self): ans = self.scheduler_config.get_to_be_downloaded_recipes() From 7c22f4ffa145e8bb3cd927f475725a676eb3e74c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 26 Feb 2011 07:16:43 +0000 Subject: [PATCH 02/70] Fix #9169: not possible to hide user category with hierarchy Fix #9166: hierarchy indicator for no children Fix removes all hidden tag browser categories. They must be re-hidden. Fix adds clicking on category nodes to search, plus four-state searching. --- resources/images/minusminus.png | Bin 0 -> 1883 bytes resources/images/plusplus.png | Bin 0 -> 4249 bytes src/calibre/gui2/tag_view.py | 167 ++++++++++++++++++++------------ src/calibre/library/caches.py | 21 ++-- 4 files changed, 114 insertions(+), 74 deletions(-) create mode 100644 resources/images/minusminus.png create mode 100644 resources/images/plusplus.png diff --git a/resources/images/minusminus.png b/resources/images/minusminus.png new file mode 100644 index 0000000000000000000000000000000000000000..71225be8d7f6e5691e4efa5135c500c140141437 GIT binary patch literal 1883 zcmb`IdpOgJAIHCzd(u|SE!Wdw!#O13C~}#GVVId|4e>k9$R&nKu}LGHFe#>?OoX`| z*~(=jhbi~uWRy$vBg!S($|XNNzvuZq&+o7E$M5%ip3mp~Jg?{T`Q!c1=Y8GN1G!IL zT^;~{eJ;)j?_H&QZ&}dpbR&qEyCO~Y_P_u;-^J*D?=A80WI1Vf zYtsc`?{i`7BeMssxeeX+SBefYOI~B zhh9d5j|le^sO~xYR{oSi$>JX=^j9f^9l`lX+=v#9w0W&R-5tOYf(VY#1I0I&f0ks1Z1vxv~|A_Z(N~&Lt_n{-L zn3l$ObH`tEIM!?rsvEOq``(Mc0teWfYF{NJ>x@srnK<*xCGUu#%F`XcEi75|7oyfb zzZYa7@ol0!6Kxq8T$g|gg&wAyRc+C4Q*0GbB|U*uR^;*zZ@4G-fO*zctvIVplJ-S$ z-St#($(IG?0AeQEL;q)DNQKXKp?n|M#}ChFiL0u6q|_6#?sh6FQrdk6clV_;>Nze&1E(7rj^k=xPS!>%*zfvRIfXp< z@nc31j0TB{{IzK!=`G?PiQ~7XyHYs4HZrDg8w@D6S3a!zOxyrCgEjQk;3vB0e$BJb zN?~~nlG!lZ4ShOGixk?AK(>*ScQQ$49i9q~E{7Ehtw##_B3mxBTUz7?hu~gALL+60 z3V)bZV%av)IhAw?HVPk#nRc*3|z7dO+J#ch5w+z3qoRv!%Ko>)~Tt<*- z$8XPn9ey@DX&GhK^NG-6{P3yB6SU+quGn;RsXrYhPM4(7=if7m?{02?5N?f)RaD`c z4V8lmdd=I-jk2^~2^*tjNBs%xXdVW}HZlIw-~ZFhObMaJb^_LT57wA=@F8|o3%Qp( z^>S);8UdQu)LdmDRVG>!&(C!IbK{Z}MWWX>QG_zYGV(q5O5qKUV5=L#r@?{vJHS+@ zo(Yue>&xzQZ~x^n%jC)sH((=ZatjkUkJqcOEk~Am7Yn*SHi&x7){tJ}7u}yx%pN+}rsO2=rJ2I^uJAMVXNLbyH39d21VB+zUAaE45hO1R9RA>u zG0_m~>&q!R0Z;m(W2Ee*G3W!}$CCuq+!BKH@eRn`)Purv=Py3FBxB4^-a#jb$LWwC zv@GAQB8>l1$`Zu2!Z|m}Dh-1RYN{$LL9h%xC8v?NZ0|d+ZnFB4y#s64<#>;~trEJt zs^uEq35O;Z$7)|pj9@RVv=vePEHvN}X)#YeQ>hD`jkgApnar2tW3DJviJviO9`KI{ zS9wFMW=9+o$CRyQNF)Dz5E{R4!+F}KmBZ3KSCvnQZPYR6jA4aCTi3ex3>8e8l0Xq?7XBf77q@^q6G_ z6eS=Y7`ZfxY=~*FV4a{qlZz|JS*mb6@9q&hy;&bDisaI48f8|#|@j!uTG#o2&k4~xeQOD_O`^UjF@^Of_Q001sIZFLn>zmdPU zZ@UN%K0NHK`cN~6}UE z>ACaNsn-&tc|7;lx)DHX<2^eI$f!I9ZymSr#iwxSy}fn0c$AE&4$qNqYWm07bfwW@ zQBk7}+Q1@5>fX<2tD=Ze%38auv$Wuw6t7a-u=D&se7rF0Y;?jc_{}#yFe)OB_Ux)(P5)cSab<5NP)1%JNG#y6kM}wKElfAq7l|{bIn-440w3|$E{k!N2mxW>Djg|g}#vV51 z7jejnnK$29fvJ}#C-!$T6@B)W#?c;2hJk&CZuTfy@yof+xw+re!s|y{B1T&b-PSU! zH_y}eM@$uaLb$E3FzrtV1^!KqP*ygovhet*R+SK2mGH3Ge84oDffwC`_whq1D3Z@+ z#5;=E2g&cgI-Nd5B>}r$JSG%ND;QN@O*!y*@kglL_05|r5_1hWMUV)X>;#vseEl{+ z&5bM}fP5i@4Dt>PTpyY?UmSZ&FPJSyK>T(O&UZ9>&CY7>z5MjB_-L_y%r|xv^H%u7 zQDBlR0UuZbnx6;HL zFIW<|+#YH0N#@YFbDHT~PKQzjMg=QcjOx)^JD-txluysA+(_aycW`L)U*VvD=v=xB zWOBjqiF#Bx1CGE%ezaz`apl}^Jv=J-9p+$$`K>0z5CeXSXEGr?;(289T!3!LpbTWEI@O!&}et$rKt4pA+E*t>n*d z0$9L*>`Jy2lIt6J0HLM2bsfa0U_6s3Hk7SeABw`DF`=iR9(wT%_d@}UP)%?Me}Az7 zG1>`2Cp{|FJ=F^SBl4ur^b#{y-U5pNBdxS(9<^S$Z!1#ul9 zAlTSqjk~8zg-C@B`~hH026)}w!f6y55bz2VUN-`-YlpT%w{1h;@<|O;vpK6s)$T&L z5(dB6DC{r(bess#gaw~b2f$!;>gqi601-xbP9 z3wx0g?Tv9lXtbObt}!nC0>N%p&_pR|VmxT?K#lpF;9-MnK#e?w*tV9MDZkPG=Wa=S zM_hgP=bh*jb|%n(Ah^CAAVLN{|3^8hpp^&Ai+;}=LZrc4)$Tw)pj!HQSnW=eL-|dk zAi2(sFb9o|_4d|bztvkSV+Wt6I8%Z5_`lh@Qn1!jie{iNOTlUCH*F`|HfjdA3z~+beC9!U*Eh zrfvOMAYK`UNZuu3^)z zT_)&@6@0#mhNFB>4H}sU{kGh}EI4&&r-zK%@TYwDoVU{hJ%bgS9Q5m6F5_JtdsUAp z*F9G*3Tf1_&Dw~W>3}48@=^WhtKAcOMY0*oGb)vKfYuIu^NXq*+FVcrdnVO;2K{Hm z_I4_A4gU@)E zQ09}@34g3GOzD}|kv3nGSI@+|&J8b5q}tdn$Cz7Z0#sX6hQ(auvhhn+{;YLa^*c&T ztSRra{crM&eiX&1OMI6sybx){f}C)r!YTeJFpR;Jq; z_gl&fBLK*AAs|bOXW_c9b=OwXIji=QLm@hi>a2FGFrs)RCz&4x`;31mK$?pVsX*A+ z5n2`VBbUXu7HS>LgZF}D{MWyi9R7ZCOS)j-RG7st%6z|zBDY3kyXOll_p(y}Yk|uG zF{$19l@S1aaeV@wqlXL-xj-9RiMk*yp3G?;@SNXy9I@o@{-@?HT-MKjH8kk@PQNwx z#abF)(E%xp5o5+(*H22cdYJ^t_sZgemDm(hfr2;{FskJ#ShrrJ6ErYQ9rq@vHkQ z!TJl%kqc#dM!;5(8Sq(wZ+y7y;h>lM$LDR+{+?AXGJQ!M4)2F=A(Fb**X=zmed6S1 zJ^53CNoc-=HqR6lJ5_=@`I13Kni7WBfa&IZ82~2IOw&|1)X0aoKPR`&cSaV!TPoI5 z(9&&&3Ol?!Y{zQBmb*72Gt&K~E3+u8?Z12aY{&KnWRdjWKkEUE9QJVG62hCf_L^|g zBxzVmA$K_>O6l^80=s%C1-s_PC|zdQbfsfk^6v=|lf z0tp<}33dnC>*}r?&*GIkI z-?P*0mn{QJy=G7J&YjM7?@a*ov#QtGpK(raV2lF_fwQd)+*R-`kh7aCKI`@f;`si zT$xkelw7g21?s8ZX;6vfM3W#{^##{N2v)!9q|oZ1a{H~ENXNFF{652@sbvmmx#Q=& zqs0%(f-M{}eny2~Kp>aJi6>K>@%!6D@$|J1OM-Hb?45md%u@Fau)UGsA#qE;t88rv zOh!`o#K?@yk`z{TIDw?yE5sU6a)1)oW1}U(bO<7nGvKAOhwG{3%Bz$FxKwW)d%TpS5_CD6Q?f%H1SZ`c=13_OV<&5 z)`6itF_%vEOBj(~F>Z^1jT%83>OWRAAS9?j*A$j~*%rKrK;ah6#?161^`9#{*$%8y z>}9uS+lg@Q<+8maop!o{m#6;&XL;F6mvJbBiH#)pa@4qW{B5P``=393L083x4;ydU z;_Q|RMQ-VakyW*lA*29|y^8ik>CP9YCg_bw9M-iWTSt6=ma2hd8%O%l4L6c$$mW5n zCO?|-P3||9cVEig+lAhU!VyJfuSPk&7C`zT33do)v%GgV=K3heD5oJUXNm0W-%L3g z2;!*{5k#_ct>MPY8M8utkr~5NjL)2J;u@6Z9nK1*k#f18u~GoEAdHlR#24Ozn;Pj} zu*ez0wtJF`20UwIrk7xy2V%-mMP!w`xiCpxGUsdiG*1@VF$6?P7Vy_1BjJJ^ZuOxw zy^C;OY2clXL}0`-Q@f%Y1K}?s%Eqj+mp8^W@RMPH`+;%2-VD8P?z*^SOg9`Mki@r< z(<*IoV9Gu-30&7CT^-7#c4GTjZiM2n@95VoGoWaE&uO-4hWv9Wu8%%@-F2HQpZmrV zI{d3^xTIQUGjStz(Mdb4Lr4q-W5` zp5l304prO`?pb2BdF+FiakSv z(o~u()WmWiX0iZoZj z(hz|4X1bvs0`I^uFAXdc+Lm3pDzoJtxU*%W-?tT__8#lCcI zc+Gn2`et`$s8RpRb#2S0Aa6bcI%_Q=)G35YdHL;Pc zCy2$)o?vWlw$P=MxDHn9>+zbsDoa0^P|xcZ2r7^$6I!6P{QfH3j37g3pO@&yVxmvK)IK zofKl1Et1zh3Dg&p)=zd84N=KfCse!vYyBsVj*E3~%Gm1A{-!duOb@0VF3 z>D<`LT*jx5k2;pdN-L62+M?|UNxz837%k-de^2jyPTOK=$UuIi((%s%KwCp!y;v0= F@*kvZ)O-K{ literal 0 HcmV?d00001 diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 2693ba8ed6..034d88e02d 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -21,6 +21,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, QFont, QSize, \ from calibre.ebooks.metadata import title_sort from calibre.gui2 import config, NONE, gprefs from calibre.library.field_metadata import TagsIcons, category_icon_map +from calibre.library.database2 import Tag from calibre.utils.config import tweaks from calibre.utils.icu import sort_key, lower, strcmp from calibre.utils.search_query_parser import saved_searches @@ -69,7 +70,8 @@ class TagDelegate(QItemDelegate): # {{{ # }}} -TAG_SEARCH_STATES = {'clear': 0, 'mark_plus': 1, 'mark_minus': 2} +TAG_SEARCH_STATES = {'clear': 0, 'mark_plus': 1, 'mark_plusplus': 2, + 'mark_minus': 3, 'mark_minusminus': 4} class TagsView(QTreeView): # {{{ @@ -127,13 +129,17 @@ class TagsView(QTreeView): # {{{ self.set_new_model(self._model.get_filter_categories_by()) def set_database(self, db, tag_match, sort_by): - self.hidden_categories = db.prefs.get('tag_browser_hidden_categories', None) + hidden_cats = db.prefs.get('tag_browser_hidden_categories', None) + self.hidden_categories = [] # migrate from config to db prefs - if self.hidden_categories is None: - self.hidden_categories = config['tag_browser_hidden_categories'] - db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories)) - else: - self.hidden_categories = set(self.hidden_categories) + if hidden_cats is None: + hidden_cats = config['tag_browser_hidden_categories'] + # strip out any non-existence field keys + for cat in hidden_cats: + if cat in db.field_metadata: + self.hidden_categories.append(cat) + db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories)) + self.hidden_categories = set(self.hidden_categories) old = getattr(self, '_model', None) if old is not None: @@ -370,14 +376,15 @@ class TagsView(QTreeView): # {{{ action='delete_user_category', key=key)) self.context_menu.addSeparator() # Hide/Show/Restore categories - if not key.startswith('@') or key.find('.') < 0: - self.context_menu.addAction(_('Hide category %s') % category, - partial(self.context_menu_handler, action='hide', - category=category)) +# if not key.startswith('@') or key.find('.') < 0: + self.context_menu.addAction(_('Hide category %s') % category, + partial(self.context_menu_handler, action='hide', + category=key)) if self.hidden_categories: m = self.context_menu.addMenu(_('Show category')) - for col in sorted(self.hidden_categories, key=sort_key): - m.addAction(col, + for col in sorted(self.hidden_categories, + key=lambda x: sort_key(self.db.field_metadata[x]['name'])): + m.addAction(self.db.field_metadata[col]['name'], partial(self.context_menu_handler, action='show', category=col)) # search by category @@ -540,6 +547,7 @@ class TagTreeItem(object): # {{{ self.id_set = set() self.is_gst = False self.boxed = False + self.icon_state_map = list(map(QVariant, icon_map)) if self.parent is not None: self.parent.append(self) if data is None: @@ -554,9 +562,11 @@ class TagTreeItem(object): # {{{ self.bold_font = QVariant(self.bold_font) self.category_key = category_key self.temporary = temporary + self.tag = Tag(data) + self.tag.is_hierarchical = category_key.startswith('@') elif self.type == self.TAG: icon_map[0] = data.icon - self.tag, self.icon_state_map = data, list(map(QVariant, icon_map)) + self.tag = data if tooltip: self.tooltip = tooltip + ' ' else: @@ -593,6 +603,8 @@ class TagTreeItem(object): # {{{ if role == Qt.EditRole: return QVariant(self.py_name) if role == Qt.DecorationRole: + if self.tag.state: + return self.icon_state_map[self.tag.state] return self.icon if role == Qt.FontRole: return self.bold_font @@ -642,11 +654,22 @@ class TagTreeItem(object): # {{{ ''' set_to: None => advance the state, otherwise a value from TAG_SEARCH_STATES ''' - if self.type == self.TAG: - if set_to is None: - self.tag.state = (self.tag.state + 1)%3 - else: - self.tag.state = set_to +# if self.type == self.TAG: + if set_to is None: + while True: + self.tag.state = (self.tag.state + 1)%5 + if self.tag.state == TAG_SEARCH_STATES['mark_plus'] or \ + self.tag.state == TAG_SEARCH_STATES['mark_minus']: + if self.tag.is_editable: + break + elif self.tag.state == TAG_SEARCH_STATES['mark_plusplus'] or\ + self.tag.state == TAG_SEARCH_STATES['mark_minusminus']: + if self.tag.is_hierarchical and len(self.children): + break + else: + break + else: + self.tag.state = set_to def child_tags(self): res = [] @@ -677,7 +700,8 @@ class TagsModel(QAbstractItemModel): # {{{ self.categories_with_ratings = ['authors', 'series', 'publisher', 'tags'] self.drag_drop_finished = drag_drop_finished - self.icon_state_map = [None, QIcon(I('plus.png')), QIcon(I('minus.png'))] + self.icon_state_map = [None, QIcon(I('plus.png')), QIcon(I('plusplus.png')), + QIcon(I('minus.png')), QIcon(I('minusminus.png'))] self.db = db self.tags_view = parent self.hidden_categories = hidden_categories @@ -691,26 +715,33 @@ class TagsModel(QAbstractItemModel): # {{{ data = self.get_node_tree(config['sort_tags_by']) gst = db.prefs.get('grouped_search_terms', {}) - self.root_item = TagTreeItem() + self.root_item = TagTreeItem(icon_map=self.icon_state_map) self.category_nodes = [] last_category_node = None category_node_map = {} self.category_node_tree = {} - for i, r in enumerate(self.row_map): - if self.hidden_categories and self.categories[i] in self.hidden_categories: - continue + for i, key in enumerate(self.row_map): + if self.hidden_categories: + if key in self.hidden_categories: + continue + found = False + for cat in self.hidden_categories: + if cat.startswith('@') and key.startswith(cat + '.'): + found = True + if found: + continue is_gst = False - if r.startswith('@') and r[1:] in gst: - tt = _(u'The grouped search term name is "{0}"').format(r[1:]) + if key.startswith('@') and key[1:] in gst: + tt = _(u'The grouped search term name is "{0}"').format(key[1:]) is_gst = True - elif r == 'news': + elif key == 'news': tt = '' else: - tt = _(u'The lookup/search name is "{0}"').format(r) + tt = _(u'The lookup/search name is "{0}"').format(key) - if r.startswith('@'): - path_parts = [p for p in r.split('.')] + if key.startswith('@'): + path_parts = [p for p in key.split('.')] path = '' last_category_node = self.root_item tree_root = self.category_node_tree @@ -719,9 +750,10 @@ class TagsModel(QAbstractItemModel): # {{{ if path not in category_node_map: node = TagTreeItem(parent=last_category_node, data=p[1:] if i == 0 else p, - category_icon=self.category_icon_map[r], - tooltip=tt if path == r else path, - category_key=path) + category_icon=self.category_icon_map[key], + tooltip=tt if path == key else path, + category_key=path, + icon_map=self.icon_state_map) last_category_node = node category_node_map[path] = node self.category_nodes.append(node) @@ -736,11 +768,12 @@ class TagsModel(QAbstractItemModel): # {{{ path += '.' else: node = TagTreeItem(parent=self.root_item, - data=self.categories[i], - category_icon=self.category_icon_map[r], - tooltip=tt, category_key=r) + data=self.categories[key], + category_icon=self.category_icon_map[key], + tooltip=tt, category_key=key, + icon_map=self.icon_state_map) node.is_gst = False - category_node_map[r] = node + category_node_map[key] = node last_category_node = node self.category_nodes.append(node) self.refresh(data=data) @@ -1015,7 +1048,7 @@ class TagsModel(QAbstractItemModel): # {{{ def get_node_tree(self, sort): old_row_map = self.row_map[:] self.row_map = [] - self.categories = [] + self.categories = {} # Get the categories if self.search_restriction: @@ -1062,7 +1095,7 @@ class TagsModel(QAbstractItemModel): # {{{ for category in tb_categories: if category in data: # The search category can come and go self.row_map.append(category) - self.categories.append(tb_categories[category]['name']) + self.categories[category] = tb_categories[category]['name'] if len(old_row_map) != 0 and len(old_row_map) != len(self.row_map): # A category has been added or removed. We must force a rebuild of @@ -1163,7 +1196,8 @@ class TagsModel(QAbstractItemModel): # {{{ sub_cat = TagTreeItem(parent=category, data = name, tooltip = None, temporary=True, category_icon = category_node.icon, - category_key=category_node.category_key) + category_key=category_node.category_key, + icon_map=self.icon_state_map) self.endInsertRows() else: # by 'first letter' cl = cl_list[idx] @@ -1173,7 +1207,8 @@ class TagsModel(QAbstractItemModel): # {{{ data = collapse_letter, category_icon = category_node.icon, tooltip = None, temporary=True, - category_key=category_node.category_key) + category_key=category_node.category_key, + icon_map=self.icon_state_map) node_parent = sub_cat else: node_parent = category @@ -1477,16 +1512,16 @@ class TagsModel(QAbstractItemModel): # {{{ def reset_all_states(self, except_=None): update_list = [] def process_tag(tag_item): - if tag_item.type != TagTreeItem.CATEGORY: - tag = tag_item.tag - if tag is except_: - tag_index = self.createIndex(tag_item.row(), 0, tag_item) - self.dataChanged.emit(tag_index, tag_index) - elif tag.state != 0 or tag in update_list: - tag_index = self.createIndex(tag_item.row(), 0, tag_item) - tag.state = 0 - update_list.append(tag) - self.dataChanged.emit(tag_index, tag_index) +# if tag_item.type != TagTreeItem.CATEGORY: + tag = tag_item.tag + if tag is except_: + tag_index = self.createIndex(tag_item.row(), 0, tag_item) + self.dataChanged.emit(tag_index, tag_index) + elif tag.state != 0 or tag in update_list: + tag_index = self.createIndex(tag_item.row(), 0, tag_item) + tag.state = 0 + update_list.append(tag) + self.dataChanged.emit(tag_index, tag_index) for t in tag_item.children: process_tag(t) @@ -1503,13 +1538,11 @@ class TagsModel(QAbstractItemModel): # {{{ ''' if not index.isValid(): return False item = index.internalPointer() - if item.type == TagTreeItem.TAG: - item.toggle(set_to=set_to) - if exclusive: - self.reset_all_states(except_=item.tag) - self.dataChanged.emit(index, index) - return True - return False + item.toggle(set_to=set_to) + if exclusive: + self.reset_all_states(except_=item.tag) + self.dataChanged.emit(index, index) + return True def tokens(self): ans = [] @@ -1523,19 +1556,31 @@ class TagsModel(QAbstractItemModel): # {{{ # into the search string only once. The nodes_seen set helps us do that nodes_seen = set() + node_searches = {TAG_SEARCH_STATES['mark_plus'] : 'true', + TAG_SEARCH_STATES['mark_plusplus'] : '.true', + TAG_SEARCH_STATES['mark_minus'] : 'false', + TAG_SEARCH_STATES['mark_minusminus'] : '.false'} + for node in self.category_nodes: + if node.tag.state: + ans.append('%s:%s'%(node.category_key, node_searches[node.tag.state])) + key = node.category_key for tag_item in node.child_tags(): tag = tag_item.tag if tag.state != TAG_SEARCH_STATES['clear']: - prefix = ' not ' if tag.state == TAG_SEARCH_STATES['mark_minus'] \ - else '' + if tag.state == TAG_SEARCH_STATES['mark_minus'] or \ + tag.state == TAG_SEARCH_STATES['mark_minusminus']: + prefix = ' not ' + else: + prefix = '' category = tag.category if key != 'news' else 'tag' if tag.name and tag.name[0] == u'\u2605': # char is a star. Assume rating ans.append('%s%s:%s'%(prefix, category, len(tag.name))) else: name = original_name(tag) - use_prefix = tag.is_hierarchical + use_prefix = tag.state in [TAG_SEARCH_STATES['mark_plusplus'], + TAG_SEARCH_STATES['mark_minusminus']] if category == 'tags': if name in tags_seen: continue diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index e626d446d2..0335c1d280 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -419,28 +419,23 @@ class ResultCache(SearchQueryParser): # {{{ def get_user_category_matches(self, location, query, candidates): res = set([]) - if self.db_prefs is None: + if self.db_prefs is None or len(query) < 2: return res user_cats = self.db_prefs.get('user_categories', []) c = set(candidates) - l = location.rfind('.') - if l > 0: - alt_loc = location[0:l] - alt_item = location[l+1:] + + if query.startswith('.'): + check_subcats = True + query = query[1:] else: - alt_loc = None + check_subcats = False + for key in user_cats: - if key == location or key.startswith(location + '.'): + if key == location or (check_subcats and key.startswith(location + '.')): for (item, category, ign) in user_cats[key]: s = self.get_matches(category, '=' + item, candidates=c) c -= s res |= s - elif key == alt_loc: - for (item, category, ign) in user_cats[key]: - if item == alt_item: - s = self.get_matches(category, '=' + item, candidates=c) - c -= s - res |= s if query == 'false': return candidates - res return res From 9462edd460acb5af2fe6b144fd21cc4200b30cd7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 26 Feb 2011 08:58:26 -0700 Subject: [PATCH 03/70] ... --- Changelog.yaml | 4 ++++ src/calibre/manual/faq.rst | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index dfc9b9efe6..b8b8f2b480 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -19,6 +19,10 @@ # new recipes: # - title: +# - title: "Launch of a new website that catalogues DRM free books. http://drmfree.calibre-ebook.com" +# description: "A growing catalogue of DRM free books. Books that you actually own after buying instead of renting." +# type: major + - version: 0.7.47 date: 2011-02-25 diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index cb7f4d62ff..8a78815751 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -81,7 +81,7 @@ Device Integration What devices does |app| support? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -At the moment |app| has full support for the SONY PRS line, Barnes & Noble Nook, Cybook Gen 3/Opus, Amazon Kindle line, Entourage Edge, Longshine ShineBook, Ectaco Jetbook, BeBook/BeBook Mini, Irex Illiad/DR1000, Foxit eSlick, PocketBook 360, Italica, eClicto, Iriver Story, Airis dBook, Hanvon N515, Binatone Readme, Teclast K3, SpringDesign Alex, Kobo Reader, various Android phones and the iPhone/iPad. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk. +At the moment |app| has full support for the SONY PRS line, Barnes & Noble Nook line, Cybook Gen 3/Opus, Amazon Kindle line, Entourage Edge, Longshine ShineBook, Ectaco Jetbook, BeBook/BeBook Mini, Irex Illiad/DR1000, Foxit eSlick, PocketBook line, Italica, eClicto, Iriver Story, Airis dBook, Hanvon N515, Binatone Readme, Teclast K3 and clones, SpringDesign Alex, Kobo Reader, various Android phones and the iPhone/iPad. In addition, using the :guilabel:`Connect to folder` function you can use it with any ebook reader that exports itself as a USB disk. How can I help get my device supported in |app|? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 2aa275dad54555f8752069fd41cc4208e6975c87 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 27 Feb 2011 11:43:10 +0000 Subject: [PATCH 04/70] 1) Fix problem with case sensitive matching when creating user categories 2) Fix problem in search where setting focus to the search box then removing it caused the item in history to replace the item in the search box case-insensitively. This broke tag state matching in the tag browser. --- src/calibre/gui2/dialogs/tag_categories.py | 6 ++++-- src/calibre/gui2/search_box.py | 4 ++++ src/calibre/gui2/tag_view.py | 13 +++++++------ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index 9bddb817cf..899d3d1920 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -178,8 +178,10 @@ class TagCategories(QDialog, Ui_TagCategories): 'multiple periods in a row or spaces before ' 'or after periods.')).exec_() return False - for c in self.categories: - if strcmp(c, cat_name) == 0: + for c in sorted(self.categories.keys(), key=sort_key): + if strcmp(c, cat_name) == 0 or \ + (icu_lower(cat_name).startswith(icu_lower(c) + '.') and\ + not cat_name.startswith(c + '.')): error_dialog(self, _('Name already used'), _('That name is already used, perhaps with different case.')).exec_() return False diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index 34be6cd276..5a4c34a5cd 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -217,11 +217,15 @@ class SearchBox2(QComboBox): # {{{ self.clear() else: self.normalize_state() + self.lineEdit().setCompleter(None) self.setEditText(txt) self.line_edit.end(False) if emit_changed: self.changed.emit() self._do_search(store_in_history=store_in_history) + c = QCompleter() + self.lineEdit().setCompleter(c) + c.setCompletionMode(c.PopupCompletion) self.focus_to_library.emit() finally: if not store_in_history: diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 034d88e02d..5986717753 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -654,7 +654,6 @@ class TagTreeItem(object): # {{{ ''' set_to: None => advance the state, otherwise a value from TAG_SEARCH_STATES ''' -# if self.type == self.TAG: if set_to is None: while True: self.tag.state = (self.tag.state + 1)%5 @@ -1319,16 +1318,19 @@ class TagsModel(QAbstractItemModel): # {{{ return False user_cats = self.db.prefs.get('user_categories', {}) + user_cat_keys_lower = [icu_lower(k) for k in user_cats] ckey = item.category_key[1:] + ckey_lower = icu_lower(ckey) dotpos = ckey.rfind('.') if dotpos < 0: nkey = val else: nkey = ckey[:dotpos+1] + val - for c in user_cats: - if c.startswith(ckey): + nkey_lower = icu_lower(nkey) + for c in sorted(user_cats.keys(), key=sort_key): + if icu_lower(c).startswith(ckey_lower): if len(c) == len(ckey): - if nkey in user_cats: + if nkey_lower in user_cat_keys_lower: error_dialog(self.tags_view, _('Rename user category'), _('The name %s is already used')%nkey, show=True) return False @@ -1336,7 +1338,7 @@ class TagsModel(QAbstractItemModel): # {{{ del user_cats[ckey] elif c[len(ckey)] == '.': rest = c[len(ckey):] - if (nkey + rest) in user_cats: + if icu_lower(nkey + rest) in user_cat_keys_lower: error_dialog(self.tags_view, _('Rename user category'), _('The name %s is already used')%(nkey+rest), show=True) return False @@ -1512,7 +1514,6 @@ class TagsModel(QAbstractItemModel): # {{{ def reset_all_states(self, except_=None): update_list = [] def process_tag(tag_item): -# if tag_item.type != TagTreeItem.CATEGORY: tag = tag_item.tag if tag is except_: tag_index = self.createIndex(tag_item.row(), 0, tag_item) From 66197505871bb75d8b888aa10ccdcbeedb774bfd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 27 Feb 2011 09:05:56 -0700 Subject: [PATCH 05/70] Dotpod by Federico Escalada --- resources/recipes/dotpod.recipe | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 resources/recipes/dotpod.recipe diff --git a/resources/recipes/dotpod.recipe b/resources/recipes/dotpod.recipe new file mode 100644 index 0000000000..b04945e6d4 --- /dev/null +++ b/resources/recipes/dotpod.recipe @@ -0,0 +1,27 @@ +__license__ = 'GPL v3' +__copyright__ = '2011-2011, Federico Escalada ' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Dotpod(BasicNewsRecipe): + __author__ = 'Federico Escalada' + description = 'Tecnologia y Comunicacion Audiovisual' + encoding = 'utf-8' + language = 'es' + max_articles_per_feed = 100 + no_stylesheets = True + oldest_article = 7 + publication_type = 'blog' + title = 'Dotpod' + authors = 'Federico Picone' + + conversion_options = { + 'authors' : authors + ,'comments' : description + ,'language' : language + } + + feeds = [('Dotpod', 'http://www.dotpod.com.ar/feed/')] + + remove_tags = [dict(name='div', attrs={'class':'feedflare'})] + From 72205310b6b455d9f8723b109d19f4dcbd584a42 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 27 Feb 2011 09:07:50 -0700 Subject: [PATCH 06/70] Buffalo News by ChappyOnIce --- resources/recipes/buffalo_news.recipe | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 resources/recipes/buffalo_news.recipe diff --git a/resources/recipes/buffalo_news.recipe b/resources/recipes/buffalo_news.recipe new file mode 100644 index 0000000000..92c96757ae --- /dev/null +++ b/resources/recipes/buffalo_news.recipe @@ -0,0 +1,56 @@ +__license__ = 'GPL v3' +__author__ = 'Todd Chapman' +__copyright__ = 'Todd Chapman' +__version__ = 'v0.1' +__date__ = '26 February 2011' + +''' +http://www.buffalonews.com/RSS/ +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1298680852(BasicNewsRecipe): + title = u'Buffalo News' + __author__ = 'ChappyOnIce' + language = 'en' + oldest_article = 2 + max_articles_per_feed = 20 + encoding = 'utf-8' + remove_javascript = True + keep_only_tags = [ + dict(name='div', attrs={'class':['main-content-left']}) + ] + + remove_tags = [ + dict(name='div', attrs={'id':['commentCount']}), + dict(name='div', attrs={'class':['story-list-links']}) + ] + + remove_tags_after = dict(name='div', attrs={'class':['body storyContent']}) + conversion_options = { + 'base_font_size' : 14, + } + feeds = [(u'City of Buffalo', u'http://www.buffalonews.com/city/communities/buffalo/?widget=rssfeed&view=feed&contentId=77944'), + (u'Southern Erie County', u'http://www.buffalonews.com/city/communities/southern-erie/?widget=rssfeed&view=feed&contentId=77944'), + (u'Eastern Erie County', u'http://www.buffalonews.com/city/communities/eastern-erie/?widget=rssfeed&view=feed&contentId=77944'), + (u'Southern Tier', u'http://www.buffalonews.com/city/communities/southern-tier/?widget=rssfeed&view=feed&contentId=77944'), + (u'Niagara County', u'http://www.buffalonews.com/city/communities/niagara-county/?widget=rssfeed&view=feed&contentId=77944'), + (u'Business', u'http://www.buffalonews.com/business/?widget=rssfeed&view=feed&contentId=77944'), + (u'MoneySmart', u'http://www.buffalonews.com/business/moneysmart/?widget=rssfeed&view=feed&contentId=77944'), + (u'Bills & NFL', u'http://www.buffalonews.com/sports/bills-nfl/?widget=rssfeed&view=feed&contentId=77944'), + (u'Sabres & NHL', u'http://www.buffalonews.com/sports/sabres-nhl/?widget=rssfeed&view=feed&contentId=77944'), + (u'Bob DiCesare', u'http://www.buffalonews.com/sports/columns/bob-dicesare/?widget=rssfeed&view=feed&contentId=77944'), + (u'Bucky Gleason', u'http://www.buffalonews.com/sports/columns/bucky-gleason/?widget=rssfeed&view=feed&contentId=77944'), + (u'Mark Gaughan', u'http://www.buffalonews.com/sports/bills-nfl/inside-the-nfl/?widget=rssfeed&view=feed&contentId=77944'), + (u'Mike Harrington', u'http://www.buffalonews.com/sports/columns/mike-harrington/?widget=rssfeed&view=feed&contentId=77944'), + (u'Jerry Sullivan', u'http://www.buffalonews.com/sports/columns/jerry-sullivan/?widget=rssfeed&view=feed&contentId=77944'), + (u'Other Sports Columns', u'http://www.buffalonews.com/sports/columns/other-sports-columns/?widget=rssfeed&view=feed&contentId=77944'), + (u'Life', u'http://www.buffalonews.com/life/?widget=rssfeed&view=feed&contentId=77944'), + (u'Bruce Andriatch', u'http://www.buffalonews.com/city/columns/bruce-andriatch/?widget=rssfeed&view=feed&contentId=77944'), + (u'Donn Esmonde', u'http://www.buffalonews.com/city/columns/donn-esmonde/?widget=rssfeed&view=feed&contentId=77944'), + (u'Rod Watson', u'http://www.buffalonews.com/city/columns/rod-watson/?widget=rssfeed&view=feed&contentId=77944'), + (u'Entertainment', u'http://www.buffalonews.com/entertainment/?widget=rssfeed&view=feed&contentId=77944'), + (u'Off Main Street', u'http://www.buffalonews.com/city/columns/off-main-street/?widget=rssfeed&view=feed&contentId=77944'), + (u'Editorials', u'http://www.buffalonews.com/editorial-page/buffalo-news-editorials/?widget=rssfeed&view=feed&contentId=77944') + ] From dab97fa3f1d4af159eb9e302e0642c27268e5415 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 27 Feb 2011 17:19:14 +0000 Subject: [PATCH 07/70] Update gui.rst to reflect 5-state searching --- src/calibre/manual/gui.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/calibre/manual/gui.rst b/src/calibre/manual/gui.rst index 882e3d5921..210bd0569e 100644 --- a/src/calibre/manual/gui.rst +++ b/src/calibre/manual/gui.rst @@ -436,25 +436,26 @@ Tag Browser .. image:: images/tag_browser.png :class: float-left-img -The Tag Browser allows you to easily browse your collection by Author/Tags/Series/etc. If you click on any Item in the Tag Browser, for example, the Author name, Isaac Asimov, then the list of books to the right is restricted to books by that author. Clicking once again on Isaac Asimov will restrict the list of books to books not by Isaac Asimov. A third click will remove the restriction. If you hold down the Ctrl or Shift keys and click on multiple items, then restrictions based on multiple items are created. For example you could Hold Ctrl and click on the tags History and Europe for find books on European history. The Tag Browser works by constructing search expressions that are automatically entered into the Search bar. It is a good way to learn how to construct basic search expressions. +The Tag Browser allows you to easily browse your collection by Author/Tags/Series/etc. If you click on any item in the Tag Browser, for example the author name Isaac Asimov, then the list of books to the right is restricted to showing books by that author. You can click on category names as well. For example, clicking on "Series" will show you all books in any series. -There is a search bar at the top of the Tag Browser that allows you to easily find any item in the Tag Browser. In addition, you can right click on any item and choose to hide it or rename it or open a "Manage x" dialog that allows you to manage items of that kind. For example the "Manage Authors" dialog allows you to rename authors and control how their names are sorted. +The first click on an item will restrict the list of books to those that contain/match the item. Continuing the above example, clicking on Isaac Asimov will show books by that author. Clicking again on the item will change what is shown, depending on whether the item has children (see sub-categories and hierarchical items below). Continuing the Isaac Asimov example, clicking again on Isaac Asimov will restrict the list of books to those not by Isaac Asimov. A third click will remove the restriction, showing all books. If you hold down the Ctrl or Shift keys and click on multiple items, then restrictions based on multiple items are created. For example you could hold Ctrl and click on the tags History and Europe for find books on European history. The Tag Browser works by constructing search expressions that are automatically entered into the Search bar. Looking at what the Tag Browser generates is a good way to learn how to construct basic search expressions. Items in the Tag browser have their icons partially colored. The amount of color depends on the average rating of the books in that category. So for example if the books by Isaac Asimov have an average of four stars, the icon for Isaac Asimov in the Tag Browser will be 4/5th colored. You can hover your mouse over the icon to see the average rating. -For convenience, you can drag and drop books from the book list to items in the Tag Browser and that item will be automatically applied to the dropped books. For example, dragging a book to Isaac Asimov will set the author of that book to Isaac Asimov or dragging it to the tag History will add the tag History to its tags. +The outer-level items in the tag browser such as Authors and Series are called categories. You can create your own categories, called User Categories, which are useful for organizing items. For example, you can use the User Categories Editor (push the Manage User Categories button) to create a user category called Favorite Authors, then put the items for your favorites into the category. User categories can have sub-categories. For example, the user category Favorites.Authors is a sub-category of Favorites. You might also have Favorites.Series, in which case there will be two sub-categories under Favorites. Sub-categories can be created by right-clicking on a user category, choosing "Add sub-category to ...", and entering the sub-category name; or by using the User Categories Editor by entering names like the Favorites example above. -The outer-level items in the tag browser such as Authors and Series are called categories. You can create your own categories, called User Categories, which are useful for organizing items. For example, you can use the user categories editor (push the Manage User Categories button) to create a user category called Favorite Authors, then put the items for your favorites into the category. User categories act like built-in categories; you can click on items to search for them. You can search for all items in a category by right-clicking on the category name and choosing "Search for books in ...". +You can search user categories in the same way as built-in categories, by clicking on them. There are four different searches cycled through by clicking: "everything matching an item in the category" indicated by a single green plus sign, "everything matching an item in the category or its sub-categories" indicated by two green plus signs, "everything not matching an item in the category" shown by a single red minus sign, and "everything not matching an item in the category or its sub-categories" shown by two red minus signs. -User categories can have sub-categories. For example, the user category Favorites.Authors is a sub-category of Favorites. You might also have Favorites.Series, in which case there will be two sub-categories under Favorites. Sub-categories can be created using Manage User Categories by entering names like the Favorites example. They can also be created by right-clicking on a user category, choosing "Add sub-category to ...", and entering the category name. +It is also possible to create hierarchies inside some of the text categories such as tags, series, and custom columns. These hierarchies show with the small triangle, permitting the sub-items to be hidden. To use hierarchies of items in a category, you must first go to Preferences / Look & Feel and enter the category name(s) into the "Categories with hierarchical items" box. Once this is done, items in that category that contain periods will be shown using the small triangle. For example, assume you create a custom column called "Genre" and indicate that it contains hierarchical items. Once done, items such as Mystery.Thriller and Mystery.English will display as Mystery with the small triangle next to it. Clicking on the triangle will show Thriller and English as sub-items. -It is also possible to create hierarchies inside some of the built-in categories (the text categories). These hierarchies show with the small triangle permitting the sub-items to be hidden. To use hierarchies in a category, you must first go to Preferences / Look & Feel and enter the category name(s) into the "Categories with hierarchical items" box. Once this is done, items in that category that contain periods will be shown using the small triangle. For example, assume you create a custom column called "Genre" and indicate that it contains hierarchical items. Once done, items such as Mystery.Thriller and Mystery.English will display as Mystery with the small triangle next to it. Clicking on the triangle will show Thriller and English as sub-items. +Hierarchical items (items with children) use the same four 'click-on' searches as user categories. Items that do not have children use two of the searches: "everything matching" and "everything not matching". -You can drag and drop items in the Tag browser onto user categories to add them to that category. +You can drag and drop items in the Tag browser onto user categories to add them to that category. If the source is a user category, holding the shift key while dragging will move the item to the new category. You can also drag and drop books from the book list onto items in the Tag Browser; dropping a book on an item causes that item to be automatically applied to the dropped books. For example, dragging a book onto Isaac Asimov will set the author of that book to Isaac Asimov. Dropping it onto the tag History will add the tag History to the book's tags. + +There is a search bar at the top of the Tag Browser that allows you to easily find any item in the Tag Browser. In addition, you can right click on any item and choose one of several operations. Some examples are to hide the it, rename it, or open a "Manage x" dialog that allows you to manage items of that kind. For example, the "Manage Authors" dialog allows you to rename authors and control how their names are sorted. You can control how items are sorted in the Tag browser via the box at the bottom of the Tag Browser. You can choose to sort by name, average rating or popularity (popularity is the number of books with an item in your library; for example; the popularity of Isaac Asimov is the number of book sin your library by Isaac Asimov). - Jobs ----- .. image:: images/jobs.png From b8f08346b52994da795a9865bf2d8697496a489d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 27 Feb 2011 10:33:24 -0700 Subject: [PATCH 08/70] Historia and Buctaras by Silviu Coatara --- resources/images/news/bucataras.png | Bin 0 -> 765 bytes resources/images/news/historiaro.png | Bin 0 -> 521 bytes resources/recipes/bucataras.recipe | 56 +++++++++++++++++++++++++++ resources/recipes/historiaro.recipe | 51 ++++++++++++++++++++++++ 4 files changed, 107 insertions(+) create mode 100644 resources/images/news/bucataras.png create mode 100644 resources/images/news/historiaro.png create mode 100644 resources/recipes/bucataras.recipe create mode 100644 resources/recipes/historiaro.recipe diff --git a/resources/images/news/bucataras.png b/resources/images/news/bucataras.png new file mode 100644 index 0000000000000000000000000000000000000000..fae90e17c4366b5accc5668256f5234effee9f84 GIT binary patch literal 765 zcmVI>y9+ zP1{Jz+)n3m&Ug8_Xo5WZ_j&Wa>igYY0YCr+q99R*%)r%!CPmgHc!yTt1O{`@(0QgR z;sh*jP#~a0u!$ij(kTj}7sLbvMFY)pu4z|-i-sX!B?@n&0f~|GttPFkL7hPG3C;(c zuW3gA{qlK5b8N~?XYk9B@nd18qHz` zMVjKOIX*_}x<*MPh@%dkne&vj!=c{(0IX5uMssC(nY@|f>WUCNK_aoPsC?j+a|{9q z31F}yf{1`pq$n0pbf5i6IiIk4@iLcJmdMivqsf$hIbonPY5^B&B621;n-EEi@bced zetWph%TdX+yvO?c7x?nd9d4|x;DaDK;$p-T(SQ~aBu1iANHEyhoQ<`0a$~suZoyaA zuF(B`haZ0a6{8{524fU(meUd=N@*IYWmZmkynD#EkG2?`PI$6k((PA#a^q9(-h0S{ zpSCE_w5%n!381k7Ax$iW7uM#-tiNN}?^PTPhlozN-`(TUUp*EUVRPpX-pey?ef$x} z?ShzP*jC=eD8s5KK4>gKTi7^%j#qv{r(tMin$hSr%}gjm}%sH?W`G{v4Aec$2X5;dPG3-*I6! zq1nDjQ8dYmMV|NmhIu5>2~m57qX7>dKjUcBN8P>uja1R`dU(QWa*lRpnL5j4Qqjs6 vXyhIGr(-rZpY!72fU#4gse;5K^_>3zn;>=X(V!n600000NkvXXu0mjfoBnR* literal 0 HcmV?d00001 diff --git a/resources/images/news/historiaro.png b/resources/images/news/historiaro.png new file mode 100644 index 0000000000000000000000000000000000000000..c9e616c8761974bdafca3acb1f3302a1ea8b5238 GIT binary patch literal 521 zcmV+k0`~ohP)O3DDx0nRYNXo8)H%u0d@zJVk*42e%coZt(nZ=p27N$UtlEeV$y zGU$Y+QYO75J=eXQ&{JD`T$8=`%G%$be_yP@V4!NH!jEc|PNzc{1gsVcY;L}0`N=W> z^;(VZhlgDLy2P?AmX;n<+$i#DeH|ppbzObj-xr`q^9yP=n=-~|s4xt5Tt5~dK;?t7 zf*_D2F$|raoCwg)yONT$`s}&#GdcBoz1Xw4mFv0y_`Xk)jD&65c%Bz~F+jtRF^K?f z0gmnb3mD3>tYm;SF-`+Gjx%YE!nT8?F)#HuJahLRXQyYp-`&M?JtPBIq*5uODB}A1 zH~D;iIzWIQ1}r{YU7`8@S|h zGb7XYeVw15Yo}BapxOHmbnx|?TCG<6-W!dEzI^%|f2I$6d%C!|kTFJ9x7$^@T&8(- z#aw=lUblzqxfC~wyeJd^s8%Y}Yc*zav!v50ZrV4jJY8XH>kWSZgq{$j2O3Ch00000 LNkvXXu0mjfVkGfv literal 0 HcmV?d00001 diff --git a/resources/recipes/bucataras.recipe b/resources/recipes/bucataras.recipe new file mode 100644 index 0000000000..b069ecc5b0 --- /dev/null +++ b/resources/recipes/bucataras.recipe @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +bucataras.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Bucataras(BasicNewsRecipe): + title = u'Bucataras' + __author__ = u'Silviu Cotoar\u0103' + description = '' + publisher = 'Bucataras' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Bucatarie,Retete' + encoding = 'utf-8' + cover_url = 'http://www.bucataras.ro/templates/default/images/pink/logo.jpg' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='h1', attrs={'class':'titlu'}) + , dict(name='div', attrs={'class':'contentL'}) + , dict(name='div', attrs={'class':'contentBottom'}) + + ] + + remove_tags = [ + dict(name='div', attrs={'class':['sociale']}) + , dict(name='div', attrs={'class':['contentR']}) + , dict(name='a', attrs={'target':['_self']}) + , dict(name='div', attrs={'class':['comentarii']}) + ] + + remove_tags_after = [ + dict(name='div', attrs={'class':['comentarii']}) + ] + + feeds = [ + (u'Feeds', u'http://www.bucataras.ro/rss/retete/') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/resources/recipes/historiaro.recipe b/resources/recipes/historiaro.recipe new file mode 100644 index 0000000000..98eb5b6dfe --- /dev/null +++ b/resources/recipes/historiaro.recipe @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +historia.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class HistoriaRo(BasicNewsRecipe): + title = u'Historia' + __author__ = u'Silviu Cotoar\u0103' + description = '' + publisher = 'Historia' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Reviste,Istorie' + encoding = 'utf-8' + cover_url = 'http://www.historia.ro/sites/all/themes/historia/images/historia.png' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='div', attrs={'class':'c_antet_title'}) + , dict(name='a', attrs={'class':'overlaybox'}) + , dict(name='div', attrs={'class':'art_content'}) + ] + + remove_tags = [ + dict(name='div', attrs={'class':['fl_left']}) + , dict(name='div', attrs={'id':['article_toolbar']}) + , dict(name='div', attrs={'class':['zoom_cont']}) + ] + + + feeds = [ + (u'Feeds', u'http://www.historia.ro/rss.xml') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) From 4fcb4fc42e125ab09e76114d215bab2d4c76affe Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 27 Feb 2011 20:36:45 +0000 Subject: [PATCH 09/70] Ignore errors when setting preferences in clean_user_categories --- src/calibre/library/database2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index bf3e9c8a14..4d6681b91d 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1195,7 +1195,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): i += 1 else: new_cats['.'.join(comps)] = user_cats[k] - self.prefs.set('user_categories', new_cats) + try: + self.prefs.set('user_categories', new_cats) + except: + pass return new_cats def get_categories(self, sort='name', ids=None, icon_map=None): From e16f04e7b7fbd3b99826f1495d7002c2826bf79c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 27 Feb 2011 16:18:56 -0700 Subject: [PATCH 10/70] When keeping x periodical issues be a little more careful about what we delete --- src/calibre/gui2/actions/fetch_news.py | 3 ++- src/calibre/library/database2.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/actions/fetch_news.py b/src/calibre/gui2/actions/fetch_news.py index fe51012e31..f7756efbab 100644 --- a/src/calibre/gui2/actions/fetch_news.py +++ b/src/calibre/gui2/actions/fetch_news.py @@ -67,7 +67,8 @@ class FetchNewsAction(InterfaceAction): keep_issues = 0 if keep_issues > 0: ids_with_tag = list(sorted(self.gui.library_view.model(). - db.tags_older_than(arg['title'], None), reverse=True)) + db.tags_older_than(arg['title'], + None, must_have_tag=_('News')), reverse=True)) ids_to_delete = ids_with_tag[keep_issues:] if ids_to_delete: self.gui.library_view.model().delete_books_by_id(ids_to_delete) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 4d6681b91d..8e90fe77bd 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1503,25 +1503,30 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ############# End get_categories - def tags_older_than(self, tag, delta): + def tags_older_than(self, tag, delta, must_have_tag=None): ''' Return the ids of all books having the tag ``tag`` that are older than than the specified time. tag comparison is case insensitive. :param delta: A timedelta object or None. If None, then all ids with the tag are returned. + :param must_have_tag: If not None the list of matches will be + restricted to books that have this tag ''' tag = tag.lower().strip() + mht = must_have_tag.lower().strip() if must_have_tag else None now = nowf() tindex = self.FIELD_MAP['timestamp'] gindex = self.FIELD_MAP['tags'] + iindex = self.FIELD_MAP['id'] for r in self.data._data: if r is not None: if delta is None or (now - r[tindex]) > delta: tags = r[gindex] - if tags and tag in [x.strip() for x in - tags.lower().split(',')]: - yield r[self.FIELD_MAP['id']] + if tags: + tags = [x.strip() for x in tags.lower().split(',')] + if tag in tags and (mht is None or mht in tags): + yield r[iindex] def get_next_series_num_for(self, series): series_id = self.conn.get('SELECT id from series WHERE name=?', From dd61e8eb879efa6039fe0749d458d67a433bc61a Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 28 Feb 2011 13:59:35 +0000 Subject: [PATCH 11/70] Implement driveinfo for devices. --- src/calibre/devices/apple/driver.py | 2 +- src/calibre/devices/bambook/driver.py | 2 +- src/calibre/devices/folder_device/driver.py | 4 +- src/calibre/devices/hanlin/driver.py | 1 + src/calibre/devices/interface.py | 2 +- src/calibre/devices/prs500/driver.py | 2 +- src/calibre/devices/prs505/driver.py | 3 -- src/calibre/devices/usbms/device.py | 3 +- src/calibre/devices/usbms/driver.py | 44 +++++++++++++++++++-- src/calibre/gui2/device.py | 17 +++++++- src/calibre/gui2/ui.py | 2 + 11 files changed, 67 insertions(+), 15 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index aaa9382612..5ead675aab 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -701,7 +701,7 @@ class ITUNES(DriverBase): self.log.info("ITUNES.get_file(): exporting '%s'" % path) outfile.write(open(self.cached_books[path]['lib_book'].location().path).read()) - def open(self): + def open(self, library_uuid): ''' Perform any device specific initialization. Called after the device is detected but before any other functions that communicate with the device. diff --git a/src/calibre/devices/bambook/driver.py b/src/calibre/devices/bambook/driver.py index 3cc0245cf7..f251310d77 100644 --- a/src/calibre/devices/bambook/driver.py +++ b/src/calibre/devices/bambook/driver.py @@ -61,7 +61,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin): detected_device=None) : self.open() - def open(self): + def open(self, library_uuid): # Make sure the Bambook library is ready if not is_bambook_lib_ready(): raise OpenFeedback(_("Unable to connect to Bambook, you need to install Bambook library first.")) diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py index d75697a6cb..c08448051d 100644 --- a/src/calibre/devices/folder_device/driver.py +++ b/src/calibre/devices/folder_device/driver.py @@ -47,6 +47,7 @@ class FOLDER_DEVICE(USBMS): #: Icon for this device icon = I('devices/folder.png') METADATA_CACHE = '.metadata.calibre' + DRIVEINFO = '.driveinfo.calibre' _main_prefix = '' _card_a_prefix = None @@ -77,7 +78,8 @@ class FOLDER_DEVICE(USBMS): only_presence=False): return self.is_connected, self - def open(self): + def open(self, library_uuid): + self.current_library_uuid = library_uuid if not self._main_prefix: return False return True diff --git a/src/calibre/devices/hanlin/driver.py b/src/calibre/devices/hanlin/driver.py index 37f8430a66..ba0cca954d 100644 --- a/src/calibre/devices/hanlin/driver.py +++ b/src/calibre/devices/hanlin/driver.py @@ -116,6 +116,7 @@ class BOOX(HANLINV3): author = 'Jesus Manuel Marinho Valcarce' supported_platforms = ['windows', 'osx', 'linux'] METADATA_CACHE = '.metadata.calibre' + DRIVEINFO = '.driveinfo.calibre' # Ordered list of supported formats FORMATS = ['epub', 'fb2', 'djvu', 'pdf', 'html', 'txt', 'rtf', 'mobi', diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index bc442f5853..90da55a9db 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -215,7 +215,7 @@ class DevicePlugin(Plugin): return True - def open(self): + def open(self, library_uuid): ''' Perform any device specific initialization. Called after the device is detected but before any other functions that communicate with the device. diff --git a/src/calibre/devices/prs500/driver.py b/src/calibre/devices/prs500/driver.py index 445ddd757b..65ecc98a81 100644 --- a/src/calibre/devices/prs500/driver.py +++ b/src/calibre/devices/prs500/driver.py @@ -240,7 +240,7 @@ class PRS500(DeviceConfig, DevicePlugin): def set_progress_reporter(self, report_progress): self.report_progress = report_progress - def open(self) : + def open(self, library_uuid) : """ Claim an interface on the device for communication. Requires write privileges to the device file. diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 3768b8be62..9f17ea22a4 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -153,9 +153,6 @@ class PRS505(USBMS): # updated on every connect self.WANTS_UPDATED_THUMBNAILS = self.settings().extra_customization[2] - def get_device_information(self, end_session=True): - return (self.gui_name, '', '', '') - def filename_callback(self, fname, mi): if getattr(mi, 'application_id', None) is not None: base = fname.rpartition('.')[0] diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index b0857de909..37b2b061e5 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -700,7 +700,7 @@ class Device(DeviceConfig, DevicePlugin): - def open(self): + def open(self, library_uuid): time.sleep(5) self._main_prefix = self._card_a_prefix = self._card_b_prefix = None if islinux: @@ -722,6 +722,7 @@ class Device(DeviceConfig, DevicePlugin): time.sleep(7) self.open_osx() + self.current_library_uuid = library_uuid self.post_open_callback() def post_open_callback(self): diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index ef654ac428..37785612b5 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -10,17 +10,18 @@ driver. It is intended to be subclassed with the relevant parts implemented for a particular device. ''' -import os -import re -import time +import os, re, time, json, uuid from itertools import cycle +from calibre.constants import numeric_version from calibre import prints, isbytestring from calibre.constants import filesystem_encoding, DEBUG from calibre.devices.usbms.cli import CLI from calibre.devices.usbms.device import Device from calibre.devices.usbms.books import BookList, Book from calibre.ebooks.metadata.book.json_codec import JsonCodec +from calibre.utils.config import from_json, to_json +from calibre.utils.date import now BASE_TIME = None def debug_print(*args): @@ -52,10 +53,45 @@ class USBMS(CLI, Device): FORMATS = [] CAN_SET_METADATA = [] METADATA_CACHE = 'metadata.calibre' + DRIVEINFO = 'driveinfo.calibre' + + def _update_driveinfo_record(self, dinfo, prefix): + if not isinstance(dinfo, dict): + dinfo = {} + if dinfo.get('device_store_uuid', None) is None: + dinfo['device_store_uuid'] = unicode(uuid.uuid4()) + dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None) + dinfo['calibre_version'] = '.'.join([unicode(i) for i in numeric_version]) + dinfo['date_last_connected'] = unicode(now()) + dinfo['prefix'] = prefix.replace('\\', '/') + return dinfo + + def _update_driveinfo_file(self, prefix): + if os.path.exists(os.path.join(prefix, self.DRIVEINFO)): + with open(os.path.join(prefix, self.DRIVEINFO), 'rb') as f: + try: + driveinfo = json.loads(f.read(), object_hook=from_json) + except: + driveinfo = None + driveinfo = self._update_driveinfo_record(driveinfo, prefix) + with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: + f.write(json.dumps(driveinfo, default=to_json)) + else: + driveinfo = self._update_driveinfo_record({}, prefix) + with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: + f.write(json.dumps(driveinfo, default=to_json)) + return driveinfo def get_device_information(self, end_session=True): self.report_progress(1.0, _('Get device information...')) - return (self.get_gui_name(), '', '', '') + self.driveinfo = {} + if self._main_prefix is not None: + self.driveinfo['main'] = self._update_driveinfo_file(self._main_prefix) + if self._card_a_prefix is not None: + self.driveinfo['A'] = self._update_driveinfo_file(self._card_a_prefix) + if self._card_b_prefix is not None: + self.driveinfo['B'] = self._update_driveinfo_file(self._card_b_prefix) + return (self.get_gui_name(), '', '', '', self.driveinfo) def books(self, oncard=None, end_session=True): from calibre.ebooks.metadata.meta import path_to_ext diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index e4096f5761..972e02a6ab 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -140,6 +140,8 @@ class DeviceManager(Thread): # {{{ self.mount_connection_requests = Queue.Queue(0) self.open_feedback_slot = open_feedback_slot self.open_feedback_msg = open_feedback_msg + self._device_information = None + self.current_library_uuid = None def report_progress(self, *args): pass @@ -159,7 +161,7 @@ class DeviceManager(Thread): # {{{ try: dev.reset(detected_device=detected_device, report_progress=self.report_progress) - dev.open() + dev.open(self.current_library_uuid) except OpenFeedback, e: if dev not in self.ejected_devices: self.open_feedback_msg(dev.get_gui_name(), e.feedback_msg) @@ -194,6 +196,7 @@ class DeviceManager(Thread): # {{{ else: self.connected_slot(False, self.connected_device_kind) self.connected_device = None + self._device_information = None def detect_device(self): self.scanner.scan() @@ -292,9 +295,13 @@ class DeviceManager(Thread): # {{{ def _get_device_information(self): info = self.device.get_device_information(end_session=False) - info = [i.replace('\x00', '').replace('\x01', '') for i in info] + if len(info) < 5: + list(info).append({}) + info = [i.replace('\x00', '').replace('\x01', '') if isinstance(i, basestring) else i + for i in info] cp = self.device.card_prefix(end_session=False) fs = self.device.free_space() + self._device_information = {'info': info, 'prefixes': cp, 'freespace': fs} return info, cp, fs def get_device_information(self, done): @@ -302,6 +309,9 @@ class DeviceManager(Thread): # {{{ return self.create_job(self._get_device_information, done, description=_('Get device information')) + def get_current_device_information(self): + return self._device_information + def _books(self): '''Get metadata from device''' mainlist = self.device.books(oncard=None, end_session=False) @@ -417,6 +427,9 @@ class DeviceManager(Thread): # {{{ return self.create_job(self._view_book, done, args=[path, target], description=_('View book on device')) + def set_current_library_uuid(self, uuid): + self.current_library_uuid = uuid + # }}} class DeviceAction(QAction): # {{{ diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 8844446de6..bf672d43ca 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -296,6 +296,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ traceback.print_exc() if ac.plugin_path is None: raise + self.device_manager.set_current_library_uuid('THIS IS A UUID') if show_gui and self.gui_debug is not None: info_dialog(self, _('Debug mode'), '

' + @@ -461,6 +462,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.memory_view.reset() self.card_a_view.reset() self.card_b_view.reset() + self.device_manager.set_current_library_uuid('THIS IS A UUID') def set_window_title(self): From 05cc7aa3f119ea6053d253ac7111bba40b0cf4d3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 28 Feb 2011 08:19:16 -0700 Subject: [PATCH 12/70] Performance improvement when updating db prefs --- src/calibre/library/prefs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/calibre/library/prefs.py b/src/calibre/library/prefs.py index 233c717897..4ef1dcb35a 100644 --- a/src/calibre/library/prefs.py +++ b/src/calibre/library/prefs.py @@ -49,8 +49,7 @@ class DBPrefs(dict): if self.disable_setting: return raw = self.to_raw(val) - self.db.conn.execute('DELETE FROM preferences WHERE key=?', (key,)) - self.db.conn.execute('INSERT INTO preferences (key,val) VALUES (?,?)', (key, + self.db.conn.execute('INSERT OR REPLACE INTO preferences (key,val) VALUES (?,?)', (key, raw)) self.db.conn.commit() dict.__setitem__(self, key, val) From 752d9f350da10cd7a8658f63c82b356dc360a52c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 28 Feb 2011 08:43:39 -0700 Subject: [PATCH 13/70] Improve performance when cleaning user categories --- src/calibre/library/database2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 8e90fe77bd..1762fd16d2 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1196,7 +1196,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: new_cats['.'.join(comps)] = user_cats[k] try: - self.prefs.set('user_categories', new_cats) + if new_cats != user_cats: + self.prefs.set('user_categories', new_cats) except: pass return new_cats From c2d85e81b772e54c7f946392ead78b4ed1c0b607 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 28 Feb 2011 08:45:31 -0700 Subject: [PATCH 14/70] MOBI Input: Ignore all ASCII control codes except CR, NL and Tab. Fixes #9219 (Instapaper magazine can't be shown in ebook-viewer) --- src/calibre/ebooks/mobi/reader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 9c52a18691..f1b1b1ef63 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -18,6 +18,7 @@ from calibre import xml_entity_to_unicode, CurrentDir, entity_to_unicode, \ replace_entities from calibre.utils.filenames import ascii_filename from calibre.utils.date import parse_date +from calibre.utils.cleantext import clean_ascii_chars from calibre.ptempfile import TemporaryDirectory from calibre.ebooks import DRMError from calibre.ebooks.chardet import ENCODING_PATS @@ -323,6 +324,7 @@ class MobiReader(object): self.cleanup_html() self.log.debug('Parsing HTML...') + self.processed_html = clean_ascii_chars(self.processed_html) try: root = html.fromstring(self.processed_html) if len(root.xpath('//html')) > 5: From 160e57ed542cef705e90164f51a331dcebc4847b Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 28 Feb 2011 16:20:42 +0000 Subject: [PATCH 15/70] Add a device name to the driveinfo structure, along with the API to set it. --- src/calibre/devices/interface.py | 9 +++++++++ src/calibre/devices/usbms/driver.py | 20 ++++++++++++++++---- src/calibre/gui2/device.py | 4 ++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 90da55a9db..86d1664271 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -447,6 +447,15 @@ class DevicePlugin(Plugin): ''' pass + def set_driveinfo_name(self, location_code, name): + ''' + Set the device name in the driveinfo file to 'name'. This setting will + persist until the file is re-created or the name is changed again. + + Non-disk devices will ignore this request. + ''' + pass + class BookList(list): ''' A list of books. Each Book object must have the fields diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 37785612b5..ef935e239a 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -55,29 +55,33 @@ class USBMS(CLI, Device): METADATA_CACHE = 'metadata.calibre' DRIVEINFO = 'driveinfo.calibre' - def _update_driveinfo_record(self, dinfo, prefix): + def _update_driveinfo_record(self, dinfo, prefix, name=None): if not isinstance(dinfo, dict): dinfo = {} if dinfo.get('device_store_uuid', None) is None: dinfo['device_store_uuid'] = unicode(uuid.uuid4()) + if dinfo.get('device_name') is None: + dinfo['device_name'] = self.get_gui_name() + if name is not None: + dinfo['device_name'] = name dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None) dinfo['calibre_version'] = '.'.join([unicode(i) for i in numeric_version]) dinfo['date_last_connected'] = unicode(now()) dinfo['prefix'] = prefix.replace('\\', '/') return dinfo - def _update_driveinfo_file(self, prefix): + def _update_driveinfo_file(self, prefix, name=None): if os.path.exists(os.path.join(prefix, self.DRIVEINFO)): with open(os.path.join(prefix, self.DRIVEINFO), 'rb') as f: try: driveinfo = json.loads(f.read(), object_hook=from_json) except: driveinfo = None - driveinfo = self._update_driveinfo_record(driveinfo, prefix) + driveinfo = self._update_driveinfo_record(driveinfo, prefix, name) with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: f.write(json.dumps(driveinfo, default=to_json)) else: - driveinfo = self._update_driveinfo_record({}, prefix) + driveinfo = self._update_driveinfo_record({}, prefix, name) with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: f.write(json.dumps(driveinfo, default=to_json)) return driveinfo @@ -93,6 +97,14 @@ class USBMS(CLI, Device): self.driveinfo['B'] = self._update_driveinfo_file(self._card_b_prefix) return (self.get_gui_name(), '', '', '', self.driveinfo) + def set_driveinfo_name(self, location_code, name): + if location_code == 'main': + self._update_driveinfo_file(self._main_prefix, name) + elif location_code == 'A': + self._update_driveinfo_file(self._card_a_prefix, name) + elif location_code == 'B': + self._update_driveinfo_file(self._card_b_prefix, name) + def books(self, oncard=None, end_session=True): from calibre.ebooks.metadata.meta import path_to_ext diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 972e02a6ab..955e287522 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -430,6 +430,10 @@ class DeviceManager(Thread): # {{{ def set_current_library_uuid(self, uuid): self.current_library_uuid = uuid + def set_driveinfo_name(self, location_code, name): + if self.connected_device: + self.connected_device.set_driveinfo_name(location_code, name) + # }}} class DeviceAction(QAction): # {{{ From 5d317eb53456b165b7dd0c755a6919b8e205e4bb Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 28 Feb 2011 16:30:27 +0000 Subject: [PATCH 16/70] Add the location code to the automatically generated device name --- src/calibre/devices/usbms/driver.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index ef935e239a..392c4c2305 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -55,13 +55,13 @@ class USBMS(CLI, Device): METADATA_CACHE = 'metadata.calibre' DRIVEINFO = 'driveinfo.calibre' - def _update_driveinfo_record(self, dinfo, prefix, name=None): + def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None): if not isinstance(dinfo, dict): dinfo = {} if dinfo.get('device_store_uuid', None) is None: dinfo['device_store_uuid'] = unicode(uuid.uuid4()) if dinfo.get('device_name') is None: - dinfo['device_name'] = self.get_gui_name() + dinfo['device_name'] = self.get_gui_name() + '_' + location_code if name is not None: dinfo['device_name'] = name dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None) @@ -70,18 +70,19 @@ class USBMS(CLI, Device): dinfo['prefix'] = prefix.replace('\\', '/') return dinfo - def _update_driveinfo_file(self, prefix, name=None): + def _update_driveinfo_file(self, prefix, location_code, name=None): if os.path.exists(os.path.join(prefix, self.DRIVEINFO)): with open(os.path.join(prefix, self.DRIVEINFO), 'rb') as f: try: driveinfo = json.loads(f.read(), object_hook=from_json) except: driveinfo = None - driveinfo = self._update_driveinfo_record(driveinfo, prefix, name) + driveinfo = self._update_driveinfo_record(driveinfo, prefix, + location_code, name) with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: f.write(json.dumps(driveinfo, default=to_json)) else: - driveinfo = self._update_driveinfo_record({}, prefix, name) + driveinfo = self._update_driveinfo_record({}, prefix, location_code, name) with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: f.write(json.dumps(driveinfo, default=to_json)) return driveinfo @@ -90,20 +91,20 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Get device information...')) self.driveinfo = {} if self._main_prefix is not None: - self.driveinfo['main'] = self._update_driveinfo_file(self._main_prefix) + self.driveinfo['main'] = self._update_driveinfo_file(self._main_prefix, 'main') if self._card_a_prefix is not None: - self.driveinfo['A'] = self._update_driveinfo_file(self._card_a_prefix) + self.driveinfo['A'] = self._update_driveinfo_file(self._card_a_prefix, 'A') if self._card_b_prefix is not None: - self.driveinfo['B'] = self._update_driveinfo_file(self._card_b_prefix) + self.driveinfo['B'] = self._update_driveinfo_file(self._card_b_prefix, 'B') return (self.get_gui_name(), '', '', '', self.driveinfo) def set_driveinfo_name(self, location_code, name): if location_code == 'main': - self._update_driveinfo_file(self._main_prefix, name) + self._update_driveinfo_file(self._main_prefix, location_code, name) elif location_code == 'A': - self._update_driveinfo_file(self._card_a_prefix, name) + self._update_driveinfo_file(self._card_a_prefix, location_code, name) elif location_code == 'B': - self._update_driveinfo_file(self._card_b_prefix, name) + self._update_driveinfo_file(self._card_b_prefix, location_code, name) def books(self, oncard=None, end_session=True): from calibre.ebooks.metadata.meta import path_to_ext From be4984f4928a00bab9bf34cffb85fe2d808a7aab Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 28 Feb 2011 16:48:10 +0000 Subject: [PATCH 17/70] Change server to use a getter for isbn instead of data[FIELD_MAP] --- src/calibre/library/server/xml.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py index efbceb9771..c6d6f9db8f 100644 --- a/src/calibre/library/server/xml.py +++ b/src/calibre/library/server/xml.py @@ -89,13 +89,16 @@ class XMLServer(object): for x in ('id', 'title', 'sort', 'author_sort', 'rating', 'size'): kwargs[x] = serialize(record[FM[x]]) - for x in ('isbn', 'formats', 'series', 'tags', 'publisher', + for x in ('formats', 'series', 'tags', 'publisher', 'comments'): y = record[FM[x]] if x == 'tags': y = format_tag_string(y, ',', ignore_max=True) kwargs[x] = serialize(y) if y else '' + isbn = self.db.isbn(record[FM['id']], index_is_id=True) + kwargs['isbn'] = serialize(isbn if isbn else '') + kwargs['safe_title'] = ascii_filename(kwargs['title']) c = kwargs.pop('comments') From a16ad224d94ae43954399e83dc5b97dcd15b1c3c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 28 Feb 2011 18:57:41 +0000 Subject: [PATCH 18/70] New formatter functions to support identifiers and date formatting --- src/calibre/utils/formatter_functions.py | 60 +++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 03491c038a..26cbe82ecd 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -12,6 +12,7 @@ import inspect, re, traceback, sys from calibre.utils.titlecase import titlecase from calibre.utils.icu import capitalize, strcmp +from calibre.utils.date import parse_date, format_date class FormatterFunctions(object): @@ -230,6 +231,15 @@ class BuiltinField(BuiltinFormatterFunction): def evaluate(self, formatter, kwargs, mi, locals, name): return formatter.get_value(name, [], kwargs) +class BuiltinRaw_field(BuiltinFormatterFunction): + name = 'raw_field' + arg_count = 1 + doc = _('raw_field(name) -- returns the metadata field named by name ' + 'without applying any formatting.') + + def evaluate(self, formatter, kwargs, mi, locals, name): + return unicode(getattr(mi, name, None)) + class BuiltinSubstr(BuiltinFormatterFunction): name = 'substr' arg_count = 3 @@ -396,6 +406,23 @@ class BuiltinListitem(BuiltinFormatterFunction): except: return '' +class BuiltinSelect(BuiltinFormatterFunction): + name = 'select' + arg_count = 2 + doc = _('select(val, key) -- interpret the value as a comma-separated list ' + 'of items, with the items being "id:value". Find the pair with the' + 'id equal to key, and return the corresponding value.' + ) + + def evaluate(self, formatter, kwargs, mi, locals, val, key): + if not val: + return '' + vals = [v.strip() for v in val.split(',')] + for v in vals: + if v.startswith(key+':'): + return v[len(key)+1:] + return '' + class BuiltinSublist(BuiltinFormatterFunction): name = 'sublist' arg_count = 4 @@ -424,6 +451,34 @@ class BuiltinSublist(BuiltinFormatterFunction): except: return '' +class BuiltinFormat_date(BuiltinFormatterFunction): + name = 'format_date' + arg_count = 2 + doc = _('format_date(val, format_string) -- format the value, which must ' + 'be a date field, using the format_string, returning a string. ' + 'The formatting codes are: ' + 'd : the day as number without a leading zero (1 to 31) ' + 'dd : the day as number with a leading zero (01 to 31) ' + 'ddd : the abbreviated localized day name (e.g. "Mon" to "Sun"). ' + 'dddd : the long localized day name (e.g. "Monday" to "Sunday"). ' + 'M : the month as number without a leading zero (1 to 12). ' + 'MM : the month as number with a leading zero (01 to 12) ' + 'MMM : the abbreviated localized month name (e.g. "Jan" to "Dec"). ' + 'MMMM : the long localized month name (e.g. "January" to "December"). ' + 'yy : the year as two digit number (00 to 99). ' + 'yyyy : the year as four digit number.') + + def evaluate(self, formatter, kwargs, mi, locals, val, format_string): + print val + if not val: + return '' + try: + dt = parse_date(val) + s = format_date(dt, format_string) + except: + s = 'BAD DATE' + return s + class BuiltinUppercase(BuiltinFormatterFunction): name = 'uppercase' arg_count = 1 @@ -464,14 +519,17 @@ builtin_contains = BuiltinContains() builtin_count = BuiltinCount() builtin_divide = BuiltinDivide() builtin_eval = BuiltinEval() -builtin_ifempty = BuiltinIfempty() +builtin_format_date = BuiltinFormat_date() builtin_field = BuiltinField() +builtin_ifempty = BuiltinIfempty() builtin_list_item = BuiltinListitem() builtin_lookup = BuiltinLookup() builtin_lowercase = BuiltinLowercase() builtin_multiply = BuiltinMultiply() builtin_print = BuiltinPrint() +builtin_raw_field = BuiltinRaw_field() builtin_re = BuiltinRe() +builtin_select = BuiltinSelect() builtin_shorten = BuiltinShorten() builtin_strcat = BuiltinStrcat() builtin_strcmp = BuiltinStrcmp() From 8c53a2ad43e8204b7e6f9992b4c4e44379496389 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 28 Feb 2011 19:34:38 -0700 Subject: [PATCH 19/70] Add database support for geenric identifiers, a library uuid and a last modified timestamp that is updated whenever book metadata/cover is changed (accessed via metadata_last_modified) --- src/calibre/ebooks/metadata/book/__init__.py | 8 +- src/calibre/ebooks/metadata/book/base.py | 46 ++-- .../ebooks/metadata/book/json_codec.py | 2 + src/calibre/ebooks/metadata/opf2.py | 38 +++- src/calibre/library/cli.py | 2 +- src/calibre/library/database2.py | 196 +++++++++++++++--- src/calibre/library/field_metadata.py | 21 +- src/calibre/library/restore.py | 5 +- src/calibre/library/schema_upgrades.py | 114 ++++++++++ src/calibre/library/sqlite.py | 20 +- src/calibre/library/sqlite_custom.c | 92 ++++++++ src/calibre/utils/date.py | 1 + 12 files changed, 472 insertions(+), 73 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py index 033a78d611..fae858aabd 100644 --- a/src/calibre/ebooks/metadata/book/__init__.py +++ b/src/calibre/ebooks/metadata/book/__init__.py @@ -18,14 +18,14 @@ SOCIAL_METADATA_FIELDS = frozenset([ 'series_index', # A floating point number # Of the form { scheme1:value1, scheme2:value2} # For example: {'isbn':'123456789', 'doi':'xxxx', ... } - 'classifiers', + 'identifiers', ]) ''' -The list of names that convert to classifiers when in get and set. +The list of names that convert to identifiers when in get and set. ''' -TOP_LEVEL_CLASSIFIERS = frozenset([ +TOP_LEVEL_IDENTIFIERS = frozenset([ 'isbn', ]) @@ -108,7 +108,7 @@ STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union( SC_FIELDS_NOT_COPIED = frozenset(['title', 'title_sort', 'authors', 'author_sort', 'author_sort_map', 'cover_data', 'tags', 'language', - 'classifiers']) + 'identifiers']) # Metadata fields that smart update should copy only if the source is not None SC_FIELDS_COPY_NOT_NULL = frozenset(['lpath', 'size', 'comments', 'thumbnail']) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index b47cc373a7..e3e9917491 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -12,7 +12,7 @@ from calibre.constants import DEBUG from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS -from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS +from calibre.ebooks.metadata.book import TOP_LEVEL_IDENTIFIERS from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS from calibre.library.field_metadata import FieldMetadata from calibre.utils.date import isoformat, format_date @@ -24,7 +24,7 @@ NULL_VALUES = { 'user_metadata': {}, 'cover_data' : (None, None), 'tags' : [], - 'classifiers' : {}, + 'identifiers' : {}, 'languages' : [], 'device_collections': [], 'author_sort_map': {}, @@ -96,8 +96,8 @@ class Metadata(object): def __getattribute__(self, field): _data = object.__getattribute__(self, '_data') - if field in TOP_LEVEL_CLASSIFIERS: - return _data.get('classifiers').get(field, None) + if field in TOP_LEVEL_IDENTIFIERS: + return _data.get('identifiers').get(field, None) if field in STANDARD_METADATA_FIELDS: return _data.get(field, None) try: @@ -123,8 +123,8 @@ class Metadata(object): def __setattr__(self, field, val, extra=None): _data = object.__getattribute__(self, '_data') - if field in TOP_LEVEL_CLASSIFIERS: - _data['classifiers'].update({field: val}) + if field in TOP_LEVEL_IDENTIFIERS: + _data['identifiers'].update({field: val}) elif field in STANDARD_METADATA_FIELDS: if val is None: val = NULL_VALUES.get(field, None) @@ -176,17 +176,21 @@ class Metadata(object): def set(self, field, val, extra=None): self.__setattr__(field, val, extra) - def get_classifiers(self): + def get_identifiers(self): ''' - Return a copy of the classifiers dictionary. + Return a copy of the identifiers dictionary. The dict is small, and the penalty for using a reference where a copy is needed is large. Also, we don't want any manipulations of the returned dict to show up in the book. ''' - return copy.deepcopy(object.__getattribute__(self, '_data')['classifiers']) + ans = object.__getattribute__(self, + '_data')['identifiers'] + if not ans: + ans = {} + return copy.deepcopy(ans) - def set_classifiers(self, classifiers): - object.__getattribute__(self, '_data')['classifiers'] = classifiers + def set_identifiers(self, identifiers): + object.__getattribute__(self, '_data')['identifiers'] = identifiers # field-oriented interface. Intended to be the same as in LibraryDatabase @@ -229,7 +233,7 @@ class Metadata(object): if v is not None: result[attr] = v # separate these because it uses the self.get(), not _data.get() - for attr in TOP_LEVEL_CLASSIFIERS: + for attr in TOP_LEVEL_IDENTIFIERS: v = self.get(attr, None) if v is not None: result[attr] = v @@ -400,8 +404,8 @@ class Metadata(object): self.set_all_user_metadata(other.get_all_user_metadata(make_copy=True)) for x in SC_FIELDS_COPY_NOT_NULL: copy_not_none(self, other, x) - if callable(getattr(other, 'get_classifiers', None)): - self.set_classifiers(other.get_classifiers()) + if callable(getattr(other, 'get_identifiers', None)): + self.set_identifiers(other.get_identifiers()) # language is handled below else: for attr in SC_COPYABLE_FIELDS: @@ -456,15 +460,15 @@ class Metadata(object): if len(other_comments.strip()) > len(my_comments.strip()): self.comments = other_comments - # Copy all the non-none classifiers - if callable(getattr(other, 'get_classifiers', None)): - d = self.get_classifiers() - s = other.get_classifiers() + # Copy all the non-none identifiers + if callable(getattr(other, 'get_identifiers', None)): + d = self.get_identifiers() + s = other.get_identifiers() d.update([v for v in s.iteritems() if v[1] is not None]) - self.set_classifiers(d) + self.set_identifiers(d) else: - # other structure not Metadata. Copy the top-level classifiers - for attr in TOP_LEVEL_CLASSIFIERS: + # other structure not Metadata. Copy the top-level identifiers + for attr in TOP_LEVEL_IDENTIFIERS: copy_not_none(self, other, attr) other_lang = getattr(other, 'language', None) diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py index c02d4e953d..f434800edf 100644 --- a/src/calibre/ebooks/metadata/book/json_codec.py +++ b/src/calibre/ebooks/metadata/book/json_codec.py @@ -119,6 +119,8 @@ class JsonCodec(object): for item in js: book = book_class(prefix, item.get('lpath', None)) for key in item.keys(): + if key == 'classifiers': + key = 'identifiers' meta = self.decode_metadata(key, item[key]) if key == 'user_metadata': book.set_all_user_metadata(meta) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index d34a563110..9c59692628 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -596,6 +596,9 @@ class OPF(object): # {{{ ans = MetaInformation(self) for n, v in self._user_metadata_.items(): ans.set_user_metadata(n, v) + + ans.set_identifiers(self.get_identifiers()) + return ans def write_user_metadata(self): @@ -855,6 +858,21 @@ class OPF(object): # {{{ return property(fget=fget, fset=fset) + def get_identifiers(self): + identifiers = {} + for x in self.XPath( + 'descendant::*[local-name() = "identifier" and text()]')( + self.metadata): + for attr, val in x.attrib.iteritems(): + if attr.endswith('scheme'): + typ = icu_lower(val) + val = etree.tostring(x, with_tail=False, encoding=unicode, + method='text').strip() + if val and typ not in ('calibre', 'uuid'): + identifiers[typ] = val + break + return identifiers + @dynamic_property def application_id(self): @@ -1166,8 +1184,8 @@ class OPFCreator(Metadata): a(DC_ELEM('description', self.comments)) if self.publisher: a(DC_ELEM('publisher', self.publisher)) - if self.isbn: - a(DC_ELEM('identifier', self.isbn, opf_attrs={'scheme':'ISBN'})) + for key, val in self.get_identifiers().iteritems(): + a(DC_ELEM('identifier', val, opf_attrs={'scheme':icu_upper(key)})) if self.rights: a(DC_ELEM('rights', self.rights)) if self.tags: @@ -1291,8 +1309,8 @@ def metadata_to_opf(mi, as_string=True): factory(DC('description'), mi.comments) if mi.publisher: factory(DC('publisher'), mi.publisher) - if mi.isbn: - factory(DC('identifier'), mi.isbn, scheme='ISBN') + for key, val in mi.get_identifiers().iteritems(): + factory(DC('identifier'), val, scheme=icu_upper(key)) if mi.rights: factory(DC('rights'), mi.rights) factory(DC('language'), mi.language if mi.language and mi.language.lower() @@ -1342,7 +1360,7 @@ def test_m2o(): mi.language = 'en' mi.comments = 'what a fun book\n\n' mi.publisher = 'publisher' - mi.isbn = 'boooo' + mi.set_identifiers({'isbn':'booo', 'dummy':'dummy'}) mi.tags = ['a', 'b'] mi.series = 's"c\'l&<>' mi.series_index = 3.34 @@ -1350,7 +1368,7 @@ def test_m2o(): mi.timestamp = nowf() mi.publication_type = 'ooooo' mi.rights = 'yes' - mi.cover = 'asd.jpg' + mi.cover = os.path.abspath('asd.jpg') opf = metadata_to_opf(mi) print opf newmi = MetaInformation(OPF(StringIO(opf))) @@ -1363,6 +1381,9 @@ def test_m2o(): o, n = getattr(mi, attr), getattr(newmi, attr) if o != n and o.strip() != n.strip(): print 'FAILED:', attr, getattr(mi, attr), '!=', getattr(newmi, attr) + if mi.get_identifiers() != newmi.get_identifiers(): + print 'FAILED:', 'identifiers', mi.get_identifiers(), + print '!=', newmi.get_identifiers() class OPFTest(unittest.TestCase): @@ -1378,6 +1399,7 @@ class OPFTest(unittest.TestCase): Next OneTwo 123456789 + dummy @@ -1405,6 +1427,8 @@ class OPFTest(unittest.TestCase): self.assertEqual(opf.rating, 4) self.assertEqual(opf.publication_type, 'test') self.assertEqual(list(opf.itermanifest())[0].get('href'), 'a ~ b') + self.assertEqual(opf.get_identifiers(), {'isbn':'123456789', + 'dummy':'dummy'}) def testWriting(self): for test in [('title', 'New & Title'), ('authors', ['One', 'Two']), @@ -1461,5 +1485,5 @@ def test_user_metadata(): if __name__ == '__main__': #test_user_metadata() - #test_m2o() + test_m2o() test() diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index e93be187f9..359f5876fd 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -20,7 +20,7 @@ from calibre.utils.date import isoformat FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', - 'formats', 'isbn', 'uuid', 'pubdate', 'cover']) + 'formats', 'isbn', 'uuid', 'pubdate', 'cover', 'last_modified']) def send_message(msg=''): prints('Notifying calibre of the change') diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 1762fd16d2..8c509e7ceb 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -6,7 +6,8 @@ __docformat__ = 'restructuredtext en' ''' The database used to store ebook metadata ''' -import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, json +import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \ + json, uuid import threading, random from itertools import repeat from math import ceil @@ -94,6 +95,31 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return property(doc=doc, fget=fget, fset=fset) + @dynamic_property + def library_id(self): + doc = ('The UUID for this library. As long as the user only operates' + ' on libraries with calibre, it will be unique') + + def fget(self): + if self._library_id_ is None: + ans = self.conn.get('SELECT uuid FROM library_id', all=False) + if ans is None: + ans = str(uuid.uuid4()) + self.library_id = ans + else: + self._library_id_ = ans + return self._library_id_ + + def fset(self, val): + self._library_id_ = unicode(val) + self.conn.executescript(''' + DELETE FROM library_id; + INSERT INTO library_id (uuid) VALUES ("%s"); + '''%self._library_id_) + self.conn.commit() + + return property(doc=doc, fget=fget, fset=fset) + def connect(self): if 'win32' in sys.platform and len(self.library_path) + 4*self.PATH_LIMIT + 10 > 259: raise ValueError('Path to library too long. Must be less than %d characters.'%(259-4*self.PATH_LIMIT-10)) @@ -120,6 +146,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def __init__(self, library_path, row_factory=False, default_prefs=None, read_only=False): self.field_metadata = FieldMetadata() + self._library_id_ = None # Create the lock to be used to guard access to the metadata writer # queues. This must be an RLock, not a Lock self.dirtied_lock = threading.RLock() @@ -148,6 +175,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.is_case_sensitive = not iswindows and not isosx and \ not os.path.exists(self.dbpath.replace('metadata.db', 'MeTAdAtA.dB')) SchemaUpgrade.__init__(self) + # Guarantee that the library_id is set + self.library_id # if we are to copy the prefs and structure from some other DB, then # we need to do it before we call initialize_dynamic @@ -293,14 +322,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): 'sort', 'author_sort', '(SELECT group_concat(format) FROM data WHERE data.book=books.id) formats', - 'isbn', 'path', - 'lccn', 'pubdate', - 'flags', 'uuid', 'has_cover', - ('au_map', 'authors', 'author', 'aum_sortconcat(link.id, authors.name, authors.sort)') + ('au_map', 'authors', 'author', + 'aum_sortconcat(link.id, authors.name, authors.sort)'), + 'last_modified', + '(SELECT identifiers_concat(type, val) FROM identifiers WHERE identifiers.book=books.id) identifiers', ] lines = [] for col in columns: @@ -318,8 +347,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'timestamp':3, 'size':4, 'rating':5, 'tags':6, 'comments':7, 'series':8, 'publisher':9, 'series_index':10, 'sort':11, 'author_sort':12, - 'formats':13, 'isbn':14, 'path':15, 'lccn':16, 'pubdate':17, - 'flags':18, 'uuid':19, 'cover':20, 'au_map':21} + 'formats':13, 'path':14, 'pubdate':15, 'uuid':16, 'cover':17, + 'au_map':18, 'last_modified':19, 'identifiers':20} for k,v in self.FIELD_MAP.iteritems(): self.field_metadata.set_field_record_index(k, v, prefer_custom=False) @@ -391,11 +420,16 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.has_id = self.data.has_id self.count = self.data.count - for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn', - 'publisher', 'rating', 'series', 'series_index', 'tags', - 'title', 'timestamp', 'uuid', 'pubdate', 'ondevice'): + for prop in ( + 'author_sort', 'authors', 'comment', 'comments', + 'publisher', 'rating', 'series', 'series_index', 'tags', + 'title', 'timestamp', 'uuid', 'pubdate', 'ondevice', + 'metadata_last_modified', + ): + fm = {'comment':'comments', 'metadata_last_modified': + 'last_modified'}.get(prop, prop) setattr(self, prop, functools.partial(self.get_property, - loc=self.FIELD_MAP['comments' if prop == 'comment' else prop])) + loc=self.FIELD_MAP[fm])) setattr(self, 'title_sort', functools.partial(self.get_property, loc=self.FIELD_MAP['sort'])) @@ -681,8 +715,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if commit: self.conn.commit() + def update_last_modified(self, book_ids, commit=False, now=None): + if now is None: + now = nowf() + if book_ids: + self.conn.executemany( + 'UPDATE books SET last_modified=? WHERE id=?', + [(now, book) for book in book_ids]) + for book_id in book_ids: + self.data.set(book_id, self.FIELD_MAP['last_modified'], now, row_is_id=True) + if commit: + self.conn.commit() + def dirtied(self, book_ids, commit=True): - changed = False + self.update_last_modified(book_ids) for book in book_ids: with self.dirtied_lock: # print 'dirtied: check id', book @@ -691,21 +737,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.dirtied_sequence += 1 continue # print 'book not already dirty' - try: - self.conn.execute( - 'INSERT INTO metadata_dirtied (book) VALUES (?)', - (book,)) - changed = True - except IntegrityError: - # Already in table - pass + + self.conn.execute( + 'INSERT OR IGNORE INTO metadata_dirtied (book) VALUES (?)', + (book,)) self.dirtied_cache[book] = self.dirtied_sequence self.dirtied_sequence += 1 + # If the commit doesn't happen, then the DB table will be wrong. This # could lead to a problem because on restart, we won't put the book back # into the dirtied_cache. We deal with this by writing the dirtied_cache # back to the table on GUI exit. Not perfect, but probably OK - if commit and changed: + if book_ids and commit: self.conn.commit() def get_a_dirtied_book(self): @@ -803,8 +846,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if mi.series: mi.series_index = row[fm['series_index']] mi.rating = row[fm['rating']] - mi.isbn = row[fm['isbn']] id = idx if index_is_id else self.id(idx) + mi.set_identifiers(self.get_identifiers(id, index_is_id=True)) mi.application_id = id mi.id = id for key, meta in self.field_metadata.custom_iteritems(): @@ -911,10 +954,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except (IOError, OSError): time.sleep(0.2) save_cover_data_to(data, path) - self.conn.execute('UPDATE books SET has_cover=1 WHERE id=?', (id,)) + now = nowf() + self.conn.execute( + 'UPDATE books SET has_cover=1,last_modified=? WHERE id=?', + (now, id)) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['cover'], True, row_is_id=True) + self.data.set(id, self.FIELD_MAP['last_modified'], now, row_is_id=True) if notify: self.notify('cover', [id]) @@ -923,8 +970,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def set_has_cover(self, id, val): dval = 1 if val else 0 - self.conn.execute('UPDATE books SET has_cover=? WHERE id=?', (dval, id,)) + now = nowf() + self.conn.execute( + 'UPDATE books SET has_cover=?,last_modified=? WHERE id=?', + (dval, now, id)) self.data.set(id, self.FIELD_MAP['cover'], val, row_is_id=True) + self.data.set(id, self.FIELD_MAP['last_modified'], now, row_is_id=True) def book_on_device(self, id): if callable(self.book_on_device_func): @@ -1222,7 +1273,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for category in tb_cats.keys(): cat = tb_cats[category] if not cat['is_category'] or cat['kind'] in ['user', 'search'] \ - or category in ['news', 'formats']: + or category in ['news', 'formats', 'identifiers']: continue # Get the ids for the item values if not cat['is_custom']: @@ -1652,8 +1703,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): doit(self.set_tags, id, mi.tags, notify=False, commit=False) if mi.comments: doit(self.set_comment, id, mi.comments, notify=False, commit=False) - if mi.isbn and mi.isbn.strip(): - doit(self.set_isbn, id, mi.isbn, notify=False, commit=False) if mi.series_index: doit(self.set_series_index, id, mi.series_index, notify=False, commit=False) @@ -1663,6 +1712,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): doit(self.set_timestamp, id, mi.timestamp, notify=False, commit=False) + mi_idents = mi.get_identifiers() + if mi_idents: + identifiers = self.get_identifiers(id, index_is_id=True) + for key, val in mi_idents.iteritems(): + if val and val.strip(): # Don't delete an existing identifier + identifiers[icu_lower(key)] = val + self.set_identifiers(id, identifiers, notify=False, commit=False) + + user_mi = mi.get_all_user_metadata(make_copy=False) for key in user_mi.iterkeys(): if key in self.field_metadata and \ @@ -2441,14 +2499,84 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if notify: self.notify('metadata', [id]) - def set_isbn(self, id, isbn, notify=True, commit=True): - self.conn.execute('UPDATE books SET isbn=? WHERE id=?', (isbn, id)) - self.dirtied([id], commit=False) + def isbn(self, idx, index_is_id=False): + row = self.data._data[idx] if index_is_id else self.data[idx] + if row is not None: + raw = row[self.FIELD_MAP['identifiers']] + if raw: + for x in raw.split(','): + if x.startswith('isbn:'): + return x[5:].strip() + + def get_identifiers(self, idx, index_is_id=False): + ans = {} + row = self.data._data[idx] if index_is_id else self.data[idx] + if row is not None: + raw = row[self.FIELD_MAP['identifiers']] + if raw: + for x in raw.split(','): + key, _, val = x.partition(':') + key, val = key.strip(), val.strip() + if key and val: + ans[key] = val + + return ans + + def _clean_identifier(self, typ, val): + typ = icu_lower(typ).strip().replace(':', '').replace(',', '') + val = val.strip().replace(',', '|').replace(':', '|') + return typ, val + + def set_identifier(self, id_, typ, val, notify=True, commit=True): + 'If val is empty, deletes identifier of type typ' + typ, val = self._clean_identifier(typ, val) + identifiers = self.get_identifiers(id_, index_is_id=True) + if not typ: + return + changed = False + if not val and typ in identifiers: + identifiers.pop(typ) + changed = True + self.conn.execute( + 'DELETE from identifiers WHERE book=? AND type=?', + (id_, typ)) + if val and identifiers.get(typ, None) != val: + changed = True + identifiers[typ] = val + self.conn.execute( + 'INSERT OR REPLACE INTO identifiers (book, type, val) VALUES (?, ?, ?)', + (id_, typ, val)) + if changed: + raw = ','.join(['%s:%s'%(k, v) for k, v in + identifiers.iteritems()]) + self.data.set(id_, self.FIELD_MAP['identifiers'], raw, + row_is_id=True) + if commit: + self.conn.commit() + if notify: + self.notify('metadata', [id_]) + + def set_identifiers(self, id_, identifiers, notify=True, commit=True): + cleaned = {} + for typ, val in identifiers.iteritems(): + typ, val = self._clean_identifier(typ, val) + if val: + cleaned[typ] = val + self.conn.execute('DELETE FROM identifiers WHERE book=?', (id_,)) + self.conn.executemany( + 'INSERT INTO identifiers (book, type, val) VALUES (?, ?, ?)', + [(id_, k, v) for k, v in cleaned.iteritems()]) + raw = ','.join(['%s:%s'%(k, v) for k, v in + cleaned.iteritems()]) + self.data.set(id_, self.FIELD_MAP['identifiers'], raw, + row_is_id=True) if commit: self.conn.commit() - self.data.set(id, self.FIELD_MAP['isbn'], isbn, row_is_id=True) if notify: - self.notify('metadata', [id]) + self.notify('metadata', [id_]) + + def set_isbn(self, id_, isbn, notify=True, commit=True): + self.set_identifier(id_, 'isbn', isbn, notify=notify, commit=commit) def add_catalog(self, path, title): format = os.path.splitext(path)[1][1:].lower() @@ -2746,7 +2874,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): prefix = self.library_path FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', - 'isbn', 'uuid', 'pubdate']) + 'uuid', 'pubdate', 'last_modified']) for x in self.custom_column_num_map: FIELDS.add(x) data = [] @@ -2761,6 +2889,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): data.append(x) x['id'] = db_id x['formats'] = [] + isbn = self.isbn(db_id, index_is_id=True) + x['isbn'] = isbn if isbn else '' if not x['authors']: x['authors'] = _('Unknown') x['authors'] = [i.replace('|', ',') for i in x['authors'].split(',')] diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index aff2803452..b0d604dc57 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -119,6 +119,15 @@ class FieldMetadata(dict): 'search_terms':['formats', 'format'], 'is_custom':False, 'is_category':True}), + ('identifiers', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':',', + 'kind':'field', + 'name':_('Identifiers'), + 'search_terms':['identifiers', 'identifier'], + 'is_custom':False, + 'is_category':True}), ('publisher', {'table':'publishers', 'column':'name', 'link_column':'publisher', @@ -296,6 +305,15 @@ class FieldMetadata(dict): 'search_terms':['date'], 'is_custom':False, 'is_category':False}), + ('last_modified', {'table':None, + 'column':None, + 'datatype':'datetime', + 'is_multiple':None, + 'kind':'field', + 'name':_('Date'), + 'search_terms':['last_modified'], + 'is_custom':False, + 'is_category':False}), ('title', {'table':None, 'column':None, 'datatype':'text', @@ -335,7 +353,8 @@ class FieldMetadata(dict): self._tb_cats[k]['display'] = {} self._tb_cats[k]['is_editable'] = True self._add_search_terms_to_map(k, v['search_terms']) - self._tb_cats['timestamp']['display'] = { + for x in ('timestamp', 'last_modified'): + self._tb_cats[x]['display'] = { 'date_format': tweaks['gui_timestamp_display_format']} self._tb_cats['pubdate']['display'] = { 'date_format': tweaks['gui_pubdate_display_format']} diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index 76f3c0333d..e03edd449a 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -13,6 +13,7 @@ from calibre.ptempfile import TemporaryDirectory from calibre.ebooks.metadata.opf2 import OPF from calibre.library.database2 import LibraryDatabase2 from calibre.constants import filesystem_encoding +from calibre.utils.date import utcfromtimestamp from calibre import isbytestring NON_EBOOK_EXTENSIONS = frozenset([ @@ -211,8 +212,8 @@ class Restore(Thread): force_id=book['id']) if book['mi'].uuid: db.set_uuid(book['id'], book['mi'].uuid, commit=False, notify=False) - db.conn.execute('UPDATE books SET path=? WHERE id=?', (book['path'], - book['id'])) + db.conn.execute('UPDATE books SET path=?,last_modified=? WHERE id=?', (book['path'], + utcfromtimestamp(book['timestamp']), book['id'])) for fmt, size, name in book['formats']: db.conn.execute(''' diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index 0b7a3f5350..d1f22d379b 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -8,6 +8,8 @@ __docformat__ = 'restructuredtext en' import os +from calibre.utils.date import isoformat, DEFAULT_DATE + class SchemaUpgrade(object): def __init__(self): @@ -468,4 +470,116 @@ class SchemaUpgrade(object): ''' self.conn.executescript(script) + def upgrade_version_18(self): + ''' + Add a library UUID. + Add an identifiers table. + Add a languages table. + Add a last_modified column. + NOTE: You cannot downgrade after this update, if you do + any changes you make to book isbns will be lost. + ''' + script = ''' + DROP TABLE IF EXISTS library_id; + CREATE TABLE library_id ( id INTEGER PRIMARY KEY, + uuid TEXT NOT NULL, + UNIQUE(uuid) + ); + + DROP TABLE IF EXISTS identifiers; + CREATE TABLE identifiers ( id INTEGER PRIMARY KEY, + book INTEGER NON NULL, + type TEXT NON NULL DEFAULT "isbn" COLLATE NOCASE, + val TEXT NON NULL COLLATE NOCASE, + UNIQUE(book, type) + ); + + DROP TABLE IF EXISTS languages; + CREATE TABLE languages ( id INTEGER PRIMARY KEY, + lang_code TEXT NON NULL COLLATE NOCASE, + UNIQUE(lang_code) + ); + + DROP TABLE IF EXISTS books_languages_link; + CREATE TABLE books_languages_link ( id INTEGER PRIMARY KEY, + book INTEGER NOT NULL, + lang_code INTEGER NOT NULL, + item_order INTEGER NOT NULL DEFAULT 0, + UNIQUE(book, lang_code) + ); + + DROP TRIGGER IF EXISTS fkc_delete_on_languages; + CREATE TRIGGER fkc_delete_on_languages + BEFORE DELETE ON languages + BEGIN + SELECT CASE + WHEN (SELECT COUNT(id) FROM books_languages_link WHERE lang_code=OLD.id) > 0 + THEN RAISE(ABORT, 'Foreign key violation: language is still referenced') + END; + END; + + DROP TRIGGER IF EXISTS fkc_delete_on_languages_link; + CREATE TRIGGER fkc_delete_on_languages_link + BEFORE INSERT ON books_languages_link + BEGIN + SELECT CASE + WHEN (SELECT id from books WHERE id=NEW.book) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: book not in books') + WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages') + END; + END; + + DROP TRIGGER IF EXISTS fkc_update_books_languages_link_a; + CREATE TRIGGER fkc_update_books_languages_link_a + BEFORE UPDATE OF book ON books_languages_link + BEGIN + SELECT CASE + WHEN (SELECT id from books WHERE id=NEW.book) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: book not in books') + END; + END; + DROP TRIGGER IF EXISTS fkc_update_books_languages_link_b; + CREATE TRIGGER fkc_update_books_languages_link_b + BEFORE UPDATE OF lang_code ON books_languages_link + BEGIN + SELECT CASE + WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages') + END; + END; + + DROP INDEX IF EXISTS books_languages_link_aidx; + CREATE INDEX books_languages_link_aidx ON books_languages_link (lang_code); + DROP INDEX IF EXISTS books_languages_link_bidx; + CREATE INDEX books_languages_link_bidx ON books_languages_link (book); + DROP INDEX IF EXISTS languages_idx; + CREATE INDEX languages_idx ON languages (lang_code COLLATE NOCASE); + + DROP TRIGGER IF EXISTS books_delete_trg; + CREATE TRIGGER books_delete_trg + AFTER DELETE ON books + BEGIN + DELETE FROM books_authors_link WHERE book=OLD.id; + DELETE FROM books_publishers_link WHERE book=OLD.id; + DELETE FROM books_ratings_link WHERE book=OLD.id; + DELETE FROM books_series_link WHERE book=OLD.id; + DELETE FROM books_tags_link WHERE book=OLD.id; + DELETE FROM books_languages_link WHERE book=OLD.id; + DELETE FROM data WHERE book=OLD.id; + DELETE FROM comments WHERE book=OLD.id; + DELETE FROM conversion_options WHERE book=OLD.id; + DELETE FROM books_plugin_data WHERE book=OLD.id; + DELETE FROM identifiers WHERE book=OLD.id; + END; + + INSERT INTO identifiers (book, val) SELECT id,isbn FROM books WHERE isbn; + + ALTER TABLE books ADD COLUMN last_modified TIMESTAMP NOT NULL DEFAULT "%s"; + + '''%isoformat(DEFAULT_DATE, sep=' ') + # Sqlite does not support non constant default values in alter + # statements + self.conn.executescript(script) + diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 622d6b8459..a57eb6b1f9 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -87,6 +87,18 @@ class SortedConcatenate(object): class SafeSortedConcatenate(SortedConcatenate): sep = '|' +class IdentifiersConcat(object): + '''String concatenation aggregator for the identifiers map''' + def __init__(self): + self.ans = [] + + def step(self, key, val): + self.ans.append(u'%s:%s'%(key, val)) + + def finalize(self): + return ','.join(self.ans) + + class AumSortedConcatenate(object): '''String concatenation aggregator for the author sort map''' def __init__(self): @@ -170,13 +182,13 @@ class DBThread(Thread): detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES) self.conn.execute('pragma cache_size=5000') encoding = self.conn.execute('pragma encoding').fetchone()[0] - c_ext_loaded = load_c_extensions(self.conn) + self.conn.create_aggregate('sortconcat', 2, SortedConcatenate) + self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate) + self.conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat) + load_c_extensions(self.conn) self.conn.row_factory = sqlite.Row if self.row_factory else lambda cursor, row : list(row) self.conn.create_aggregate('concat', 1, Concatenate) self.conn.create_aggregate('aum_sortconcat', 3, AumSortedConcatenate) - if not c_ext_loaded: - self.conn.create_aggregate('sortconcat', 2, SortedConcatenate) - self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate) self.conn.create_collation('PYNOCASE', partial(pynocase, encoding=encoding)) self.conn.create_function('title_sort', 1, title_sort) diff --git a/src/calibre/library/sqlite_custom.c b/src/calibre/library/sqlite_custom.c index 650c474c2c..dee17c79d4 100644 --- a/src/calibre/library/sqlite_custom.c +++ b/src/calibre/library/sqlite_custom.c @@ -77,6 +77,7 @@ static void sort_concat_free(SortConcatList *list) { free(list->vals[i]->val); free(list->vals[i]); } + free(list->vals); } static int sort_concat_cmp(const void *a_, const void *b_) { @@ -142,11 +143,102 @@ static void sort_concat_finalize2(sqlite3_context *context) { // }}} +// identifiers_concat {{{ + +typedef struct { + char *val; + size_t length; +} IdentifiersConcatItem; + +typedef struct { + IdentifiersConcatItem **vals; + size_t count; + size_t length; +} IdentifiersConcatList; + +static void identifiers_concat_step(sqlite3_context *context, int argc, sqlite3_value **argv) { + const char *key, *val; + size_t len = 0; + IdentifiersConcatList *list; + + assert(argc == 2); + + list = (IdentifiersConcatList*) sqlite3_aggregate_context(context, sizeof(*list)); + if (list == NULL) return; + + if (list->vals == NULL) { + list->vals = (IdentifiersConcatItem**)calloc(100, sizeof(IdentifiersConcatItem*)); + if (list->vals == NULL) return; + list->length = 100; + list->count = 0; + } + + if (list->count == list->length) { + list->vals = (IdentifiersConcatItem**)realloc(list->vals, list->length + 100); + if (list->vals == NULL) return; + list->length = list->length + 100; + } + + list->vals[list->count] = (IdentifiersConcatItem*)calloc(1, sizeof(IdentifiersConcatItem)); + if (list->vals[list->count] == NULL) return; + + key = (char*) sqlite3_value_text(argv[0]); + val = (char*) sqlite3_value_text(argv[1]); + if (key == NULL || val == NULL) {return;} + len = strlen(key) + strlen(val) + 1; + + list->vals[list->count]->val = (char*)calloc(len+1, sizeof(char)); + if (list->vals[list->count]->val == NULL) return; + snprintf(list->vals[list->count]->val, len+1, "%s:%s", key, val); + list->vals[list->count]->length = len; + + list->count = list->count + 1; + +} + + +static void identifiers_concat_finalize(sqlite3_context *context) { + IdentifiersConcatList *list; + IdentifiersConcatItem *item; + char *ans, *pos; + size_t sz = 0, i; + + list = (IdentifiersConcatList*) sqlite3_aggregate_context(context, sizeof(*list)); + if (list == NULL || list->vals == NULL || list->count < 1) return; + + for (i = 0; i < list->count; i++) { + sz += list->vals[i]->length; + } + sz += list->count; // Space for commas + ans = (char*)calloc(sz+2, sizeof(char)); + if (ans == NULL) return; + + pos = ans; + + for (i = 0; i < list->count; i++) { + item = list->vals[i]; + if (item == NULL || item->val == NULL) continue; + memcpy(pos, item->val, item->length); + pos += item->length; + *pos = ','; + pos += 1; + free(item->val); + free(item); + } + *(pos-1) = 0; // Remove trailing comma + sqlite3_result_text(context, ans, -1, SQLITE_TRANSIENT); + free(ans); + free(list->vals); +} + +// }}} + MYEXPORT int sqlite3_extension_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); sqlite3_create_function(db, "sortconcat", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize); sqlite3_create_function(db, "sort_concat", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize2); + sqlite3_create_function(db, "identifiers_concat", 2, SQLITE_UTF8, NULL, NULL, identifiers_concat_step, identifiers_concat_finalize); return 0; } diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index 31c770bea5..eaf68df904 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -45,6 +45,7 @@ utc_tz = _utc_tz = tzutc() local_tz = _local_tz = SafeLocalTimeZone() UNDEFINED_DATE = datetime(101,1,1, tzinfo=utc_tz) +DEFAULT_DATE = datetime(2000,1,1, tzinfo=utc_tz) def is_date_undefined(qt_or_dt): d = qt_or_dt From c42ba307a130a255902082dc420c8c86f4f12608 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 28 Feb 2011 22:12:45 -0700 Subject: [PATCH 20/70] ... --- src/calibre/gui2/widgets.py | 7 ++--- src/calibre/manual/templates/layout.html | 34 ++++++++++++------------ 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index f6c4cce3ef..3622cc6c39 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -13,7 +13,7 @@ from PyQt4.Qt import QIcon, QFont, QLabel, QListWidget, QAction, \ QRegExp, QSettings, QSize, QSplitter, \ QPainter, QLineEdit, QComboBox, QPen, \ QMenu, QStringListModel, QCompleter, QStringList, \ - QTimer, QRect + QTimer, QRect, QFontDatabase from calibre.gui2 import NONE, error_dialog, pixmap_to_data, gprefs from calibre.gui2.filename_pattern_ui import Ui_Form @@ -299,8 +299,6 @@ class ImageView(QWidget): # }}} - - class FontFamilyModel(QAbstractListModel): def __init__(self, *args): @@ -312,6 +310,9 @@ class FontFamilyModel(QAbstractListModel): self.families = [] print 'WARNING: Could not load fonts' traceback.print_exc() + # Restrict to Qt families as Qt tends to crash + qt_families = set([unicode(x) for x in QFontDatabase().families()]) + self.families = list(qt_families.intersection(set(self.families))) self.families.sort() self.families[:0] = [_('None')] diff --git a/src/calibre/manual/templates/layout.html b/src/calibre/manual/templates/layout.html index b427482947..8f35a9a6c5 100644 --- a/src/calibre/manual/templates/layout.html +++ b/src/calibre/manual/templates/layout.html @@ -1,23 +1,6 @@ {% extends "!layout.html" %} {% block extrahead %} - {% if not embedded %} - - {% endif %} -