From 42726c5e0f87b80956b1300d7c14275ffa113958 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Mon, 10 Oct 2022 16:54:21 +0100 Subject: [PATCH 1/8] Python templates, as discussed at length in https://www.mobileread.com/forums/showthread.php?t=349525. I think this is stable, usable, and very close to finished, but I will probably need to react fo feedback. --- manual/images/python_template_example.png | Bin 0 -> 13795 bytes manual/template_lang.rst | 50 +++++- src/calibre/gui2/dialogs/template_dialog.py | 153 +++++++++++++++--- src/calibre/gui2/dialogs/template_dialog.ui | 6 +- .../gui2/preferences/template_functions.py | 91 +++++++---- src/calibre/utils/formatter.py | 109 ++++++++++--- src/calibre/utils/formatter_functions.py | 101 ++++++++---- 7 files changed, 388 insertions(+), 122 deletions(-) create mode 100644 manual/images/python_template_example.png diff --git a/manual/images/python_template_example.png b/manual/images/python_template_example.png new file mode 100644 index 0000000000000000000000000000000000000000..90aaf337624f2161387b8f54bcd0446559746ffe GIT binary patch literal 13795 zcmZ{LWl$VJ`{zQC1%f*ScPCho5Zo4D2<|K{Ay^0wf#9%s@Zj$5?j!_v2<{L(_)Xsb zdskOibswg7YIe4_r)TErpLLj;iX0|789E3A!c>r#hJru{LBPfcMg)!=V>p=tA81bU zx-K9PR^Pu3A(0J>90Z~UDM(9bdZZs`=)>9NK7W6%I1pZ$D$L)fi$0OlrcdB8LFy=~ zstyi7ex)P$+H~_!7Vj_$(~Dx&txwke-SL-e(<$GxuT%oFSZ43n6@QD^m~sygC1DCM zaH7~Fu_#ibQ7C{{Z1aRfrwR*NZfFkUW(?o^q+}MK>_1}!|5S3=NQ=|)|=P1C=&elJnHhg6HEITKyrzoI|7{-feU3>`~nEQ^3Ea;E9*tL1?#9l7daW9w1kacd07YL^$qGSzU97GMQTbVw3#^R5yR5RwEANC{}K4E!w;2T2z# zweu==B|MpHp=R}ZK;w;p(4q1h;N1Rilf0f5A3UCuQ3mnR1F6%-E=?CtDBtykQV+#` zarxH((*HETO-cib<7z$1-ayAy*Avx6cLq@7;w%`ac#JQxAE+Qs#j| z4~*qltRB9}@9>0m_VA0hXHd&OT}TdNr9dRP=C@(G6Ex&~vGjBq5zP6w_I$Hzv1N0I z&5{9ljs~o{9b$P%Q=;r|ZUzN8POc0XCtnpTeIlKzv#%%!ST#EER|TiJ znl5W`ninOJe{z`A+^90r9vVJ}?D^yCL%iBh=|$R&2LCD){BUPnmCChB&?1i@k_Zr@sLiYeq;`9oSBsew_Ex$&aM1Kt-h-0~e+Gi#B zOn!;~-8yvMAX!7ds&1L6tjZfbNP32Fa_V;34Ea`mleiYsY%L%sFyFkP=mcy5X8d`qO&u7!}g>$ zv;v#BOHI>luGE!_CC4$YK20BL%n%KWD7JTW5J9BW-0v(C5|e55C1qfX)NBq(e-Mxh z|Dv)}pwZ-ma6bAC?rPw@3S<`}B{i?ptyM?6bMpU7wbB9wi3`nH1i7fhP5x#6X;Ts4 zak;}oGqZRdA-)hUEt^DwhX~=wZly_i9q@zwL^b2=b?ti_5{NDwgG8dB>JmmUZg{9m zs?DtPmOQrDFQ>4f>#^%burO#3A*0p(LRwD65s`d1Gxko$od3hK9~yC~WCuo*pAi-Z9;Rh2O_{*SabS%hHA<=%p1I z39WojTzu?LnH~11Uu+BdiY|^j5C%m~3HvNmekW0CDvi zV<`!uCLL9lWo?g5By^*<#Vs3BI54TI4|kcZm7Ui!+D5+0R;8^M z=ZZ0JuYXOAzx{Y${-NsF-i!8M(RjuG1N-T+GJmbDjr`$uuuYEAYmd(Fs#WZ2)!3eo zj9>OE^^j~Vd9>l;@#E3G@BU)x2k*UgKVM+0U}FRd8vLx)rckBc_n^IV-qAtyk17>; z)YtkxplYB%o$Q{+u1kIAeQrKw`t77xjZ^uF1DC+qKThhiMj^Hgod2{%H~0!OkeXbP zlQc^9Ol7uQ>wY%P`2Hbcd=?iEs8XTGpx*Ug&l_uxr)#%V&lhi=M^7JCEBzj0xng8< zd6%qiP`xl3O7d*P^sNIXODg9DJ@MQ=%q+oYhPf(V6ZeeWQZj}RRm>q&qQF59B-?R=v`!LhMoM?6N6n<$>vTlQT@G$S$iJjLK&xc6||f6`JtEU_PFyQ zP0ahX$7%!d{p~hE4J14`X{0jS*}7PM2YsU-kFh@-`UTHXZ;-A>`9OZwoK=@{&!vB7 zSPlw#T)js=e0;Uz9vt43#eP;up~}^AG^M#n>cSaDacZ{FqR{Tx!%gGspnI6 zG5+>R)ws#8?DE3+DWtN5%T?95MLxb@a_r_@aXXPOs(OK{Nx}R`XT0dB3!gDbSxs_C zb4;3<0a*ZvAs=$6Hl7>SE(0+yrdm(m@m3uWs=4_gZ(iftUIU|0xa)W$uZ1F;aQbm0 zv}tOcyOF_S@R(r)VaBYQsnIT#GSR--aq-85A&q|1k`j@-sXdE>Bz9!{HOqV$V%;AC z^H{-@=o)tV31#n&jH|?e3X$35&bx)0xr(=5KQ=JgSn{CsbKSA*g=I-&Kc2haL{&Er zj9|Z2Fw=d}^H^q=lc(9oH`d?Jstu91Z+JCjLByD2P|13zrOx$dPa5M4mxHD^?5g5x zzq&4@J{Y~-WN;U^DCtnR>(dt#ra!3EEgwDODUyY%rH4=P7fgFlLYo>@sf7gm&?Uvu z%&lc5$4^}l^9xC0`oh~?32DNzcTbVJqg6So+d3$Dh-YiZMGdbPF?Stx^M?Q8=g_;X z7>z4?u`QY;fX&?sZI*2l$jpEAguhUY(`oQ;>Sgx#!v9i`L(i*QfP`-gtp++jt++~g z@Yyxh7@{CIQFZXB*+6?wiH!#G=g=Yln<+;J0^vDR_DkX|juSY-zCF&6YU^66eVEH7 za%uE(lpCrO(eMp7p#Prt;?iGp8WrRe2)nNztDLH?nhtpS~fxQaE7 zZFBOsEY7hYW($10_rcQl@31A@myMDY(swH`510Ah5O(|-m}|eqv6}*4T_(f|U{hom zf8_0}NpRtLq9j`2h~i;b&SbM`KHBGo2V@=97Vz7H8WO~9czU1>fxj@6vdb%us*I9{ zLni}vk8fAsDMT(p7J@Y2ovG3u9N+n7l?dRw-?E&?`j=l4y~w{p0D({MyDQ`*YEBd$ zkoB<3PQ7}ug?^mL+^$R|ckEw@Rp~^WwK2!HY{sQcgUss^skGj|KJIcO7*#Pfd(j+d zq$sSAprzmv3lqjo>}WP0420T-JEz0J3H(ICrYnsJJOc(6tiJp5U{ODD8)QpBkQRojy1fs|TRhWDD4Af7_D2SN@Ho!s z{1e~l+`mwY@V(aUZgB1KqO$#1IpU2%(+3YzLWhzjSp=c zADjH0tU?l_Q;$y;TSj#4Wp1A4j>G*1D%B5jFAW!B(4J2z3rv5S2Vr{Y>PlX2;Z`n) z_T;K~QkB&m`K5QKDuVyZx9<|FBW8ErHouX>VoupZLh#RQXH;oPRA z)#ABh!yd*UJUqzw(7hx|+9`f0znST^IaAfU?)2eV@=&vN;nkGhaj6sMs14o{g@Ys!9Z+!lL&EjX8*J|_7vraU z=J1Z)R=+>(r*}fj)BpnBq*0XZUH>!wbZ*qnBA+|qadFc@!3$v8L0JK%`{m@PcPp8D zaVHJFD0mV$TOOhD6RS_bo%e$=`S~kTOA!J()mTzBq;r1Mq`~71R5dsfTCswpKHJ&j z$H&IQX6j-O~h|L3~|LDH1_L4ogQUr+Cz80tUhg`QJI;1@?2QUc(jy~6(& z-&@LmZIEJsdj4!^jK7T~TJhd=&^G?O!b7Ir8uLU{w$4kfp6@xP{t`VsxR`w>77T{j&oHA$tkZ;dMDqx#u;Yn7{vI z0ssjjWe|xhYN1FnA6XPrruQZG)6uhZ@80sv8>7#+^=^Xzc0s|tiL@VmCkTGtsPq#f zlLbYkeP}8_=>j+uJ3ELZN*0vs_cS;D?s=mYL*(*++-PB$_t< zaX7(lOxO%0a$h?|v$P(TJc*M)5mKsGF=4|aSr?p^V3jgz+Fk+Qr?Kvr)k-L{s+-|7 z`sN-f-wjnL-w$T5Fkq=j93xmQ*pFyd$|IFm*okJ6;wYpKD*MKy4hiw}K?nW*M|d7o(*dTG*V?jN7@f9Fr_Y@>c;(Pt6%t%Vt2gBa$*HriPK@^Now>G*7~IIQ9BYDsmjg3mlDl{X zk9F(q$uKaQQj38;hqe`-U@R-SR98C%^2W?vr*ApC8Q-nF)gbEbzSK=GUhl zR(aG~VQlJ0|#@B zVu+ZCeFn>%JcTJGfACCbvR0G}itSF56flG&dHZW0I;P@rC|qw!cscee=!pxkII|&;Ncw z_e}NhK}U3-AIJdRgRut9-YRSykPiaK!e*@dhKt-v3_o!LZz?AdXfN^Zcvfqr8z|<@ zdoY-8(o7GaEy(-y6%1+=E=IeQ?C1b5l1Uj#S9f>!e!1gCQ>K6<4pK@VqCfX5dt*-& zCpqA)I_M2VF_>Qxt@UwNz*MoeT2VOFrGeDwBGspTEet--!WW4jwF`JRw3^Z-VlBZWB ztzzQMT=9UjE`xzo6V>wy`$ccX?A=VRRG4Gs93jq*7Dj0j9Cufb98aB$Wg&fk%Fw_>gKx~+*{%Y+aua>$?;NTI zU#x#Qx0{aVH$=E!Rl2Lg(h{>2mb_wrU8i0K-%e*#ko^#& zk*2jBvXanh)Bg|A#W!2ygB2XBQ(VS^3l??NI2zT+6tI{->O6dp@BPNRq$wPj zU}orc%MVOSuhx6#g4DQTTZud@%p-hj6mdcRdT+#zUgLfAOAg&)Y!7`1yVvj>G zYPZ!mc1>_ixcz2v4HNHsG^`MAhk-&xVyQ#&&ZLeSeJ>qOsiI`*{3pOBNb$T2MHw+xkU7l|90v0YjP2T}!v&@%OpNnkRD1ya)SDofBm?^1U3CpoQ1QQ=p`FN$;ltjarmEIfUbRE8M5j@r z&)*KkEOjxU6Vpd~glt!=32N8j_C)boWFQ?!E(L{#vY4xap~!y+sb^u@)@Wg&PGTQ2 z5~0eMy6B!%kO4dM6|)zx4lDX#I}-cBxs`%Ldl%7Kp0wk61k2A9iZ1HFFDZvt>Jq>8 zCebRqfP~xR=nj7y((+;*cD^0`v&4vpUsSWex}J&}nmWkG>?I=*^T6$(TP%MQ@bU{d z<5<;YIX672j&LBiZ$8Pbd%#ETjBODQUPPzp&~Vml?d)5dzc~2y-|qsKxo%?LhKPnd z(Vd0jsNjpa%hV>&1CLw^{B}g)jDp0~eJgdULQ<;+J@B(&3#56rvBA{%qX2sZDDSO!l`o#EIfg%vDcC{@Vw zEhX-9H7*S7#(7q-QR!2Bn))}olmjJ?xu;a>T-84jvsbs;lmZePayZsDKfgY{6xwl4 z<5=$)uZ^F3YZX+Z4SQkpaCa)kA2=@{GWm01!aBjbRN|ODkXr7?-x41H5$looIU>8! zf&-@m-={lsV*?1Mjb1c&`AalgwE*yal-YUqAc>MR@Z>Pb)e2w&HC_ec4ai2+gNkSP z^Z!1T)W`SK$b~Jh;mhdZu2%!glX6qtNyaIrp^zXw`-uV8J&VzA+9LW|on>us1qp}x zZTHyT#JyD2q+P1lmDcF=ifqZvY*B-=n=-s)o619pxlXp}zs|G#6QfD5E8QE)e06V;`U4l8M~|Jcv$BhUNg#h% z@@;Pri|#q9ZherFDzQ2l!v__`qF)jG?6^1AVfGB7$2NnO>JAEcL|jSitz$LcMk^Vl z+`qkSZ_9Cc`_<+{CG3`SHT~zt!@Y)kRK!E*DDxgEzwU|bc0$2JQsL&W_~8kaItZHmsiubXf`}qcM-4(|%+YKHsAm@INL zvj7T%WA?lpeNc%&m*o!^G0Hgg4>^-92f*!D5YgB1YjF0*lQ_07)bCm66|hhec#kd6 z_>TwqJWepcz(Q!6GP0Cn0-P|cD~RZZMY1l@#id;=M|RDVz3$74j`j=T2rD7?YO0T< z8bl+UtD@oCG7y&h=^$o~*|zifMG3)Om<`H&{q%o_{ps$|?!A7O5zjc{cD(U0Jwc

nvhy1LMiFRa%e2drGhKdr6)*U2qVNQ$}#+7g7-UO7lNLe1hu{sV1K4{M%;mFU3=6Y=P!_6G7dU z4g*F0Vuebpqdwv41E*Iq#4OX91D_X?|9b5HnsR4Q-TeqEI%FZ5YeAoHpK9G>Nr4TI z%hoK5+D_LbH?cC0x~#GI?<@0;E~IXmQ$0=*_0{4+7S;k@JeoyW1Pw*;hp64mAJxcq zy;}qd!`01oX>UWh-<9jQrvO7|6p?@hH}#z=k0aUf@xs2?WEOERrre@4#EOIE(h30_ zW%{B7xoUX%4DV&tVNNoKxzVw`?&G<;qFcXEms8;#kW)>bd3B`4j{Au5g1 z@kVJis3K37WGaTk>S#Xa-*{f?@$tI7DE&z;)n4s;xGWY;TKQ3za%nD&s0A5SH9XP?;43}FuMMEr=S?AI6cYueD@AGAXso$T@F<4BWq)u7*W5%c85 zc2Zu>s`arB&qroO*Wkae*QzK^j6}1j6L2As3+MRu!(26rLq`!|Yx&-nS*F+$3>zer zsvNTC>W9^Wos|+8a`Uk#>ZbXJ=xbT63h9?sUse~Fysiw*=U%eP-2(dd-o`c&@`blf zesHsh-@TS}?wfjL78ti@zL=FFg|&6)`yy&mY%+|PFM@h7I5tvt|2Z0*pNA9Kj7;`0 z&nvC8FtiaqpM(FtPAj{3=3{^DIVZleY3C`T8rM>jW%vO5>`yDobXH7c3MV}5J*NRO zU5aSCZw2&+%@t}v&f{4}YD)8S(Ff8Ol1)^9DR&caL@T#4{l`DpdtMivc&}gL)Zpo; zEs*PJ$tnC7;lZ5?jGy+yb&B0FIGfOZHfn3@?|Jy_pil`nRB#=$ON&NISh#z2x$*u<3op*uyDKQXf? zrygC|p@prHW-y;XYgD~7`vsOml07#YtH3p%&v^XZC#2%AKp5Z1V=X;EuNF$(yHt+N za89gHKX0a`GjJhfXTC(t`iwb14W(ZPUt7Z^8Y0Tn4DzCB`Z zte=r{x5O2q8#<`!!LZm2NThV`ziA>`^K)n}FQ=}V;=Sp@30W*ekciI8f${0Z22%%l zHNVEo&TRA$vjk3|tUGmKUViaHVm8qJTha+~P9sXdVu%eu*YMchDgh=^{18*_3`r)S zh6LzX+@f9pe$1f4B<~1^68SSRAdwYShx1|;&;VdE`3&tyeR%F7JI|C=-=1{Gc}}1H zP-DlM%a?Of@IjE+Ps!HG%K4|Q-6Cr_R06J>Qls`99&Zc+5r+NC0=U!A^J)dNun_6( zb!eKxAxUFD+eY|5Ve*@@O20ke{#0^2hTFqPABWg(J#?pDB{F2{63;?E#nxOypl10g zDoO$nG4MI;XGv4EG16k>A%vAq{I%*Sty*YrI1|~|sK`E1MiJk53Yl;9$#8I1^E+6@ zIkW~Jn&i{bY#GTXXndlTfj5W6Pk@tFB%o0FeEEtm8jhcoA+Fe8BM+mn3WJrMahYn_ z$2kUZ7FmbmrfGD?`OWd|*)>Rs>zn$tAWT!-o+81j+KT%)@4j#O=MW<4O4VdeI9o<^ z86fYpv}dM0Jyuq=7VItaH+Z<=a+H1cnBT~%mXC+T{7l{3j_xDPH(8ODr`k-kZc<7{j7?H{=n!{k^s!IjBp2z7~1lA021<*7q zQHce1VKD!Ib**J?&0+Z5OzNnec^bzq-^7A5^ncnCqRWzl*uTHbO>iYZi)Th5NS{gQ zqLC+!#}?(>rlFG*6}89{%agN`OD_qI+O62^zgzs)V8R#jX_J$)ngI|wI8}|JaRiDQ z?3O)yZR}y~L&_#;diB9B>fXc|3M(J82ZV2JTkTjvCVEO}2OTOP0j=&>W@+W}0up$V zMF(|m^mzJ!_}>27OH9R_&aVG_u7JUJ&b3oo9z|i>eU62L^LudAMDtL3{%gn6#1=aZ zz|>9bPjbL6mipaKdX0mp&!87~l`62A18QW7*@?yWaL&-nrGXJiM84GD z7Lva$jZ#O2G@QKplU>q?C4^$iP^gC#RHQOG!w?C30&fzBV16no0 zpN|eO%&MPakAM3k?!SFedFjHkfCF=@!pix=l;zuY!VP~*PivxFG|jqY%kV=<9q1am zsau1J-{Hd>5L2X9xMXpMEf4XHw3G__5Aw>^tKmYbUbI|7x{cKi*H+FWM)Cl`F$cEA zo86S?Sx7@OmIS=Ty5sKa66N0`rmwlCx#~Ii61D*^A54ZkS2E?%|2o12Bk7ljV2vKV zIm<}yd&pmUQ(JMjbgWmMPLj0UKXDp(*Y{e_^Zp{>y2luyXei~<7tx-}T&b^qA%MPi3`Eey`_=0tp7QoqSCTb#jH zo}s0E1ikr0XD5W+{?*7Kal{%JQ=#Bh{N48R!@fJ~v{|7WEOL&DwQ^<|x2A31s> z-5~4Uvh-bbWm4S=d#92BR!c`-YjtUmu#L~bF@ftIkiJlPm~OI#fLTc6%KAz!-z-Q5 zOH}1@@50A0l9j-NDcJ{r%nTjgLk%2;L=&IsDOw^k1l>z^@OOrLlQh)#yHODLd&V~_YO zFAFw2Qp0!}8tTyxbu1PL^DW)%Chsj0^Y~zsh@p9|;NusdY?`WqjaaC=1zuRBiiYN7 zfU`~mA^H4eNI*+b9mM2e`JdQ@a zVNydrqSA4KoX?G5Eh@x$$kPn?Wfu1$Nl_y{QmgmmvMg;J`LZqqfhNBb$inQdehPY} zX2xPeYFGt_%Atz}^J(j&Ovb|2vQp7w6PXT45w0UuxNG7a_H>%Az(U{qOoty?hK1vf zE=qLM5ts%F>}zeYJ~8Nv^$7Be{H4KAJ-*!1Wt{o-p>_%MR)MdvW@_J=b=36`bgM<+Ear zg`sN>l-(EDsLDg>dqTM{KqTwjn7Sy;zEVWFXgFe*9ysen6772#Q4Prx!-uLCqLw``^=InbZW+O@mnN}rAs51cLQ3AQIE zi)VCTjF}f8*XDlrA>E^US%%0P3|O}@ky`HTRrD*S{ZyUX=wpoS)*4Hn=I0SjW}G)M z49x7C3+nR4A%*8Sc_{24Yx*y>kKL&* z4OkZxov=uxS@OQLB|&mi?_truRO>dh>`yp42HS%Zw?1;7iYt#Ts0-V7O+;AV5ah9y zhJD<24lP%wNrgjuqve$zb;8t@gz@UW({1>kTo~IJv)M#_C5Qt}Chg9aC^KvrF=GhJ zZEI`l=#S`#8>8&!z1xUsdI-gY z7$E!6)um3*e;o1CwnLxfjOMma`xb7fUqO2^$2G-WNqFazG23MQ5Pd~i+kFz{5dK0f z2NdQtZA4UMa%Fsw{gN+>$CH`IS%S*}$ES`9V5>bTf+IPLn<_%!IG#wNmTo!Qss&%)9Qt(T`1tMp^wx&JT{@3)rO?li`y}DDZ)NBFBpdSUi zO-;}9jg;Z^`NDu}-COv<2?!f6SdqAok?U=IXZr>hQ zVq(iSz!YM)HjpYXMd;cu7o78ccoKUj*!6=DzWrr@AI3@qGMJoM(eii-{@{1ltvXAF z?aV{d=9h4&0*?ox2+3DJ3)cL0x>Y+u&49dUGzA9eXU75SX!r-euc*XSe+Oe`$o?De z(m(!5XjQ2Y@$}y3DL;ew00T_|u1tS|6RW=pCz4UWObnqcW1)lz{bogl9RsF(n$ff6 zl4C*sn#aQ-fBzS2AmLIQ+)5B{niA(=^s-zo`-1+{z~{^?zQ*B<+zbmt;u_cnQk~Df z#oF^g>%05l`>RH-&5}n#@h}bvrYxrPTlInWfq#ls-}anyI&iGf(w~s>lSMVQZ=aM_ zHmUfvUO=3(pCLZ#G~DrSWnB!G7>#V2ZSs=$nYGW~UW6eDf6z@+9MVW_^%}VB|aB=dAB$<(36HgXj!aXB#ga zJQ?5m^8b@6(?h65Do8wAevzYpLSWD@HYgwYt`*J!+d>tQP~p*cN7ru()1xLSGAQ7* zw|YJE3W}X{t9)~rPxX*ueE(bZc@fq3LG0;ot1s4E@r-i4o4xyR%D;@#>fg|q>o4C` zO;s3{B%J|QICUX_Hr~#ER3(;eoX=+an7trX>L6R_x_zd~vE_-sJ}8ht9OYF*%JLZ_ zrLoosVpaT1xx!UWnl;28Z9YaU+_g(Vn@XmA{-h`YnLhmtVgLL>>ne1RYkE9mn$PKa`OXXhuhTHC~cb=Zo{PdD?iOllaIw;P8w_)W= z=zXh;K3}wHyVnHE^xr_=EooEb%D`z1QYR=k76jQmaXzq~@YoCg_Ul0kftjMs`0TX7 zt|);;AqiI&(ZK7VaCTfJSv&DCH~W??r7(`{a~HOqTA>n__tG|eS#ifhTeefG*`u3( zc$D=%i$s*{z=ek}r!J3q(;JOM(aX&(;%Whg73!#Z>|zn4+=0_^IN_)#*-oMIz3;u2 zY_WlZz?Vh#1fx__xd=gQ(~&L*8esVGqr6T;u|eHGF|zK)?|+qt>U>`bK5ld!(Y;YS&e8i8$gkHbiS%c3Sum1FDnRb}(7 zn|VIKF)pI!R_@N(Hu0B+K1#M{=z#E!kc^lYN>V<(qOq5xWwCtkRQ5n<@h#hP+T&H* z%Nb`cilOKOeD57El``4hyWyLs5D_D2^FX5Sv!yjUI-PjLvhvV(SZ<};JdB`JD z?76`}1teof#)4&YggibhkLE^NSPY1@uhn&%d6b3mBu|2+g`5M@&5S1OV+rim!(%8~ z5B4paO5_XeWb5)Q=5$_=cp8CgB;|r5r;z+-i?3DNLfQ}IRy)1cw}IHGYKARBS1?9p zy0e~JNXQm7JtWxl9dan~+NZo(ya0}QfK4#_;~6&2ot2E>2av{yC%3}c6B>hGK7OYX zyxfIHo7)! zpdG{YwT486Y?98i-H95}<-lM`#oqS1SX} z3x4qr7b|qN9pqG53G`YN{-P+__<;+zfWjlMQN8@`_wPCP9gY1?W(KKn;>>(0UvBV8 z|8?x5gJ)3cUn12uErfpzszNw|y%z<6LX&bd21FRN`ON8&^#tEZmDOgHo`c_g&&eea zUofSo{jH=~l^9;t$|thAmeW+q_ zoxZx7FZ00jgtGZ~r8sZBlQ?ah$ZXpIL%N;T5W)|}O?=IG4yzZf$~4K5 zIm~x`g-@5polm!R6)jIc-?Y~kz~MG|R7?OI&J_zF8{Vi+19G(2LL6LTL3GLxRO@et z8(WT1KZYY?32fJa_FU~YwcGzTpEFOSPO?icFQHEGJBkR8ak$&lTJVY8cF6$)@}i5a zwYzui?|}=~TbG5DQNPruLQ8zASRDoHwcMrQGPylupM_CboCL`Ss4q8E)965++}t3g zKu=P3nF>GACRbna{M}f&P8vtta&V!Zphly>D2{z}sX#%aBv)t}2jNg`0M$#~$|Xhb z4`+fccg%g?5gOMVe4h;~KX0y_KBP=#D2_=cl$O5$)aUOe(1J2=OSP1*uP}aCPF1rX zPM)NsRdilO1F=saN0sAF^jdwA(N*o=dX&Q3wdaeCKTsTt!^`;QdG+vXu83~2LYw6gl};D)SZ4M}^JiYR>J&?t`#X!J zq+X`H_xOoJ{~~@u`1Q;YP4JH|fWmv$+>d@9%Q#z$+znt{MLe7Dl$0`qD#c n!0pJC#X)i#^rr?RK2vc8p|WWr#BBo{1f(FNB3&VA68OIWy&hn> literal 0 HcmV?d00001 diff --git a/manual/template_lang.rst b/manual/template_lang.rst index c677cba8d1..343f5861a9 100644 --- a/manual/template_lang.rst +++ b/manual/template_lang.rst @@ -630,7 +630,7 @@ the value of a custom field #genre. You cannot do this in the :ref:`Single Funct The example shows several things: -* `TPM` is used if the expression begins with ``:'`` and ends with ``'``. Anything else is assumed to be in :ref:`Single Function Mode `. +* `TPM` is used if the expression begins with ``:'`` and ends with ``'}``. Anything else is assumed to be in :ref:`Single Function Mode `. * the variable ``$`` stands for the field named in the template: the expression is operating upon, ``#series`` in this case. * functions must be given all their arguments. There is no default value. For example, the standard built-in functions must be given an additional initial parameter indicating the source field. * white space is ignored and can be used anywhere within the expression. @@ -642,16 +642,56 @@ In `TPM`, using ``{`` and ``}`` characters in string literals can lead to errors As with `General Program Mode`, for functions documented under :ref:`Single Function Mode ` you must supply the value the function is to act upon as the first parameter in addition to the documented parameters. In `TPM` you can use ``$`` to access the value specified by the ``lookup name`` for the template expression. -Stored general program mode templates +.. _python_mode: + +Python Template Mode +----------------------------------- + +Python Template Mode (PTM) lets you write templates using native python and the `calibre API `_. The database API will be of most use; further discussion is beyond the scope of this manual. PTM templates are faster and can do more complicated operations but you must know how to write code in python using the calibre API. + +A PTM template begins with:: + + python: + def evaluate(book, db, globals, arguments, **kwargs): + # book is a calibre metadata object + # db is a calibre legacy database object + # globals is the template global variable dictionary + # arguments is a list of arguments if the template is called by a GPM template, otherwise None + # kwargs is a dictionary provided for future use + + # Python code goes here + return 'a string' + +You can add the above text to your template using the context menu, usually accessed with a right click. The comments are not significant and can be removed. You must use python indenting. + +Here is an example of a PTM template that produces a list of all the authors for a series. The list is stored in a `Column built from other columns, behaves like tags`. It shows in :guilabel:`Book details` and has the `on separate lines` checked (in :guilabel:`Preferences->Look & feel->Book details`). That option requires the list to be comma-separated. To satisfy that requirement the template converts commas in author names to semicolons then builds a comma-separated list of authors. The authors are then sorted, which is why the template uses author_sort.:: + + python: + def evaluate(book, db, globals, arguments, **kwargs): + if book.series is None: + return '' + ans = set() + for id_ in db.search_getting_ids(f'series:"={book.series}"', ''): + ans.update([v.strip() for v in db.new_api.field_for('author_sort', id_).split('&')]) + return ', '.join(v.replace(',', ';') for v in sorted(ans)) + +The output in :guilabel:`Book details` looks like this: + +.. image:: images/python_template_example.png + :align: center + :alt: E-book conversion dialog + :class: half-width-img + +Stored templates ---------------------------------------- -:ref:`General Program Mode ` supports saving templates and calling those templates from another template, much like calling stored functions. You save templates using :guilabel:`Preferences->Advanced->Template functions`. More information is provided in that dialog. You call a template the same way you call a function, passing positional arguments if desired. An argument can be any expression. Examples of calling a template, assuming the stored template is named ``foo``: +Both :ref:`General Program Mode ` and :ref:`Python Template Mode ` support saving templates and calling those templates from another template, much like calling stored functions. You save templates using :guilabel:`Preferences->Advanced->Template functions`. More information is provided in that dialog. You call a template the same way you call a function, passing positional arguments if desired. An argument can be any expression. Examples of calling a template, assuming the stored template is named ``foo``: * ``foo()`` -- call the template passing no arguments. * ``foo(a, b)`` call the template passing the values of the two variables ``a`` and ``b``. * ``foo(if field('series') then field('series_index') else 0 fi)`` -- if the book has a ``series`` then pass the ``series_index``, otherwise pass the value ``0``. -You retrieve the arguments passed in the call to the stored template using the ``arguments`` function. It both declares and initializes local variables, effectively parameters. The variables are positional; they get the value of the parameter given in the call in the same position. If the corresponding parameter is not provided in the call then ``arguments`` assigns that variable the provided default value. If there is no default value then the variable is set to the empty string. For example, the following ``arguments`` function declares 2 variables, ``key``, ``alternate``:: +In GPM you retrieve the arguments passed in the call to the stored template using the ``arguments`` function. It both declares and initializes local variables, effectively parameters. The variables are positional; they get the value of the parameter given in the call in the same position. If the corresponding parameter is not provided in the call then ``arguments`` assigns that variable the provided default value. If there is no default value then the variable is set to the empty string. For example, the following ``arguments`` function declares 2 variables, ``key``, ``alternate``:: arguments(key, alternate='series') @@ -661,6 +701,8 @@ Examples, again assuming the stored template is named ``foo``: * ``foo('series', '#genre')`` the variable ``key`` is assigned the value ``'series'`` and the variable ``alternate`` is assigned the value ``'#genre'``. * ``foo()`` -- the variable ``key`` is assigned the empty string and the variable ``alternate`` is assigned the value ``'series'``. +In PTM the arguments are passed in the ``arguments`` parameter, which is a list of strings. There isn't any way to specify default values. You must check the length of the ``arguments`` list to be sure that the number of arguments is what you expect. + An easy way to test stored templates is using the ``Template tester`` dialog. For ease of access give it a keyboard shortcut in :guilabel:`Preferences->Advanced->Keyboard shortcuts->Template tester`. Giving the ``Stored templates`` dialog a shortcut will help switching more rapidly between the tester and editing the stored template's source code. Providing additional information to templates diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index d8c5ed7068..36f3eac3ea 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -23,7 +23,7 @@ from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog from calibre.library.coloring import (displayable_columns, color_row_key) from calibre.utils.config_base import tweaks from calibre.utils.date import DEFAULT_DATE -from calibre.utils.formatter_functions import formatter_functions +from calibre.utils.formatter_functions import formatter_functions, StoredObjectType from calibre.utils.formatter import StopException from calibre.utils.icu import sort_key from calibre.utils.localization import localize_user_manual_link @@ -42,49 +42,89 @@ class ParenPosition: class TemplateHighlighter(QSyntaxHighlighter): + # Code in this class is liberally borrowed from gui2.widgets.PythonHighlighter BN_FACTOR = 1000 - KEYWORDS = ["program", 'if', 'then', 'else', 'elif', 'fi', 'for', 'rof', - 'separator', 'break', 'continue', 'return', 'in', 'inlist', - 'def', 'fed', 'limit'] + KEYWORDS_GPM = ["program", 'if', 'then', 'else', 'elif', 'fi', 'for', 'rof', + 'separator', 'break', 'continue', 'return', 'in', 'inlist', + 'def', 'fed', 'limit'] + + KEYWORDS_PYTHON = ["and", "as", "assert", "break", "class", "continue", "def", + "del", "elif", "else", "except", "exec", "finally", "for", "from", + "global", "if", "import", "in", "is", "lambda", "not", "or", + "pass", "print", "raise", "return", "try", "while", "with", + "yield"] + + BUILTINS_PYTHON = ["abs", "all", "any", "basestring", "bool", "callable", "chr", + "classmethod", "cmp", "compile", "complex", "delattr", "dict", + "dir", "divmod", "enumerate", "eval", "execfile", "exit", "file", + "filter", "float", "frozenset", "getattr", "globals", "hasattr", + "hex", "id", "int", "isinstance", "issubclass", "iter", "len", + "list", "locals", "long", "map", "max", "min", "object", "oct", + "open", "ord", "pow", "property", "range", "reduce", "repr", + "reversed", "round", "set", "setattr", "slice", "sorted", + "staticmethod", "str", "sum", "super", "tuple", "type", "unichr", + "unicode", "vars", "xrange", "zip"] + + CONSTANTS_PYTHON = ["False", "True", "None", "NotImplemented", "Ellipsis"] def __init__(self, parent=None, builtin_functions=None): super().__init__(parent) self.initialize_formats() - self.initialize_rules(builtin_functions) + self.initialize_rules(builtin_functions, for_python=False) self.regenerate_paren_positions() self.highlighted_paren = False - def initialize_rules(self, builtin_functions): + def initialize_rules(self, builtin_functions, for_python=False): + self.for_python = for_python r = [] def a(a, b): r.append((re.compile(a), b)) - a( - r"\b[a-zA-Z]\w*\b(?!\(|\s+\()" - r"|\$+#?[a-zA-Z]\w*", - "identifier") + if not for_python: + a( + r"\b[a-zA-Z]\w*\b(?!\(|\s+\()" + r"|\$+#?[a-zA-Z]\w*", + "identifier") - a( - "|".join([r"\b%s\b" % keyword for keyword in self.KEYWORDS]), - "keyword") + a( + "|".join([r"\b%s\b" % keyword for keyword in self.KEYWORDS_GPM]), + "keyword") - a( - "|".join([r"\b%s\b" % builtin for builtin in - (builtin_functions if builtin_functions else - formatter_functions().get_builtins())]), - "builtin") + a( + "|".join([r"\b%s\b" % builtin for builtin in + (builtin_functions if builtin_functions else + formatter_functions().get_builtins())]), + "builtin") + a(r"""(? -1: + self.setCurrentBlockState(state) + self.setFormat(i, len(text), self.Formats["string"]) + if self.generate_paren_positions: t = str(text) i = 0 @@ -327,6 +399,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): self.set_mi(mi, fm) self.last_text = '' + self.highlighting_gpm = True self.highlighter = TemplateHighlighter(self.textbox.document(), builtin_functions=self.builtins) self.textbox.cursorPositionChanged.connect(self.text_cursor_changed) self.textbox.textChanged.connect(self.textbox_changed) @@ -494,9 +567,12 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): ca.setIcon(QIcon.ic('ok.png')) ca.triggered.connect(partial(self.set_word_wrap, not word_wrapping)) m.addSeparator() - ca = m.addAction(_('Load template from the Template tester')) - ca.triggered.connect(self.load_last_template_text) + ca = m.addAction(_('Add python template definition text')) + ca.triggered.connect(self.add_python_template_header_text) m.addSeparator() + ca = m.addAction(_('Load template from the Template tester')) + m.addSeparator() + ca.triggered.connect(self.load_last_template_text) ca = m.addAction(_('Load template from file')) ca.setIcon(QIcon.ic('document_open.png')) ca.triggered.connect(self.load_template_from_file) @@ -505,6 +581,19 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): ca.triggered.connect(self.save_template) m.exec(self.textbox.mapToGlobal(point)) + def add_python_template_header_text(self): + self.textbox.setPlainText('python:\n' + 'def evaluate(book, db, globals, arguments, **kwargs):\n' + '\t# book is a calibre metadata object\n' + '\t# db is a calibre legacy database object\n' + '\t# globals is the template global variable dictionary\n' + '\t# arguments is a list of arguments if the template is ' + 'called by a GPM template, otherwise None\n' + '\t# kwargs is a dictionary provided for future use' + '\n\n\t# Python code goes here\n' + "\treturn 'a string'" + + self.textbox.toPlainText()) + def set_word_wrap(self, to_what): gprefs['gpm_template_editor_word_wrap_mode'] = to_what self.textbox.setWordWrapMode(QTextOption.WrapMode.WordWrap if to_what else QTextOption.WrapMode.NoWrap) @@ -673,6 +762,17 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): def textbox_changed(self): cur_text = str(self.textbox.toPlainText()) + if cur_text.startswith('python:'): + if self.highlighting_gpm == True: + self.highlighter.initialize_rules(self.builtins, True) + self.highlighting_gpm = False + self.break_box.setChecked(False) + self.break_box.setEnabled(False) + print(self.break_box.isEnabled()) + elif not self.highlighting_gpm: + self.highlighter.initialize_rules(self.builtins, False) + self.highlighting_gpm = True + self.break_box.setEnabled(True) if self.last_text != cur_text: self.last_text = cur_text self.highlighter.regenerate_paren_positions() @@ -707,14 +807,15 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): pos_in_block) def function_type_string(self, name, longform=True): - if self.all_functions[name].is_python: + if self.all_functions[name].object_type is StoredObjectType.PythonFunction: if name in self.builtins: return (_('Built-in template function') if longform else _('Built-in function')) return (_('User defined Python template function') if longform else _('User function')) - else: - return (_('Stored user defined template') if longform else _('Stored template')) + elif self.all_functions[name].object_type is StoredObjectType.StoredPythonTemplate: + return (_('Stored user defined python template') if longform else _('Stored template')) + return (_('Stored user defined GPM template') if longform else _('Stored template')) def function_changed(self, toWhat): name = str(self.function.itemData(toWhat)) diff --git a/src/calibre/gui2/dialogs/template_dialog.ui b/src/calibre/gui2/dialogs/template_dialog.ui index e97d602e26..e384531ad4 100644 --- a/src/calibre/gui2/dialogs/template_dialog.ui +++ b/src/calibre/gui2/dialogs/template_dialog.ui @@ -344,8 +344,10 @@ you the value as well as all the local variables</p> <p>The text of the template program goes in this box. - Don't forget that a General Program Mode template must begin with - the word "program:".</p> + A General Program Mode template must begin with the word "program:". + A python template must begin with the word "python:" followed by a + function definition line. There is a context menu item you can use + to enter the first lines of a python template.</p> diff --git a/src/calibre/gui2/preferences/template_functions.py b/src/calibre/gui2/preferences/template_functions.py index 59b80a76e8..29982a5d45 100644 --- a/src/calibre/gui2/preferences/template_functions.py +++ b/src/calibre/gui2/preferences/template_functions.py @@ -13,7 +13,8 @@ from calibre.gui2.preferences.template_functions_ui import Ui_Form from calibre.gui2.widgets import PythonHighlighter from calibre.utils.formatter_functions import ( compile_user_function, compile_user_template_functions, formatter_functions, - function_pref_is_python, function_pref_name, load_user_template_functions + function_object_type, function_pref_name, load_user_template_functions, + StoredObjectType ) from polyglot.builtins import iteritems @@ -48,9 +49,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):

  • your parameters: you must supply one or more formal parameters. The number must match the arg count box, unless arg count is -1 (variable number or arguments), in which case the last argument must - be *args. At least one argument is required, and is usually the value of - the field being operated upon. Note that when writing in basic template - mode, the user does not provide this first argument. Instead it is + be *args. Note that when a function is called in basic template + mode at least one argument is always passed. It is supplied by the formatter.

    • @@ -87,10 +87,12 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): in template processing. You use a stored template in another template as if it were a template function, for example 'some_name(arg1, arg2...)'.

    -

    Stored templates must use General Program Mode -- they must begin with - the text '{0}'. You retrieve arguments passed to a stored template using - the '{1}()' template function, as in '{1}(var1, var2, ...)'. The passed - arguments are copied to the named variables.

    +

    Stored templates must use either General Program Mode -- they must + either begin with the text '{0}' or be {1}. You retrieve arguments + passed to a GPM stored template using the '{2}()' template function, as + in '{2}(var1, var2, ...)'. The passed arguments are copied to the named + variables. Arguments passed to a python template are in the '{2}' + parameter. Arguments are always strings.

    For example, this stored template checks if any items are in a list, returning '1' if any are found and '' if not.

    @@ -112,7 +114,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): See the template language tutorial for more information.

    ''') - self.st_textBrowser.setHtml(help_text.format('program:', 'arguments')) + self.st_textBrowser.setHtml(help_text.format('program:', 'python templates', 'arguments')) self.st_textBrowser.adjustSize() self.st_show_hide_help_button.clicked.connect(self.st_show_hide_help) self.st_textBrowser_height = self.st_textBrowser.height() @@ -150,14 +152,14 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.builtin_source_dict = {} self.funcs = {k:v for k,v in formatter_functions().get_functions().items() - if v.is_python} + if v.object_type is StoredObjectType.PythonFunction} self.builtins = formatter_functions().get_builtins_and_aliases() self.st_funcs = {} try: for v in self.db.prefs.get('user_template_functions', []): - if not function_pref_is_python(v): + if function_object_type(v) is not StoredObjectType.PythonFunction: self.st_funcs.update({function_pref_name(v):compile_user_function(*v)}) except: if question_dialog(self, _('Template functions'), @@ -281,34 +283,53 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): error_dialog(self.gui, _('Template functions'), _('Function not defined'), show=True) - def create_button_clicked(self, use_name=None): - self.changed_signal.emit() - name = use_name if use_name else str(self.function_name.currentText()) - name = name.split(' -- ')[0] + def check_errors_before_save(self, name, for_replace=False): + # Returns True if there is an error if not name: error_dialog(self.gui, _('Template functions'), _('Name cannot be empty'), show=True) - return - if name in self.funcs: + return True + if not for_replace and name in self.funcs: error_dialog(self.gui, _('Template functions'), _('Name %s already used')%(name,), show=True) - return + return True if name in {function_pref_name(v) for v in self.db.prefs.get('user_template_functions', []) - if not function_pref_is_python(v)}: + if function_object_type(v) is not StoredObjectType.PythonFunction}: error_dialog(self.gui, _('Template functions'), _('The name {} is already used for stored template').format(name), show=True) - return + return True if self.argument_count.value() == 0: - box = warning_dialog(self.gui, _('Template functions'), - _('Argument count should be -1 or greater than zero. ' - 'Setting it to zero means that this function cannot ' - 'be used in single function mode.'), det_msg='', - show=False, show_copy_button=False) - box.bb.setStandardButtons(box.bb.standardButtons() | QDialogButtonBox.StandardButton.Cancel) - box.det_msg_toggle.setVisible(False) - if not box.exec(): - return + if not question_dialog(self.gui, _('Template functions'), + _('Setting argument count to to zero means that this ' + 'function cannot be used in single function mode. ' + 'Is this OK?'), + det_msg='', + show_copy_button=False, + default_yes=False, + skip_dialog_name='template_functions_zero_args_warning', + skip_dialog_msg='Ask this question again', + yes_text=_('Save the function'), + no_text=_('Cancel the save')): + print('cancelled') + return True + try: + prog = str(self.program.toPlainText()) + cls = compile_user_function(name, str(self.documentation.toPlainText()), + self.argument_count.value(), prog) + except: + error_dialog(self.gui, _('Template functions'), + _('Exception while compiling function'), show=True, + det_msg=traceback.format_exc()) + return True + return False + + def create_button_clicked(self, use_name=None, need_error_checks=True): + name = use_name if use_name else str(self.function_name.currentText()) + name = name.split(' -- ')[0] + if need_error_checks and self.check_errors_before_save(name, for_replace=False): + return + self.changed_signal.emit() try: prog = str(self.program.toPlainText()) cls = compile_user_function(name, str(self.documentation.toPlainText()), @@ -364,8 +385,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): def replace_button_clicked(self): name = str(self.function_name.itemData(self.function_name.currentIndex())) + if self.check_errors_before_save(name, for_replace=True): + return self.delete_button_clicked() - self.create_button_clicked(use_name=name) + self.create_button_clicked(use_name=name, need_error_checks=False) def refresh_gui(self, gui): pass @@ -428,14 +451,14 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.changed_signal.emit() name = use_name if use_name else str(self.te_name.currentText()) for k,v in formatter_functions().get_functions().items(): - if k == name and v.is_python: + if k == name and v.object_type is StoredObjectType.PythonFunction: error_dialog(self.gui, _('Stored templates'), - _('The name {} is already used for template function').format(name), show=True) + _('The name {} is already used by a template function').format(name), show=True) try: prog = str(self.te_textbox.toPlainText()) - if not prog.startswith('program:'): + if not prog.startswith(('program:', 'python:')): error_dialog(self.gui, _('Stored templates'), - _('The stored template must begin with "program:"'), show=True) + _("The stored template must begin with '{0}' or '{1}'").format('program:', 'python:'), show=True) cls = compile_user_function(name, str(self.template_editor.new_doc.toPlainText()), 0, prog) diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index 938ac696d8..7aaab09fb3 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -11,12 +11,14 @@ __docformat__ = 'restructuredtext en' import re, string, traceback, numbers from functools import partial from math import modf +from sys import exc_info from calibre import prints from calibre.constants import DEBUG from calibre.ebooks.metadata.book.base import field_metadata from calibre.utils.config import tweaks -from calibre.utils.formatter_functions import formatter_functions +from calibre.utils.formatter_functions import ( + formatter_functions, get_database, function_object_type, StoredObjectType) from calibre.utils.icu import strcmp from polyglot.builtins import error_message @@ -137,7 +139,8 @@ class StoredTemplateCallNode(Node): def __init__(self, line_number, name, function, expression_list): Node.__init__(self, line_number, 'call template: ' + name + '()') self.node_type = self.NODE_CALL_STORED_TEMPLATE - self.function = function + self.name = name + self.function = function # instance of the definition class self.expression_list = expression_list @@ -579,16 +582,20 @@ class _Parser: return LocalFunctionCallNode(self.line_number, name, arguments) def call_expression(self, name, arguments): - subprog = self.funcs[name].cached_parse_tree - if subprog is None: + compiled_func = self.funcs[name].cached_compiled_text + if compiled_func is None: text = self.funcs[name].program_text - if not text.startswith('program:'): - self.error(_("A stored template must begin with '{0}'").format('program:')) - text = text[len('program:'):] - subprog = _Parser().program(self.parent, self.funcs, - self.parent.lex_scanner.scan(text)) - self.funcs[name].cached_parse_tree = subprog - return StoredTemplateCallNode(self.line_number, name, subprog, arguments) + if function_object_type(text) is StoredObjectType.StoredGPMTemplate: + text = text[len('program:'):] + compiled_func = _Parser().program(self.parent, self.funcs, + self.parent.lex_scanner.scan(text)) + elif function_object_type(text) is StoredObjectType.StoredPythonTemplate: + text = text[len('python:'):] + compiled_func = self.parent.compile_python_template(text) + else: + self.error(_("A stored template must begin with '{0}' or {1}").format('program:', 'python:')) + self.funcs[name].cached_compiled_text = compiled_func + return StoredTemplateCallNode(self.line_number, name, self.funcs[name], arguments) def top_expr(self): return self.or_expr() @@ -775,7 +782,7 @@ class _Parser: if id_ in self.local_functions: return self.local_call_expression(id_, arguments) # Check for calling a stored template - if id_ in self.func_names and not self.funcs[id_].is_python: + if id_ in self.func_names and self.funcs[id_].object_type is not StoredObjectType.PythonFunction: return self.call_expression(id_, arguments) # We must have a reference to a formatter function. Check if # the right number of arguments were supplied @@ -846,7 +853,8 @@ class _Interpreter: try: if is_call: - ret = self.do_node_stored_template_call(StoredTemplateCallNode(1, prog, None), args=args) + # prog is an instance of the function definition class + ret = self.do_node_stored_template_call(StoredTemplateCallNode(1, prog.name, prog, None), args=args) else: ret = self.expression_list(prog) except ReturnExecuted as e: @@ -1014,7 +1022,10 @@ class _Interpreter: else: saved_line_number = None try: - val = self.expression_list(prog.function) + if function_object_type(prog.function.program_text) is StoredObjectType.StoredGPMTemplate: + val = self.expression_list(prog.function.cached_compiled_text) + else: + val = self.parent._run_python_template(prog.function.cached_compiled_text, args) except ReturnExecuted as e: val = e.get_value() self.override_line_number = saved_line_number @@ -1526,14 +1537,62 @@ class TemplateFormatter(string.Formatter): def _eval_sfm_call(self, template_name, args, global_vars): func = self.funcs[template_name] - tree = func.cached_parse_tree - if tree is None: - tree = self.gpm_parser.program(self, self.funcs, - self.lex_scanner.scan(func.program_text[len('program:'):])) - func.cached_parse_tree = tree - return self.gpm_interpreter.program(self.funcs, self, tree, None, - is_call=True, args=args, - global_vars=global_vars) + compiled_text = func.cached_compiled_text + if func.object_type is StoredObjectType.StoredGPMTemplate: + if compiled_text is None: + compiled_text = self.gpm_parser.program(self, self.funcs, + self.lex_scanner.scan(func.program_text[len('program:'):])) + func.cached_compiled_text = compiled_text + return self.gpm_interpreter.program(self.funcs, self, func, None, + is_call=True, args=args, + global_vars=global_vars) + elif function_object_type(func) is StoredObjectType.StoredPythonTemplate: + if compiled_text is None: + compiled_text = self.compile_python_template(func.program_text[len('python:'):]) + func.cached_compiled_text = compiled_text + print(args) + return self._run_python_template(compiled_text, args) + + def _eval_python_template(self, template, column_name): + if column_name is not None and self.template_cache is not None: + func = self.template_cache.get(column_name + '::python', None) + if not func: + func = self.compile_python_template(template) + self.template_cache[column_name + '::python'] = func + else: + func = self.compile_python_template(template) + return self._run_python_template(func, arguments=None) + + def _run_python_template(self, compiled_template, arguments): + try: + return compiled_template(book=self.book, + db=get_database(self.book, get_database(self.book, None)), + globals=self.global_vars, + arguments=arguments) + except Exception as e: + ss = traceback.extract_tb(exc_info()[2])[-1] + raise ValueError(_('Error in function {0} on line {1} : {2} - {3}').format( + ss.name, ss.lineno, type(e).__name__, str(e))) + + def compile_python_template(self, template): + def replace_func(mo): + return mo.group().replace('\t', ' ') + + prog ='\n'.join([re.sub(r'^\t*', replace_func, line) + for line in template.splitlines()]) + locals_ = {} + if DEBUG and tweaks.get('enable_template_debug_printing', False): + print(prog) + try: + exec(prog, locals_) + func = locals_['evaluate'] + return func + except SyntaxError as e: + raise(ValueError( + _('Syntax error on line {0} column {1}: text {2}').format(e.lineno, e.offset, e.text))) + except KeyError: + raise(ValueError(_("Error: the {0} function is not defined in the template").format('evaluate'))) + # ################# Override parent classes methods ##################### def get_value(self, key, args, kwargs): @@ -1587,7 +1646,7 @@ class TemplateFormatter(string.Formatter): else: args = self.arg_parser.scan(fmt[p+1:])[0] args = [self.backslash_comma_to_comma.sub(',', a) for a in args] - if not func.is_python: + if func.object_type is not StoredObjectType.PythonFunction: args.insert(0, val) val = self._eval_sfm_call(fname, args, self.global_vars) else: @@ -1615,6 +1674,8 @@ class TemplateFormatter(string.Formatter): if fmt.startswith('program:'): ans = self._eval_program(kwargs.get('$', None), fmt[8:], self.column_name, global_vars, break_reporter) + elif fmt.startswith('python:'): + ans = self._eval_python_template(fmt[7:], self.column_name) else: ans = self.vformat(fmt, args, kwargs) if self.strip_results: @@ -1728,7 +1789,7 @@ class TemplateFormatter(string.Formatter): traceback.print_exc() if column_name: prints('Error evaluating column named:', column_name) - ans = error_value + ' ' + error_message(e) + ans = str(e) return ans finally: self.restore_state(state) diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 6edf05d77b..0f1238c498 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -14,6 +14,7 @@ __docformat__ = 'restructuredtext en' import inspect, re, traceback, numbers from contextlib import suppress from datetime import datetime, timedelta +from enum import Enum from functools import partial from math import trunc, floor, ceil, modf @@ -28,6 +29,12 @@ from calibre.utils.localization import calibre_langcode_to_name, canonicalize_la from polyglot.builtins import iteritems, itervalues +class StoredObjectType(Enum): + PythonFunction = 1 + StoredGPMTemplate = 2 + StoredPythonTemplate = 3 + + class FormatterFunctions: error_function_body = ('def evaluate(self, formatter, kwargs, mi, locals):\n' @@ -123,6 +130,39 @@ def formatter_functions(): return _ff +def only_in_gui_error(name): + raise ValueError(_('The function {} can be used only in the GUI').format(name)) + + +def get_database(mi, name): + proxy = mi.get('_proxy_metadata', None) + if proxy is None: + if name is not None: + only_in_gui_error(name) + return None + wr = proxy.get('_db', None) + if wr is None: + if name is not None: + raise ValueError(_('In function {}: The database has been closed').format(name)) + return None + cache = wr() + if cache is None: + if name is not None: + raise ValueError(_('In function {}: The database has been closed').format(name)) + return None + wr = getattr(cache, 'library_database_instance', None) + if wr is None: + if name is not None: + only_in_gui_error() + return None + db = wr() + if db is None: + if name is not None: + raise ValueError(_('In function {}: The database has been closed').format(name)) + return None + return db + + class FormatterFunction: doc = _('No documentation provided') @@ -130,7 +170,7 @@ class FormatterFunction: category = 'Unknown' arg_count = 0 aliases = [] - is_python = True + object_type = StoredObjectType.PythonFunction def evaluate(self, formatter, kwargs, mi, locals, *args): raise NotImplementedError() @@ -145,25 +185,10 @@ class FormatterFunction: return str(ret) def only_in_gui_error(self): - raise ValueError(_('The function {} can be used only in the GUI').format(self.name)) + only_in_gui_error(self.name) def get_database(self, mi): - proxy = mi.get('_proxy_metadata', None) - if proxy is None: - self.only_in_gui_error() - wr = proxy.get('_db', None) - if wr is None: - raise ValueError(_('In function {}: The database has been closed').format(self.name)) - cache = wr() - if cache is None: - raise ValueError(_('In function {}: The database has been closed').format(self.name)) - wr = getattr(cache, 'library_database_instance', None) - if wr is None: - self.only_in_gui_error() - db = wr() - if db is None: - raise ValueError(_('In function {}: The database has been closed').format(self.name)) - return db + return get_database(mi, self.name) class BuiltinFormatterFunction(FormatterFunction): @@ -2368,13 +2393,17 @@ _formatter_builtins = [ class FormatterUserFunction(FormatterFunction): - def __init__(self, name, doc, arg_count, program_text, is_python): - self.is_python = is_python + def __init__(self, name, doc, arg_count, program_text, object_type): + self.object_type = object_type self.name = name self.doc = doc self.arg_count = arg_count self.program_text = program_text - self.cached_parse_tree = None + self.cached_compiled_text = None + # Keep this for external code compatibility. Set it to True if we have a + # python template function, otherwise false. This might break something + # if the code depends on stored templates being in GPM. + self.is_python = True if object_type is StoredObjectType.PythonFunction else False def to_pref(self): return [self.name, self.doc, self.arg_count, self.program_text] @@ -2383,13 +2412,20 @@ class FormatterUserFunction(FormatterFunction): tabs = re.compile(r'^\t*') -def function_pref_is_python(pref): - if isinstance(pref, list): - pref = pref[3] - if pref.startswith('def'): - return True - if pref.startswith('program'): - return False +def function_object_type(thing): + # 'thing' can be a preference instance, program text, or an already-compiled function + if isinstance(thing, FormatterUserFunction): + return thing.object_type + if isinstance(thing, list): + text = thing[3] + else: + text = thing + if text.startswith('def'): + return StoredObjectType.PythonFunction + if text.startswith('program'): + return StoredObjectType.StoredGPMTemplate + if text.startswith('python'): + return StoredObjectType.StoredPythonTemplate raise ValueError('Unknown program type in formatter function pref') @@ -2398,8 +2434,9 @@ def function_pref_name(pref): def compile_user_function(name, doc, arg_count, eval_func): - if not function_pref_is_python(eval_func): - return FormatterUserFunction(name, doc, arg_count, eval_func, False) + typ = function_object_type(eval_func) + if typ is not StoredObjectType.PythonFunction: + return FormatterUserFunction(name, doc, arg_count, eval_func, typ) def replace_func(mo): return mo.group().replace('\t', ' ') @@ -2415,7 +2452,7 @@ class UserFunction(FormatterUserFunction): if DEBUG and tweaks.get('enable_template_debug_printing', False): print(prog) exec(prog, locals_) - cls = locals_['UserFunction'](name, doc, arg_count, eval_func, True) + cls = locals_['UserFunction'](name, doc, arg_count, eval_func, typ) return cls @@ -2432,7 +2469,7 @@ def compile_user_template_functions(funcs): # then white space differences don't cause them to compare differently cls = compile_user_function(*func) - cls.is_python = function_pref_is_python(func) + cls.object_type = function_object_type(func) compiled_funcs[cls.name] = cls except Exception: try: From 715f6d9b81264feddf17c9271bbcc4fc5a7f57e4 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Mon, 10 Oct 2022 17:14:03 +0100 Subject: [PATCH 2/8] Fix failing test. --- src/calibre/utils/formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index 7aaab09fb3..c052df3a9f 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -1789,7 +1789,7 @@ class TemplateFormatter(string.Formatter): traceback.print_exc() if column_name: prints('Error evaluating column named:', column_name) - ans = str(e) + ans = error_value + ' ' + error_message(e) return ans finally: self.restore_state(state) From b0413ab3aac5516eab59789e2a7626e6cb3ce51c Mon Sep 17 00:00:00 2001 From: un-pogaz <46523284+un-pogaz@users.noreply.github.com> Date: Mon, 10 Oct 2022 19:33:39 +0200 Subject: [PATCH 3/8] improve Highlighter --- src/calibre/gui2/dialogs/template_dialog.py | 68 ++++++++++++--------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index 36f3eac3ea..74c3e76824 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -46,7 +46,7 @@ class TemplateHighlighter(QSyntaxHighlighter): BN_FACTOR = 1000 - KEYWORDS_GPM = ["program", 'if', 'then', 'else', 'elif', 'fi', 'for', 'rof', + KEYWORDS_GPM = ['if', 'then', 'else', 'elif', 'fi', 'for', 'rof', 'separator', 'break', 'continue', 'return', 'in', 'inlist', 'def', 'fed', 'limit'] @@ -89,6 +89,7 @@ class TemplateHighlighter(QSyntaxHighlighter): r"|\$+#?[a-zA-Z]\w*", "identifier") + a(r"^\bprogram\b", "keyword") a( "|".join([r"\b%s\b" % keyword for keyword in self.KEYWORDS_GPM]), "keyword") @@ -101,22 +102,25 @@ class TemplateHighlighter(QSyntaxHighlighter): a(r"""(? Date: Mon, 10 Oct 2022 19:02:00 +0100 Subject: [PATCH 4/8] Some fixes and improvements from un_pogaz --- src/calibre/gui2/dialogs/template_dialog.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index 74c3e76824..c6854f7e2c 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -89,7 +89,7 @@ class TemplateHighlighter(QSyntaxHighlighter): r"|\$+#?[a-zA-Z]\w*", "identifier") - a(r"^\bprogram\b", "keyword") + a(r"^\bprogram\b:", "keyword") a( "|".join([r"\b%s\b" % keyword for keyword in self.KEYWORDS_GPM]), "keyword") @@ -102,7 +102,7 @@ class TemplateHighlighter(QSyntaxHighlighter): a(r"""(? Date: Mon, 10 Oct 2022 21:06:09 +0100 Subject: [PATCH 5/8] Only detect the template type keyword on the first line. Remove an extra space from the default formatter exception message. --- src/calibre/gui2/dialogs/template_dialog.py | 9 ++++++++- src/calibre/utils/formatter.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index c6854f7e2c..3356dc97f8 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -216,7 +216,14 @@ class TemplateHighlighter(QSyntaxHighlighter): pp = self.find_paren(bn, i) if pp and pp.highlight: self.setFormat(i, length, self.Formats[format_]) + elif format_ == 'keyword': + if bn > 0 and i == 0: + if text[i:i+length] == ('python:' if self.for_python else 'program:'): + continue + print('bn', bn, format_, text[i:i+length]) + self.setFormat(i, length, self.Formats[format_]) else: + print('bn', bn, format_, text[i:i+length]) self.setFormat(i, length, self.Formats[format_]) self.setCurrentBlockState(NORMAL) @@ -796,7 +803,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): w = tv.cellWidget(r, 0) w.setText(mi.title) w.setCursorPosition(0) - v = SafeFormat().safe_format(txt, mi, _('EXCEPTION: '), + v = SafeFormat().safe_format(txt, mi, _('EXCEPTION:'), mi, global_vars=self.global_vars, template_functions=self.all_functions, break_reporter=self.break_reporter if r == break_on_mi else None) diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index c052df3a9f..44f9da012e 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -1591,7 +1591,7 @@ class TemplateFormatter(string.Formatter): raise(ValueError( _('Syntax error on line {0} column {1}: text {2}').format(e.lineno, e.offset, e.text))) except KeyError: - raise(ValueError(_("Error: the {0} function is not defined in the template").format('evaluate'))) + raise(ValueError(_("The {0} function is not defined in the template").format('evaluate'))) # ################# Override parent classes methods ##################### From 3f856408d70439062e156d86523373aec7948ea4 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Mon, 10 Oct 2022 21:25:06 +0100 Subject: [PATCH 6/8] Add some tests for python templates --- src/calibre/db/tests/reading.py | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 5a46bcca4e..30c68cdaf4 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -811,3 +811,39 @@ class ReadingTest(BaseTest): v = formatter.safe_format('program: book_values("rating", "title:true", ",", 0)', {}, 'TEMPLATE ERROR', mi) self.assertEqual(set(v.split(',')), {'4', '6'}) # }}} + + def test_python_templates(self): # {{{ + from calibre.ebooks.metadata.book.formatter import SafeFormat + formatter = SafeFormat() + + # need an empty metadata object to pass to the formatter + db = self.init_legacy(self.library_path) + mi = db.get_metadata(1) + + # test counting books matching a search + template = '''python: +def evaluate(book, db, **kwargs): + ids = db.new_api.search("series:true") + return str(len(ids)) +''' + v = formatter.safe_format(template, {}, 'TEMPLATE ERROR', mi) + self.assertEqual(v, '2') + + # test counting books when none match the search + template = '''python: +def evaluate(book, db, **kwargs): + ids = db.new_api.search("series:afafaf") + return str(len(ids)) +''' + v = formatter.safe_format(template, {}, 'TEMPLATE ERROR', mi) + self.assertEqual(v, '0') + + # test is_multiple values + template = '''python: +def evaluate(book, db, **kwargs): + tags = db.new_api.all_field_names('tags') + return ','.join(list(tags)) +''' + v = formatter.safe_format(template, {}, 'TEMPLATE ERROR', mi) + self.assertEqual(set(v.split(',')), {'Tag One', 'News', 'Tag Two'}) + # }}} From 985c189bd4c706cf2c727c0c6ca3350d6eed7ee0 Mon Sep 17 00:00:00 2001 From: un-pogaz <46523284+un-pogaz@users.noreply.github.com> Date: Tue, 11 Oct 2022 09:46:33 +0200 Subject: [PATCH 7/8] sligh improvement to syntax to word mode and regex --- src/calibre/gui2/dialogs/template_dialog.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index 3356dc97f8..24eff84f9f 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -89,7 +89,7 @@ class TemplateHighlighter(QSyntaxHighlighter): r"|\$+#?[a-zA-Z]\w*", "identifier") - a(r"^\bprogram\b:", "keyword") + a(r"^program:", "keymode") a( "|".join([r"\b%s\b" % keyword for keyword in self.KEYWORDS_GPM]), "keyword") @@ -102,7 +102,7 @@ class TemplateHighlighter(QSyntaxHighlighter): a(r"""(? 0 and i == 0: - if text[i:i+length] == ('python:' if self.for_python else 'program:'): - continue - print('bn', bn, format_, text[i:i+length]) - self.setFormat(i, length, self.Formats[format_]) + continue + self.setFormat(i, length, self.Formats['keyword']) else: - print('bn', bn, format_, text[i:i+length]) self.setFormat(i, length, self.Formats[format_]) self.setCurrentBlockState(NORMAL) From 1b7cbccdafbaaa0492706a1b42ac84c1925ea57e Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Tue, 11 Oct 2022 11:41:51 +0100 Subject: [PATCH 8/8] Add a test calling a stored python template from a GPM template --- src/calibre/db/tests/reading.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 30c68cdaf4..12dc79edcc 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -846,4 +846,23 @@ def evaluate(book, db, **kwargs): ''' v = formatter.safe_format(template, {}, 'TEMPLATE ERROR', mi) self.assertEqual(set(v.split(',')), {'Tag One', 'News', 'Tag Two'}) + + # test calling a python stored template from a GPM template + from calibre.utils.formatter_functions import ( + load_user_template_functions, unload_user_template_functions) + load_user_template_functions('aaaaa', + [['python_stored_template', + "", + 0, + '''python: +def evaluate(book, db, globals, arguments): + tags = set(db.new_api.all_field_names('tags')) + tags.add(arguments[0]) + return ','.join(list(tags)) +''' + ]], None) + v = formatter.safe_format('program: python_stored_template("one argument")', {}, + 'TEMPLATE ERROR', mi) + unload_user_template_functions('aaaaa') + self.assertEqual(set(v.split(',')), {'Tag One', 'News', 'Tag Two', 'one argument'}) # }}}