From 48363e9d69d9c3c9434298aceb175af22b8f6726 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Sep 2011 11:00:52 -0600 Subject: [PATCH] Allow downloading metadata from amazon.co.jp. To use it, configure the amazon metadata source to use the Japanese amazon site. Fixes #842447 ([improvement]add support for Amazon Japan metadata download) --- src/calibre/db/tests/metadata.db | Bin 224256 -> 230400 bytes src/calibre/db/tests/reading.py | 16 +++- src/calibre/ebooks/metadata/sources/amazon.py | 72 ++++++++++++++++-- 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/src/calibre/db/tests/metadata.db b/src/calibre/db/tests/metadata.db index 174345276e205e17b15923cc641588517f7d4822..8fb89cbb8e87ff051c764bb653fef03e64094284 100644 GIT binary patch delta 8468 zcmc&(4RBP|6@K@gx0{e;L&7Ey5(t}J8bA`*pKP*AF(Cp%Ab|w1qM%`uJRn1|3)zhz zmH|dPW1ZT8*-mfO8Aryp({a?1#7SoQW0PQQr$en&MMTt~w8JPe&`CRtwYKNp_p@(* z5>ThpOy2H$@1A?^Ip6uccaNO29eFhWSefNxNs=xecQlZXPwqGYHuzMcFet&FVGsrf z3-f*k+e*PARk&=8{juGVKF5~*5&W-msE?)8Jhm~c;J-eGjEl?4Y=kYvC(}JCC^IaS@7cA?~}vOAYG8=&BL#^ z@$2Y%G9AAf8c2i9c00L?v~1~)^(aDU{l@F|c&gpOYIm)}UAxTf#{b3QQymCYyZtEx z)TFSZ(qU4u@vs5PQe!2@LeC=7Sz8x!2SZA=5~}f4*L2nGtnR4u*Hm};0)d)3pC{<| zcqKy=zr*WU<_#|MyO(%U}7m?ba&+F^*)>U_PdIQxp-VT3tz~k$vKCFa-oo=@$ zSW_G7O3}w1SmyN{T~E11rcmJeFr?i;Bvg$CRr{w2X+(C4mGEn-8HQ2MY$ABHpV zg(M9Y6ij3}r_(1SxB?g9G&~Os_Cp7>KsA)m8}ws(ioQsnKxd88rS|Etm~{3>W0BtW z&PY#xZ@AqMul!uNdhp?EX!Y2h|*Lc1bnE30ksM( zF`zc7lLB>#3JrlIs5wMCgkYy7_#2!?-A7jV3#l5TVKK# z@KN&!W)MD-hG3pq_<>d24zVv?2*F1ZT!)WvrL%CGja#+YPGU+l7TX`OanlytDSwQF z6_w{F5_|&xK>810W{ymXiJ+$9R#1|GXEI<5lD$jmw-WqT!n-oK3fKt}&@B8$0y6M? zcD4+ki<S+4tIe>yd5W)YfjQDHRnfYQ|- zRYHB8yOroehvHA2nnPVP;s;K-ke^-WccRJFsaeFi%n|TZIdUBM=j;o`y2HEhRb4=T z<$6$wh9mgO<8~YG3_6b|es*NN1vPu4XyDZ@uoP9m8iEqfwagTE-#st%$Cn5LGq5LdQ^1zaYU?xPbHCz~J1*~Vzbte$yPovZ3S>AG6J zzvXq4a^X-j>V#D@2~&YQ5w2^NUQnTRGzV9aH&M~_%9Xs+bJ`BFYs$t%=Gnns6yhgB;eUBl(Azvb<$<2J`29Zm$D7?w{aIElEGB)VQvqGQ% zGuQ%zKCzRVjd?7%Qnb73rb)Xu(pn0OYVKA^9B3&7NoEdY2-^@jU5?;aV)B0n(@o_P zY(g;Dc_qWvvj9QkFeVol6U7Wo!2|&rYDFl_T*}jFCU2Pu7%j7A`gGSJM$4S$Iwsf0 zgxrOaBqwCkI3+_pR3w}c3C3XrHGUbI%X#QF5Mv+fN!fmv^N?+z#(3)X+>l65ZzV6dUd^+#U*vG69f z!R5SrNRW$WLVHAVGW2}f3RP3j;Jz@)ULmn9GwGhPSE$s|N!nh4ORg0zY#PwIddpxc zNosmYDO{p6GnY5?lvk--rIsd>U9ek6z(m(>AZ4~3FJ8PH@o$ceise)kUn2vgZ6k+d z(7pa^B-Ec5K-G`W5qX_O@=Ex>K~kiLBt~CBG(W)WeY`H<^&VdDY8T#rgRR4~FDr|z z(xRhZc-T{WAjGbtwL^#x|;D}T8(Wp~@uO4~ukq1IO{R(Sa4#mXFPTU0* zD(Ch_m}gPPR&9bc^t(>B^iG%s$btP}6O>uScIFT}(F_%=X)6@UoS2Q?1NMUCW0g85 zJF^a)EU*HHx})t}q$sYuGt?7R*l)IgV_nW6VS0Xa^B#Uk(iyxi;x&%fhj^K>=lF9w z@OOM{94}7G=e)t%*24U3b=%@9=9{v~0B+;kznTA=?OL-HWjU{|kNZ*SEw---cCnoe z@Fe@sT4-QTZh=|D_pSg&XZUJ#oA5m)62HxkZ-*jgUkOjjuC7pwo!kMH2_yTbH1aCk zuuf@Y>aAnf+rU9w%&`TR+1m*DWx~$Jw4P9h(!*T?7gyNPHE7@m8({9#R`w^XTp@@1 zdwRHKD{++<_rPrl>AgaF|0FKDN!=XlfMjw~cKr@;X1UbuG277y<(ikCTn<4lXCA}o znJ$D~Y|;yXJ&W7#Sg4~%1VplrdNt`w+MtTPuo5=26%DYERjq<|EUxZoN0?o&H%D4P zsNv3}M$_gyHW*N5(t!Hhx&_+qzljCJ(z*rmvhhS=ZzS3sV+~uOY|?-WBurtCFFoi} zL)8%v9h{vfq84%J4edeG-VP;|_@J{@+M|)a7`J>(IS_MJA!hHW#}{bhdYg zyA)jW-2*76w?-gotZ7PGPu~se^rXe+Z3PQ^d=r#pS2}MwZ~06xPB-N(u7iNW>WcD{ zi4;w1ru1;NQ#2IH=pSCc?jQFy!hFrWA8FLxyS!0z?@1k?#_Y|d&2XF>={-j3!Np#0 zhBFwI^EzN2yS5E%xhR5KJv?q9IiP;*u@0!1o=G_3V~ux1MV4!4q^}oq6Gz;th8ZjD z1UK8b5~hhToXlAZlOxaLHo=VEI;&9S96h9iG>PLnr*Pa9r(GC-VKj^Vq6Lm{Af&i+ zv?gCufEOY>c8hNqotwjxEzb=ecf4b?cDv@t8ik&dc-c*T#iLZ^V^AqNril|h;Vk-k z$3}PNX-SOF^u@1^mTfjD>I^=*NVUcreX*eNMgt7_c#UJmX!k*`N==-M&8{$OVpJQ| zd4)Xw`2Az$Wr<@*V|h*2+IYce5%I-iW6t<qjXyqyNa`R3 z;=dcavr3b6o1T1P{P#b@ylNvL#_p?f#6KKk+)5YN#U@xd+`0jd5UVO$N)E0S?&{5y zQtnZrOE*OH6)?Z?-**rWQ?o<80TE?7Z(d2}nCb<3mr)1B`K(b0^JIQT%vNlL(n%w* z)*ONRZij5zJR7~vN>Z#MoT32GVy>TDd*tIsOuxXYO znePBEd2`tx*JGU<*$%a8#rxq}$mbQWfZ3bC!QNd5`4-V$4HvY*_i*=s-4e^)2&>sM z_dpHH-wc0eXFFidFtz}Atd7GU;_t~kk*#5^t6>5AVg)vA*P389kM`fn4ris>{taU$ z{fwaWm{qD1Rox>@@$F2e!0fIc0O|I?<;FAGgtdpjnXP+QQf?_|fn`Qo5x;XxZ2*tS z0K<9r!USQFc2F21RmvXT4grqIMfR)haLginmpfMp8)<>OgjzwNR>ms7n#Z>uftmaZ z6J^^5sL+^m@0zU3;*pbZu3Y}8@b)GWo?6AADRH*)S^Rg*NFR@AJ<`yL`;;zSPVxK0 zu0BNxC%x&w{$H(%4hIIH8l+h7XKmEuRWKK^k-{9ebxEvnyO;dD8&@-Ui=xTa-v{Fy g{4>Lg?}xVu+q)j}hiB}77J`tS!MekahTysX0?&Ogc>n+a delta 1278 zcmd5*T}&KR6rMZZo!tSU>=p`0Y1~3;OTcuEAc@vef&p70U1FFsS=cNL-R2)LZfgO@?_*d|uwn4Dw5!Dm+Bu$VvU%7=@{jBnxNhs=HNVZj_VcHfu!>J{ zQmI!IyYlKqMlA-ybjmoHLlq;Lt5`3X4R2-5|ofYqQ!58U#0^4YN z0+l5Cv57JXl+a=V74$&@<+?dIW=OYYx}QKNecX>BwO*sUU8v7xuGLmnfGZ6_Gk((O zi<(9}ePX~c$1}OewW``0#+B!{98aZ^W;C8QqE>p)wBnXz>W=^N^*(6&c62d1+lg(O z-xKe3Nl~jt3!}+BsL0KI*YSj^1&=mK#hQO6!fn3tuu-sM4EiiT#w4B1e3mycc}VTa zQNX)5;xQH|;MO`WcrzFm+G*x=c(Gpdgm#lTUQ1^AGLz-q5;D8Zr&5tB=i*jph-zRG;eLQQ4&Glnef|s_McwC4B{7R#h&1(^CHTb@DQ1z2b$9 zDZPrh z4xLKVC=*BbgH(F;AnK^P8=uf{1aG*GKhj(Tj_5puhpMaJq~6ytLiStep?xvvqT~o# z4w~rF_22r zd^>i~!3pfAPzo>8MGN1$ehVTs3SAY4PvI{)rHZmh?Qf%sV&lkD%@LWq{5?!kXadWE zXK~B(QN-dTjxuU^SMu@Z`#4Xx+wqK>y_bgL2+(2=N`(BuR;Ctba(}c|Z1%~e#IN`s t;{Ujz&;G}T8diGLD3z=*oKbMb4(^&l(T&}@PidIUcTj#0;x1=5{sA`}eBl59 diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index e9c8be83b8..6149839a88 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -8,7 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, shutil#, unittest +import os, shutil, unittest, tempfile def create_db(library_path): from calibre.library.database2 import LibraryDatabase2 @@ -17,5 +17,17 @@ def create_db(library_path): src = os.path.join(os.path.dirname(__file__), 'metadata.db') db = os.path.join(library_path, 'metadata.db') shutil.copyfile(src, db) - db = LibraryDatabase2(library_path) + return db + +class ReadingTest(unittest.TestCase): + + def setUp(self): + self.library_path = tempfile.mkdtemp() + create_db(self.library_path) + + def tearDown(self): + shutil.rmtree(self.library_path) + + def test_read(self): + pass diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index aaa13d5769..fa72766a0a 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -74,6 +74,20 @@ class Worker(Thread): # Get details {{{ 9: ['sept'], 12: ['déc'], }, + 'jp': { + 1: [u'1月'], + 2: [u'2月'], + 3: [u'3月'], + 4: [u'4月'], + 5: [u'5月'], + 6: [u'6月'], + 7: [u'7月'], + 8: [u'8月'], + 9: [u'9月'], + 10: [u'10月'], + 11: [u'11月'], + 12: [u'12月'], + }, } @@ -86,13 +100,15 @@ class Worker(Thread): # Get details {{{ text()="Produktinformation" or \ text()="Dettagli prodotto" or \ text()="Product details" or \ - text()="Détails sur le produit"]/../div[@class="content"] + text()="Détails sur le produit" or \ + text()="登録情報"]/../div[@class="content"] ''' self.publisher_xpath = ''' descendant::*[starts-with(text(), "Publisher:") or \ starts-with(text(), "Verlag:") or \ starts-with(text(), "Editore:") or \ - starts-with(text(), "Editeur")] + starts-with(text(), "Editeur") or \ + starts-with(text(), "出版社:")] ''' self.language_xpath = ''' descendant::*[ @@ -101,10 +117,11 @@ class Worker(Thread): # Get details {{{ or text() = "Sprache:" \ or text() = "Lingua:" \ or starts-with(text(), "Langue") \ + or starts-with(text(), "言語") \ ] ''' self.ratings_pat = re.compile( - r'([0-9.]+) (out of|von|su|étoiles sur) (\d+)( (stars|Sternen|stelle)){0,1}') + r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち) ([\d\.]+)( (stars|Sternen|stelle)){0,1}') lm = { 'eng': ('English', 'Englisch'), @@ -112,6 +129,7 @@ class Worker(Thread): # Get details {{{ 'ita': ('Italian', 'Italiano'), 'deu': ('German', 'Deutsch'), 'spa': ('Spanish', 'Espa\xf1ol', 'Espaniol'), + 'jpn': ('Japanese', u'日本語'), } self.lang_map = {} for code, names in lm.iteritems(): @@ -403,6 +421,7 @@ class Amazon(Source): 'de' : _('Germany'), 'uk' : _('UK'), 'it' : _('Italy'), + 'jp' : _('Japan'), } options = ( @@ -411,6 +430,22 @@ class Amazon(Source): 'country\'s Amazon website.'), choices=AMAZON_DOMAINS), ) + def __init__(self, *args, **kwargs): + Source.__init__(self, *args, **kwargs) + self.set_amazon_id_touched_fields() + + def save_settings(self, *args, **kwargs): + Source.save_settings(self, *args, **kwargs) + self.set_amazon_id_touched_fields() + + def set_amazon_id_touched_fields(self): + ident_name = "identifier:amazon" + if self.domain != 'com': + ident_name += '_' + self.domain + tf = [x for x in self.touched_fields if not + x.startswith('identifier:amazon')] + [ident_name] + self.touched_fields = frozenset(tf) + def get_domain_and_asin(self, identifiers): for key, val in identifiers.iteritems(): key = key.lower() @@ -488,13 +523,23 @@ class Amazon(Source): # Insufficient metadata to make an identify query return None, None - latin1q = dict([(x.encode('latin1', 'ignore'), y.encode('latin1', + # magic parameter to enable Japanese Shift_JIS encoding. + if domain == 'jp': + q['__mk_ja_JP'] = u'カタカナ' + + if domain == 'jp': + encode_to = 'Shift_JIS' + else: + encode_to = 'latin1' + encoded_q = dict([(x.encode(encode_to, 'ignore'), y.encode(encode_to, 'ignore')) for x, y in q.iteritems()]) udomain = domain if domain == 'uk': udomain = 'co.uk' - url = 'http://www.amazon.%s/s/?'%udomain + urlencode(latin1q) + elif domain == 'jp': + udomain = 'co.jp' + url = 'http://www.amazon.%s/s/?'%udomain + urlencode(encoded_q) return url, domain # }}} @@ -663,7 +708,7 @@ if __name__ == '__main__': # tests {{{ # To run these test use: calibre-debug -e # src/calibre/ebooks/metadata/sources/amazon.py from calibre.ebooks.metadata.sources.test import (test_identify_plugin, - title_test, authors_test) + isbn_test, title_test, authors_test) com_tests = [ # {{{ ( # Description has links @@ -744,6 +789,21 @@ if __name__ == '__main__': # tests {{{ ), ] # }}} + jp_tests = [ # {{{ + ( # isbn -> title, authors + {'identifiers':{'isbn': '9784101302720' }}, + [title_test(u'精霊の守り人', + exact=True), authors_test([u'上橋 菜穂子']) + ] + ), + ( # title, authors -> isbn (will use Shift_JIS encoding in query.) + {'title': u'考えない練習', + 'authors': [u'小池 龍之介']}, + [isbn_test('9784093881067'), ] + ), + ] # }}} + test_identify_plugin(Amazon.name, com_tests) + #test_identify_plugin(Amazon.name, jp_tests) # }}}