From 31e2bc3818b888503b14c025230c7468e3dcad78 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 09:53:28 -0600 Subject: [PATCH 01/17] Fix #781135 (ParseException when pressing "Next" or "Previous" in plugin search) --- src/calibre/gui2/preferences/plugins.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index 8888a64e84..4f88e5aa1d 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -75,6 +75,8 @@ class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{ def find(self, query): query = query.strip() + if not query: + return QModelIndex() matches = self.parse(query) if not matches: return QModelIndex() @@ -87,6 +89,8 @@ class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{ def find_next(self, idx, query, backwards=False): query = query.strip() + if not query: + return idx matches = self.parse(query) if not matches: return idx From 231df586c08b8d155a7996f6b000db65f7b438c3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 14:56:10 -0600 Subject: [PATCH 02/17] ODT Input: Handle inline special styles defined on tags. Fixes #780250 (Italic text not converting from ODT source) --- src/odf/odf2xhtml.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/odf/odf2xhtml.py b/src/odf/odf2xhtml.py index a04aa48bf7..0ae89b1663 100644 --- a/src/odf/odf2xhtml.py +++ b/src/odf/odf2xhtml.py @@ -1415,18 +1415,34 @@ ol, ul { padding-left: 2em; } self.writedata() c = attrs.get( (TEXTNS,'style-name'), None) htmlattrs = {} + # Changed by Kovid to handle inline apecial styles defined on tags. + # Apparently LibreOffice does this. + special = 'span' if c: c = c.replace(".","_") special = special_styles.get("S-"+c) - if special is None and self.generate_css: - htmlattrs['class'] = "S-%s" % c - self.opentag('span', htmlattrs) + if special is None: + special = 'span' + if self.generate_css: + htmlattrs['class'] = "S-%s" % c + + self.opentag(special, htmlattrs) self.purgedata() def e_text_span(self, tag, attrs): """ End the """ self.writedata() - self.closetag('span', False) + c = attrs.get( (TEXTNS,'style-name'), None) + # Changed by Kovid to handle inline apecial styles defined on tags. + # Apparently LibreOffice does this. + special = 'span' + if c: + c = c.replace(".","_") + special = special_styles.get("S-"+c) + if special is None: + special = 'span' + + self.closetag(special, False) self.purgedata() def s_text_tab(self, tag, attrs): From fa0c8702b0d6d51ae7d96f1c14e43c27beb9f7c7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 16:33:23 -0600 Subject: [PATCH 03/17] ... --- recipes/divahair.recipe | 53 ++++++++++++++++++++++++++++++++++ recipes/icons/divahair.png | Bin 0 -> 675 bytes recipes/icons/mayra.png | Bin 0 -> 620 bytes recipes/icons/moldovaazi.png | Bin 0 -> 243 bytes recipes/icons/newsmoldova.png | Bin 0 -> 837 bytes recipes/mayra.recipe | 51 ++++++++++++++++++++++++++++++++ recipes/moldovaazi.recipe | 50 ++++++++++++++++++++++++++++++++ recipes/newsmoldova.recipe | 50 ++++++++++++++++++++++++++++++++ 8 files changed, 204 insertions(+) create mode 100644 recipes/divahair.recipe create mode 100644 recipes/icons/divahair.png create mode 100644 recipes/icons/mayra.png create mode 100644 recipes/icons/moldovaazi.png create mode 100644 recipes/icons/newsmoldova.png create mode 100644 recipes/mayra.recipe create mode 100644 recipes/moldovaazi.recipe create mode 100644 recipes/newsmoldova.recipe diff --git a/recipes/divahair.recipe b/recipes/divahair.recipe new file mode 100644 index 0000000000..978ac19808 --- /dev/null +++ b/recipes/divahair.recipe @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +divahair.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class DivaHair(BasicNewsRecipe): + title = u'Diva Hair' + language = 'ro' + __author__ = u'Silviu Cotoar\u0103' + description = u'Coafuri, frizuri, tunsori ..' + publisher = u'Diva Hair' + category = u'Ziare,Stiri,Coafuri,Femei' + oldest_article = 5 + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + encoding = 'utf-8' + remove_javascript = True + cover_url = 'http://www.divahair.ro/imgs/logo.jpg' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='td', attrs={'class':'spatiuart'}) + , dict(name='div', attrs={'class':'spatiuart'}) + ] + + + remove_tags = [ + dict(name='div', attrs={'class':'categorie'}) + , dict(name='div', attrs={'class':'gri gri2 detaliiart'}) + , dict(name='div', attrs={'class':'articol_box_bottom'}) + ] + + remove_tags_after = [ + dict(name='div', attrs={'class':'articol_box_bottom'}) + ] + + feeds = [ (u'\u0218tiri', u'http://www.divahair.ro/feed') ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/recipes/icons/divahair.png b/recipes/icons/divahair.png new file mode 100644 index 0000000000000000000000000000000000000000..4cb2964687b0acf7fc5d93332ca5963bb1d96a63 GIT binary patch literal 675 zcmV;U0$lxxP)jf%BN7SpBJEufALZZ58cfd7ZMb2yJMDZK6 zM5*`@t`{VWwuMAuqDiB<_wCS=ru5g%!$ZHyVdT;~+ zXl^&cZbt~gLy@wn3HviY&dyF;!5|I6Aa=hWt-1M87(m**J{%n#I9~;E1w+`|+R$2B z5Kh;lFo1Lf0yw+7afL$I12528Tak^82>Cb+Af4SWafL!S+S{?YpCX?$A*B2-7(lv$ zA-?_m#mCuMrjto(C={@`xQJmG7={5rxm;#xX$hqiySuyW?Ckv8psR~7$s}*1QLd_0 z#>dA2n3|d*m&-9fKaWz1csx$ES_NQiY>aq3{#Ri4`LJv@qW%4hj*g=1I{ADauh&aB z97fl5EXyL5N-;P%$nx?s(P)%Hp@32fSp+iHQjY1_rn)7O5N`BQx17kw^r9xw$#|`uaFLJgm=z5X{WXFfuYiI-N$7lCRU# zoSmHT)fGxxTU4`IsMYEn+e4tby`_43N@;VG;_3?Jg9EHm=}z~KKv@>GT#oB>n&RpT zSKHg%oS&mi^WNX63zSk=l?uOgozmJG#q~8Rx{j(;>esk0e*pJl2G9Ddg zkKy*DcOS*DZoy&@d|KO>tUb-!fe4*7xRmEB>!Q?lw*0vS5Osp0EVNOb;cSQV}sH!*s zwp1o^G}FA&9%YOnc69{TOH+N61Hk#cgNb(RDQ=lKhOTQP)E~b@t;JE*=H%BlQL+DJ z5*a>>E^i_~SBMY_C`i75f2fB~Q!|8Ni0ADi8tvpqex4IU$C;V_!az1dskFxM>66eI z!7i0?Xrche5C#ph*?!XL6xC{tLZL_~!rM3RNvBg(DjO76iXaxOLc|ckbwDgm-+q$O z4&uo;$y5SVSzIhIIGAO9y-fc=hFYygDwV`8t)qn0@4cNFfj(aLY z?BW?+fa>a+8<_IPn1; oXT0iAyAf3;VL#XQd-Z>|sWM#kdXX$=fEF@%y85}Sb4q9e0J&XVnE(I) literal 0 HcmV?d00001 diff --git a/recipes/icons/newsmoldova.png b/recipes/icons/newsmoldova.png new file mode 100644 index 0000000000000000000000000000000000000000..a4faaa46357560501f42c45788b34e86896ff51e GIT binary patch literal 837 zcmV-L1G@Z)P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!~g&e!~vBn4jTXf0@_JLK~y+Tm66#` zQ$ZBQ@8998&n9Z3PyPoc#>A)*g#=A7CPp6=7YY~!0!ly-1VSmB>_MSn4TYAarIc1$ z*3#bY_jWI|R4k>He$EVmg!tr5CU?#^=lsrhW;p!E_vB*_#x(0^D!A@R9S#mt7+>6g z(;I=LNzUS-Sre?DAo>*syr><-^s*V9<4ZV5rda%?U&(T~%Qg%AoFhjFj41=sQ_ z@pW((&#L-ak)%^oB)0$lE^FXVvf!L zXySu+tqgFdK#Ki?V+YjYsCpeE3v0}QZqtc#*I(nt%Z7gey~Gx~gCZF&W|y*rtYR51 zWR+mc>0Fki$N4cria5}c)ebh zM>VU}icly7PNht_((*TQJi`!Qe&fe$CZ6;3!&yC4!V0GZAYg=H6YYLNlj zmi_2m38IFa?~;!78ZTOjCL^n6UWCHB3$A)zU|lFgz8;1_h@(R3IR#9ZcTl2mu`yb7 zK1i2*XeZ4YwLWx^KSzVAS%KNARC&;0piX>P^2E`!=*8zbfpw*Ez1quM(J^v*%qX5D zm)K1#PV-Up7=vga1#~28Eq~OGS7g Date: Wed, 11 May 2011 16:34:59 -0600 Subject: [PATCH 04/17] Fix #780728 (RTF to mobi conversion fails) --- src/calibre/ebooks/rtf/input.py | 2 +- src/calibre/ebooks/rtf2xml/process_tokens.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/rtf/input.py b/src/calibre/ebooks/rtf/input.py index 23c16f473d..f08aa76605 100644 --- a/src/calibre/ebooks/rtf/input.py +++ b/src/calibre/ebooks/rtf/input.py @@ -86,7 +86,7 @@ class RTFInput(InputFormatPlugin): run_lev = 4 self.log('Running RTFParser in debug mode') except: - pass + self.log.warn('Impossible to run RTFParser in debug mode') parser = ParseRtf( in_file = stream, out_file = ofile, diff --git a/src/calibre/ebooks/rtf2xml/process_tokens.py b/src/calibre/ebooks/rtf2xml/process_tokens.py index 65162d0d37..11aab48588 100755 --- a/src/calibre/ebooks/rtf2xml/process_tokens.py +++ b/src/calibre/ebooks/rtf2xml/process_tokens.py @@ -197,8 +197,8 @@ class ProcessTokens: # character info => ci 'b' : ('ci', 'bold______', self.bool_st_func), 'blue' : ('ci', 'blue______', self.color_func), - 'caps' : ('ci', 'caps______', self.bool_st_func), - 'cf' : ('ci', 'font-color', self.default_func), + 'caps' : ('ci', 'caps______', self.bool_st_func), + 'cf' : ('ci', 'font-color', self.colorz_func), 'chftn' : ('ci', 'footnot-mk', self.bool_st_func), 'dn' : ('ci', 'font-down_', self.divide_by_2), 'embo' : ('ci', 'emboss____', self.bool_st_func), @@ -624,6 +624,11 @@ class ProcessTokens: num = 'true' return 'cw<%s<%s Date: Wed, 11 May 2011 17:24:20 -0600 Subject: [PATCH 05/17] Fix #780804 (not found gdbm on Windows in 0.8.0 and Japanese) --- setup/resources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/resources.py b/setup/resources.py index 84ff136371..1501e28017 100644 --- a/setup/resources.py +++ b/setup/resources.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' import os, cPickle, re, anydbm, shutil, marshal, zipfile, glob from zlib import compress -from setup import Command, basenames, __appname__ +from setup import Command, basenames, __appname__, iswindows def get_opts_from_parser(parser): def do_opt(opt): @@ -128,7 +128,7 @@ class Resources(Command): if not os.path.exists(base): os.makedirs(base) - if self.newer(dest, src): + if self.newer(dest, src) or iswindows: self.info('\tGenerating Kanwadict') for line in open(src, "r"): From 1a38be575d68278caff8647446255f92e3eabf79 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 17:35:27 -0600 Subject: [PATCH 06/17] ... --- setup/resources.py | 4 ++-- src/odf/odf2xhtml.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/resources.py b/setup/resources.py index 1501e28017..afa8829988 100644 --- a/setup/resources.py +++ b/setup/resources.py @@ -139,7 +139,7 @@ class Resources(Command): dest = self.j(self.RESOURCES, 'localization', 'pykakasi','itaijidict2.pickle') - if self.newer(dest, src): + if self.newer(dest, src) or iswindows: self.info('\tGenerating Itaijidict') self.mkitaiji(src, dest) @@ -147,7 +147,7 @@ class Resources(Command): dest = self.j(self.RESOURCES, 'localization', 'pykakasi','kanadict2.pickle') - if self.newer(dest, src): + if self.newer(dest, src) or iswindows: self.info('\tGenerating kanadict') self.mkkanadict(src, dest) diff --git a/src/odf/odf2xhtml.py b/src/odf/odf2xhtml.py index 0ae89b1663..b1dbebb775 100644 --- a/src/odf/odf2xhtml.py +++ b/src/odf/odf2xhtml.py @@ -1415,7 +1415,7 @@ ol, ul { padding-left: 2em; } self.writedata() c = attrs.get( (TEXTNS,'style-name'), None) htmlattrs = {} - # Changed by Kovid to handle inline apecial styles defined on tags. + # Changed by Kovid to handle inline special styles defined on tags. # Apparently LibreOffice does this. special = 'span' if c: @@ -1433,7 +1433,7 @@ ol, ul { padding-left: 2em; } """ End the """ self.writedata() c = attrs.get( (TEXTNS,'style-name'), None) - # Changed by Kovid to handle inline apecial styles defined on tags. + # Changed by Kovid to handle inline special styles defined on tags. # Apparently LibreOffice does this. special = 'span' if c: From 48166bd87764c26ccb6401482048679d558a3d93 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 18:08:00 -0600 Subject: [PATCH 07/17] Ensure kakasi databases are always rebuilt on windows --- setup/commands.py | 5 +- setup/installer/windows/__init__.py | 1 + setup/installer/windows/wix-template.xml | 3 - setup/resources.py | 209 ++++++++++++----------- 4 files changed, 114 insertions(+), 104 deletions(-) diff --git a/setup/commands.py b/setup/commands.py index 7e22ff14f3..febc684c08 100644 --- a/setup/commands.py +++ b/setup/commands.py @@ -11,7 +11,7 @@ __all__ = [ 'build', 'build_pdf2xml', 'server', 'gui', 'develop', 'install', - 'resources', + 'kakasi', 'resources', 'check', 'sdist', 'manual', 'tag_release', @@ -49,8 +49,9 @@ gui = GUI() from setup.check import Check check = Check() -from setup.resources import Resources +from setup.resources import Resources, Kakasi resources = Resources() +kakasi = Kakasi() from setup.publish import Manual, TagRelease, Stage1, Stage2, \ Stage3, Stage4, Publish diff --git a/setup/installer/windows/__init__.py b/setup/installer/windows/__init__.py index b51eccc832..59042ac56c 100644 --- a/setup/installer/windows/__init__.py +++ b/setup/installer/windows/__init__.py @@ -32,6 +32,7 @@ class Win32(VMInstaller): FREEZE_TEMPLATE = 'python -OO setup.py {freeze_command} --no-ice' INSTALLER_EXT = 'msi' SHUTDOWN_CMD = ['shutdown.exe', '-s', '-f', '-t', '0'] + BUILD_BUILD = ['python setup.py kakasi',] + VMInstaller.BUILD_BUILD def download_installer(self): installer = self.installer() diff --git a/setup/installer/windows/wix-template.xml b/setup/installer/windows/wix-template.xml index b5d2f4b292..5de08e155f 100644 --- a/setup/installer/windows/wix-template.xml +++ b/setup/installer/windows/wix-template.xml @@ -11,9 +11,6 @@ SummaryCodepage='1252' /> - - Date: Wed, 11 May 2011 20:10:00 -0400 Subject: [PATCH 08/17] Store: Search selected book's title and author. Put stores into sub menu. --- src/calibre/gui2/actions/store.py | 41 +++++++++++++++++++++++-- src/calibre/gui2/library/models.py | 3 ++ src/calibre/gui2/store/search/search.py | 14 +++++++-- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index 1989250bc8..db505cf590 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -26,16 +26,51 @@ class StoreAction(InterfaceAction): def load_menu(self): self.store_menu.clear() self.store_menu.addAction(_('Search'), self.search) + self.store_menu.addAction(_('Search Author'), self.search_author) + self.store_menu.addAction(_('Search Title'), self.search_title) self.store_menu.addSeparator() + self.store_list_menu = self.store_menu.addMenu(_('Stores')) for n, p in sorted(self.gui.istores.items(), key=lambda x: x[0].lower()): - self.store_menu.addAction(n, partial(self.open_store, p)) + self.store_list_menu.addAction(n, partial(self.open_store, p)) self.qaction.setMenu(self.store_menu) - def search(self): + def search(self, query=''): self.show_disclaimer() from calibre.gui2.store.search.search import SearchDialog - sd = SearchDialog(self.gui.istores, self.gui) + sd = SearchDialog(self.gui.istores, self.gui, query) sd.exec_() + + def search_author(self): + rows = self.gui.current_view().selectionModel().selectedRows() + if not rows or len(rows) == 0: + return + row = rows[0].row() + + author = '' + if self.gui.current_view() is self.gui.library_view: + author = self.gui.library_view.model().authors(row) + else: + mi = self.gui.current_view().model().get_book_display_info(row) + author = ' & '.join(mi.authors) + + query = 'author:"%s"' % author + self.search(query) + + def search_title(self): + rows = self.gui.current_view().selectionModel().selectedRows() + if not rows or len(rows) == 0: + return + row = rows[0].row() + + title = '' + if self.gui.current_view() is self.gui.library_view: + title = self.gui.library_view.model().title(row) + else: + mi = self.gui.current_view().model().get_book_display_info(row) + title = mi.title + + query = 'title:"%s"' % title + self.search(query) def open_store(self, store_plugin): self.show_disclaimer() diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index dd5082c27f..fc1117167d 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -506,6 +506,9 @@ class BooksModel(QAbstractTableModel): # {{{ def id(self, row): return self.db.id(getattr(row, 'row', lambda:row)()) + def authors(self, row_number): + return self.db.authors(row_number) + def title(self, row_number): return self.db.title(row_number) diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index 62e4e97f11..f7e8c88cd9 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -23,8 +23,8 @@ TIMEOUT = 75 # seconds class SearchDialog(QDialog, Ui_Dialog): - def __init__(self, istores, *args): - QDialog.__init__(self, *args) + def __init__(self, istores, parent=None, query=''): + QDialog.__init__(self, parent) self.setupUi(self) self.config = JSONConfig('store/search') @@ -54,6 +54,9 @@ class SearchDialog(QDialog, Ui_Dialog): setattr(self, 'store_check_' + x, cbox) stores_group_layout.addStretch() + # Set the search query + self.search_edit.setText(query) + # Create and add the progress indicator self.pi = ProgressIndicator(self, 24) self.top_layout.addWidget(self.pi) @@ -93,7 +96,7 @@ class SearchDialog(QDialog, Ui_Dialog): # Store / Formats self.results_view.setColumnWidth(4, int(total*.25)) - def do_search(self, checked=False): + def do_search(self): # Stop all running threads. self.checker.stop() self.search_pool.abort() @@ -252,4 +255,9 @@ class SearchDialog(QDialog, Ui_Dialog): self.search_pool.abort() self.cache_pool.abort() self.save_state() + + def exec_(self): + if unicode(self.search_edit.text()).strip(): + self.do_search() + return QDialog.exec_(self) From abf31ec8262905669bb4b3e0323c3430be00c813 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 18:20:44 -0600 Subject: [PATCH 09/17] Use Qt 4.7.3 for windows build --- setup/installer/windows/freeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index f666427598..7fb60968e7 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -14,7 +14,7 @@ from setup.build_environment import msvc, MT, RC from setup.installer.windows.wix import WixMixIn OPENSSL_DIR = r'Q:\openssl' -QT_DIR = 'Q:\\Qt\\4.7.2' +QT_DIR = 'Q:\\Qt\\4.7.3' QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns'] LIBUSB_DIR = 'C:\\libusb' LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll' From 6259914a9c402f5a8f25ebdc40737a55c30c6375 Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 11 May 2011 20:41:07 -0400 Subject: [PATCH 10/17] Store: Search by author and title menu option. --- src/calibre/gui2/actions/store.py | 53 ++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index db505cf590..c629b2c52a 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -10,6 +10,7 @@ from functools import partial from PyQt4.Qt import QMenu +from calibre.gui2 import error_dialog from calibre.gui2.actions import InterfaceAction from calibre.gui2.dialogs.confirm_delete import confirm @@ -25,9 +26,10 @@ class StoreAction(InterfaceAction): def load_menu(self): self.store_menu.clear() - self.store_menu.addAction(_('Search'), self.search) - self.store_menu.addAction(_('Search Author'), self.search_author) - self.store_menu.addAction(_('Search Title'), self.search_title) + self.store_menu.addAction(_('Search for ebooks'), self.search) + self.store_menu.addAction(_('Search by this author'), self.search_author) + self.store_menu.addAction(_('Search by this title'), self.search_title) + self.store_menu.addAction(_('Search by this author and title'), self.search_author_title) self.store_menu.addSeparator() self.store_list_menu = self.store_menu.addMenu(_('Stores')) for n, p in sorted(self.gui.istores.items(), key=lambda x: x[0].lower()): @@ -40,12 +42,13 @@ class StoreAction(InterfaceAction): sd = SearchDialog(self.gui.istores, self.gui, query) sd.exec_() - def search_author(self): + def _get_selected_row(self): rows = self.gui.current_view().selectionModel().selectedRows() if not rows or len(rows) == 0: - return - row = rows[0].row() - + return None + return rows[0].row() + + def _get_author(self, row): author = '' if self.gui.current_view() is self.gui.library_view: author = self.gui.library_view.model().authors(row) @@ -53,23 +56,43 @@ class StoreAction(InterfaceAction): mi = self.gui.current_view().model().get_book_display_info(row) author = ' & '.join(mi.authors) - query = 'author:"%s"' % author + return author + + def search_author(self): + row = self._get_selected_row() + if row == None: + error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True) + return + + query = 'author:"%s"' % self._get_author(row) self.search(query) - def search_title(self): - rows = self.gui.current_view().selectionModel().selectedRows() - if not rows or len(rows) == 0: - return - row = rows[0].row() - + def _get_title(self, row): title = '' if self.gui.current_view() is self.gui.library_view: title = self.gui.library_view.model().title(row) else: mi = self.gui.current_view().model().get_book_display_info(row) title = mi.title + + return title + + def search_title(self): + row = self._get_selected_row() + if row == None: + error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True) + return + + query = 'title:"%s"' % self._get_title(row) + self.search(query) - query = 'title:"%s"' % title + def search_author_title(self): + row = self._get_selected_row() + if row == None: + error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True) + return + + query = 'author:"%s" title:"%s"' % (self._get_author(row), self._get_title(row)) self.search(query) def open_store(self, store_plugin): From 0fa971937519ab39801831d2135f69c7128a070f Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 11 May 2011 21:02:15 -0400 Subject: [PATCH 11/17] Store: Fix B&N plugin. Fix cleaning of query. --- src/calibre/gui2/store/bn_plugin.py | 4 ++-- src/calibre/gui2/store/search/download_thread.py | 13 +++++++++---- src/calibre/gui2/store/search/search.py | 9 ++++++--- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index f26a60c89d..62826e825d 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' import random import re -import urllib2 +import urllib from contextlib import closing from lxml import html @@ -48,7 +48,7 @@ class BNStore(BasicStoreConfig, StorePlugin): def search(self, query, max_results=10, timeout=60): url = 'http://productsearch.barnesandnoble.com/search/results.aspx?STORE=EBOOK&SZE=%s&WRD=' % max_results - url += urllib2.quote(query) + url += urllib.quote_plus(query) br = browser() diff --git a/src/calibre/gui2/store/search/download_thread.py b/src/calibre/gui2/store/search/download_thread.py index 6dd59cc5a7..97279d7773 100644 --- a/src/calibre/gui2/store/search/download_thread.py +++ b/src/calibre/gui2/store/search/download_thread.py @@ -12,6 +12,7 @@ from threading import Thread from Queue import Queue from calibre import browser +from calibre.constants import DEBUG from calibre.utils.magick.draw import thumbnail class GenericDownloadThreadPool(object): @@ -119,7 +120,8 @@ class SearchThread(Thread): self.results.put((res, store_plugin)) self.tasks.task_done() except: - traceback.print_exc() + if DEBUG: + traceback.print_exc() class CoverThreadPool(GenericDownloadThreadPool): @@ -157,7 +159,8 @@ class CoverThread(Thread): callback() self.tasks.task_done() except: - continue + if DEBUG: + traceback.print_exc() class DetailsThreadPool(GenericDownloadThreadPool): @@ -191,7 +194,8 @@ class DetailsThread(Thread): callback(result) self.tasks.task_done() except: - continue + if DEBUG: + traceback.print_exc() class CacheUpdateThreadPool(GenericDownloadThreadPool): @@ -221,4 +225,5 @@ class CacheUpdateThread(Thread): store_plugin, timeout = self.tasks.get() store_plugin.update_cache(timeout=timeout, suppress_progress=True) except: - traceback.print_exc() + if DEBUG: + traceback.print_exc() diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index f7e8c88cd9..eea1a692de 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -139,14 +139,17 @@ class SearchDialog(QDialog, Ui_Dialog): query = query.replace('>', '') query = query.replace('<', '') # Remove the prefix. - for loc in ( 'all', 'author', 'authors', 'title'): - query = re.sub(r'%s:"?(?P[^\s"]+)"?' % loc, '\g', query) + for loc in ('all', 'author', 'authors', 'title'): + query = re.sub(r'%s:"(?P[^\s"]+)"' % loc, '\g', query) + query = query.replace('%s:' % loc, '') # Remove the prefix and search text. for loc in ('cover', 'drm', 'format', 'formats', 'price', 'store'): query = re.sub(r'%s:"[^"]"' % loc, '', query) query = re.sub(r'%s:[^\s]*' % loc, '', query) # Remove logic. - query = re.sub(r'(^|\s)(and|not|or)(\s|$)', ' ', query) + query = re.sub(r'(^|\s)(and|not|or|a|the|is|of)(\s|$)', ' ', query) + # Remove " + query = query.replace('"', '') # Remove excess whitespace. query = re.sub(r'\s{2,}', ' ', query) query = query.strip() From 8c6c5c668fd6d95a6a035d72b7eb0fe4413641bc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 May 2011 22:28:24 -0600 Subject: [PATCH 12/17] ... --- recipes/arcamax.recipe | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/recipes/arcamax.recipe b/recipes/arcamax.recipe index db4d753cef..d1c1c6766d 100644 --- a/recipes/arcamax.recipe +++ b/recipes/arcamax.recipe @@ -93,7 +93,7 @@ class Arcamax(BasicNewsRecipe): for page in pages: page_soup = self.index_to_soup(url) if page_soup: - title = page_soup.find(name='div', attrs={'class':'comics-header'}).h1.contents[0] + title = self.tag_to_string(page_soup.find(name='div', attrs={'class':'comics-header'}).h1.contents[0]) page_url = url # orig prev_page_url = 'http://www.arcamax.com' + page_soup.find('a', attrs={'class':'prev'}, text='Previous').parent['href'] prev_page_url = 'http://www.arcamax.com' + page_soup.find('span', text='Previous').parent.parent['href'] @@ -127,4 +127,3 @@ class Arcamax(BasicNewsRecipe): p{font-family:Arial,Helvetica,sans-serif;font-size:small;} body{font-family:Helvetica,Arial,sans-serif;font-size:small;} ''' - From da90da98e3dcf73c855ceac5edc9092608dafed3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 09:07:29 -0600 Subject: [PATCH 13/17] Handle people with legacy databases that have custom columns with non ascii search names. Fixes #781490 (KeyError happens whenever I click on a book in the main screen (list)) --- src/calibre/ebooks/metadata/book/base.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 7927517b22..ceb6751238 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -112,10 +112,15 @@ class Metadata(object): Be careful with numeric fields since this will return True for zero as well as None. + + Also returns True if the field does not exist. ''' - null_val = NULL_VALUES.get(field, None) - val = getattr(self, field, None) - return not val or val == null_val + try: + null_val = NULL_VALUES.get(field, None) + val = getattr(self, field, None) + return not val or val == null_val + except: + return True def __getattribute__(self, field): _data = object.__getattribute__(self, '_data') From c7c9ade376c0461e4bac6bb3fd531437e6d14955 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 09:40:52 -0600 Subject: [PATCH 14/17] Driver for Dell Streak on windows --- src/calibre/devices/android/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index ca84271778..db473a755e 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -109,7 +109,7 @@ class ANDROID(USBMS): 'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H', 'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD', '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2', - 'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB'] + 'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD'] From 16c92f3d23728b76c1cf4240cfe9aa02f60f9918 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 10:10:14 -0600 Subject: [PATCH 15/17] Updated Ming Pao and Various Taiwanese news sources by Eddie Lau --- recipes/china_times.recipe | 42 +++ recipes/liberty_times.recipe | 44 +++ recipes/ming_pao.recipe | 656 +++++++++++++++++++---------------- recipes/united_daily.recipe | 67 ++++ 4 files changed, 505 insertions(+), 304 deletions(-) create mode 100644 recipes/china_times.recipe create mode 100644 recipes/liberty_times.recipe create mode 100644 recipes/united_daily.recipe diff --git a/recipes/china_times.recipe b/recipes/china_times.recipe new file mode 100644 index 0000000000..8c1493d71f --- /dev/null +++ b/recipes/china_times.recipe @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +__license__ = 'GPL v3' +# dug from http://www.mobileread.com/forums/showthread.php?p=1012294 + +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1277443634(BasicNewsRecipe): + title = u'中時電子報' + oldest_article = 1 + max_articles_per_feed = 100 + + feeds = [(u'焦點', u'http://rss.chinatimes.com/rss/focus-u.rss'), + (u'政治', u'http://rss.chinatimes.com/rss/Politic-u.rss'), + (u'社會', u'http://rss.chinatimes.com/rss/social-u.rss'), + (u'國際', u'http://rss.chinatimes.com/rss/international-u.rss'), + (u'兩岸', u'http://rss.chinatimes.com/rss/mainland-u.rss'), + (u'地方', u'http://rss.chinatimes.com/rss/local-u.rss'), + (u'言論', u'http://rss.chinatimes.com/rss/comment-u.rss'), + (u'科技', u'http://rss.chinatimes.com/rss/technology-u.rss'), + (u'運動', u'http://rss.chinatimes.com/rss/sport-u.rss'), + (u'藝文', u'http://rss.chinatimes.com/rss/philology-u.rss'), + #(u'旺報', u'http://rss.chinatimes.com/rss/want-u.rss'), + #(u'財經', u'http://rss.chinatimes.com/rss/finance-u.rss'), # broken links + #(u'股市', u'http://rss.chinatimes.com/rss/stock-u.rss') # broken links + ] + + __author__ = 'einstuerzende, updated by Eddie Lau' + __version__ = '1.0' + language = 'zh' + publisher = 'China Times Group' + description = 'China Times (Taiwan)' + category = 'News, Chinese, Taiwan' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + encoding = 'big5' + conversion_options = {'linearize_tables':True} + masthead_url = 'http://www.fcuaa.org/gif/chinatimeslogo.gif' + cover_url = 'http://www.fcuaa.org/gif/chinatimeslogo.gif' + keep_only_tags = [dict(name='div', attrs={'class':['articlebox','articlebox clearfix']})] + remove_tags = [dict(name='div', attrs={'class':['focus-news']})] + diff --git a/recipes/liberty_times.recipe b/recipes/liberty_times.recipe new file mode 100644 index 0000000000..c3a9d106da --- /dev/null +++ b/recipes/liberty_times.recipe @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +__license__ = 'GPL v3' +# dug from http://www.mobileread.com/forums/showthread.php?p=1012294 + + +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1277443634(BasicNewsRecipe): + title = u'自由電子報' + oldest_article = 1 + max_articles_per_feed = 100 + + feeds = [(u'焦點新聞', u'http://www.libertytimes.com.tw/rss/fo.xml'), + (u'政治新聞', u'http://www.libertytimes.com.tw/rss/p.xml'), + (u'生活新聞', u'http://www.libertytimes.com.tw/rss/life.xml'), + (u'國際新聞', u'http://www.libertytimes.com.tw/rss/int.xml'), + (u'自由廣場', u'http://www.libertytimes.com.tw/rss/o.xml'), + (u'社會新聞', u'http://www.libertytimes.com.tw/rss/so.xml'), + (u'體育新聞', u'http://www.libertytimes.com.tw/rss/sp.xml'), + (u'財經焦點', u'http://www.libertytimes.com.tw/rss/e.xml'), + (u'證券理財', u'http://www.libertytimes.com.tw/rss/stock.xml'), + (u'影視焦點', u'http://www.libertytimes.com.tw/rss/show.xml'), + (u'北部新聞', u'http://www.libertytimes.com.tw/rss/north.xml'), + (u'中部新聞', u'http://www.libertytimes.com.tw/rss/center.xml'), + (u'南部新聞', u'http://www.libertytimes.com.tw/rss/south.xml'), + (u'大台北新聞', u'http://www.libertytimes.com.tw/rss/taipei.xml'), + (u'藝術文化', u'http://www.libertytimes.com.tw/rss/art.xml'), + ] + extra_css = '''span[class='insubject1'][id='newtitle'] {font-size:200%; font-weight:bold;}''' + __author__ = 'einstuerzende, updated by Eddie Lau' + __version__ = '1.1' + language = 'zh' + publisher = 'Liberty Times Group' + description = 'Liberty Times (Taiwan)' + category = 'News, Chinese, Taiwan' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + encoding = 'big5' + conversion_options = {'linearize_tables':True} + masthead_url = 'http://www.libertytimes.com.tw/2008/images/img_auto/005/logo_new.gif' + cover_url = 'http://www.libertytimes.com.tw/2008/images/img_auto/005/logo_new.gif' + keep_only_tags = [dict(name='td', attrs={'id':['newsContent']})] + diff --git a/recipes/ming_pao.recipe b/recipes/ming_pao.recipe index 4a405a59dd..08ee20cb15 100644 --- a/recipes/ming_pao.recipe +++ b/recipes/ming_pao.recipe @@ -1,15 +1,18 @@ +# -*- coding: utf-8 -*- __license__ = 'GPL v3' __copyright__ = '2010-2011, Eddie Lau' -# Users of Kindle 3 (with limited system-level CJK support) +# Users of Kindle 3 with limited system-level CJK support # please replace the following "True" with "False". __MakePeriodical__ = True -# Turn it to True if your device supports display of CJK titles +# Turn below to true if your device supports display of CJK titles __UseChineseTitle__ = False - +# Trun below to true if you wish to use life.mingpao.com as the main article source +__UseLife__ = True ''' Change Log: +2011/05/12: switch the main parse source to life.mingpao.com, which has more photos on the article pages 2011/03/06: add new articles for finance section, also a new section "Columns" 2011/02/28: rearrange the sections [Disabled until Kindle has better CJK support and can remember last (section,article) read in Sections & Articles @@ -32,41 +35,43 @@ import os, datetime, re from calibre.web.feeds.recipes import BasicNewsRecipe from contextlib import nested + from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.ebooks.metadata.toc import TOC from calibre.ebooks.metadata import MetaInformation class MPHKRecipe(BasicNewsRecipe): - title = 'Ming Pao - Hong Kong' - oldest_article = 1 - max_articles_per_feed = 100 - __author__ = 'Eddie Lau' - description = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)' - publisher = 'MingPao' - category = 'Chinese, News, Hong Kong' - remove_javascript = True - use_embedded_content = False - no_stylesheets = True - language = 'zh' - encoding = 'Big5-HKSCS' - recursions = 0 - conversion_options = {'linearize_tables':True} - timefmt = '' - extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}' - masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif' - keep_only_tags = [dict(name='h1'), + title = 'Ming Pao - Hong Kong' + oldest_article = 1 + max_articles_per_feed = 100 + __author__ = 'Eddie Lau' + description = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)' + publisher = 'MingPao' + category = 'Chinese, News, Hong Kong' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + language = 'zh' + encoding = 'Big5-HKSCS' + recursions = 0 + conversion_options = {'linearize_tables':True} + timefmt = '' + extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}' + masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif' + keep_only_tags = [dict(name='h1'), dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page title dict(name='font', attrs={'color':['AA0000']}), # for column articles title dict(attrs={'id':['newscontent']}), # entertainment and column page content dict(attrs={'id':['newscontent01','newscontent02']}), - dict(attrs={'class':['photo']}) + dict(attrs={'class':['photo']}), + dict(name='img', attrs={'width':['180'], 'alt':['按圖放大']}) # images for source from life.mingpao.com ] - remove_tags = [dict(name='style'), - dict(attrs={'id':['newscontent135']}), # for the finance page - dict(name='table')] # for content fetched from life.mingpao.com - remove_attributes = ['width'] - preprocess_regexps = [ + remove_tags = [dict(name='style'), + dict(attrs={'id':['newscontent135']}), # for the finance page from mpfinance.com + dict(name='table')] # for content fetched from life.mingpao.com + remove_attributes = ['width'] + preprocess_regexps = [ (re.compile(r'
', re.DOTALL|re.IGNORECASE), lambda match: '

'), (re.compile(r'

', re.DOTALL|re.IGNORECASE), @@ -80,10 +85,10 @@ class MPHKRecipe(BasicNewsRecipe): lambda match: "") ] - def image_url_processor(cls, baseurl, url): - # trick: break the url at the first occurance of digit, add an additional - # '_' at the front - # not working, may need to move this to preprocess_html() method + def image_url_processor(cls, baseurl, url): + # trick: break the url at the first occurance of digit, add an additional + # '_' at the front + # not working, may need to move this to preprocess_html() method # minIdx = 10000 # i0 = url.find('0') # if i0 >= 0 and i0 < minIdx: @@ -115,314 +120,357 @@ class MPHKRecipe(BasicNewsRecipe): # i9 = url.find('9') # if i9 >= 0 and i9 < minIdx: # minIdx = i9 - return url + return url - def get_dtlocal(self): - dt_utc = datetime.datetime.utcnow() - # convert UTC to local hk time - at around HKT 6.00am, all news are available - dt_local = dt_utc - datetime.timedelta(-2.0/24) - return dt_local + def get_dtlocal(self): + dt_utc = datetime.datetime.utcnow() + # convert UTC to local hk time - at around HKT 6.00am, all news are available + dt_local = dt_utc - datetime.timedelta(-2.0/24) + return dt_local - def get_fetchdate(self): - return self.get_dtlocal().strftime("%Y%m%d") + def get_fetchdate(self): + return self.get_dtlocal().strftime("%Y%m%d") - def get_fetchformatteddate(self): - return self.get_dtlocal().strftime("%Y-%m-%d") + def get_fetchformatteddate(self): + return self.get_dtlocal().strftime("%Y-%m-%d") - def get_fetchday(self): - # convert UTC to local hk time - at around HKT 6.00am, all news are available - return self.get_dtlocal().strftime("%d") + def get_fetchday(self): + # dt_utc = datetime.datetime.utcnow() + # convert UTC to local hk time - at around HKT 6.00am, all news are available + # dt_local = dt_utc - datetime.timedelta(-2.0/24) + return self.get_dtlocal().strftime("%d") - def get_cover_url(self): - cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg' - br = BasicNewsRecipe.get_browser() - try: - br.open(cover) - except: - cover = None - return cover + def get_cover_url(self): + cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + cover = None + return cover - def parse_index(self): - feeds = [] - dateStr = self.get_fetchdate() + def parse_index(self): + feeds = [] + dateStr = self.get_fetchdate() - for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), - (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), - (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]: - articles = self.parse_section(url) - if articles: - feeds.append((title, articles)) + if __UseLife__: + for title, url, keystr in [(u'\u8981\u805e Headline', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalga', 'nal'), + (u'\u6e2f\u805e Local', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgb', 'nal'), + (u'\u6559\u80b2 Education', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgf', 'nal'), + (u'\u793e\u8a55/\u7b46\u9663 Editorial', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr', 'nal'), + (u'\u8ad6\u58c7 Forum', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalfa', 'nal'), + (u'\u4e2d\u570b China', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalca', 'nal'), + (u'\u570b\u969b World', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalta', 'nal'), + (u'\u7d93\u6fdf Finance', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea', 'nal'), + (u'\u9ad4\u80b2 Sport', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalsp', 'nal'), + (u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal'), + (u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl')]: + articles = self.parse_section2(url, keystr) + if articles: + feeds.append((title, articles)) - # special- editorial - ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr') - if ed_articles: - feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles)) + for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'), + (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) + else: + for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), + (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), + (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) - for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), - (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'), - (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]: - articles = self.parse_section(url) - if articles: - feeds.append((title, articles)) + # special- editorial + ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr') + if ed_articles: + feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles)) - # special - finance - #fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm') - fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea') - if fin_articles: - feeds.append((u'\u7d93\u6fdf Finance', fin_articles)) + for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), + (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'), + (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) - for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), - (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]: - articles = self.parse_section(url) - if articles: - feeds.append((title, articles)) + # special - finance + #fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm') + fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea') + if fin_articles: + feeds.append((u'\u7d93\u6fdf Finance', fin_articles)) - # special - entertainment - ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm') - if ent_articles: - feeds.append((u'\u5f71\u8996 Film/TV', ent_articles)) + for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), + (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) - for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'), - (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]: - articles = self.parse_section(url) - if articles: - feeds.append((title, articles)) + # special - entertainment + ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm') + if ent_articles: + feeds.append((u'\u5f71\u8996 Film/TV', ent_articles)) + + for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'), + (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) - # special- columns - col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn') - if col_articles: - feeds.append((u'\u5c08\u6b04 Columns', col_articles)) + # special- columns + col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn') + if col_articles: + feeds.append((u'\u5c08\u6b04 Columns', col_articles)) - return feeds + return feeds - def parse_section(self, url): - dateStr = self.get_fetchdate() - soup = self.index_to_soup(url) - divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']}) - current_articles = [] - included_urls = [] - divs.reverse() - for i in divs: - a = i.find('a', href = True) - title = self.tag_to_string(a) - url = a.get('href', False) - url = 'http://news.mingpao.com/' + dateStr + '/' +url - if url not in included_urls and url.rfind('Redirect') == -1: - current_articles.append({'title': title, 'url': url, 'description':'', 'date':''}) - included_urls.append(url) - current_articles.reverse() - return current_articles + # parse from news.mingpao.com + def parse_section(self, url): + dateStr = self.get_fetchdate() + soup = self.index_to_soup(url) + divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']}) + current_articles = [] + included_urls = [] + divs.reverse() + for i in divs: + a = i.find('a', href = True) + title = self.tag_to_string(a) + url = a.get('href', False) + url = 'http://news.mingpao.com/' + dateStr + '/' +url + if url not in included_urls and url.rfind('Redirect') == -1: + current_articles.append({'title': title, 'url': url, 'description':'', 'date':''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def parse_ed_section(self, url): - self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href=True) - a.reverse() - current_articles = [] - included_urls = [] - for i in a: - title = self.tag_to_string(i) - url = 'http://life.mingpao.com/cfm/' + i.get('href', False) - if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('nal') == -1): - current_articles.append({'title': title, 'url': url, 'description': ''}) - included_urls.append(url) - current_articles.reverse() - return current_articles + # parse from life.mingpao.com + def parse_section2(self, url, keystr): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind(keystr) == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def parse_fin_section(self, url): - self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href= True) - current_articles = [] - included_urls = [] - for i in a: - #url = 'http://www.mpfinance.com/cfm/' + i.get('href', False) - url = 'http://life.mingpao.com/cfm/' + i.get('href', False) - #if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1: - if url not in included_urls and (not url.rfind('txt') == -1) and (not url.rfind('nal') == -1): - title = self.tag_to_string(i) - current_articles.append({'title': title, 'url': url, 'description':''}) - included_urls.append(url) - return current_articles + def parse_ed_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('nal') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def parse_ent_section(self, url): - self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href=True) - a.reverse() - current_articles = [] - included_urls = [] - for i in a: - title = self.tag_to_string(i) - url = 'http://ol.mingpao.com/cfm/' + i.get('href', False) - if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1): - current_articles.append({'title': title, 'url': url, 'description': ''}) - included_urls.append(url) - current_articles.reverse() - return current_articles + def parse_fin_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href= True) + current_articles = [] + included_urls = [] + for i in a: + #url = 'http://www.mpfinance.com/cfm/' + i.get('href', False) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + #if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1: + if url not in included_urls and (not url.rfind('txt') == -1) and (not url.rfind('nal') == -1): + title = self.tag_to_string(i) + current_articles.append({'title': title, 'url': url, 'description':''}) + included_urls.append(url) + return current_articles - def parse_col_section(self, url): - self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href=True) - a.reverse() - current_articles = [] - included_urls = [] - for i in a: - title = self.tag_to_string(i) - url = 'http://life.mingpao.com/cfm/' + i.get('href', False) - if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('ncl') == -1): - current_articles.append({'title': title, 'url': url, 'description': ''}) - included_urls.append(url) - current_articles.reverse() - return current_articles + def parse_ent_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://ol.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - for item in soup.findAll(style=True): - del item['width'] - for item in soup.findAll(stype=True): - del item['absmiddle'] - return soup + def parse_col_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('ncl') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def create_opf(self, feeds, dir=None): - if dir is None: - dir = self.output_dir - if __UseChineseTitle__ == True: - title = u'\u660e\u5831 (\u9999\u6e2f)' - else: - title = self.short_title() - # if not generating a periodical, force date to apply in title - if __MakePeriodical__ == False: - title = title + ' ' + self.get_fetchformatteddate() - if True: - mi = MetaInformation(title, [self.publisher]) - mi.publisher = self.publisher - mi.author_sort = self.publisher - if __MakePeriodical__ == True: - mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() - else: - mi.publication_type = self.publication_type+':'+self.short_title() - #mi.timestamp = nowf() - mi.timestamp = self.get_dtlocal() - mi.comments = self.description - if not isinstance(mi.comments, unicode): - mi.comments = mi.comments.decode('utf-8', 'replace') - #mi.pubdate = nowf() - mi.pubdate = self.get_dtlocal() - opf_path = os.path.join(dir, 'index.opf') - ncx_path = os.path.join(dir, 'index.ncx') - opf = OPFCreator(dir, mi) - # Add mastheadImage entry to section - mp = getattr(self, 'masthead_path', None) - if mp is not None and os.access(mp, os.R_OK): - from calibre.ebooks.metadata.opf2 import Guide - ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu()) - ref.type = 'masthead' - ref.title = 'Masthead Image' - opf.guide.append(ref) + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll(style=True): + del item['width'] + for item in soup.findAll(stype=True): + del item['absmiddle'] + return soup - manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))] - manifest.append(os.path.join(dir, 'index.html')) - manifest.append(os.path.join(dir, 'index.ncx')) + def create_opf(self, feeds, dir=None): + if dir is None: + dir = self.output_dir + if __UseChineseTitle__ == True: + title = u'\u660e\u5831 (\u9999\u6e2f)' + else: + title = self.short_title() + # if not generating a periodical, force date to apply in title + if __MakePeriodical__ == False: + title = title + ' ' + self.get_fetchformatteddate() + if True: + mi = MetaInformation(title, [self.publisher]) + mi.publisher = self.publisher + mi.author_sort = self.publisher + if __MakePeriodical__ == True: + mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() + else: + mi.publication_type = self.publication_type+':'+self.short_title() + #mi.timestamp = nowf() + mi.timestamp = self.get_dtlocal() + mi.comments = self.description + if not isinstance(mi.comments, unicode): + mi.comments = mi.comments.decode('utf-8', 'replace') + #mi.pubdate = nowf() + mi.pubdate = self.get_dtlocal() + opf_path = os.path.join(dir, 'index.opf') + ncx_path = os.path.join(dir, 'index.ncx') + opf = OPFCreator(dir, mi) + # Add mastheadImage entry to section + mp = getattr(self, 'masthead_path', None) + if mp is not None and os.access(mp, os.R_OK): + from calibre.ebooks.metadata.opf2 import Guide + ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu()) + ref.type = 'masthead' + ref.title = 'Masthead Image' + opf.guide.append(ref) - # Get cover - cpath = getattr(self, 'cover_path', None) - if cpath is None: - pf = open(os.path.join(dir, 'cover.jpg'), 'wb') - if self.default_cover(pf): - cpath = pf.name - if cpath is not None and os.access(cpath, os.R_OK): - opf.cover = cpath - manifest.append(cpath) + manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))] + manifest.append(os.path.join(dir, 'index.html')) + manifest.append(os.path.join(dir, 'index.ncx')) - # Get masthead - mpath = getattr(self, 'masthead_path', None) - if mpath is not None and os.access(mpath, os.R_OK): - manifest.append(mpath) + # Get cover + cpath = getattr(self, 'cover_path', None) + if cpath is None: + pf = open(os.path.join(dir, 'cover.jpg'), 'wb') + if self.default_cover(pf): + cpath = pf.name + if cpath is not None and os.access(cpath, os.R_OK): + opf.cover = cpath + manifest.append(cpath) - opf.create_manifest_from_files_in(manifest) - for mani in opf.manifest: - if mani.path.endswith('.ncx'): - mani.id = 'ncx' - if mani.path.endswith('mastheadImage.jpg'): - mani.id = 'masthead-image' - entries = ['index.html'] - toc = TOC(base_path=dir) - self.play_order_counter = 0 - self.play_order_map = {} + # Get masthead + mpath = getattr(self, 'masthead_path', None) + if mpath is not None and os.access(mpath, os.R_OK): + manifest.append(mpath) - def feed_index(num, parent): - f = feeds[num] - for j, a in enumerate(f): - if getattr(a, 'downloaded', False): - adir = 'feed_%d/article_%d/'%(num, j) - auth = a.author - if not auth: - auth = None - desc = a.text_summary - if not desc: - desc = None - else: - desc = self.description_limiter(desc) - entries.append('%sindex.html'%adir) - po = self.play_order_map.get(entries[-1], None) - if po is None: - self.play_order_counter += 1 - po = self.play_order_counter - parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'), + opf.create_manifest_from_files_in(manifest) + for mani in opf.manifest: + if mani.path.endswith('.ncx'): + mani.id = 'ncx' + if mani.path.endswith('mastheadImage.jpg'): + mani.id = 'masthead-image' + entries = ['index.html'] + toc = TOC(base_path=dir) + self.play_order_counter = 0 + self.play_order_map = {} + + def feed_index(num, parent): + f = feeds[num] + for j, a in enumerate(f): + if getattr(a, 'downloaded', False): + adir = 'feed_%d/article_%d/'%(num, j) + auth = a.author + if not auth: + auth = None + desc = a.text_summary + if not desc: + desc = None + else: + desc = self.description_limiter(desc) + entries.append('%sindex.html'%adir) + po = self.play_order_map.get(entries[-1], None) + if po is None: + self.play_order_counter += 1 + po = self.play_order_counter + parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'), play_order=po, author=auth, description=desc) - last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep)) - for sp in a.sub_pages: - prefix = os.path.commonprefix([opf_path, sp]) - relp = sp[len(prefix):] - entries.append(relp.replace(os.sep, '/')) - last = sp + last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep)) + for sp in a.sub_pages: + prefix = os.path.commonprefix([opf_path, sp]) + relp = sp[len(prefix):] + entries.append(relp.replace(os.sep, '/')) + last = sp - if os.path.exists(last): - with open(last, 'rb') as fi: - src = fi.read().decode('utf-8') - soup = BeautifulSoup(src) - body = soup.find('body') - if body is not None: - prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last)))) - templ = self.navbar.generate(True, num, j, len(f), + if os.path.exists(last): + with open(last, 'rb') as fi: + src = fi.read().decode('utf-8') + soup = BeautifulSoup(src) + body = soup.find('body') + if body is not None: + prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last)))) + templ = self.navbar.generate(True, num, j, len(f), not self.has_single_feed, a.orig_url, self.publisher, prefix=prefix, center=self.center_navbar) - elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') - body.insert(len(body.contents), elem) - with open(last, 'wb') as fi: - fi.write(unicode(soup).encode('utf-8')) - if len(feeds) == 0: - raise Exception('All feeds are empty, aborting.') + elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') + body.insert(len(body.contents), elem) + with open(last, 'wb') as fi: + fi.write(unicode(soup).encode('utf-8')) + if len(feeds) == 0: + raise Exception('All feeds are empty, aborting.') - if len(feeds) > 1: - for i, f in enumerate(feeds): - entries.append('feed_%d/index.html'%i) - po = self.play_order_map.get(entries[-1], None) - if po is None: - self.play_order_counter += 1 - po = self.play_order_counter - auth = getattr(f, 'author', None) - if not auth: - auth = None - desc = getattr(f, 'description', None) - if not desc: - desc = None - feed_index(i, toc.add_item('feed_%d/index.html'%i, None, + if len(feeds) > 1: + for i, f in enumerate(feeds): + entries.append('feed_%d/index.html'%i) + po = self.play_order_map.get(entries[-1], None) + if po is None: + self.play_order_counter += 1 + po = self.play_order_counter + auth = getattr(f, 'author', None) + if not auth: + auth = None + desc = getattr(f, 'description', None) + if not desc: + desc = None + feed_index(i, toc.add_item('feed_%d/index.html'%i, None, f.title, play_order=po, description=desc, author=auth)) - else: - entries.append('feed_%d/index.html'%0) - feed_index(0, toc) + else: + entries.append('feed_%d/index.html'%0) + feed_index(0, toc) - for i, p in enumerate(entries): - entries[i] = os.path.join(dir, p.replace('/', os.sep)) - opf.create_spine(entries) - opf.set_toc(toc) + for i, p in enumerate(entries): + entries[i] = os.path.join(dir, p.replace('/', os.sep)) + opf.create_spine(entries) + opf.set_toc(toc) + + with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): + opf.render(opf_file, ncx_file) - with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): - opf.render(opf_file, ncx_file) diff --git a/recipes/united_daily.recipe b/recipes/united_daily.recipe new file mode 100644 index 0000000000..6954a7e725 --- /dev/null +++ b/recipes/united_daily.recipe @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +__license__ = 'GPL v3' + +from calibre.web.feeds.news import BasicNewsRecipe + +class UnitedDaily(BasicNewsRecipe): + title = u'聯合新聞網' + oldest_article = 1 + max_articles_per_feed = 100 + + feeds = [(u'焦點', u'http://udn.com/udnrss/focus.xml'), + (u'政治', u'http://udn.com/udnrss/politics.xml'), + (u'社會', u'http://udn.com/udnrss/social.xml'), + (u'生活', u'http://udn.com/udnrss/life.xml'), + (u'綜合', u'http://udn.com/udnrss/education.xml'), + (u'意見評論', u'http://udn.com/udnrss/opinion.xml'), + (u'大台北', u'http://udn.com/udnrss/local_taipei.xml'), + (u'桃竹苗', u'http://udn.com/udnrss/local_tyhcml.xml'), + (u'中彰投', u'http://udn.com/udnrss/local_tcchnt.xml'), + (u'雲嘉南', u'http://udn.com/udnrss/local_ylcytn.xml'), + (u'高屏離島', u'http://udn.com/udnrss/local_ksptisland.xml'), + (u'基宜花東', u'http://udn.com/udnrss/local_klilhltt.xml'), + (u'台灣百寶鄉', u'http://udn.com/udnrss/local_oddlyenough.xml'), + (u'兩岸要聞', u'http://udn.com/udnrss/mainland.xml'), + (u'國際焦點', u'http://udn.com/udnrss/international.xml'), + (u'台商經貿', u'http://udn.com/udnrss/financechina.xml'), + (u'國際財經', u'http://udn.com/udnrss/financeworld.xml'), + (u'財經焦點', u'http://udn.com/udnrss/financesfocus.xml'), + (u'股市要聞', u'http://udn.com/udnrss/stock.xml'), + (u'股市快訊', u'http://udn.com/udnrss/stklatest.xml'), + (u'稅務法務', u'http://udn.com/udnrss/tax.xml'), + (u'房市情報', u'http://udn.com/udnrss/houses.xml'), + (u'棒球', u'http://udn.com/udnrss/baseball.xml'), + (u'籃球', u'http://udn.com/udnrss/basketball.xml'), + (u'體壇動態', u'http://udn.com/udnrss/sportsfocus.xml'), + (u'熱門星聞', u'http://udn.com/udnrss/starsfocus.xml'), + (u'廣電港陸', u'http://udn.com/udnrss/tv.xml'), + (u'海外星球', u'http://udn.com/udnrss/starswestern.xml'), + (u'日韓星情', u'http://udn.com/udnrss/starsjk.xml'), + (u'電影世界', u'http://udn.com/udnrss/movie.xml'), + (u'流行音樂', u'http://udn.com/udnrss/music.xml'), + (u'觀點專題', u'http://udn.com/udnrss/starssubject.xml'), + (u'食樂指南', u'http://udn.com/udnrss/food.xml'), + (u'折扣好康', u'http://udn.com/udnrss/shopping.xml'), + (u'醫藥新聞', u'http://udn.com/udnrss/health.xml'), + (u'家婦繽紛', u'http://udn.com/udnrss/benfen.xml'), + (u'談星論命', u'http://udn.com/udnrss/astrology.xml'), + (u'文化副刊', u'http://udn.com/udnrss/reading.xml'), + ] + + extra_css = '''div[id='story_title'] {font-size:200%; font-weight:bold;}''' + + __author__ = 'Eddie Lau' + __version__ = '1.0' + language = 'zh' + publisher = 'United Daily News Group' + description = 'United Daily (Taiwan)' + category = 'News, Chinese, Taiwan' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + encoding = 'big5' + conversion_options = {'linearize_tables':True} + masthead_url = 'http://udn.com/NEWS/2004/images/logo_udn.gif' + cover_url = 'http://udn.com/NEWS/2004/images/logo_udn.gif' + keep_only_tags = [dict(name='div', attrs={'id':['story_title','story_author', 'story']})] + remove_tags = [dict(name='div', attrs={'id':['mvouter']})] From a204568f2ee507292716ff42bc3c03bd0fa85755 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 10:38:07 -0600 Subject: [PATCH 16/17] Fix #779994 (Metadata download API priority only partially considered) --- .../ebooks/metadata/sources/identify.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index 31998dfcb2..b084f86294 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -372,6 +372,18 @@ def identify(log, abort, # {{{ longest, lp = -1, '' for plugin, presults in results.iteritems(): presults.sort(key=plugin.identify_results_keygen(**sort_kwargs)) + + # Throw away lower priority results from the same source that have exactly the same + # title and authors as a higher priority result + filter_results = set() + filtered_results = [] + for r in presults: + key = (r.title, tuple(r.authors)) + if key not in filter_results: + filtered_results.append(r) + filter_results.add(key) + presults = filtered_results + plog = logs[plugin].getvalue().strip() log('\n'+'*'*30, plugin.name, '*'*30) log('Request extra headers:', plugin.browser.addheaders) @@ -479,7 +491,7 @@ if __name__ == '__main__': # tests {{{ ( {'title':'Magykal Papers', 'authors':['Sage']}, - [title_test('The Magykal Papers', exact=True)], + [title_test('Septimus Heap: The Magykal Papers', exact=True)], ), @@ -506,12 +518,6 @@ if __name__ == '__main__': # tests {{{ exact=True), authors_test(['Dan Brown'])] ), - ( # No ISBN - {'title':'Justine', 'authors':['Durrel']}, - [title_test('Justine', exact=True), - authors_test(['Lawrence Durrel'])] - ), - ( # A newer book {'identifiers':{'isbn': '9780316044981'}}, [title_test('The Heroes', exact=True), From 0f7272d1b4fafcfa1be45a78d02017a0ab3ca789 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 12 May 2011 10:58:54 -0600 Subject: [PATCH 17/17] Fix #781759 (Identifiers lost by 0.8 metadata download) --- src/calibre/gui2/actions/edit_metadata.py | 4 ++++ src/calibre/gui2/metadata/single.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 4ab4950179..ac475cb027 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -478,6 +478,10 @@ class EditMetadataAction(InterfaceAction): try: set_title = not mi.is_null('title') set_authors = not mi.is_null('authors') + idents = db.get_identifiers(i, index_is_id=True) + if mi.identifiers: + idents.update(mi.identifiers) + mi.identifiers = idents db.set_metadata(i, mi, commit=False, set_title=set_title, set_authors=set_authors, notify=False) self.applied_ids.append(i) diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index 5c4e241bba..099831ccba 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -336,7 +336,9 @@ class MetadataSingleDialogBase(ResizableDialog): if not mi.is_null('tags'): self.tags.current_val = mi.tags if not mi.is_null('identifiers'): - self.identifiers.current_val = mi.identifiers + current = self.identifiers.current_val + current.update(mi.identifiers) + self.identifiers.current_val = current if not mi.is_null('pubdate'): self.pubdate.current_val = mi.pubdate if not mi.is_null('series') and mi.series.strip():