From 925b3ea1fcdf76b98b9048f09c8d5333f85deead Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jan 2010 13:45:50 -0700 Subject: [PATCH 1/4] EPUB Output: Fix play order in generated NCX being uniformly zero when the input HTML file has a name with special characters. Fixes #4397 (Play order) --- src/calibre/ebooks/oeb/base.py | 8 ++++---- src/calibre/manual/faq.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index c8e66b70bb..c57dfc3706 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -1806,7 +1806,7 @@ class OEBBook(object): return {OPF_MIME: ('content.opf', package)} def _update_playorder(self, ncx): - hrefs = set(xpath(ncx, '//ncx:content/@src')) + hrefs = set(map(urlnormalize, xpath(ncx, '//ncx:content/@src'))) playorder = {} next = 1 selector = XPath('h:body//*[@id or @name]') @@ -1828,9 +1828,9 @@ class OEBBook(object): if added: next += 1 selector = XPath('ncx:content/@src') - for elem in xpath(ncx, '//*[@playOrder and ./ncx:content[@src]]'): - href = selector(elem)[0] - order = playorder.get(href, 0) + for i, elem in enumerate(xpath(ncx, '//*[@playOrder and ./ncx:content[@src]]')): + href = urlnormalize(selector(elem)[0]) + order = playorder.get(href, i) elem.attrib['playOrder'] = str(order) return diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 9e0b2a85de..18feb2a519 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -86,7 +86,7 @@ At the moment |app| has full support for the SONY PRS 300/500/505/600/700/900, B How can I help get my device supported in |app|? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If your device appears as a USB disk to the operating system. Adding support for it to |app| is very easy. +If your device appears as a USB disk to the operating system, adding support for it to |app| is very easy. We just need some information from you: * What e-book formats does your device support? From c5fa78d8689732f252a1812a18448764301ac7d2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jan 2010 13:48:21 -0700 Subject: [PATCH 2/4] New recipe for The New England Journal of Medicine by Krittika Goyal --- resources/recipes/nejm.recipe | 86 +++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 resources/recipes/nejm.recipe diff --git a/resources/recipes/nejm.recipe b/resources/recipes/nejm.recipe new file mode 100644 index 0000000000..8337ece34b --- /dev/null +++ b/resources/recipes/nejm.recipe @@ -0,0 +1,86 @@ +import string, re +from calibre import strftime +from calibre.web.feeds.recipes import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup + +class NYTimes(BasicNewsRecipe): + + title = 'New England Journal of Medicine' + __author__ = 'Krittika Goyal' + description = 'Medical news' + timefmt = ' [%d %b, %Y]' + needs_subscription = True + + no_stylesheets = True + #remove_tags_before = dict(name='h1', attrs={'class':'heading'}) + #remove_tags_after = dict(name='td', attrs={'class':'newptool1'}) + remove_tags = [ + dict(name='iframe'), + #dict(name='div', attrs={'class':'related-articles'}), + #dict(name='div', attrs={'id':['qrformdiv', 'inSection', 'alpha-inner']}), + dict(name='form', attrs={'onsubmit':"return verifySearch(this.w,'Keyword, citation, or author')"}), + dict(name='table', attrs={'cellspacing':'0'}), + ] + + def preprocess_html(self, soup): + table = soup.find('table') + if table is not None: + table.extract() + return soup + + #TO LOGIN + def get_browser(self): + br = BasicNewsRecipe.get_browser() + br.open('http://content.nejm.org/cgi/login?uri=/') + br.select_form(nr=0) + br['username'] = self.username + br['code'] = self.password + response = br.submit() + raw = response.read() + if 'Welcome' not in raw: + raise Exception('Login failed. Check your username and password') + return br + + #TO GET ARTICLE TOC + def nejm_get_index(self): + return self.index_to_soup('http://content.nejm.org/current.dtl') + + # To parse artice toc + def parse_index(self): + soup = self.nejm_get_index() + + div = soup.find(id='centerTOC') + + current_section = None + current_articles = [] + feeds = [] + for x in div.findAll(True): + if x.name == 'img' and '/toc/' in x.get('src', '') and 'uarrow.gif' not in x.get('src', ''): + # Section heading found + if current_articles and current_section and 'Week in the' not in current_section: + feeds.append((current_section, current_articles)) + current_section = x.get('alt') + current_articles = [] + self.log('\tFound section:', current_section) + if current_section is not None and x.name == 'strong': + title = self.tag_to_string(x) + a = x.parent.find('a', href=lambda x: x and '/full/' in x) + if a is None: + continue + url = a.get('href', False) + if not url or not title: + continue + if url.startswith('/'): + url = 'http://content.nejm.org'+url + self.log('\t\tFound article:', title) + self.log('\t\t\t', url) + if url.startswith('/'): + url = 'http://online.wsj.com'+url + current_articles.append({'title': title, 'url':url, + 'description':'', 'date':''}) + + if current_articles and current_section: + feeds.append((current_section, current_articles)) + + return feeds + From 42765deb103b83eae87243f2a467dff3b50b6785 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jan 2010 14:10:51 -0700 Subject: [PATCH 3/4] Fix #4357 (iRiver Story not recognized by Calibre) --- src/calibre/devices/usbms/device.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 8072cd4874..e852ae7f62 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -260,7 +260,8 @@ class Device(DeviceConfig, DevicePlugin): # if the device is connected without a card, the above # will incorrectly identify the main mem as carda # See for example the driver for the Nook - if 'main' not in drives and 'carda' in drives: + if drives.get('carda', None) is not None and \ + drives.get('main', None) is None: drives['main'] = drives.pop('carda') drives = self.windows_open_callback(drives) From bed548d9dfd0018f6ea707719787465dc8099b56 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jan 2010 14:47:18 -0700 Subject: [PATCH 4/4] Implement #4234 (Sort tags for an item alphabetically.) and improved recipe for The National Post --- resources/recipes/national_post.recipe | 93 +++++++++++++++++---- src/calibre/gui2/library.py | 2 +- src/calibre/web/feeds/recipes/collection.py | 3 +- 3 files changed, 80 insertions(+), 18 deletions(-) diff --git a/resources/recipes/national_post.recipe b/resources/recipes/national_post.recipe index 660f638e63..7fe00032a5 100644 --- a/resources/recipes/national_post.recipe +++ b/resources/recipes/national_post.recipe @@ -1,20 +1,81 @@ -from calibre.web.feeds.news import BasicNewsRecipe +import string, re +from calibre import strftime +from calibre.web.feeds.recipes import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup -class AdvancedUserRecipe1261379503(BasicNewsRecipe): - title = u'National Post' - language = 'en_CA' - __author__ = 'Nick Redding' - description = u"News from Canada" - oldest_article = 2 - max_articles_per_feed = 25 +class NYTimes(BasicNewsRecipe): - keep_only_tags = [dict(name='div', attrs={'id':'content'})] - remove_tags = [dict(name='div', attrs={'class':'story-tools'}),dict(name='div', attrs={'class':'newsblock'}),dict(name='p', attrs={'class':'border-top'}),dict(name='div', attrs={'id':'footer'})] + title = 'National Post' + __author__ = 'Krittika Goyal' + description = 'Canadian national newspaper' + timefmt = ' [%d %b, %Y]' + needs_subscription = False + + no_stylesheets = True + #remove_tags_before = dict(name='h1', attrs={'class':'heading'}) + #remove_tags_after = dict(name='td', attrs={'class':'newptool1'}) + remove_tags = [ + dict(name='iframe'), + dict(name='div', attrs={'class':'story-tools'}), + #dict(name='div', attrs={'id':['qrformdiv', 'inSection', 'alpha-inner']}), + #dict(name='form', attrs={'onsubmit':''}), + #dict(name='table', attrs={'cellspacing':'0'}), + ] - feeds = [(u'News Headlines', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=section| news'), - (u'FP Headlines', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=section| financial%20post|storytype|business'), - (u'Arts & Life Headlines', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=section| arts%20%26%20life|storytype|news'), - (u'Canada News', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=storytyp e|webcanada&feed=rss'), - (u'World News', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=storytyp e|webworld&feed=rss'),(u'Editorial', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=section| editorial'), - (u'FP Opinion', u'http://www.nationalpost.com/scripts/columnists.aspx?publication=national+post&columnty pe=fp')] + # def preprocess_html(self, soup): + # table = soup.find('table') + # if table is not None: + # table.extract() + # return soup + + + #TO GET ARTICLE TOC + def nejm_get_index(self): + return self.index_to_soup('http://www.nationalpost.com/todays-paper/index.html') + + # To parse artice toc + def parse_index(self): + soup = self.nejm_get_index() + + div = soup.find(id='LegoText4') + + current_section = None + current_articles = [] + feeds = [] + for x in div.findAll(True): + if x.name == 'h4': + # Section found + if current_articles and current_section: + feeds.append((current_section, current_articles)) + current_section = self.tag_to_string(x) + current_articles = [] + self.log('\tFound section:', current_section) + if current_section is not None and x.name == 'h3': + # Article found + title = self.tag_to_string(x) + a = x.find('a', href=lambda x: x and 'story' in x) + if a is None: + continue + url = a.get('href', False) + if not url or not title: + continue + if url.startswith('story'): + url = 'http://www.nationalpost.com/todays-paper/'+url + self.log('\t\tFound article:', title) + self.log('\t\t\t', url) + current_articles.append({'title': title, 'url':url, + 'description':'', 'date':''}) + + if current_articles and current_section: + feeds.append((current_section, current_articles)) + + return feeds + def preprocess_html(self, soup): + story = soup.find(name='div', attrs={'class':'triline'}) + #td = heading.findParent(name='td') + #td.extract() + soup = BeautifulSoup('t') + body = soup.find(name='body') + body.insert(0, story) + return soup diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index 8c43e25a42..488d535c73 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -580,7 +580,7 @@ class BooksModel(QAbstractTableModel): def tags(r): tags = self.db.data[r][tgdx] if tags: - return ', '.join(tags.split(',')) + return ', '.join(sorted(tags.split(','))) def series(r): series = self.db.data[r][srdx] diff --git a/src/calibre/web/feeds/recipes/collection.py b/src/calibre/web/feeds/recipes/collection.py index fb59dd08f7..478947fcd9 100644 --- a/src/calibre/web/feeds/recipes/collection.py +++ b/src/calibre/web/feeds/recipes/collection.py @@ -20,7 +20,8 @@ NS = 'http://calibre-ebook.com/recipe_collection' E = ElementMaker(namespace=NS, nsmap={None:NS}) def iterate_over_builtin_recipe_files(): - exclude = ['craigslist', 'iht', 'outlook_india', 'toronto_sun'] + exclude = ['craigslist', 'iht', 'outlook_india', 'toronto_sun', + 'indian_express', 'india_today', 'toi'] d = os.path.dirname base = os.path.join(d(d(d(d(d(d(os.path.abspath(__file__))))))), 'resources', 'recipes') for x in os.walk(base):