From 0c25a25eb4617f501843391bd79d203bbf513950 Mon Sep 17 00:00:00 2001 From: Oliver Graf Date: Sat, 28 Jul 2012 20:34:26 +0200 Subject: [PATCH 1/4] Fix to MOBI 'Unknown' problem (replacing all empty strings with the string Unknown) --- src/calibre/ebooks/mobi/utils.py | 7 +++++-- src/calibre/ebooks/mobi/writer2/serializer.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/mobi/utils.py b/src/calibre/ebooks/mobi/utils.py index ae8e583a1b..dfdb73fdff 100644 --- a/src/calibre/ebooks/mobi/utils.py +++ b/src/calibre/ebooks/mobi/utils.py @@ -302,7 +302,7 @@ def encode_tbs(val, extra, flag_size=4): ans += encint(extra[0b0001]) return ans -def utf8_text(text): +def utf8_text(text, empty=False): ''' Convert a possibly null string to utf-8 bytes, guaranteeing to return a non empty, normalized bytestring. @@ -313,7 +313,10 @@ def utf8_text(text): text = text.decode('utf-8', 'replace') text = normalize(text).encode('utf-8') else: - text = _('Unknown').encode('utf-8') + if not empty: + text = _('Unknown').encode('utf-8') + else: + text = u''.encode('utf-8') # yeah, stupid return text def align_block(raw, multiple=4, pad=b'\0'): diff --git a/src/calibre/ebooks/mobi/writer2/serializer.py b/src/calibre/ebooks/mobi/writer2/serializer.py index 5251bf934f..87b42cd601 100644 --- a/src/calibre/ebooks/mobi/writer2/serializer.py +++ b/src/calibre/ebooks/mobi/writer2/serializer.py @@ -355,7 +355,7 @@ class Serializer(object): text = text.replace(u'\u00AD', '') # Soft-hyphen if quot: text = text.replace('"', '"') - self.buf.write(utf8_text(text)) + self.buf.write(utf8_text(text, empty=True)) def fixup_links(self): ''' From 3245270e9e24e1b249a664b782ab2391b8afdde1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 29 Jul 2012 01:36:28 +0530 Subject: [PATCH 2/4] Fix inappropriate use of utf8_text() --- src/calibre/ebooks/mobi/writer2/serializer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/mobi/writer2/serializer.py b/src/calibre/ebooks/mobi/writer2/serializer.py index 5251bf934f..1e8a204ad5 100644 --- a/src/calibre/ebooks/mobi/writer2/serializer.py +++ b/src/calibre/ebooks/mobi/writer2/serializer.py @@ -11,8 +11,9 @@ import re from calibre.ebooks.oeb.base import (OEB_DOCS, XHTML, XHTML_NS, XML_NS, namespace, prefixname, urlnormalize) +from calibre.ebooks import normalize from calibre.ebooks.mobi.mobiml import MBP_NS -from calibre.ebooks.mobi.utils import is_guide_ref_start, utf8_text +from calibre.ebooks.mobi.utils import is_guide_ref_start from collections import defaultdict from urlparse import urldefrag @@ -355,7 +356,7 @@ class Serializer(object): text = text.replace(u'\u00AD', '') # Soft-hyphen if quot: text = text.replace('"', '"') - self.buf.write(utf8_text(text)) + self.buf.write(normalize(text).encode('utf-8')) def fixup_links(self): ''' From cca78313c15cb2698c8e86e71b81470d9574845b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 29 Jul 2012 09:39:52 +0530 Subject: [PATCH 3/4] Philosophy Now by Rick Shang --- recipes/phillosophy_now.recipe | 75 ++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 recipes/phillosophy_now.recipe diff --git a/recipes/phillosophy_now.recipe b/recipes/phillosophy_now.recipe new file mode 100644 index 0000000000..7c12832c70 --- /dev/null +++ b/recipes/phillosophy_now.recipe @@ -0,0 +1,75 @@ +import re +from calibre.web.feeds.recipes import BasicNewsRecipe +from collections import OrderedDict + +class PhilosophyNow(BasicNewsRecipe): + + title = 'Philosophy Now' + __author__ = 'Rick Shang' + + description = '''Philosophy Now is a lively magazine for everyone + interested in ideas. It isn't afraid to tackle all the major questions of + life, the universe and everything. Published every two months, it tries to + corrupt innocent citizens by convincing them that philosophy can be + exciting, worthwhile and comprehensible, and also to provide some enjoyable + reading matter for those already ensnared by the muse, such as philosophy + students and academics.''' + language = 'en' + category = 'news' + encoding = 'UTF-8' + + keep_only_tags = [dict(attrs={'id':'fullMainColumn'})] + remove_tags = [dict(attrs={'class':'articleTools'})] + no_javascript = True + no_stylesheets = True + needs_subscription = True + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + br.open('https://philosophynow.org/auth/login') + br.select_form(nr = 1) + br['username'] = self.username + br['password'] = self.password + br.submit() + return br + + def parse_index(self): + #Go to the issue + soup0 = self.index_to_soup('http://philosophynow.org/') + issue = soup0.find('div',attrs={'id':'navColumn'}) + + #Find date & cover + cover = issue.find('div', attrs={'id':'cover'}) + date = self.tag_to_string(cover.find('h3')).strip() + self.timefmt = u' [%s]'%date + img=cover.find('img',src=True)['src'] + self.cover_url = 'http://philosophynow.org' + re.sub('medium','large',img) + issuenum = re.sub('/media/images/covers/medium/issue','',img) + issuenum = re.sub('.jpg','',issuenum) + + #Go to the main body + current_issue_url = 'http://philosophynow.org/issues/' + issuenum + soup = self.index_to_soup(current_issue_url) + div = soup.find ('div', attrs={'class':'articlesColumn'}) + + feeds = OrderedDict() + + for post in div.findAll('h3'): + articles = [] + a=post.find('a',href=True) + if a is not None: + url="http://philosophynow.org" + a['href'] + title=self.tag_to_string(a).strip() + s=post.findPrevious('h4') + section_title = self.tag_to_string(s).strip() + d=post.findNext('p') + desc = self.tag_to_string(d).strip() + articles.append({'title':title, 'url':url, 'description':desc, 'date':''}) + + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles + ans = [(key, val) for key, val in feeds.iteritems()] + return ans + From 45deedd546aba851beadf46f8eb0abb046dac2e3 Mon Sep 17 00:00:00 2001 From: Oliver Graf Date: Sun, 29 Jul 2012 08:26:29 +0200 Subject: [PATCH 4/4] Removed uneeded utf8_text, after trunk fix. --- src/calibre/ebooks/mobi/utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/mobi/utils.py b/src/calibre/ebooks/mobi/utils.py index dfdb73fdff..ae8e583a1b 100644 --- a/src/calibre/ebooks/mobi/utils.py +++ b/src/calibre/ebooks/mobi/utils.py @@ -302,7 +302,7 @@ def encode_tbs(val, extra, flag_size=4): ans += encint(extra[0b0001]) return ans -def utf8_text(text, empty=False): +def utf8_text(text): ''' Convert a possibly null string to utf-8 bytes, guaranteeing to return a non empty, normalized bytestring. @@ -313,10 +313,7 @@ def utf8_text(text, empty=False): text = text.decode('utf-8', 'replace') text = normalize(text).encode('utf-8') else: - if not empty: - text = _('Unknown').encode('utf-8') - else: - text = u''.encode('utf-8') # yeah, stupid + text = _('Unknown').encode('utf-8') return text def align_block(raw, multiple=4, pad=b'\0'):