Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

pep8 compliance

Browse Source
This commit is contained in:
Kovid Goyal 2013-04-09 13:53:05 +05:30
parent 98ebcf2a05
commit 6bfcd2536a
1 changed files with 37 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
src/calibre/web/feeds/news.py
View File

@ -219,7 +219,7 @@ class BasicNewsRecipe(Recipe):
#: } #: }
#: #:
#: All keys are optional. For a full explanantion of the search criteria, see #: All keys are optional. For a full explanantion of the search criteria, see
#: `Beautiful Soup <http://www.crummy.com/software/BeautifulSoup/documentation.html#The basic find method: findAll(name, attrs, recursive, text, limit, **kwargs)>`_ #: `Beautiful Soup <http://www.crummy.com/software/BeautifulSoup/bs3/documentation.html#Searching%20the%20Parse%20Tree>`_
#: A common example:: #: A common example::
#: #:
#: remove_tags = [dict(name='div', attrs={'class':'advert'})] #: remove_tags = [dict(name='div', attrs={'class':'advert'})]
@ -556,7 +556,7 @@ class BasicNewsRecipe(Recipe):
url = article[key] url = article[key]
if url and url.startswith('http://'): if url and url.startswith('http://'):
return url return url
ans = article.get('link', None) ans = article.get('link', None)
if not ans and getattr(article, 'links', None): if not ans and getattr(article, 'links', None):
for item in article.links: for item in article.links:
if item.get('rel', 'alternate') == 'alternate': if item.get('rel', 'alternate') == 'alternate':
@ -725,7 +725,7 @@ class BasicNewsRecipe(Recipe):
`weights`: A dictionary that maps weights to titles. If any titles `weights`: A dictionary that maps weights to titles. If any titles
in index are not in weights, they are assumed to have a weight of 0. in index are not in weights, they are assumed to have a weight of 0.
''' '''
weights = defaultdict(lambda : 0, weights) weights = defaultdict(lambda: 0, weights)
index.sort(cmp=lambda x, y: cmp(weights[x], weights[y])) index.sort(cmp=lambda x, y: cmp(weights[x], weights[y]))
return index return index
@ -860,8 +860,8 @@ class BasicNewsRecipe(Recipe):
if isinstance(self.feeds, basestring): if isinstance(self.feeds, basestring):
self.feeds = [self.feeds] self.feeds = [self.feeds]
if self.needs_subscription and (\ if self.needs_subscription and (
self.username is None or self.password is None or \ self.username is None or self.password is None or
(not self.username and not self.password)): (not self.username and not self.password)):
if self.needs_subscription != 'optional': if self.needs_subscription != 'optional':
raise ValueError(_('The "%s" recipe needs a username and password.')%self.title) raise ValueError(_('The "%s" recipe needs a username and password.')%self.title)
@ -870,7 +870,7 @@ class BasicNewsRecipe(Recipe):
self.image_map, self.image_counter = {}, 1 self.image_map, self.image_counter = {}, 1
self.css_map = {} self.css_map = {}
web2disk_cmdline = [ 'web2disk', web2disk_cmdline = ['web2disk',
'--timeout', str(self.timeout), '--timeout', str(self.timeout),
'--max-recursions', str(self.recursions), '--max-recursions', str(self.recursions),
'--delay', str(self.delay), '--delay', str(self.delay),
@ -913,7 +913,6 @@ class BasicNewsRecipe(Recipe):
self.failed_downloads = [] self.failed_downloads = []
self.partial_failures = [] self.partial_failures = []
def _postprocess_html(self, soup, first_fetch, job_info): def _postprocess_html(self, soup, first_fetch, job_info):
if self.no_stylesheets: if self.no_stylesheets:
for link in list(soup.findAll('link', type=re.compile('css')))+list(soup.findAll('style')): for link in list(soup.findAll('link', type=re.compile('css')))+list(soup.findAll('style')):
@ -923,7 +922,8 @@ class BasicNewsRecipe(Recipe):
head = soup.find('body') head = soup.find('body')
if not head: if not head:
head = soup.find(True) head = soup.find(True)
style = BeautifulSoup(u'<style type="text/css" title="override_css">%s</style>'%(self.template_css +'\n\n'+(self.extra_css if self.extra_css else ''))).find('style') style = BeautifulSoup(u'<style type="text/css" title="override_css">%s</style>'%(
self.template_css +'\n\n'+(self.extra_css if self.extra_css else ''))).find('style')
head.insert(len(head.contents), style) head.insert(len(head.contents), style)
if first_fetch and job_info: if first_fetch and job_info:
url, f, a, feed_len = job_info url, f, a, feed_len = job_info
@ -969,7 +969,6 @@ class BasicNewsRecipe(Recipe):
self.populate_article_metadata(article, ans, first_fetch) self.populate_article_metadata(article, ans, first_fetch)
return ans return ans
def download(self): def download(self):
''' '''
Download and pre-process all articles from the feeds in this recipe. Download and pre-process all articles from the feeds in this recipe.
@ -1041,12 +1040,12 @@ class BasicNewsRecipe(Recipe):
def feed2index(self, f, feeds): def feed2index(self, f, feeds):
feed = feeds[f] feed = feeds[f]
if feed.image_url is not None: # Download feed image if feed.image_url is not None: # Download feed image
imgdir = os.path.join(self.output_dir, 'images') imgdir = os.path.join(self.output_dir, 'images')
if not os.path.isdir(imgdir): if not os.path.isdir(imgdir):
os.makedirs(imgdir) os.makedirs(imgdir)
if self.image_map.has_key(feed.image_url): if feed.image_url in self.image_map:
feed.image_url = self.image_map[feed.image_url] feed.image_url = self.image_map[feed.image_url]
else: else:
bn = urlparse.urlsplit(feed.image_url).path bn = urlparse.urlsplit(feed.image_url).path
@ -1065,7 +1064,6 @@ class BasicNewsRecipe(Recipe):
if isinstance(feed.image_url, str): if isinstance(feed.image_url, str):
feed.image_url = feed.image_url.decode(sys.getfilesystemencoding(), 'strict') feed.image_url = feed.image_url.decode(sys.getfilesystemencoding(), 'strict')
templ = (templates.TouchscreenFeedTemplate if self.touchscreen else templ = (templates.TouchscreenFeedTemplate if self.touchscreen else
templates.FeedTemplate) templates.FeedTemplate)
templ = templ(lang=self.lang_for_html) templ = templ(lang=self.lang_for_html)
@ -1074,7 +1072,6 @@ class BasicNewsRecipe(Recipe):
return templ.generate(f, feeds, self.description_limiter, return templ.generate(f, feeds, self.description_limiter,
extra_css=css).render(doctype='xhtml') extra_css=css).render(doctype='xhtml')
def _fetch_article(self, url, dir_, f, a, num_of_feeds): def _fetch_article(self, url, dir_, f, a, num_of_feeds):
br = self.browser br = self.browser
if self.get_browser.im_func is BasicNewsRecipe.get_browser.im_func: if self.get_browser.im_func is BasicNewsRecipe.get_browser.im_func:
@ -1117,7 +1114,7 @@ class BasicNewsRecipe(Recipe):
with PersistentTemporaryFile('_feeds2disk.html') as pt: with PersistentTemporaryFile('_feeds2disk.html') as pt:
pt.write(raw) pt.write(raw)
url = ('file:'+pt.name) if iswindows else ('file://'+pt.name) url = ('file:'+pt.name) if iswindows else ('file://'+pt.name)
return self._fetch_article(url, dir, f, a, num_of_feeds) return self._fetch_article(url, dir, f, a, num_of_feeds)
def remove_duplicate_articles(self, feeds): def remove_duplicate_articles(self, feeds):
seen_keys = defaultdict(set) seen_keys = defaultdict(set)
@ -1223,9 +1220,9 @@ class BasicNewsRecipe(Recipe):
if not url: if not url:
continue continue
func, arg = (self.fetch_embedded_article, article) \ func, arg = (self.fetch_embedded_article, article) \
if self.use_embedded_content or (self.use_embedded_content == None and feed.has_embedded_content()) \ if self.use_embedded_content or (self.use_embedded_content is None and feed.has_embedded_content()) \
else \ else \
((self.fetch_obfuscated_article if self.articles_are_obfuscated \ ((self.fetch_obfuscated_article if self.articles_are_obfuscated
else self.fetch_article), url) else self.fetch_article), url)
req = WorkRequest(func, (arg, art_dir, f, a, len(feed)), req = WorkRequest(func, (arg, art_dir, f, a, len(feed)),
{}, (f, a), self.article_downloaded, {}, (f, a), self.article_downloaded,
@ -1235,13 +1232,11 @@ class BasicNewsRecipe(Recipe):
req.feed_dir = feed_dir req.feed_dir = feed_dir
self.jobs.append(req) self.jobs.append(req)
self.jobs_done = 0 self.jobs_done = 0
tp = ThreadPool(self.simultaneous_downloads) tp = ThreadPool(self.simultaneous_downloads)
for req in self.jobs: for req in self.jobs:
tp.putRequest(req, block=True, timeout=0) tp.putRequest(req, block=True, timeout=0)
self.report_progress(0, _('Starting download [%d thread(s)]...')%self.simultaneous_downloads) self.report_progress(0, _('Starting download [%d thread(s)]...')%self.simultaneous_downloads)
while True: while True:
try: try:
@ -1328,7 +1323,6 @@ class BasicNewsRecipe(Recipe):
if os.path.exists(mpath): if os.path.exists(mpath):
os.remove(mpath) os.remove(mpath)
def download_masthead(self, url): def download_masthead(self, url):
try: try:
self._download_masthead(url) self._download_masthead(url)
@ -1455,7 +1449,6 @@ class BasicNewsRecipe(Recipe):
self.play_order_counter = 0 self.play_order_counter = 0
self.play_order_map = {} self.play_order_map = {}
def feed_index(num, parent): def feed_index(num, parent):
f = feeds[num] f = feeds[num]
for j, a in enumerate(f): for j, a in enumerate(f):
@ -1595,7 +1588,6 @@ class BasicNewsRecipe(Recipe):
parsed_feeds.append(feed) parsed_feeds.append(feed)
self.log.exception(msg) self.log.exception(msg)
remove = [f for f in parsed_feeds if len(f) == 0 and remove = [f for f in parsed_feeds if len(f) == 0 and
self.remove_empty_feeds] self.remove_empty_feeds]
for f in remove: for f in remove:
@ -1629,8 +1621,11 @@ class BasicNewsRecipe(Recipe):
res = self.tag_to_string(item) res = self.tag_to_string(item)
if res: if res:
strings.append(res) strings.append(res)
elif use_alt and item.has_key('alt'): elif use_alt:
strings.append(item['alt']) try:
strings.append(item['alt'])
except KeyError:
pass
ans = u''.join(strings) ans = u''.join(strings)
if normalize_whitespace: if normalize_whitespace:
ans = re.sub(r'\s+', ' ', ans) ans = re.sub(r'\s+', ' ', ans)
@ -1646,24 +1641,26 @@ class BasicNewsRecipe(Recipe):
@classmethod @classmethod
def adeify_images(cls, soup): def adeify_images(cls, soup):
''' '''
If your recipe when converted to EPUB has problems with images when If your recipe when converted to EPUB has problems with images when
viewed in Adobe Digital Editions, call this method from within viewed in Adobe Digital Editions, call this method from within
:meth:`postprocess_html`. :meth:`postprocess_html`.
''' '''
for item in soup.findAll('img'): for item in soup.findAll('img'):
for attrib in ['height','width','border','align','style']: for attrib in ['height','width','border','align','style']:
if item.has_key(attrib): try:
del item[attrib] del item[attrib]
oldParent = item.parent except KeyError:
myIndex = oldParent.contents.index(item) pass
item.extract() oldParent = item.parent
divtag = Tag(soup,'div') myIndex = oldParent.contents.index(item)
brtag = Tag(soup,'br') item.extract()
oldParent.insert(myIndex,divtag) divtag = Tag(soup,'div')
divtag.append(item) brtag = Tag(soup,'br')
divtag.append(brtag) oldParent.insert(myIndex,divtag)
return soup divtag.append(item)
divtag.append(brtag)
return soup
class CustomIndexRecipe(BasicNewsRecipe): class CustomIndexRecipe(BasicNewsRecipe):