mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Remove not working recipe
Fixes #864 (Deleted bloomberg recipe, change in service)
This commit is contained in:
Kovid Goyal
parent
e9ef5296e7
commit
d4c95b0df6
@ -1,211 +0,0 @@
|
||||
#!/usr/bin/env python2
|
||||
# -*- coding: utf-8 -*-
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2018, Dale Furrow dkfurrow@gmail.com'
|
||||
'''
|
||||
chron.com
|
||||
'''
|
||||
from datetime import datetime, timedelta
|
||||
from lxml import html, etree
|
||||
import io
|
||||
from calibre.web.feeds.recipes import BasicNewsRecipe
|
||||
import urllib2
|
||||
from collections import OrderedDict
|
||||
import calendar
|
||||
from calibre.ebooks.BeautifulSoup import Tag
|
||||
|
||||
|
||||
contributors_url = "https://www.bloomberg.com/view/contributors"
|
||||
output_date_format = "%d %b, %H:%M"
|
||||
|
||||
hdr = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
|
||||
'Accept-Encoding': 'none',
|
||||
'Accept-Language': 'en-US,en;q=0.8',
|
||||
'Connection': 'keep-alive'}
|
||||
|
||||
|
||||
def get_article_parsed(this_url):
|
||||
req = urllib2.Request(this_url, headers=hdr)
|
||||
page = urllib2.urlopen(req)
|
||||
content = page.read()
|
||||
parser = etree.HTMLParser()
|
||||
parsed = html.parse(io.BytesIO(bytes(content)), parser)
|
||||
return parsed
|
||||
|
||||
|
||||
class BloombergContributor:
|
||||
_name = None
|
||||
_url_name = None
|
||||
_url_code = None
|
||||
_article_list = None # article is title, link, date, description
|
||||
date_formats = ["%B %d, %Y %I:%M %p", "%b %d, %Y %I:%M %p"]
|
||||
|
||||
def __init__(self, name, url_name, url_code):
|
||||
self._name = name
|
||||
self._url_name = url_name
|
||||
self._url_code = url_code
|
||||
self._article_list = []
|
||||
|
||||
def __str__(self):
|
||||
return "{0} ({1}): {2:d} articles".format(self._name, self._url_name, len(self._article_list))
|
||||
|
||||
def populate_article_list(self):
|
||||
list_url = "{0}/{1}/{2}/articles".format(
|
||||
contributors_url, self._url_code, self._url_name)
|
||||
parsed_list = get_article_parsed(list_url)
|
||||
articles = parsed_list.xpath("//li[contains(@class, 'item_lwCjl')]")
|
||||
for article in articles:
|
||||
headline = article.find('a')
|
||||
link = headline.attrib['href']
|
||||
title = headline.text.strip()
|
||||
article_date_eles = article.xpath(
|
||||
".//span[contains(@class, 'time_3qQJR')]")
|
||||
if len(article_date_eles) > 0:
|
||||
article_date_str = article_date_eles[0].text.strip()
|
||||
article_date = self.parse_date_str(article_date_str)
|
||||
else:
|
||||
article_date = None
|
||||
summary_eles = article.xpath(
|
||||
".//p[contains(@class, 'summary_17SO6')]")
|
||||
if len(summary_eles) > 0:
|
||||
summary = summary_eles[0].text.strip()
|
||||
else:
|
||||
summary = "No summary..."
|
||||
self._article_list.append((title.encode('ascii', 'ignore'), link, summary.encode('ascii', 'ignore'),
|
||||
article_date, self.get_article_timestamp(article_date)))
|
||||
|
||||
@staticmethod
|
||||
def get_article_timestamp(article_date):
|
||||
# assume all times Eastern...
|
||||
# 2nd sunday March, 1st Sunday Nov
|
||||
c = calendar.Calendar(firstweekday=calendar.SUNDAY)
|
||||
march_cal = c.monthdatescalendar(article_date.year, 3)
|
||||
dst_start = [
|
||||
day for week in march_cal for day in week
|
||||
if day.weekday() == calendar.SUNDAY and day.month == 3
|
||||
][1]
|
||||
nov_cal = c.monthdatescalendar(article_date.year, 11)
|
||||
dst_end = [day for week in nov_cal for day in week
|
||||
if day.weekday() == calendar.SUNDAY and day.month == 11
|
||||
][0]
|
||||
dst_start = datetime(dst_start.year, dst_start.month, dst_start.day, 2)
|
||||
dst_end = datetime(dst_end.year, dst_end.month, dst_end.day, 1)
|
||||
if dst_start > article_date > dst_end:
|
||||
shift = timedelta(hours=4)
|
||||
else:
|
||||
shift = timedelta(hours=5)
|
||||
return float((article_date + shift - datetime.utcfromtimestamp(0)).total_seconds())
|
||||
|
||||
def parse_date_str(self, date_str):
|
||||
parsed = None
|
||||
for date_format in self.date_formats:
|
||||
try:
|
||||
parsed = datetime.strptime(date_str[0:-4], date_format)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return parsed
|
||||
|
||||
def get_article_list(self):
|
||||
return self._article_list
|
||||
|
||||
def get_ordered_article_feed(self):
|
||||
output = OrderedDict()
|
||||
for article in self._article_list:
|
||||
article_date = article[3]
|
||||
article_dict = {'title': article[0], 'url': article[1],
|
||||
'description': "{0}: {1}".format(self.get_name(), article[2]),
|
||||
'author': self.get_name() + ": " + article[3].strftime(output_date_format),
|
||||
'date': self.get_name() + ": " + article[3].strftime(output_date_format),
|
||||
'timestamp': article[4]}
|
||||
output[article_date] = article_dict
|
||||
return OrderedDict(sorted(output.items(), key=lambda t: t[0], reverse=True))
|
||||
|
||||
def get_name(self):
|
||||
return self._name
|
||||
|
||||
|
||||
def new_tag(soup, name, attrs=()):
|
||||
impl = getattr(soup, 'new_tag', None)
|
||||
if impl is not None:
|
||||
return impl(name, attrs=dict(attrs))
|
||||
return Tag(soup, name, attrs=attrs or None)
|
||||
|
||||
|
||||
class BloombergContributors(BasicNewsRecipe):
|
||||
title = u'Bloomberg, Editorial Contributors'
|
||||
description = 'Articles from Bloomberg.com contributors'
|
||||
__author__ = 'Dale Furrow'
|
||||
xpath_contributor_list = """//li[contains(@class, 'item_2zsS8')]/a"""
|
||||
language = 'en'
|
||||
no_stylesheets = True
|
||||
remove_attributes = ['style', 'xmlns']
|
||||
keep_only_tags = [dict(name='article', attrs={'data-type': 'article'})]
|
||||
# note space...
|
||||
remove_tags = [
|
||||
dict(name='div', attrs={'class': ['share-article-button ', 'text-to-speech']})]
|
||||
oldest_article = 7.0
|
||||
ignore_duplicate_articles = {'url'}
|
||||
recursions = 0
|
||||
category = 'news, USA, world, economy, politics'
|
||||
language = 'en'
|
||||
|
||||
def get_contributors_list(self):
|
||||
page_doc = get_article_parsed(contributors_url)
|
||||
els = page_doc.xpath(self.xpath_contributor_list)
|
||||
contributor_list = []
|
||||
for el in els:
|
||||
name = el.find("span").text.strip() # name
|
||||
contibutor_items = el.attrib['href'].split('/')
|
||||
contributor = BloombergContributor(
|
||||
name, contibutor_items[4], contibutor_items[3])
|
||||
contributor_list.append(contributor)
|
||||
for contributor in contributor_list:
|
||||
contributor.populate_article_list()
|
||||
return contributor_list
|
||||
|
||||
def postprocess_html(self, soup, first_fetch):
|
||||
'''
|
||||
:param soup: A `BeautifulSoup
|
||||
<https://www.crummy.com/software/BeautifulSoup/bs3/documentation.html>
|
||||
`_ instance containing the downloaded :term:`HTML`.
|
||||
:param first_fetch: True if this is the first page of an article.
|
||||
Remember: BeautifulSoup3! Interface is much different than bs4
|
||||
'''
|
||||
time_eles = soup.findAll("time", {"class": "article-timestamp"})
|
||||
if len(time_eles) > 0:
|
||||
time_stamp = time_eles[0].get('datetime')
|
||||
try:
|
||||
parsed_time = datetime.strptime(time_stamp, "%Y-%m-%dT%H:%M:%S.%fZ")\
|
||||
.strftime("%B %d, %Y %I:%M %p") + " UTC"
|
||||
except:
|
||||
parsed_time = time_stamp
|
||||
insert_tag = new_tag(soup, "p", [("class", "user-inserted")])
|
||||
insert_tag.insert(0, parsed_time)
|
||||
soup.time.replaceWith(insert_tag)
|
||||
|
||||
return soup
|
||||
|
||||
def parse_index(self):
|
||||
self.timefmt = ' [%a, %d %b, %Y]'
|
||||
self.log('starting parse_index: {0}'.format(
|
||||
datetime.now().strftime("%B %d, %Y %I:%M %p")))
|
||||
feeds = []
|
||||
feed_dict = OrderedDict()
|
||||
contributor_list = self.get_contributors_list()
|
||||
self.log("Found {0:d} contibutors on main page".format(
|
||||
len(contributor_list)))
|
||||
for contributor in contributor_list:
|
||||
articles = contributor.get_ordered_article_feed()
|
||||
feed_dict.update(articles)
|
||||
feed_dict = OrderedDict(
|
||||
sorted(feed_dict.items(), key=lambda t: t[0], reverse=True))
|
||||
self.log("Found {0:d} linked articles from contributors".format(
|
||||
len(feed_dict)))
|
||||
feeds.append(("Columns", list(feed_dict.values())))
|
||||
# self.log("Total of {0:d} {1} articles".format(len(article_list), cat))
|
||||
self.log('finishing parse_index: {0}'.format(
|
||||
datetime.now().strftime("%B %d, %Y %I:%M %p")))
|
||||
return feeds
|
||||
Loading…
x
Reference in New Issue
Block a user