diff --git a/recipes/scientific_american.recipe b/recipes/scientific_american.recipe index 92bcffe5c2..fc3b2a6f13 100644 --- a/recipes/scientific_american.recipe +++ b/recipes/scientific_american.recipe @@ -3,14 +3,6 @@ __license__ = 'GPL v3' from calibre.web.feeds.news import BasicNewsRecipe -try: - from calibre.web.feeds.jsnews import CSSSelect -except ImportError: - def CSSSelect(expr): - from cssselect import HTMLTranslator - from lxml.etree import XPath - return XPath(HTMLTranslator().css_to_xpath(expr)) - def absurl(url): if url.startswith('/'): url = 'http://www.scientificamerican.com' + url @@ -46,8 +38,8 @@ class ScientificAmerican(BasicNewsRecipe): def parse_index(self): # Get the cover, date and issue URL root = self.index_to_soup('http://www.scientificamerican.com/sciammag/', as_tree=True) - for a in CSSSelect('.archiveIssues a.cover[href]')(root): - self.cover_url = absurl(CSSSelect('img[src]')(a)[0].get('src')) + for a in root.xpath('''descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), ' archiveIssues ')]/descendant-or-self::*/a[@class and contains(concat(' ', normalize-space(@class), ' '), ' cover ') and (@href)]'''): + self.cover_url = absurl(a.xpath('descendant-or-self::img[@src]')[0].get('src')) root = self.index_to_soup(absurl(a.get('href')), as_tree=True) for a in a.xpath('following-sibling::a[@href]'): self.timefmt = self.tag_to_string(a).strip() @@ -58,7 +50,7 @@ class ScientificAmerican(BasicNewsRecipe): # Now parse the actual issue to get the list of articles feeds = [] - for i, div in enumerate(CSSSelect('div.toc-features, div.toc-departments')(root)): + for i, div in enumerate(root.xpath('''descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' toc-features ')] | descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' toc-departments ')]''')): if i == 0: feeds.append(('Features', list(self.parse_sciam_features(div)))) else: @@ -67,7 +59,7 @@ class ScientificAmerican(BasicNewsRecipe): return feeds def parse_sciam_features(self, div): - for h4 in CSSSelect('li a[href] h4')(div): + for h4 in div.xpath('''descendant-or-self::li/descendant-or-self::*/a[@href]/descendant-or-self::*/h4'''): title = self.tag_to_string(h4) a = h4.getparent() url = absurl(a.get('href')) @@ -81,7 +73,7 @@ class ScientificAmerican(BasicNewsRecipe): def parse_sciam_departments(self, div): section_title, articles = 'Unknown', [] - for x in CSSSelect('li a[href] h3, li span.deptTitle a[href]')(div): + for x in div.xpath('''descendant-or-self::li/descendant-or-self::*/a[@href]/descendant-or-self::*/h3 | descendant-or-self::li/descendant-or-self::*/span[@class and contains(concat(' ', normalize-space(@class), ' '), ' deptTitle ')]/descendant-or-self::*/a[@href]'''): if x.tag == 'a': if articles: yield section_title, list(articles) diff --git a/recipes/wsj.recipe b/recipes/wsj.recipe index 442a03d250..f9c095fe0f 100644 --- a/recipes/wsj.recipe +++ b/recipes/wsj.recipe @@ -1,13 +1,21 @@ from calibre.web.feeds.jsnews import JavascriptRecipe from calibre.web.jsbrowser.browser import NotAFile -try: - from calibre.web.feeds.jsnews import CSSSelect -except ImportError: - def CSSSelect(expr): - from cssselect import HTMLTranslator - from lxml.etree import XPath - return XPath(HTMLTranslator().css_to_xpath(expr)) +def CSSSelect(expr): + expr = { + 'div.whatsNews-simple': '''descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' whatsNews-simple ')]''', + 'a.mjLinkItem[href]': '''descendant-or-self::a[@class and contains(concat(' ', normalize-space(@class), ' '), ' mjLinkItem ') and (@href)]''', + '.meta_sectionName': '''descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), ' meta_sectionName ')]''', + 'p':'descendant-or-self::p', + 'div.whatsNews-simple.whatsNews-itp': '''descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' whatsNews-simple ') and (@class and contains(concat(' ', normalize-space(@class), ' '), ' whatsNews-itp '))]''', + 'a[href]': 'descendant-or-self::a[@href]', + 'span.date-date':"descendant-or-self::span[@class and contains(concat(' ', normalize-space(@class), ' '), ' date-date ')]", + 'div.itpSectionHeaderPdf a[href]': "descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' itpSectionHeaderPdf ')]/descendant-or-self::*/a[@href]", + 'div.itpHeader ul.tab a[href]': "descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' itpHeader ')]/descendant-or-self::*/ul[@class and contains(concat(' ', normalize-space(@class), ' '), ' tab ')]/descendant-or-self::*/a[@href]", + + }[expr] + from lxml.etree import XPath + return XPath(expr) class WSJ(JavascriptRecipe): diff --git a/recipes/wsj_free.recipe b/recipes/wsj_free.recipe index 8205cc1a4d..6172e0befb 100644 --- a/recipes/wsj_free.recipe +++ b/recipes/wsj_free.recipe @@ -1,13 +1,21 @@ from calibre.web.feeds.jsnews import JavascriptRecipe from calibre.web.jsbrowser.browser import NotAFile -try: - from calibre.web.feeds.jsnews import CSSSelect -except ImportError: - def CSSSelect(expr): - from cssselect import HTMLTranslator - from lxml.etree import XPath - return XPath(HTMLTranslator().css_to_xpath(expr)) +def CSSSelect(expr): + expr = { + 'div.whatsNews-simple': '''descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' whatsNews-simple ')]''', + 'a.mjLinkItem[href]': '''descendant-or-self::a[@class and contains(concat(' ', normalize-space(@class), ' '), ' mjLinkItem ') and (@href)]''', + '.meta_sectionName': '''descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), ' meta_sectionName ')]''', + 'p':'descendant-or-self::p', + 'div.whatsNews-simple.whatsNews-itp': '''descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' whatsNews-simple ') and (@class and contains(concat(' ', normalize-space(@class), ' '), ' whatsNews-itp '))]''', + 'a[href]': 'descendant-or-self::a[@href]', + 'span.date-date':"descendant-or-self::span[@class and contains(concat(' ', normalize-space(@class), ' '), ' date-date ')]", + 'div.itpSectionHeaderPdf a[href]': "descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' itpSectionHeaderPdf ')]/descendant-or-self::*/a[@href]", + 'div.itpHeader ul.tab a[href]': "descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' itpHeader ')]/descendant-or-self::*/ul[@class and contains(concat(' ', normalize-space(@class), ' '), ' tab ')]/descendant-or-self::*/a[@href]", + + }[expr] + from lxml.etree import XPath + return XPath(expr) class WSJ(JavascriptRecipe): diff --git a/src/calibre/web/feeds/jsnews.py b/src/calibre/web/feeds/jsnews.py index cd29f87c3a..c5a6d764d6 100644 --- a/src/calibre/web/feeds/jsnews.py +++ b/src/calibre/web/feeds/jsnews.py @@ -30,17 +30,6 @@ def image_data_to_url(data, base='cover'): ans.name = 'cover.' + ext return ans -css_select_cache = {} - -def CSSSelect(expr): - try: - return css_select_cache[expr] - except KeyError: - from cssselect import HTMLTranslator - from lxml.etree import XPath - ans = css_select_cache[expr] = XPath(HTMLTranslator().css_to_xpath(expr)) - return ans - class JavascriptRecipe(BasicNewsRecipe):