Please enter your username and password for %s
If you do not have one, please subscribe to get access to the articles.
Click OK to proceed.')%(recipe.title,))
- d.exec_()
- if d.result() == QDialog.Accepted:
- username, password = d.username(), d.password()
- else:
- fetch = False
- if fetch:
- data = dict(title=recipe.title, username=username, password=password,
- script=getattr(recipe, 'gui_recipe_script', None))
- self.emit(SIGNAL('fetch_news(PyQt_PyObject)'), data)
-
- def set_custom_feeds(self, feeds):
- self.custom_menu.set_feeds(feeds)
-
-class CustomNewMenuItem(QAction):
-
- def __init__(self, title, script, parent):
- QAction.__init__(self, QIcon(':/images/user_profile.svg'), title, parent)
- self.title = title
- self.recipe = compile_recipe(script)
- self.recipe.gui_recipe_script = script
-
-class CustomNewsMenu(QMenu):
-
- def __init__(self):
- QMenu.__init__(self)
- self.setTitle(_('Download custom news'))
- self.connect(self, SIGNAL('triggered(QAction*)'), self.launch)
-
- def launch(self, action):
- self.emit(SIGNAL('start_news_fetch(PyQt_PyObject, PyQt_PyObject)'),
- action.recipe, None)
-
- def set_feeds(self, feeds):
- self.clear()
- for title, src in feeds:
- self.addAction(CustomNewMenuItem(title, src, self))
diff --git a/src/calibre/web/feeds/main.py b/src/calibre/web/feeds/main.py
index 4ef7d89dd4..ebc2c937ed 100644
--- a/src/calibre/web/feeds/main.py
+++ b/src/calibre/web/feeds/main.py
@@ -8,8 +8,7 @@ CLI for downloading feeds.
import sys, os, logging
from calibre.web.feeds.recipes import get_builtin_recipe, compile_recipe, titles
from calibre.web.fetch.simple import option_parser as _option_parser
-from calibre.web.feeds.news import Profile2Recipe, BasicNewsRecipe
-from calibre.ebooks.lrf.web.profiles import DefaultProfile, FullContentProfile
+from calibre.web.feeds.news import BasicNewsRecipe
from calibre.utils.config import Config, StringConfig
def config(defaults=None):
@@ -119,23 +118,19 @@ def run_recipe(opts, recipe_arg, parser, notification=None, handler=None):
pb = ProgressBar(term, _('Fetching feeds...'), no_progress_bar=opts.no_progress_bar)
notification = pb.update
- recipe, is_profile = None, False
+ recipe = None
if opts.feeds is not None:
recipe = BasicNewsRecipe
else:
try:
if os.access(recipe_arg, os.R_OK):
- recipe = compile_recipe(open(recipe_arg).read())
- is_profile = DefaultProfile in recipe.__bases__ or \
- FullContentProfile in recipe.__bases__
+ recipe = compile_recipe(open(recipe_arg).read())
else:
raise Exception('not file')
except:
- recipe, is_profile = get_builtin_recipe(recipe_arg)
+ recipe = get_builtin_recipe(recipe_arg)
if recipe is None:
recipe = compile_recipe(recipe_arg)
- is_profile = DefaultProfile in recipe.__bases__ or \
- FullContentProfile in recipe.__bases__
if recipe is None:
raise RecipeError(recipe_arg+ ' is an invalid recipe')
@@ -148,10 +143,7 @@ def run_recipe(opts, recipe_arg, parser, notification=None, handler=None):
handler.setFormatter(ColoredFormatter('%(levelname)s: %(message)s\n')) # The trailing newline is need because of the progress bar
logging.getLogger('feeds2disk').addHandler(handler)
- if is_profile:
- recipe = Profile2Recipe(recipe, opts, parser, notification)
- else:
- recipe = recipe(opts, parser, notification)
+ recipe = recipe(opts, parser, notification)
if not os.path.exists(recipe.output_dir):
os.makedirs(recipe.output_dir)
diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py
index fe621f9bfa..24fe30fa6f 100644
--- a/src/calibre/web/feeds/news.py
+++ b/src/calibre/web/feeds/news.py
@@ -22,7 +22,6 @@ from calibre.web.feeds import feed_from_xml, templates, feeds_from_index, Feed
from calibre.web.fetch.simple import option_parser as web2disk_option_parser
from calibre.web.fetch.simple import RecursiveFetcher
from calibre.utils.threadpool import WorkRequest, ThreadPool, NoResultsPending
-from calibre.ebooks.lrf.web.profiles import FullContentProfile
from calibre.ptempfile import PersistentTemporaryFile
@@ -359,19 +358,19 @@ class BasicNewsRecipe(object, LoggingInterface):
'''
if re.match(r'\w+://', url_or_raw):
f = self.browser.open(url_or_raw)
- raw = f.read()
+ _raw = f.read()
f.close()
- if not raw:
+ if not _raw:
raise RuntimeError('Could not fetch index from %s'%url_or_raw)
else:
- raw = url_or_raw
+ _raw = url_or_raw
if raw:
- return raw
- if not isinstance(raw, unicode) and self.encoding:
- raw = raw.decode(self.encoding)
+ return _raw
+ if not isinstance(_raw, unicode) and self.encoding:
+ _raw = _raw.decode(self.encoding)
massage = list(BeautifulSoup.MARKUP_MASSAGE)
massage.append((re.compile(r'&(\S+?);'), lambda match: entity_to_unicode(match, encoding=self.encoding)))
- return BeautifulSoup(raw, markupMassage=massage)
+ return BeautifulSoup(_raw, markupMassage=massage)
def sort_index_by(self, index, weights):
@@ -943,34 +942,6 @@ class BasicNewsRecipe(object, LoggingInterface):
nmassage.extend(entity_replace)
return BeautifulSoup(raw, markupMassage=nmassage)
-class Profile2Recipe(BasicNewsRecipe):
- '''
- Used to migrate the old news Profiles to the new Recipes. Uses the settings
- from the old Profile to populate the settings in the Recipe. Also uses, the
- Profile's get_browser and parse_feeds.
- '''
- def __init__(self, profile_class, options, parser, progress_reporter):
- self.old_profile = profile_class(logging.getLogger('feeds2disk'),
- username=options.username,
- password=options.password,
- lrf=options.lrf)
- for attr in ('preprocess_regexps', 'oldest_article', 'delay', 'timeout',
- 'match_regexps', 'filter_regexps', 'html2lrf_options',
- 'timefmt', 'needs_subscription', 'summary_length',
- 'max_articles_per_feed', 'title','no_stylesheets', 'encoding'):
- setattr(self, attr, getattr(self.old_profile, attr))
-
- self.simultaneous_downloads = 1
- BasicNewsRecipe.__init__(self, options, parser, progress_reporter)
- self.browser = self.old_profile.browser
- self.use_embedded_content = isinstance(self.old_profile, FullContentProfile)
-
- def parse_index(self):
- feeds = []
- for key, val in self.old_profile.parse_feeds().items():
- feeds.append((key, val))
- return feeds
-
class CustomIndexRecipe(BasicNewsRecipe):
def custom_index(self):
diff --git a/src/calibre/web/feeds/recipes/__init__.py b/src/calibre/web/feeds/recipes/__init__.py
index 3337353597..736d2a2249 100644
--- a/src/calibre/web/feeds/recipes/__init__.py
+++ b/src/calibre/web/feeds/recipes/__init__.py
@@ -12,13 +12,13 @@ recipe_modules = [
'discover_magazine', 'scientific_american', 'new_york_review_of_books',
'daily_telegraph', 'guardian', 'el_pais', 'new_scientist', 'b92',
'politika', 'moscow_times', 'latimes', 'japan_times', 'san_fran_chronicle',
- 'demorgen_be', 'de_standaard'
+ 'demorgen_be', 'de_standaard', 'ap', 'barrons', 'chr_mon', 'cnn', 'faznet',
+ 'jpost', 'jutarnji', 'nasa', 'reuters', 'spiegelde', 'wash_post', 'zeitde',
]
import re, imp, inspect, time, os
from calibre.web.feeds.news import BasicNewsRecipe, CustomIndexRecipe, AutomaticNewsRecipe
from calibre.ebooks.lrf.web.profiles import DefaultProfile, FullContentProfile
-from calibre.ebooks.lrf.web import builtin_profiles
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.path import path
from calibre.ptempfile import PersistentTemporaryDirectory
@@ -95,13 +95,10 @@ def get_builtin_recipe(title):
'''
for r in recipes:
if r.title == title:
- return r, False
- for p in builtin_profiles:
- if p.title == title:
- return p, True
- return None, False
+ return r
+ return None
-_titles = list(frozenset([r.title for r in recipes] + [p.title for p in builtin_profiles]))
+_titles = [r.title for r in recipes]
_titles.sort(cmp=english_sort)
titles = _titles
diff --git a/src/calibre/web/feeds/recipes/ap.py b/src/calibre/web/feeds/recipes/ap.py
new file mode 100644
index 0000000000..cbd9055b61
--- /dev/null
+++ b/src/calibre/web/feeds/recipes/ap.py
@@ -0,0 +1,39 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class AssociatedPress(BasicNewsRecipe):
+
+ title = u'Associated Press'
+ description = 'Global news'
+ __author__ = 'Kovid Goyal'
+ use_embedded_content = False
+ max_articles_per_feed = 15
+ html2lrf_options = ['--force-page-break-before-tag="chapter"']
+
+
+ preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in
+[
+ (r'
', lambda match : '
'), + (r'
', lambda match : '
'), + (r'Learn more about our Privacy Policy.*?', lambda match : '