diff --git a/src/libprs500/ebooks/lrf/web/convert_from.py b/src/libprs500/ebooks/lrf/web/convert_from.py
index 38939e8ade..9a42945f9e 100644
--- a/src/libprs500/ebooks/lrf/web/convert_from.py
+++ b/src/libprs500/ebooks/lrf/web/convert_from.py
@@ -104,16 +104,20 @@ def process_profile(args, options, logger=None):
if len(args) < 2:
args.append(name)
args[1] = name
+ index = -1
if len(args) == 2:
try:
- index = -1
- if args[1] != 'default':
- index = available_profiles.index(args[1])
+ if isinstance(args[1], basestring):
+ if args[1] != 'default':
+ index = available_profiles.index(args[1])
except ValueError:
raise CommandLineError('Unknown profile: %s\nValid profiles: %s'%(args[1], available_profiles))
else:
raise CommandLineError('Only one profile at a time is allowed.')
- profile = DefaultProfile if index == -1 else builtin_profiles[index]
+ if isinstance(args[1], basestring):
+ profile = DefaultProfile if index == -1 else builtin_profiles[index]
+ else:
+ profile = args[1]
profile = profile(logger, options.verbose, options.username, options.password)
if profile.browser is not None:
options.browser = profile.browser
@@ -170,7 +174,11 @@ def process_profile(args, options, logger=None):
def main(args=sys.argv, logger=None):
parser = option_parser()
- options, args = parser.parse_args(args)
+ if not isinstance(args[-1], basestring): # Called from GUI
+ options, args2 = parser.parse_args(args[:-1])
+ args = args2 + [args[-1]]
+ else:
+ options, args = parser.parse_args(args)
if len(args) > 2 or (len(args) == 1 and not options.user_profile):
parser.print_help()
return 1
diff --git a/src/libprs500/ebooks/lrf/web/profiles/__init__.py b/src/libprs500/ebooks/lrf/web/profiles/__init__.py
index e815dbb88f..66d8d2a7c2 100644
--- a/src/libprs500/ebooks/lrf/web/profiles/__init__.py
+++ b/src/libprs500/ebooks/lrf/web/profiles/__init__.py
@@ -37,6 +37,7 @@ class DefaultProfile(object):
url_search_order = ['guid', 'link'] # THe order of elements to search for a URL when parssing the RSS feed
pubdate_fmt = None # The format string used to parse the publication date in the RSS feed. If set to None some default heuristics are used, these may fail, in which case set this to the correct string or re-implement strptime in your subclass.
use_pubdate = True, # If True will look for a publication date for each article. If False assumes the publication date is the current time.
+ summary_length = 500 # Max number of characters in the short description (ignored in DefaultProfile)
no_stylesheets = False # Download stylesheets only if False
allow_duplicates = False # If False articles with the same title in the same feed are not downloaded multiple times
needs_subscription = False # If True the GUI will ask the userfor a username and password to use while downloading
@@ -52,14 +53,17 @@ class DefaultProfile(object):
preprocess_regexps = []
# See the built-in profiles for examples of these settings.
-
+
+ feeds = []
def get_feeds(self):
'''
Return a list of RSS feeds to fetch for this profile. Each element of the list
must be a 2-element tuple of the form (title, url).
'''
- raise NotImplementedError
+ if not self.feeds:
+ raise NotImplementedError
+ return self.feeds
@classmethod
def print_version(cls, url):
@@ -134,8 +138,8 @@ class DefaultProfile(object):
prefix = 'file:' if iswindows else ''
clist += u'
\n'%(prefix+cfile, category)
src = build_sub_index(category, articles[category])
- open(cfile, 'wb').write(src.encode('utf-8'))
-
+ open(cfile, 'wb').write(src.encode('utf-8'))
+
src = '''\
@@ -225,7 +229,6 @@ class DefaultProfile(object):
added_articles[title].append(d['title'])
if delta > self.oldest_article*3600*24:
continue
-
except Exception, err:
if self.verbose:
self.logger.exception('Error parsing article:\n%s'%(item,))
@@ -325,19 +328,16 @@ class FullContentProfile(DefaultProfile):
This profile is designed for feeds that embed the full article content in the RSS file.
'''
-
- summary_length = 500 # Max number of characters in the short description
-
max_recursions = 0
article_counter = 0
+ html_description = True
+
def build_index(self):
'''Build an RSS based index.html'''
import os
articles = self.parse_feeds(require_url=False)
-
-
def build_sub_index(title, items):
ilist = ''
li = u'
%(title)s[%(date)s] \n'+\
@@ -347,7 +347,7 @@ class FullContentProfile(DefaultProfile):
if not content:
self.logger.debug('Skipping article as it has no content:%s'%item['title'])
continue
- item['description'] = item['description'][:self.summary_length]+'…'
+ item['description'] = cutoff(item['description'], self.summary_length)+'…'
self.article_counter = self.article_counter + 1
url = os.path.join(self.temp_dir, 'article%d.html'%self.article_counter)
item['url'] = url
@@ -383,7 +383,8 @@ class FullContentProfile(DefaultProfile):
prefix = 'file:' if iswindows else ''
clist += u'
\n'%(prefix+cfile, category)
src = build_sub_index(category, articles[category])
- open(cfile, 'wb').write(src.encode('utf-8'))
+ open(cfile, 'wb').write(src.encode('utf-8'))
+ open('/tmp/category'+str(cnum)+'.html', 'wb').write(src.encode('utf-8'))
src = '''\
@@ -401,4 +402,15 @@ class FullContentProfile(DefaultProfile):
open(index, 'wb').write(src.encode('utf-8'))
return index
+def cutoff(src, pos, fuzz=50):
+ si = src.find(';', pos)
+ if si > 0 and si-pos > fuzz:
+ si = -1
+ gi = src.find('>', pos)
+ if gi > 0 and gi-pos > fuzz:
+ gi = -1
+ npos = max(si, gi)
+ if npos < 0:
+ npos = pos
+ return src[:npos+1]
\ No newline at end of file
diff --git a/src/libprs500/ebooks/lrf/web/profiles/dilbert.py b/src/libprs500/ebooks/lrf/web/profiles/dilbert.py
index 56025205bd..927fb4ef6f 100644
--- a/src/libprs500/ebooks/lrf/web/profiles/dilbert.py
+++ b/src/libprs500/ebooks/lrf/web/profiles/dilbert.py
@@ -23,8 +23,6 @@ Fetch Dilbert.
from libprs500.ebooks.lrf.web.profiles import DefaultProfile
-import re
-
class Dilbert(DefaultProfile):
title = 'Dilbert'
diff --git a/src/libprs500/gui2/dialogs/user_profiles.py b/src/libprs500/gui2/dialogs/user_profiles.py
new file mode 100644
index 0000000000..245583fff2
--- /dev/null
+++ b/src/libprs500/gui2/dialogs/user_profiles.py
@@ -0,0 +1,187 @@
+## Copyright (C) 2008 Kovid Goyal kovid@kovidgoyal.net
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License along
+## with this program; if not, write to the Free Software Foundation, Inc.,
+## 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+import time
+
+from PyQt4.QtCore import SIGNAL
+from PyQt4.QtGui import QDialog, QMessageBox
+
+from libprs500.ebooks.lrf.web.profiles import FullContentProfile, DefaultProfile
+from libprs500.gui2.dialogs.user_profiles_ui import Ui_Dialog
+from libprs500.gui2 import qstring_to_unicode, error_dialog, question_dialog
+
+class UserProfiles(QDialog, Ui_Dialog):
+
+ def __init__(self, parent, feeds):
+ QDialog.__init__(self, parent)
+ Ui_Dialog.__init__(self)
+ self.setupUi(self)
+
+ self.connect(self.remove_feed_button, SIGNAL('clicked(bool)'),
+ self.added_feeds.remove_selected_items)
+ self.connect(self.remove_profile_button, SIGNAL('clicked(bool)'),
+ self.available_profiles.remove_selected_items)
+ self.connect(self.add_feed_button, SIGNAL('clicked(bool)'),
+ self.add_feed)
+ self.connect(self.add_profile_button, SIGNAL('clicked(bool)'),
+ self.add_profile)
+ self.connect(self.feed_url, SIGNAL('returnPressed()'), self.add_feed)
+ self.connect(self.feed_title, SIGNAL('returnPressed()'), self.add_feed)
+ self.connect(self.available_profiles,
+ SIGNAL('currentItemChanged(QListWidgetItem*, QListWidgetItem*)'),
+ self.edit_profile)
+ self.connect(self.toggle_mode_button, SIGNAL('clicked(bool)'), self.toggle_mode)
+ self.clear()
+ for title, src in feeds:
+ self.available_profiles.add_item(title, (title, src), replace=True)
+
+
+ def edit_profile(self, current, previous):
+ if not current:
+ current = previous
+ src = current.user_data[1]
+ if 'class BasicUserProfile' in src:
+ profile = self.create_class(src)
+ self.populate_options(profile)
+ self.stacks.setCurrentIndex(0)
+ self.toggle_mode_button.setText('Switch to Advanced mode')
+ else:
+ self.source_code.setPlainText(src)
+ self.stacks.setCurrentIndex(1)
+ self.toggle_mode_button.setText('Switch to Basic mode')
+
+ def toggle_mode(self, *args):
+ if self.stacks.currentIndex() == 1:
+ self.stacks.setCurrentIndex(0)
+ self.toggle_mode_button.setText('Switch to Advanced mode')
+ else:
+ self.stacks.setCurrentIndex(1)
+ self.toggle_mode_button.setText('Switch to Basic mode')
+ if not qstring_to_unicode(self.source_code.toPlainText()).strip():
+ src = self.options_to_profile()[0]
+ self.source_code.setPlainText(src.replace('BasicUserProfile', 'AdvancedUserProfile'))
+
+
+ def add_feed(self, *args):
+ title = qstring_to_unicode(self.feed_title.text()).strip()
+ if not title:
+ d = error_dialog(self, 'Feed must have a title', 'The feed must have a title')
+ d.exec_()
+ return
+ url = qstring_to_unicode(self.feed_url.text()).strip()
+ if not url:
+ d = error_dialog(self, 'Feed must have a URL', 'The feed %s must have a URL'%title)
+ d.exec_()
+ return
+ try:
+ self.added_feeds.add_item(title+' - '+url, (title, url))
+ except ValueError:
+ error_dialog(self, 'Already in list', 'This feed has already been added to the profile').exec_()
+ return
+ self.feed_title.setText('')
+ self.feed_url.setText('')
+
+ def options_to_profile(self):
+ classname = 'BasicUserProfile'+str(int(time.time()))
+ title = qstring_to_unicode(self.profile_title.text()).strip()
+ if not title:
+ title = classname
+ self.profile_title.setText(title)
+ summary_length = self.summary_length.value()
+ oldest_article = self.oldest_article.value()
+ max_articles = self.max_articles.value()
+ feeds = [i.user_data for i in self.added_feeds.items()]
+
+ src = '''\
+class %(classname)s(%(base_class)s):
+ title = %(title)s
+ summary_length = %(summary_length)d
+ oldest_article = %(oldest_article)d
+ max_articles_per_feed = %(max_articles)d
+
+ feeds = %(feeds)s
+'''%dict(classname=classname, title=repr(title), summary_length=summary_length,
+ feeds=repr(feeds), oldest_article=oldest_article,
+ max_articles=max_articles,
+ base_class='DefaultProfile' if self.full_articles.isChecked() else 'FullContentProfile')
+ return src, title
+
+
+ def populate_source_code(self):
+ src = self.options_to_profile().replace('BasicUserProfile', 'AdvancedUserProfile')
+ self.source_code.setPlainText(src)
+
+ @classmethod
+ def create_class(cls, src):
+ environment = {'FullContentProfile':FullContentProfile, 'DefaultProfile':DefaultProfile}
+ exec src in environment
+ for item in environment.values():
+ if hasattr(item, 'build_index'):
+ if item.__name__ not in ['DefaultProfile', 'FullContentProfile']:
+ return item
+
+
+ def add_profile(self, clicked):
+ if self.stacks.currentIndex() == 0:
+ src, title = self.options_to_profile()
+
+ try:
+ self.create_class(src)
+ except Exception, err:
+ error_dialog(self, 'Invalid input',
+ '