diff --git a/resources/images/mimetypes/opml.png b/resources/images/mimetypes/opml.png new file mode 100644 index 0000000000..96c565db19 Binary files /dev/null and b/resources/images/mimetypes/opml.png differ diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index a2dd2cfc7a..a270d62cdd 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -525,6 +525,7 @@ class FileIconProvider(QFileIconProvider): 'xps' : 'xps', 'oxps' : 'xps', 'docx' : 'docx', + 'opml' : 'opml', } def __init__(self): diff --git a/src/calibre/gui2/dialogs/opml.py b/src/calibre/gui2/dialogs/opml.py new file mode 100644 index 0000000000..8c64589583 --- /dev/null +++ b/src/calibre/gui2/dialogs/opml.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2014, Kovid Goyal ' + +from collections import defaultdict, namedtuple +from operator import itemgetter + +from PyQt4.Qt import ( + QDialog, QFormLayout, QHBoxLayout, QLineEdit, QToolButton, QIcon, + QDialogButtonBox, Qt, QSpinBox, QCheckBox) + +from lxml import etree + +from calibre.gui2 import choose_files, error_dialog +from calibre.utils.icu import sort_key + +Group = namedtuple('Group', 'title feeds') + +def uniq(vals, kmap=lambda x:x): + ''' Remove all duplicates from vals, while preserving order. kmap must be a + callable that returns a hashable value for every item in vals ''' + vals = vals or () + lvals = (kmap(x) for x in vals) + seen = set() + seen_add = seen.add + return tuple(x for x, k in zip(vals, lvals) if k not in seen and not seen_add(k)) + +def import_opml(raw, preserve_groups=True): + root = etree.fromstring(raw) + groups = defaultdict(list) + ax = etree.XPath('ancestor::outline[@title or @text]') + for outline in root.xpath('//outline[@type="rss" and @xmlUrl]'): + url = outline.get('xmlUrl') + parent = outline.get('title', '') or url + title = parent if ('title' in outline.attrib and parent) else None + if preserve_groups: + for ancestor in ax(outline): + if ancestor.get('type', None) != 'rss': + text = ancestor.get('title') or ancestor.get('text') + if text: + parent = text + break + groups[parent].append((title, url)) + + for title in sorted(groups.iterkeys(), key=sort_key): + yield Group(title, uniq(groups[title], kmap=itemgetter(1))) + + +class ImportOPML(QDialog): + + def __init__(self, parent=None): + QDialog.__init__(self, parent=parent) + self.l = l = QFormLayout(self) + self.setLayout(l) + self.setWindowTitle(_('Import OPML file')) + self.setWindowIcon(QIcon(I('opml.png'))) + + self.h = h = QHBoxLayout() + self.path = p = QLineEdit(self) + p.setMinimumWidth(300) + p.setPlaceholderText(_('Path to OPML file')) + h.addWidget(p) + self.cfb = b = QToolButton(self) + b.setIcon(QIcon(I('document_open.png'))) + b.setToolTip(_('Browse for OPML file')) + b.clicked.connect(self.choose_file) + h.addWidget(b) + l.addRow(_('&OPML file:'), h) + l.labelForField(h).setBuddy(p) + b.setFocus(Qt.OtherFocusReason) + + self._articles_per_feed = a = QSpinBox(self) + a.setMinimum(1), a.setMaximum(1000), a.setValue(100) + a.setToolTip(_('Maximum number of articles to download per RSS feed')) + l.addRow(_('&Maximum articles per feed:'), a) + + self._oldest_article = o = QSpinBox(self) + o.setMinimum(1), o.setMaximum(3650), o.setValue(7) + o.setSuffix(_(' days')) + o.setToolTip(_('Articles in the RSS feeds older than this will be ignored')) + l.addRow(_('&Oldest article:'), o) + + self.preserve_groups = g = QCheckBox(_('Preserve groups in the OPML file')) + g.setToolTip('

' + _( + 'If enabled, every group of feeds in the OPML file will be converted into a single recipe. Otherwise every feed becomes its own recipe')) + g.setChecked(True) + l.addRow(g) + + self._replace_existing = r = QCheckBox(_('Replace existing recipes')) + r.setToolTip('

' + _( + 'If enabled, any existing recipes with the same titles as entries in the OPML file will be replaced.' + ' Otherwise, new entries with modified titles will be created')) + r.setChecked(True) + l.addRow(r) + + self.bb = bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel) + bb.accepted.connect(self.accept), bb.rejected.connect(self.reject) + l.addRow(bb) + + self.recipes = () + + @property + def articles_per_feed(self): + return self._articles_per_feed.value() + + @property + def oldest_article(self): + return self._oldest_article.value() + + @property + def replace_existing(self): + return self._replace_existing.isChecked() + + def choose_file(self): + opml_files = choose_files( + self, 'opml-select-dialog', _('Select OPML file'), filters=[(_('OPML files'), ['opml'])], + all_files=False, select_only_single_file=True) + if opml_files: + self.path.setText(opml_files[0]) + + def accept(self): + path = unicode(self.path.text()) + if not path: + return error_dialog(self, _('Path not specified'), _( + 'You must specify the path to the OPML file to import'), show=True) + with open(path, 'rb') as f: + raw = f.read() + self.recipes = tuple(import_opml(raw, self.preserve_groups.isChecked())) + if len(self.recipes) == 0: + return error_dialog(self, _('No feeds found'), _( + 'No importable RSS feeds found in the OPML file'), show=True) + + QDialog.accept(self) + +if __name__ == '__main__': + import sys + for group in import_opml(open(sys.argv[-1], 'rb').read()): + print (group.title) + for title, url in group.feeds: + print ('\t%s - %s' % (title, url)) + print () diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index 48404f5c03..2d927bb14a 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -8,8 +8,9 @@ from PyQt4.Qt import (QUrl, QAbstractListModel, Qt, QVariant, QFont) from calibre.web.feeds.recipes import compile_recipe, custom_recipes from calibre.web.feeds.news import AutomaticNewsRecipe from calibre.gui2.dialogs.user_profiles_ui import Ui_Dialog -from calibre.gui2 import error_dialog, question_dialog, open_url, \ - choose_files, ResizableDialog, NONE, open_local_file +from calibre.gui2 import ( + error_dialog, question_dialog, open_url, choose_files, ResizableDialog, + NONE, open_local_file) from calibre.gui2.widgets import PythonHighlighter from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.icu import sort_key @@ -59,10 +60,28 @@ class CustomRecipeModel(QAbstractListModel): self.recipe_model.update_custom_recipe(urn, title, script) self.reset() + def replace_many_by_title(self, scriptmap): + script_urn_map = {} + for title, script in scriptmap.iteritems(): + urn = None + for x in self.recipe_model.custom_recipe_collection: + if x.get('title', False) == title: + urn = x.get('id') + if urn is not None: + script_urn_map.update({urn: (title, script)}) + + if script_urn_map: + self.recipe_model.update_custom_recipes(script_urn_map) + self.reset() + def add(self, title, script): self.recipe_model.add_custom_recipe(title, script) self.reset() + def add_many(self, scriptmap): + self.recipe_model.add_custom_recipes(scriptmap) + self.reset() + def remove(self, rows): urns = [] for r in rows: @@ -90,6 +109,7 @@ class UserProfiles(ResizableDialog, Ui_Dialog): self.remove_profile_button.clicked[(bool)].connect(self.remove_selected_items) self.add_feed_button.clicked[(bool)].connect(self.add_feed) self.load_button.clicked[()].connect(self.load) + self.opml_button.clicked[()].connect(self.opml_import) self.builtin_recipe_button.clicked[()].connect(self.add_builtin_recipe) self.share_button.clicked[()].connect(self.share) self.show_recipe_files_button.clicked.connect(self.show_recipe_files) @@ -201,15 +221,17 @@ class UserProfiles(ResizableDialog, Ui_Dialog): self.feed_title.setText('') self.feed_url.setText('') - def options_to_profile(self): + def options_to_profile(self, **kw): classname = 'BasicUserRecipe'+str(int(time.time())) - title = unicode(self.profile_title.text()).strip() + title = kw.get('title', self.profile_title.text()) + title = unicode(title).strip() if not title: title = classname self.profile_title.setText(title) - oldest_article = self.oldest_article.value() - max_articles = self.max_articles.value() - feeds = [i.user_data for i in self.added_feeds.items()] + oldest_article = kw.get('oldest_article', self.oldest_article.value()) + max_articles = kw.get('max_articles', self.max_articles.value()) + feeds = kw.get('feeds', + [i.user_data for i in self.added_feeds.items()]) src = '''\ class %(classname)s(%(base_class)s): @@ -346,6 +368,50 @@ class %(classname)s(%(base_class)s): self.model.add(title, profile) self.clear() + def opml_import(self): + from calibre.gui2.dialogs.opml import ImportOPML + d = ImportOPML(parent=self) + if d.exec_() != d.Accepted: + return + oldest_article, max_articles_per_feed, replace_existing = d.oldest_article, d.articles_per_feed, d.replace_existing + failed_recipes, replace_recipes, add_recipes = {}, {}, {} + + for group in d.recipes: + title = base_title = group.title or _('Unknown') + if not replace_existing: + c = 0 + while self._model.has_title(title): + c += 1 + title = u'%s %d' % (base_title, c) + src, title = self.options_to_profile(**{ + 'title':title, + 'feeds':group.feeds, + 'oldest_article':oldest_article, + 'max_articles':max_articles_per_feed, + }) + try: + compile_recipe(src) + except Exception: + import traceback + failed_recipes[title] = traceback.format_exc() + continue + + if replace_existing and self._model.has_title(title): + replace_recipes[title] = src + else: + add_recipes[title] = src + + if add_recipes: + self.model.add_many(add_recipes) + if replace_recipes: + self.model.replace_many_by_title(replace_recipes) + if failed_recipes: + det_msg = '\n'.join('%s\n%s\n' % (title, tb) for title, tb in failed_recipes.iteritems()) + error_dialog(self, _('Failed to create recipes'), _( + 'Failed to create some recipes, click "Show details" for details'), show=True, + det_msg=det_msg) + self.clear() + def populate_options(self, profile): self.oldest_article.setValue(profile.oldest_article) self.max_articles.setValue(profile.max_articles_per_feed) diff --git a/src/calibre/gui2/dialogs/user_profiles.ui b/src/calibre/gui2/dialogs/user_profiles.ui index 1e29477e6c..c615569eb4 100644 --- a/src/calibre/gui2/dialogs/user_profiles.ui +++ b/src/calibre/gui2/dialogs/user_profiles.ui @@ -34,8 +34,8 @@ 0 0 - 730 - 601 + 726 + 595 @@ -135,6 +135,22 @@ + + + + Import a collection of RSS feeds in OPML format +Many RSS readers can export their subscribed RSS feeds +in OPML format + + + Import &OPML + + + + :/images/mimetypes/opml.png:/images/mimetypes/opml.png + + + diff --git a/src/calibre/web/feeds/recipes/collection.py b/src/calibre/web/feeds/recipes/collection.py index 5cfbb7899c..5092e3ee4c 100644 --- a/src/calibre/web/feeds/recipes/collection.py +++ b/src/calibre/web/feeds/recipes/collection.py @@ -120,23 +120,29 @@ def get_custom_recipe_collection(*args): def update_custom_recipe(id_, title, script): + update_custom_recipes( [(id_, title, script)] ) + +def update_custom_recipes(script_ids): from calibre.web.feeds.recipes import custom_recipes, \ custom_recipe_filename - id_ = str(int(id_)) - existing = custom_recipes.get(id_, None) + bdir = os.path.dirname(custom_recipes.file_path) + for id_, title, script in script_ids: - if existing is None: - fname = custom_recipe_filename(id_, title) - else: - fname = existing[1] - if isinstance(script, unicode): - script = script.encode('utf-8') + id_ = str(int(id_)) + existing = custom_recipes.get(id_, None) - custom_recipes[id_] = (title, fname) + if existing is None: + fname = custom_recipe_filename(id_, title) + else: + fname = existing[1] + if isinstance(script, unicode): + script = script.encode('utf-8') - with open(os.path.join(bdir, fname), 'wb') as f: - f.write(script) + custom_recipes[id_] = (title, fname) + + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) def add_custom_recipe(title, script): @@ -149,10 +155,10 @@ def add_custom_recipes(script_map): keys = tuple(map(int, custom_recipes.iterkeys())) if keys: id_ = max(keys)+1 + bdir = os.path.dirname(custom_recipes.file_path) with custom_recipes: for title, script in script_map.iteritems(): fid = str(id_) - bdir = os.path.dirname(custom_recipes.file_path) fname = custom_recipe_filename(fid, title) if isinstance(script, unicode): diff --git a/src/calibre/web/feeds/recipes/model.py b/src/calibre/web/feeds/recipes/model.py index b5c123bd4f..f1a1213c72 100644 --- a/src/calibre/web/feeds/recipes/model.py +++ b/src/calibre/web/feeds/recipes/model.py @@ -17,8 +17,8 @@ from calibre.utils.localization import get_language from calibre.web.feeds.recipes.collection import \ get_builtin_recipe_collection, get_custom_recipe_collection, \ SchedulerConfig, download_builtin_recipe, update_custom_recipe, \ - add_custom_recipe, remove_custom_recipe, get_custom_recipe, \ - get_builtin_recipe + update_custom_recipes, add_custom_recipe, add_custom_recipes, \ + remove_custom_recipe, get_custom_recipe, get_builtin_recipe from calibre.utils.search_query_parser import ParseException class NewsTreeItem(object): @@ -171,13 +171,28 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser): update_custom_recipe(id_, title, script) self.custom_recipe_collection = get_custom_recipe_collection() + def update_custom_recipes(self, script_urn_map): + script_ids = [] + for urn, title_script in script_urn_map.iteritems(): + id_ = int(urn[len('custom:'):]) + (title, script) = title_script + script_ids.append((id_, title, script)) + + update_custom_recipes(script_ids) + self.custom_recipe_collection = get_custom_recipe_collection() + def add_custom_recipe(self, title, script): add_custom_recipe(title, script) self.custom_recipe_collection = get_custom_recipe_collection() + def add_custom_recipes(self, scriptmap): + add_custom_recipes(scriptmap) + self.custom_recipe_collection = get_custom_recipe_collection() + def remove_custom_recipes(self, urns): ids = [int(x[len('custom:'):]) for x in urns] - for id_ in ids: remove_custom_recipe(id_) + for id_ in ids: + remove_custom_recipe(id_) self.custom_recipe_collection = get_custom_recipe_collection() def do_refresh(self, restrict_to_urns=set([])):