Fetch news: Allow automatically creating custom news sources by import OPML files. Many RSS news reader programs can export their list of RSS feeds in the OPML format.

Merge branch 'master' of https://github.com/ingkebil/calibre

src/calibre/gui2/__init__.py

@@ -525,6 +525,7 @@ class FileIconProvider(QFileIconProvider):
              'xps'     : 'xps',
              'oxps'    : 'xps',
              'docx'    : 'docx',
+             'opml'    : 'opml',
              }
 
     def __init__(self):

src/calibre/gui2/dialogs/opml.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
+
+from collections import defaultdict, namedtuple
+from operator import itemgetter
+
+from PyQt4.Qt import (
+    QDialog, QFormLayout, QHBoxLayout, QLineEdit, QToolButton, QIcon,
+    QDialogButtonBox, Qt, QSpinBox, QCheckBox)
+
+from lxml import etree
+
+from calibre.gui2 import choose_files, error_dialog
+from calibre.utils.icu import sort_key
+
+Group = namedtuple('Group', 'title feeds')
+
+def uniq(vals, kmap=lambda x:x):
+    ''' Remove all duplicates from vals, while preserving order. kmap must be a
+    callable that returns a hashable value for every item in vals '''
+    vals = vals or ()
+    lvals = (kmap(x) for x in vals)
+    seen = set()
+    seen_add = seen.add
+    return tuple(x for x, k in zip(vals, lvals) if k not in seen and not seen_add(k))
+
+def import_opml(raw, preserve_groups=True):
+    root = etree.fromstring(raw)
+    groups = defaultdict(list)
+    ax = etree.XPath('ancestor::outline[@title or @text]')
+    for outline in root.xpath('//outline[@type="rss" and @xmlUrl]'):
+        url = outline.get('xmlUrl')
+        parent = outline.get('title', '') or url
+        title = parent if ('title' in outline.attrib and parent) else None
+        if preserve_groups:
+            for ancestor in ax(outline):
+                if ancestor.get('type', None) != 'rss':
+                    text = ancestor.get('title') or ancestor.get('text')
+                    if text:
+                        parent = text
+                        break
+        groups[parent].append((title, url))
+
+    for title in sorted(groups.iterkeys(), key=sort_key):
+        yield Group(title, uniq(groups[title], kmap=itemgetter(1)))
+
+
+class ImportOPML(QDialog):
+
+    def __init__(self, parent=None):
+        QDialog.__init__(self, parent=parent)
+        self.l = l = QFormLayout(self)
+        self.setLayout(l)
+        self.setWindowTitle(_('Import OPML file'))
+        self.setWindowIcon(QIcon(I('opml.png')))
+
+        self.h = h = QHBoxLayout()
+        self.path = p = QLineEdit(self)
+        p.setMinimumWidth(300)
+        p.setPlaceholderText(_('Path to OPML file'))
+        h.addWidget(p)
+        self.cfb = b = QToolButton(self)
+        b.setIcon(QIcon(I('document_open.png')))
+        b.setToolTip(_('Browse for OPML file'))
+        b.clicked.connect(self.choose_file)
+        h.addWidget(b)
+        l.addRow(_('&OPML file:'), h)
+        l.labelForField(h).setBuddy(p)
+        b.setFocus(Qt.OtherFocusReason)
+
+        self._articles_per_feed = a = QSpinBox(self)
+        a.setMinimum(1), a.setMaximum(1000), a.setValue(100)
+        a.setToolTip(_('Maximum number of articles to download per RSS feed'))
+        l.addRow(_('&Maximum articles per feed:'), a)
+
+        self._oldest_article = o = QSpinBox(self)
+        o.setMinimum(1), o.setMaximum(3650), o.setValue(7)
+        o.setSuffix(_(' days'))
+        o.setToolTip(_('Articles in the RSS feeds older than this will be ignored'))
+        l.addRow(_('&Oldest article:'), o)
+
+        self.preserve_groups = g = QCheckBox(_('Preserve groups in the OPML file'))
+        g.setToolTip('<p>' + _(
+            'If enabled, every group of feeds in the OPML file will be converted into a single recipe. Otherwise every feed becomes its own recipe'))
+        g.setChecked(True)
+        l.addRow(g)
+
+        self._replace_existing = r = QCheckBox(_('Replace existing recipes'))
+        r.setToolTip('<p>' + _(
+            'If enabled, any existing recipes with the same titles as entries in the OPML file will be replaced.'
+            ' Otherwise, new entries with modified titles will be created'))
+        r.setChecked(True)
+        l.addRow(r)
+
+        self.bb = bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel)
+        bb.accepted.connect(self.accept), bb.rejected.connect(self.reject)
+        l.addRow(bb)
+
+        self.recipes = ()
+
+    @property
+    def articles_per_feed(self):
+        return self._articles_per_feed.value()
+
+    @property
+    def oldest_article(self):
+        return self._oldest_article.value()
+
+    @property
+    def replace_existing(self):
+        return self._replace_existing.isChecked()
+
+    def choose_file(self):
+        opml_files = choose_files(
+            self, 'opml-select-dialog', _('Select OPML file'), filters=[(_('OPML files'), ['opml'])],
+            all_files=False, select_only_single_file=True)
+        if opml_files:
+            self.path.setText(opml_files[0])
+
+    def accept(self):
+        path = unicode(self.path.text())
+        if not path:
+            return error_dialog(self, _('Path not specified'), _(
+                'You must specify the path to the OPML file to import'), show=True)
+        with open(path, 'rb') as f:
+            raw = f.read()
+        self.recipes = tuple(import_opml(raw, self.preserve_groups.isChecked()))
+        if len(self.recipes) == 0:
+            return error_dialog(self, _('No feeds found'), _(
+                'No importable RSS feeds found in the OPML file'), show=True)
+
+        QDialog.accept(self)
+
+if __name__ == '__main__':
+    import sys
+    for group in import_opml(open(sys.argv[-1], 'rb').read()):
+        print (group.title)
+        for title, url in group.feeds:
+            print ('\t%s - %s' % (title, url))
+        print ()

src/calibre/gui2/dialogs/user_profiles.py

@@ -8,8 +8,9 @@ from PyQt4.Qt import (QUrl, QAbstractListModel, Qt, QVariant, QFont)
 from calibre.web.feeds.recipes import compile_recipe, custom_recipes
 from calibre.web.feeds.news import AutomaticNewsRecipe
 from calibre.gui2.dialogs.user_profiles_ui import Ui_Dialog
-from calibre.gui2 import error_dialog, question_dialog, open_url, \
-                         choose_files, ResizableDialog, NONE, open_local_file
+from calibre.gui2 import (
+    error_dialog, question_dialog, open_url, choose_files, ResizableDialog,
+    NONE, open_local_file)
 from calibre.gui2.widgets import PythonHighlighter
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.icu import sort_key
@@ -59,10 +60,28 @@ class CustomRecipeModel(QAbstractListModel):
             self.recipe_model.update_custom_recipe(urn, title, script)
             self.reset()
 
+    def replace_many_by_title(self, scriptmap):
+        script_urn_map = {}
+        for title, script in scriptmap.iteritems():
+            urn = None
+            for x in self.recipe_model.custom_recipe_collection:
+                if x.get('title', False) == title:
+                    urn = x.get('id')
+            if urn is not None:
+                script_urn_map.update({urn: (title, script)})
+
+        if script_urn_map:
+            self.recipe_model.update_custom_recipes(script_urn_map)
+            self.reset()
+
     def add(self, title, script):
         self.recipe_model.add_custom_recipe(title, script)
         self.reset()
 
+    def add_many(self, scriptmap):
+        self.recipe_model.add_custom_recipes(scriptmap)
+        self.reset()
+
     def remove(self, rows):
         urns = []
         for r in rows:
@@ -90,6 +109,7 @@ class UserProfiles(ResizableDialog, Ui_Dialog):
         self.remove_profile_button.clicked[(bool)].connect(self.remove_selected_items)
         self.add_feed_button.clicked[(bool)].connect(self.add_feed)
         self.load_button.clicked[()].connect(self.load)
+        self.opml_button.clicked[()].connect(self.opml_import)
         self.builtin_recipe_button.clicked[()].connect(self.add_builtin_recipe)
         self.share_button.clicked[()].connect(self.share)
         self.show_recipe_files_button.clicked.connect(self.show_recipe_files)
@@ -201,15 +221,17 @@ class UserProfiles(ResizableDialog, Ui_Dialog):
         self.feed_title.setText('')
         self.feed_url.setText('')
 
-    def options_to_profile(self):
+    def options_to_profile(self, **kw):
         classname = 'BasicUserRecipe'+str(int(time.time()))
-        title = unicode(self.profile_title.text()).strip()
+        title = kw.get('title', self.profile_title.text())
+        title = unicode(title).strip()
         if not title:
             title = classname
         self.profile_title.setText(title)
-        oldest_article = self.oldest_article.value()
-        max_articles   = self.max_articles.value()
-        feeds = [i.user_data for i in self.added_feeds.items()]
+        oldest_article = kw.get('oldest_article', self.oldest_article.value())
+        max_articles   = kw.get('max_articles', self.max_articles.value())
+        feeds = kw.get('feeds',
+                [i.user_data for i in self.added_feeds.items()])
 
         src = '''\
 class %(classname)s(%(base_class)s):
@@ -346,6 +368,50 @@ class %(classname)s(%(base_class)s):
                 self.model.add(title, profile)
             self.clear()
 
+    def opml_import(self):
+        from calibre.gui2.dialogs.opml import ImportOPML
+        d = ImportOPML(parent=self)
+        if d.exec_() != d.Accepted:
+            return
+        oldest_article, max_articles_per_feed, replace_existing = d.oldest_article, d.articles_per_feed, d.replace_existing
+        failed_recipes, replace_recipes, add_recipes = {}, {}, {}
+
+        for group in d.recipes:
+            title = base_title = group.title or _('Unknown')
+            if not replace_existing:
+                c = 0
+                while self._model.has_title(title):
+                    c += 1
+                    title = u'%s %d' % (base_title, c)
+            src, title = self.options_to_profile(**{
+                    'title':title,
+                    'feeds':group.feeds,
+                    'oldest_article':oldest_article,
+                    'max_articles':max_articles_per_feed,
+                })
+            try:
+                compile_recipe(src)
+            except Exception:
+                import traceback
+                failed_recipes[title] = traceback.format_exc()
+                continue
+
+            if replace_existing and self._model.has_title(title):
+                replace_recipes[title] = src
+            else:
+                add_recipes[title] = src
+
+        if add_recipes:
+            self.model.add_many(add_recipes)
+        if replace_recipes:
+            self.model.replace_many_by_title(replace_recipes)
+        if failed_recipes:
+            det_msg = '\n'.join('%s\n%s\n' % (title, tb) for title, tb in failed_recipes.iteritems())
+            error_dialog(self, _('Failed to create recipes'), _(
+                'Failed to create some recipes, click "Show details" for details'), show=True,
+                         det_msg=det_msg)
+        self.clear()
+
     def populate_options(self, profile):
         self.oldest_article.setValue(profile.oldest_article)
         self.max_articles.setValue(profile.max_articles_per_feed)

src/calibre/gui2/dialogs/user_profiles.ui

@@ -34,8 +34,8 @@
        <rect>
         <x>0</x>
         <y>0</y>
-        <width>730</width>
-        <height>601</height>
+        <width>726</width>
+        <height>595</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_3">
@@ -135,6 +135,22 @@
                </property>
               </widget>
              </item>
+             <item>
+              <widget class="QPushButton" name="opml_button">
+               <property name="toolTip">
+                <string>Import a collection of RSS feeds in OPML format
+Many RSS readers can export their subscribed RSS feeds
+in OPML format</string>
+               </property>
+               <property name="text">
+                <string>Import &amp;OPML</string>
+               </property>
+               <property name="icon">
+                <iconset resource="../../../../resources/images.qrc">
+                 <normaloff>:/images/mimetypes/opml.png</normaloff>:/images/mimetypes/opml.png</iconset>
+               </property>
+              </widget>
+             </item>
             </layout>
            </widget>
           </item>

src/calibre/web/feeds/recipes/collection.py

@@ -120,23 +120,29 @@ def get_custom_recipe_collection(*args):
 
 
 def update_custom_recipe(id_, title, script):
+    update_custom_recipes( [(id_, title, script)] )
+
+def update_custom_recipes(script_ids):
     from calibre.web.feeds.recipes import custom_recipes, \
             custom_recipe_filename
-    id_ = str(int(id_))
-    existing = custom_recipes.get(id_, None)
+
     bdir = os.path.dirname(custom_recipes.file_path)
+    for id_, title, script in script_ids:
 
-    if existing is None:
-        fname = custom_recipe_filename(id_, title)
-    else:
-        fname = existing[1]
-    if isinstance(script, unicode):
-        script = script.encode('utf-8')
+        id_ = str(int(id_))
+        existing = custom_recipes.get(id_, None)
 
-    custom_recipes[id_] = (title, fname)
+        if existing is None:
+            fname = custom_recipe_filename(id_, title)
+        else:
+            fname = existing[1]
+        if isinstance(script, unicode):
+            script = script.encode('utf-8')
 
-    with open(os.path.join(bdir, fname), 'wb') as f:
-        f.write(script)
+        custom_recipes[id_] = (title, fname)
+
+        with open(os.path.join(bdir, fname), 'wb') as f:
+            f.write(script)
 
 
 def add_custom_recipe(title, script):
@@ -149,10 +155,10 @@ def add_custom_recipes(script_map):
     keys = tuple(map(int, custom_recipes.iterkeys()))
     if keys:
         id_ = max(keys)+1
+    bdir = os.path.dirname(custom_recipes.file_path)
     with custom_recipes:
         for title, script in script_map.iteritems():
             fid = str(id_)
-            bdir = os.path.dirname(custom_recipes.file_path)
 
             fname = custom_recipe_filename(fid, title)
             if isinstance(script, unicode):

src/calibre/web/feeds/recipes/model.py

@@ -17,8 +17,8 @@ from calibre.utils.localization import get_language
 from calibre.web.feeds.recipes.collection import \
         get_builtin_recipe_collection, get_custom_recipe_collection, \
         SchedulerConfig, download_builtin_recipe, update_custom_recipe, \
-        add_custom_recipe, remove_custom_recipe, get_custom_recipe, \
-        get_builtin_recipe
+        update_custom_recipes, add_custom_recipe, add_custom_recipes, \
+        remove_custom_recipe, get_custom_recipe, get_builtin_recipe
 from calibre.utils.search_query_parser import ParseException
 
 class NewsTreeItem(object):
@@ -171,13 +171,28 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
         update_custom_recipe(id_, title, script)
         self.custom_recipe_collection = get_custom_recipe_collection()
 
+    def update_custom_recipes(self, script_urn_map):
+        script_ids = []
+        for urn, title_script in script_urn_map.iteritems():
+            id_ = int(urn[len('custom:'):])
+            (title, script) = title_script
+            script_ids.append((id_, title, script))
+
+        update_custom_recipes(script_ids)
+        self.custom_recipe_collection = get_custom_recipe_collection()
+
     def add_custom_recipe(self, title, script):
         add_custom_recipe(title, script)
         self.custom_recipe_collection = get_custom_recipe_collection()
 
+    def add_custom_recipes(self, scriptmap):
+        add_custom_recipes(scriptmap)
+        self.custom_recipe_collection = get_custom_recipe_collection()
+
     def remove_custom_recipes(self, urns):
         ids = [int(x[len('custom:'):]) for x in urns]
-        for id_ in ids: remove_custom_recipe(id_)
+        for id_ in ids:
+            remove_custom_recipe(id_)
         self.custom_recipe_collection = get_custom_recipe_collection()
 
     def do_refresh(self, restrict_to_urns=set([])):