IGN:Initial implementation of recipe scheduler

src/calibre/ebooks/html.py

@@ -675,7 +675,7 @@ class Processor(Parser):
                 text = (u''.join(link.xpath('string()'))).strip()
                 if text:
                     href = link.get('href', '')
-                    if href:
+                    if href and not (href.startswith('http://') or href.startswith('https://')):
                         href = 'content/'+href
                         parts = href.split('#')
                         href, fragment = parts[0], None

src/calibre/ebooks/lit/reader.py

@@ -771,15 +771,21 @@ class LitReader(object):
                     raise("Reset table entry out of bounds")
                 if bytes_remaining >= window_bytes:
                     lzx.reset()
+                    try:
                         result.append(
                             lzx.decompress(content[base:size], window_bytes))
+                    except lzx.LzxError:
+                        self._warn("LZX decompression error; skipping chunk")
                     bytes_remaining -= window_bytes
                     base = size
             accum += int32(reset_table[RESET_INTERVAL:])
             ofs_entry += 8
         if bytes_remaining < window_bytes and bytes_remaining > 0:
             lzx.reset()
+            try:
                 result.append(lzx.decompress(content[base:], bytes_remaining))
+            except lzx.LzxError:
+                self._warn("LZX decompression error; skipping chunk")
             bytes_remaining = 0
         if bytes_remaining > 0:
             raise LitError("Failed to completely decompress section")
@@ -826,6 +832,9 @@ class LitReader(object):
         if not os.path.isdir(dir):
             os.makedirs(dir)
 
+    def _warn(self, msg):
+        print "WARNING: %s" % (msg,)
+
 def option_parser():
     from calibre.utils.config import OptionParser
     parser = OptionParser(usage=_('%prog [options] LITFILE'))

src/calibre/gui2/dialogs/scheduler.py

@@ -0,0 +1,344 @@
+from __future__ import with_statement
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+Scheduler for automated recipe downloads
+'''
+
+import sys, copy
+from threading import RLock
+from datetime import datetime, timedelta
+from PyQt4.Qt import QDialog, QApplication, QLineEdit, QPalette, SIGNAL, QBrush, \
+                     QColor, QAbstractListModel, Qt, QVariant, QFont, QIcon, \
+                     QFile, QObject, QTimer
+
+from calibre import english_sort
+from calibre.gui2.dialogs.scheduler_ui import Ui_Dialog
+from calibre.web.feeds.recipes import recipes, recipe_modules, compile_recipe
+from calibre.utils.search_query_parser import SearchQueryParser
+from calibre.utils.pyparsing import ParseException
+from calibre.gui2 import dynamic, NONE, error_dialog
+
+class Recipe(object):
+    
+    def __init__(self, id, recipe_class, builtin):
+        self.id              = id
+        self.title           = recipe_class.title
+        self.description     = recipe_class.description
+        self.last_downloaded = datetime.fromordinal(1)
+        self.downloading     = False
+        self.builtin         = builtin
+        self.schedule        = None
+        self.needs_subscription = recipe_class.needs_subscription
+        
+    def __cmp__(self, other):
+        if self.id == getattr(other, 'id', None):
+            return 0
+        if self.schedule is None and getattr(other, 'schedule', None) is not None:
+            return 1
+        if self.schedule is not None and getattr(other, 'schedule', None) is None:
+            return -1
+        if self.builtin and not getattr(other, 'builtin', True):
+            return 1
+        if not self.builtin and getattr(other, 'builtin', True):
+            return -1
+        return english_sort(self.title, getattr(other, 'title', ''))
+    
+    def __hash__(self):
+        return hash(self.id)
+    
+    def __eq__(self, other):
+        return self.id == getattr(other, 'id', None)
+    
+    def __repr__(self):
+        return u'%s:%s'%(self.id, self.title)
+    
+builtin_recipes = [Recipe(m, r, True) for r, m in zip(recipes, recipe_modules)]
+
+class RecipeModel(QAbstractListModel, SearchQueryParser):
+    
+    LOCATIONS = ['all']
+    
+    def __init__(self, db, *args):
+        QAbstractListModel.__init__(self, *args)
+        SearchQueryParser.__init__(self)
+        self.default_icon = QIcon(':/images/news.svg')
+        self.custom_icon = QIcon(':/images/user_profile.svg')
+        self.recipes = copy.deepcopy(builtin_recipes)
+        for x in db.get_recipes():
+            recipe = compile_recipe(x[1])
+            self.recipes.append(Recipe(x[0], recipe, False))
+            
+        sr = dynamic['scheduled_recipes']
+        if not sr:
+            sr = []
+        for recipe in self.recipes:
+            if recipe in sr:
+                recipe.schedule = sr[sr.index(recipe)].schedule
+        
+        self.recipes.sort()
+        self._map = list(range(len(self.recipes)))
+    
+    def universal_set(self):
+        return set(self.recipes)
+    
+    def get_matches(self, location, query):
+        query = query.strip().lower()
+        if not query:
+            return set(self.recipes)
+        results = set([])
+        for recipe in self.recipes:
+            if query in recipe.title.lower() or query in recipe.description.lower():
+                results.add(recipe) 
+        return results
+    
+    def search(self, query):
+        try:
+            results = self.parse(unicode(query))
+        except ParseException:
+            self._map = list(range(len(self.recipes)))
+        else:
+            self._map = []
+            for i, recipe in enumerate(self.recipes):
+                if recipe in results:
+                    self._map.append(i)
+        self.reset()
+    
+    def resort(self):
+        self.recipes.sort()
+        self.reset()
+        
+    def columnCount(self, *args):
+        return 1
+    
+    def rowCount(self, *args):
+        return len(self._map)
+    
+    def data(self, index, role):
+        recipe = self.recipes[self._map[index.row()]]
+        if role == Qt.FontRole:
+            if recipe.schedule is not None:
+                font = QFont()
+                font.setBold(True)
+                return QVariant(font)
+            if not recipe.builtin:
+                font = QFont()
+                font.setItalic(True)
+                return QVariant(font)
+        elif role == Qt.DisplayRole:
+            return QVariant(recipe.title)
+        elif role == Qt.UserRole:
+            return recipe
+        elif role == Qt.DecorationRole:
+            icon = self.default_icon
+            if not recipe.builtin:
+                icon = self.custom_icon
+            elif QFile(':/images/news/%s.png'%recipe.id).exists():
+                icon = QIcon(':/images/news/%s.png'%recipe.id)
+            return QVariant(icon)
+        
+        return NONE
+            
+
+class Search(QLineEdit):
+    
+    HELP_TEXT = _('Search')
+    INTERVAL = 500 #: Time to wait before emitting search signal
+    
+    def __init__(self, *args):
+        QLineEdit.__init__(self, *args)
+        self.default_palette = QApplication.palette(self)
+        self.gray = QPalette(self.default_palette)
+        self.gray.setBrush(QPalette.Text, QBrush(QColor('gray')))
+        self.connect(self, SIGNAL('editingFinished()'),
+                     lambda : self.emit(SIGNAL('goto(PyQt_PyObject)'), unicode(self.text())))
+        self.clear_to_help_mode()
+        self.timer = None
+        self.connect(self, SIGNAL('textEdited(QString)'), self.text_edited_slot)
+            
+    def focusInEvent(self, ev):
+        self.setPalette(QApplication.palette(self))
+        if self.in_help_mode():
+            self.setText('')
+        return QLineEdit.focusInEvent(self, ev)
+    
+    def in_help_mode(self):
+        return unicode(self.text()) == self.HELP_TEXT
+    
+    def clear_to_help_mode(self):
+        self.setPalette(self.gray)
+        self.setText(self.HELP_TEXT)
+        
+    def text_edited_slot(self, text):
+        text = unicode(text)
+        self.timer = self.startTimer(self.INTERVAL)
+
+    def timerEvent(self, event):
+        self.killTimer(event.timerId())
+        if event.timerId() == self.timer:
+            text = unicode(self.text())
+            self.emit(SIGNAL('search(PyQt_PyObject)'), text)
+
+    
+
+class SchedulerDialog(QDialog, Ui_Dialog):
+    
+    def __init__(self, db, *args):
+        QDialog.__init__(self, *args)
+        self.setupUi(self)
+        self.search = Search(self)
+        self.recipe_box.layout().insertWidget(0, self.search)
+        self.detail_box.setVisible(False)
+        self._model = RecipeModel(db)
+        self.current_recipe = None
+        self.recipes.setModel(self._model)
+        self.connect(self.recipes, SIGNAL('activated(QModelIndex)'), self.show_recipe)
+        self.connect(self.recipes, SIGNAL('clicked(QModelIndex)'), self.show_recipe)
+        self.connect(self.username, SIGNAL('textEdited(QString)'), self.set_account_info)
+        self.connect(self.password, SIGNAL('textEdited(QString)'), self.set_account_info)
+        self.connect(self.schedule, SIGNAL('stateChanged(int)'), self.do_schedule)
+        self.connect(self.schedule, SIGNAL('stateChanged(int)'), 
+                     lambda state: self.interval.setEnabled(state == Qt.Checked))
+        self.connect(self.show_password, SIGNAL('stateChanged(int)'),
+                     lambda state: self.password.setEchoMode(self.password.Normal if state == Qt.Checked else self.password.Password))
+        self.connect(self.interval, SIGNAL('valueChanged(int)'), self.do_schedule)
+        self.connect(self.search, SIGNAL('search(PyQt_PyObject)'), self._model.search)
+        self.connect(self._model, SIGNAL('modelReset()'), lambda : self.detail_box.setVisible(False))
+        self.connect(self.download, SIGNAL('clicked()'), self.download_now)
+        self.search.setFocus(Qt.OtherFocusReason)
+        
+    def download_now(self):
+        recipe = self._model.data(self.recipes.currentIndex(), Qt.UserRole)
+        self.emit(SIGNAL('download_now(PyQt_PyObject)'), recipe)
+        
+    def set_account_info(self, *args):
+        username, password = map(unicode, (self.username.text(), self.password.text()))
+        username, password = username.strip(), password.strip()
+        recipe = self._model.data(self.recipes.currentIndex(), Qt.UserRole)
+        key = 'recipe_account_info_%s'%recipe.id
+        dynamic[key] = (username, password) if username and password else None
+        
+    def do_schedule(self, *args):
+        recipe = self.recipes.currentIndex()
+        if not recipe.isValid():
+            return
+        recipe = self._model.data(recipe, Qt.UserRole)
+        recipes = dynamic['scheduled_recipes'] 
+        if self.schedule.checkState() == Qt.Checked:
+            if recipe in recipes:
+                recipe = recipes[recipes.index(recipe)]
+            else:
+                recipes.append(recipe)
+            recipes.schedule = self.interval.value()
+            if recipes.schedule == 0.0:
+                recipes.schedule = 1/24.
+            if recipe.need_subscription and not dynamic['recipe_account_info_%s'%recipe.id]:
+                error_dialog(self, _('Must set account information'), _('This recipe requires a username and password')).exec_()
+                self.schedule.setCheckState(Qt.Unchecked)
+                return
+        else:
+            if recipe in recipes:
+                recipes.remove(recipe)
+        dynamic['scheduled_recipes'] = recipes
+        self.emit(SIGNAL('new_schedule(PyQt_PyObject)'), recipes)
+        self._model.resort()
+                
+    def show_recipe(self, index):
+        recipe = self._model.data(index, Qt.UserRole)
+        self.current_recipe = recipe
+        self.title.setText(recipe.title)
+        self.description.setText(recipe.description if recipe.description else '')
+        self.schedule.setChecked(recipe.schedule is not None)
+        self.interval.setValue(recipe.schedule if recipe.schedule is not None else 1)
+        self.detail_box.setVisible(True)
+        self.account.setVisible(recipe.needs_subscription)
+        self.interval.setEnabled(self.schedule.checkState == Qt.Checked)
+        key = 'recipe_account_info_%s'%recipe.id
+        account_info = dynamic[key]
+        self.show_password.setChecked(False)
+        if account_info:
+            self.username.blockSignals(True)
+            self.password.blockSignals(True)
+            self.username.setText(account_info[0])
+            self.password.setText(account_info[1])
+            self.username.blockSignals(False)
+            self.password.blockSignals(False)
+            
+class Scheduler(QObject):
+    
+    INTERVAL = 5 # minutes
+    
+    def __init__(self, main):
+        self.main = main
+        QObject.__init__(self)
+        self.lock = RLock()
+        self.queue = set([])
+        recipes = dynamic['scheduled_recipes']
+        if not recipes:
+            recipes = []
+        self.refresh_schedule(recipes)
+        self.timer = QTimer()
+        self.connect(self.timer, SIGNAL('timeout()'), self.check)
+        self.timer.start(self.INTERVAL * 60000)
+    
+    def check(self):
+        db = self.main.library_view.model().db
+        now = datetime.utcnow()
+        needs_downloading = set([])
+        for recipe in self.recipes:
+            delta = now - recipe.last_downloaded
+            if delta > timedelta(days=recipe.schedule):
+                needs_downloading.add(recipe)
+        with self.lock:
+            needs_downloading = [r for r in needs_downloading if r not in self.queue]
+            for recipe in needs_downloading:
+                try:
+                    id = int(recipe.id)
+                    script = db.get_recipe(id)
+                    if script is None:
+                        self.recipes.remove(recipe)
+                        dynamic['scheduled_recipes'] = self.recipes
+                        continue
+                except ValueError:
+                    script = recipe.title
+                self.main.download_scheduled_recipe(recipe, script, self.recipe_downloaded)
+                self.queue.add(recipe)
+        
+    def recipe_downloaded(self, recipe):
+        with self.lock:
+            self.queue.remove(recipe)
+        recipe = self.recipes[self.recipes.index(recipe)]
+        now = datetime.utcnow()
+        d = now - recipe.last_downloaded
+        interval = timedelta(days=recipe.schedule)
+        if abs(d - interval) < timedelta(hours=1):
+            recipe.last_downloaded += interval
+        else:
+            recipe.last_downloaded = now
+        dynamic['scheduled_recipes'] = self.recipes
+    
+    def download(self, recipe):
+        if recipe in self.recipes:
+            recipe = self.recipes[self.recipes.index(recipe)]
+        raise NotImplementedError
+    
+    def refresh_schedule(self, recipes):
+        self.recipes = recipes
+    
+    def show_dialog(self):
+        d = SchedulerDialog(self.main.library_view.model().db)
+        self.connect(d, SIGNAL('new_schedule(PyQt_PyObject)'), self.refresh_schedule)
+        self.connect(d, SIGNAL('download_now(PyQt_PyObject)'), self.download)
+        d.exec_()
+
+def main(args=sys.argv):
+    app = QApplication([])
+    from calibre.library.database2 import LibraryDatabase2
+    d = SchedulerDialog(LibraryDatabase2('/home/kovid/documents/library'))
+    d.exec_()
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

src/calibre/gui2/dialogs/scheduler.ui

@@ -0,0 +1,307 @@
+<ui version="4.0" >
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog" >
+  <property name="geometry" >
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>726</width>
+    <height>551</height>
+   </rect>
+  </property>
+  <property name="windowTitle" >
+   <string>Schedule recipes for download</string>
+  </property>
+  <property name="windowIcon" >
+   <iconset resource="../images.qrc" >
+    <normaloff>:/images/news.svg</normaloff>:/images/news.svg</iconset>
+  </property>
+  <layout class="QGridLayout" name="gridLayout" >
+   <item rowspan="2" row="0" column="0" >
+    <widget class="QGroupBox" name="recipe_box" >
+     <property name="title" >
+      <string>Recipes</string>
+     </property>
+     <layout class="QVBoxLayout" name="verticalLayout" >
+      <item>
+       <widget class="QListView" name="recipes" >
+        <property name="alternatingRowColors" >
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item row="0" column="1" >
+    <layout class="QVBoxLayout" name="verticalLayout_3" >
+     <item>
+      <spacer name="horizontalSpacer_3" >
+       <property name="orientation" >
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="sizeHint" stdset="0" >
+        <size>
+         <width>40</width>
+         <height>0</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+     <item>
+      <widget class="QGroupBox" name="detail_box" >
+       <property name="title" >
+        <string>Schedule for download</string>
+       </property>
+       <layout class="QVBoxLayout" name="verticalLayout_2" >
+        <item>
+         <widget class="QLabel" name="title" >
+          <property name="font" >
+           <font>
+            <weight>75</weight>
+            <bold>true</bold>
+           </font>
+          </property>
+          <property name="text" >
+           <string>title</string>
+          </property>
+          <property name="textFormat" >
+           <enum>Qt::PlainText</enum>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QLabel" name="description" >
+          <property name="text" >
+           <string>description</string>
+          </property>
+          <property name="wordWrap" >
+           <bool>true</bool>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <spacer name="verticalSpacer_2" >
+          <property name="orientation" >
+           <enum>Qt::Vertical</enum>
+          </property>
+          <property name="sizeHint" stdset="0" >
+           <size>
+            <width>20</width>
+            <height>40</height>
+           </size>
+          </property>
+         </spacer>
+        </item>
+        <item>
+         <widget class="QCheckBox" name="schedule" >
+          <property name="text" >
+           <string>&amp;Schedule for download every:</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <layout class="QHBoxLayout" name="horizontalLayout" >
+          <item>
+           <spacer name="horizontalSpacer" >
+            <property name="orientation" >
+             <enum>Qt::Horizontal</enum>
+            </property>
+            <property name="sizeHint" stdset="0" >
+             <size>
+              <width>40</width>
+              <height>20</height>
+             </size>
+            </property>
+           </spacer>
+          </item>
+          <item>
+           <widget class="QDoubleSpinBox" name="interval" >
+            <property name="sizePolicy" >
+             <sizepolicy vsizetype="Fixed" hsizetype="Expanding" >
+              <horstretch>0</horstretch>
+              <verstretch>0</verstretch>
+             </sizepolicy>
+            </property>
+            <property name="toolTip" >
+             <string>Interval at which to download this recipe. A value of zero means that the recipe will be downloaded every hour.</string>
+            </property>
+            <property name="suffix" >
+             <string> days</string>
+            </property>
+            <property name="decimals" >
+             <number>1</number>
+            </property>
+            <property name="minimum" >
+             <double>0.000000000000000</double>
+            </property>
+            <property name="maximum" >
+             <double>365.100000000000023</double>
+            </property>
+            <property name="singleStep" >
+             <double>1.000000000000000</double>
+            </property>
+            <property name="value" >
+             <double>1.000000000000000</double>
+            </property>
+           </widget>
+          </item>
+          <item>
+           <spacer name="horizontalSpacer_2" >
+            <property name="orientation" >
+             <enum>Qt::Horizontal</enum>
+            </property>
+            <property name="sizeHint" stdset="0" >
+             <size>
+              <width>40</width>
+              <height>20</height>
+             </size>
+            </property>
+           </spacer>
+          </item>
+         </layout>
+        </item>
+        <item>
+         <widget class="QGroupBox" name="account" >
+          <property name="title" >
+           <string>&amp;Account</string>
+          </property>
+          <layout class="QGridLayout" name="gridLayout_2" >
+           <item row="0" column="1" >
+            <widget class="QLineEdit" name="username" />
+           </item>
+           <item row="0" column="0" >
+            <widget class="QLabel" name="label_2" >
+             <property name="text" >
+              <string>&amp;Username:</string>
+             </property>
+             <property name="buddy" >
+              <cstring>username</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="1" column="0" >
+            <widget class="QLabel" name="label_3" >
+             <property name="text" >
+              <string>&amp;Password:</string>
+             </property>
+             <property name="buddy" >
+              <cstring>password</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="1" column="1" >
+            <widget class="QLineEdit" name="password" >
+             <property name="echoMode" >
+              <enum>QLineEdit::Password</enum>
+             </property>
+            </widget>
+           </item>
+           <item row="2" column="0" >
+            <widget class="QCheckBox" name="show_password" >
+             <property name="text" >
+              <string>&amp;Show password</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </widget>
+        </item>
+        <item>
+         <spacer name="verticalSpacer_3" >
+          <property name="orientation" >
+           <enum>Qt::Vertical</enum>
+          </property>
+          <property name="sizeHint" stdset="0" >
+           <size>
+            <width>20</width>
+            <height>40</height>
+           </size>
+          </property>
+         </spacer>
+        </item>
+        <item>
+         <widget class="QLabel" name="label" >
+          <property name="text" >
+           <string>For the scheduling to work, you must leave calibre running.</string>
+          </property>
+          <property name="wordWrap" >
+           <bool>true</bool>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QPushButton" name="download" >
+          <property name="text" >
+           <string>&amp;Download now</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <spacer name="verticalSpacer" >
+          <property name="orientation" >
+           <enum>Qt::Vertical</enum>
+          </property>
+          <property name="sizeHint" stdset="0" >
+           <size>
+            <width>20</width>
+            <height>40</height>
+           </size>
+          </property>
+         </spacer>
+        </item>
+       </layout>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="1" column="1" >
+    <widget class="QDialogButtonBox" name="buttonBox" >
+     <property name="orientation" >
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons" >
+      <set>QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources>
+  <include location="../images.qrc" />
+ </resources>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/main.py

@@ -34,7 +34,8 @@ from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
 from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
 from calibre.gui2.dialogs.jobs import JobsDialog
 from calibre.gui2.dialogs.conversion_error import ConversionErrorDialog
-from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebooks, set_conversion_defaults, fetch_news
+from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebooks, \
+                                set_conversion_defaults, fetch_news, fetch_scheduled_recipe
 from calibre.gui2.dialogs.config import ConfigDialog
 from calibre.gui2.dialogs.search import SearchDialog
 from calibre.gui2.dialogs.user_profiles import UserProfiles
@@ -809,6 +810,13 @@ class Main(MainWindow, Ui_MainWindow):
             self.library_view.model().db.set_feeds(feeds)
             self.news_menu.set_custom_feeds(feeds)
 
+    def download_scheduled_recipe(self, recipe, script, callback):
+        func, args, desc, fmt, temp_files = fetch_scheduled_recipe(recipe, script)
+        job = self.job_manager.run_job(Dispatcher(self.scheduled_recipe_fetched), func, args=args,
+                                            description=desc)
+        self.conversion_jobs[job] = (temp_files, fmt, recipe, callback)
+        self.status_bar.showMessage(_('Fetching news from ')+recipe.title, 2000)
+    
     def fetch_news(self, data):
         func, args, desc, fmt, temp_files = fetch_news(data)
         self.status_bar.showMessage(_('Fetching news from ')+data['title'], 2000)
@@ -817,6 +825,19 @@ class Main(MainWindow, Ui_MainWindow):
         self.conversion_jobs[job] = (temp_files, fmt)
         self.status_bar.showMessage(_('Fetching news from ')+data['title'], 2000)
         
+    def scheduled_recipe_fetched(self, job):
+        temp_files, fmt, recipe, callback = self.conversion_jobs.pop(job)
+        pt = temp_files[0]
+        if job.exception is not None:
+            self.job_exception(job)
+            return
+        mi = get_metadata(open(pt.name, 'rb'), fmt, use_libprs_metadata=False)
+        mi.tags = ['news', recipe.title]
+        paths, formats, metadata = [pt.name], [fmt], [mi]
+        self.library_view.model().add_books(paths, formats, metadata, add_duplicates=True)
+        callback(recipe)
+        self.status_bar.showMessage(recipe.title + _(' fetched.'), 3000)
+            
     def news_fetched(self, job):
         temp_files, fmt = self.conversion_jobs.pop(job)
         pt = temp_files[0]

src/calibre/gui2/tools.py

@@ -13,7 +13,7 @@ from calibre.utils.config import prefs
 from calibre.gui2.dialogs.lrf_single import LRFSingleDialog, LRFBulkDialog
 from calibre.gui2.dialogs.epub import Config as EPUBConvert
 import calibre.gui2.dialogs.comicconf as ComicConf
-from calibre.gui2 import warning_dialog
+from calibre.gui2 import warning_dialog, dynamic
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.ebooks.lrf import preferred_source_formats as LRF_PREFERRED_SOURCE_FORMATS
 from calibre.ebooks.metadata.opf import OPFCreator
@@ -360,6 +360,20 @@ def _fetch_news(data, fmt):
     return 'feeds2'+fmt.lower(), [args], _('Fetch news from ')+data['title'], fmt.upper(), [pt]
     
 
+def fetch_scheduled_recipe(recipe, script):
+    fmt = prefs['output_format'].lower()
+    pt = PersistentTemporaryFile(suffix='_feeds2%s.%s'%(fmt.lower(), fmt.lower()))
+    pt.close()
+    args = ['feeds2%s'%fmt.lower(), '--output', pt.name, '--debug']
+    if recipe.needs_subscription:
+        x = dynamic['recipe_account_info_%s'%recipe.id]
+        if not x:
+            raise ValueError(_('You must set a username and password for %s')%recipe.title)
+        args.extend(['--username', x[0], '--password', x[1]])
+    args.append(script)
+    return 'feeds2'+fmt, [args], _('Fetch news from ')+recipe.title, fmt.upper(), [pt]
+            
+
 def convert_single_ebook(*args):
     fmt = prefs['output_format'].lower()
     if fmt == 'lrf':

src/calibre/library/database2.py

@@ -715,6 +715,12 @@ class LibraryDatabase2(LibraryDatabase):
         self.conn.execute(st%dict(ltable='series', table='series', ltable_col='series'))
         self.conn.commit()
     
+    def get_recipes(self):
+        return self.conn.get('SELECT id, title FROM feeds')
+    
+    def get_recipe(self, id):
+        return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False)
+    
     def get_categories(self, sort_on_count=False):
         categories = {}
         def get(name, category, field='name'):

src/calibre/trac/plugins/templates/download.html

@@ -34,7 +34,6 @@
       </tr>
       </table>
       <br />
-      <object width="250" height="250"><param name="movie" value="http://widget.chipin.com/widget/id/328a348be996a273"></param><param name="allowScriptAccess" value="always"></param><param name="wmode" value="transparent"></param><embed src="http://widget.chipin.com/widget/id/328a348be996a273" flashVars="" type="application/x-shockwave-flash" allowScriptAccess="always" wmode="transparent" width="250" height="250"></embed></object>
       
       </center>
     </div>

src/calibre/web/feeds/news.py

@@ -519,6 +519,8 @@ class BasicNewsRecipe(object, LoggingInterface):
         if self.remove_javascript:
             for script in list(soup.findAll('script')):
                 script.extract()
+        for script in list(soup.findAll('noscript')):
+                script.extract()
         return self.postprocess_html(soup, first_fetch)
         
     

src/calibre/web/feeds/templates.py

@@ -130,7 +130,7 @@ class FeedTemplate(Template):
         </style>
     </head>
     <body style="page-break-before:always">
-        <h2>${feed.title}</h2>
+        <h2 class="feed_title">${feed.title}</h2>
         <py:if test="getattr(feed, 'image', None)">
         <div class="feed_image">
             <img alt="${feed.image_alt}" src="${feed.image_url}" />