Custom recipes: Store custom recipes in the calibre config directory instead of the library database. This allows scheduling of custom recipes to work with multiple libraries. Note that you may have to re-schedule any existing custom recipes

src/calibre/gui2/dialogs/scheduler.py

@@ -411,7 +411,8 @@ class Scheduler(QObject):
         QObject.__init__(self, parent)
         self.internet_connection_failed = False
         self._parent = parent
-        self.recipe_model = RecipeModel(db)
+        self.recipe_model = RecipeModel()
+        self.db = db
         self.lock = QMutex(QMutex.Recursive)
         self.download_queue = set([])
 
@@ -433,7 +434,9 @@ class Scheduler(QObject):
         self.timer.timeout.connect(self.check)
         self.oldest = gconf['oldest_news']
         QTimer.singleShot(5 * 1000, self.oldest_check)
-        self.database_changed = self.recipe_model.database_changed
+
+    def database_changed(self, db):
+        self.db = db
 
     def oldest_check(self):
         if self.oldest > 0:
@@ -549,7 +552,6 @@ class Scheduler(QObject):
 if __name__ == '__main__':
     from calibre.gui2 import is_ok_to_use_qt
     is_ok_to_use_qt()
-    from calibre.library.database2 import LibraryDatabase2
+    d = SchedulerDialog(RecipeModel())
-    d = SchedulerDialog(RecipeModel(LibraryDatabase2('/home/kovid/documents/library')))
     d.exec_()
 

src/calibre/gui2/dialogs/user_profiles.py

@@ -366,8 +366,7 @@ class %(classname)s(%(base_class)s):
 if __name__ == '__main__':
     from calibre.gui2 import is_ok_to_use_qt
     is_ok_to_use_qt()
-    from calibre.library.database2 import LibraryDatabase2
     from calibre.web.feeds.recipes.model import RecipeModel
-    d=UserProfiles(None, RecipeModel(LibraryDatabase2('/home/kovid/documents/library')))
+    d=UserProfiles(None, RecipeModel())
     d.exec_()
 

src/calibre/library/database2.py

@@ -1187,12 +1187,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.clean_custom()
         self.conn.commit()
 
-    def get_recipes(self):
-        return self.conn.get('SELECT id, script FROM feeds')
-
-    def get_recipe(self, id):
-        return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False)
-
     def get_books_for_category(self, category, id_):
         ans = set([])
 
@@ -3113,8 +3107,4 @@ books_series_link      feeds
         s = self.conn.get('''SELECT book FROM books_plugin_data WHERE name=?''', (name,))
         return [x[0] for x in s]
 
-    def get_custom_recipes(self):
-        for id, title, script in self.conn.get('SELECT id,title,script FROM feeds'):
-            yield id, title, script
-
 

src/calibre/library/schema_upgrades.py

@@ -582,4 +582,22 @@ class SchemaUpgrade(object):
         # statements
         self.conn.executescript(script)
 
+    def upgrade_version_19(self):
+        recipes = self.conn.get('SELECT id,title,script FROM feeds')
+        if recipes:
+            from calibre.web.feeds.recipes import custom_recipes, \
+                    custom_recipe_filename
+            bdir = os.path.dirname(custom_recipes.file_path)
+            for id_, title, script in recipes:
+                existing = frozenset(map(int, custom_recipes.iterkeys()))
+                if id_ in existing:
+                    id_ = max(existing) + 1000
+                id_ = str(id_)
+                fname = custom_recipe_filename(id_, title)
+                custom_recipes[id_] = (title, fname)
+                if isinstance(script, unicode):
+                    script = script.encode('utf-8')
+                with open(os.path.join(bdir, fname), 'wb') as f:
+                    f.write(script)
+
 

src/calibre/web/feeds/recipes/__init__.py

@@ -10,6 +10,7 @@ from calibre.web.feeds.news import BasicNewsRecipe, CustomIndexRecipe, \
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre import __appname__, english_sort
+from calibre.utils.config import JSONConfig
 
 BeautifulSoup, time, english_sort
 
@@ -17,6 +18,14 @@ basic_recipes = (BasicNewsRecipe, AutomaticNewsRecipe, CustomIndexRecipe,
         CalibrePeriodical)
 _tdir = None
 _crep = 0
+
+custom_recipes = JSONConfig('custom_recipes/index.json')
+
+def custom_recipe_filename(id_, title):
+    from calibre.utils.filenames import ascii_filename
+    return ascii_filename(title[:50]) + \
+                        ('_%s.recipe'%id_)
+
 def compile_recipe(src):
     '''
     Compile the code in src and return the first object that is a recipe or profile.

src/calibre/web/feeds/recipes/collection.py

@@ -88,19 +88,89 @@ def serialize_builtin_recipes():
 def get_builtin_recipe_collection():
     return etree.parse(P('builtin_recipes.xml')).getroot()
 
-def get_custom_recipe_collection(db):
+def get_custom_recipe_collection(*args):
-    from calibre.web.feeds.recipes import compile_recipe
+    from calibre.web.feeds.recipes import compile_recipe, \
+            custom_recipes
+    bdir = os.path.dirname(custom_recipes.file_path)
     rmap = {}
-    for id, title, recipe in db.get_custom_recipes():
+    for id_, x in custom_recipes.iteritems():
+        title, fname = x
+        recipe = os.path.join(bdir, fname)
         try:
+            recipe = open(recipe, 'rb').read().decode('utf-8')
             recipe_class = compile_recipe(recipe)
             if recipe_class is not None:
-                rmap['custom:%d'%id] = recipe_class
+                rmap['custom:%s'%id_] = recipe_class
         except:
+            import traceback
+            traceback.print_exc()
             continue
-
     return etree.fromstring(serialize_collection(rmap))
 
+
+def update_custom_recipe(id_, title, script):
+    from calibre.web.feeds.recipes import custom_recipes, \
+            custom_recipe_filename
+    id_ = str(int(id_))
+    existing = custom_recipes.get(id_, None)
+    bdir = os.path.dirname(custom_recipes.file_path)
+
+    if existing is None:
+        fname = custom_recipe_filename(id_, title)
+    else:
+        fname = existing[1]
+    if isinstance(script, unicode):
+        script = script.encode('utf-8')
+
+    custom_recipes[id_] = (title, fname)
+
+    with open(os.path.join(bdir, fname), 'wb') as f:
+        f.write(script)
+
+
+def add_custom_recipe(title, script):
+    from calibre.web.feeds.recipes import custom_recipes, \
+            custom_recipe_filename
+    id_ = 1000
+    keys = tuple(map(int, custom_recipes.iterkeys()))
+    if keys:
+        id_ = max(keys)+1
+    id_ = str(id_)
+    bdir = os.path.dirname(custom_recipes.file_path)
+
+    fname = custom_recipe_filename(id_, title)
+    if isinstance(script, unicode):
+        script = script.encode('utf-8')
+
+    custom_recipes[id_] = (title, fname)
+
+    with open(os.path.join(bdir, fname), 'wb') as f:
+        f.write(script)
+
+
+def remove_custom_recipe(id_):
+    from calibre.web.feeds.recipes import custom_recipes
+    id_ = str(int(id_))
+    existing = custom_recipes.get(id_, None)
+    if existing is not None:
+        bdir = os.path.dirname(custom_recipes.file_path)
+        fname = existing[1]
+        del custom_recipes[id_]
+        try:
+            os.remove(os.path.join(bdir, fname))
+        except:
+            pass
+
+def get_custom_recipe(id_):
+    from calibre.web.feeds.recipes import custom_recipes
+    id_ = str(int(id_))
+    existing = custom_recipes.get(id_, None)
+    if existing is not None:
+        bdir = os.path.dirname(custom_recipes.file_path)
+        fname = existing[1]
+        with open(os.path.join(bdir, fname), 'rb') as f:
+            return f.read().decode('utf-8')
+
 def get_builtin_recipe_titles():
     return [r.get('title') for r in get_builtin_recipe_collection()]
 

src/calibre/web/feeds/recipes/model.py

@@ -9,14 +9,15 @@ __docformat__ = 'restructuredtext en'
 import os, copy
 
 from PyQt4.Qt import QAbstractItemModel, QVariant, Qt, QColor, QFont, QIcon, \
-        QModelIndex, QMetaObject, pyqtSlot, pyqtSignal
+        QModelIndex, pyqtSignal
 
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.gui2 import NONE
 from calibre.utils.localization import get_language
 from calibre.web.feeds.recipes.collection import \
         get_builtin_recipe_collection, get_custom_recipe_collection, \
-        SchedulerConfig, download_builtin_recipe
+        SchedulerConfig, download_builtin_recipe, update_custom_recipe, \
+        add_custom_recipe, remove_custom_recipe, get_custom_recipe
 from calibre.utils.pyparsing import ParseException
 
 class NewsTreeItem(object):
@@ -122,26 +123,15 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
     LOCATIONS = ['all']
     searched = pyqtSignal(object)
 
-    def __init__(self, db, *args):
+    def __init__(self, *args):
         QAbstractItemModel.__init__(self, *args)
         SearchQueryParser.__init__(self, locations=['all'])
-        self.db = db
         self.default_icon = QVariant(QIcon(I('news.png')))
         self.custom_icon = QVariant(QIcon(I('user_profile.png')))
         self.builtin_recipe_collection = get_builtin_recipe_collection()
         self.scheduler_config = SchedulerConfig()
         self.do_refresh()
 
-    @pyqtSlot()
-    def do_database_change(self):
-        self.db = self.newdb
-        self.newdb = None
-        self.do_refresh()
-
-    def database_changed(self, db):
-        self.newdb = db
-        QMetaObject.invokeMethod(self, 'do_database_change', Qt.QueuedConnection)
-
     def get_builtin_recipe(self, urn, download=True):
         if download:
             try:
@@ -158,23 +148,24 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
             if recipe.get('id', False) == urn:
                 if coll is self.builtin_recipe_collection:
                     return self.get_builtin_recipe(urn[8:], download=download)
-                return self.db.get_feed(int(urn[len('custom:'):]))
+                return get_custom_recipe(int(urn[len('custom:'):]))
 
     def update_custom_recipe(self, urn, title, script):
-        self.db.update_feed(int(urn[len('custom:'):]), script, title)
+        id_ = int(urn[len('custom:'):])
-        self.custom_recipe_collection = get_custom_recipe_collection(self.db)
+        update_custom_recipe(id_, title, script)
+        self.custom_recipe_collection = get_custom_recipe_collection()
 
     def add_custom_recipe(self, title, script):
-        self.db.add_feed(title, script)
+        add_custom_recipe(title, script)
-        self.custom_recipe_collection = get_custom_recipe_collection(self.db)
+        self.custom_recipe_collection = get_custom_recipe_collection()
 
     def remove_custom_recipes(self, urns):
         ids = [int(x[len('custom:'):]) for x in urns]
-        self.db.remove_feeds(ids)
+        for id_ in ids: remove_custom_recipe(id_)
-        self.custom_recipe_collection = get_custom_recipe_collection(self.db)
+        self.custom_recipe_collection = get_custom_recipe_collection()
 
     def do_refresh(self, restrict_to_urns=set([])):
-        self.custom_recipe_collection = get_custom_recipe_collection(self.db)
+        self.custom_recipe_collection = get_custom_recipe_collection()
 
         def factory(cls, parent, *args):
             args = list(args)