From 1f1128d5a4e8469ea4a947e0fae75d63ad7fea4b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 00:08:02 -0700 Subject: [PATCH] Custom recipes: Store custom recipes in the calibre config directory instead of the library database. This allows scheduling of custom recipes to work with multiple libraries. Note that you may have to re-schedule any existing custom recipes --- src/calibre/gui2/dialogs/scheduler.py | 10 +-- src/calibre/gui2/dialogs/user_profiles.py | 3 +- src/calibre/library/database2.py | 10 --- src/calibre/library/schema_upgrades.py | 18 +++++ src/calibre/web/feeds/recipes/__init__.py | 9 +++ src/calibre/web/feeds/recipes/collection.py | 80 +++++++++++++++++++-- src/calibre/web/feeds/recipes/model.py | 35 ++++----- 7 files changed, 122 insertions(+), 43 deletions(-) diff --git a/src/calibre/gui2/dialogs/scheduler.py b/src/calibre/gui2/dialogs/scheduler.py index 67a13813df..b2561342b8 100644 --- a/src/calibre/gui2/dialogs/scheduler.py +++ b/src/calibre/gui2/dialogs/scheduler.py @@ -411,7 +411,8 @@ class Scheduler(QObject): QObject.__init__(self, parent) self.internet_connection_failed = False self._parent = parent - self.recipe_model = RecipeModel(db) + self.recipe_model = RecipeModel() + self.db = db self.lock = QMutex(QMutex.Recursive) self.download_queue = set([]) @@ -433,7 +434,9 @@ class Scheduler(QObject): self.timer.timeout.connect(self.check) self.oldest = gconf['oldest_news'] QTimer.singleShot(5 * 1000, self.oldest_check) - self.database_changed = self.recipe_model.database_changed + + def database_changed(self, db): + self.db = db def oldest_check(self): if self.oldest > 0: @@ -549,7 +552,6 @@ class Scheduler(QObject): if __name__ == '__main__': from calibre.gui2 import is_ok_to_use_qt is_ok_to_use_qt() - from calibre.library.database2 import LibraryDatabase2 - d = SchedulerDialog(RecipeModel(LibraryDatabase2('/home/kovid/documents/library'))) + d = SchedulerDialog(RecipeModel()) d.exec_() diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index fe64deb430..290982caaf 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -366,8 +366,7 @@ class %(classname)s(%(base_class)s): if __name__ == '__main__': from calibre.gui2 import is_ok_to_use_qt is_ok_to_use_qt() - from calibre.library.database2 import LibraryDatabase2 from calibre.web.feeds.recipes.model import RecipeModel - d=UserProfiles(None, RecipeModel(LibraryDatabase2('/home/kovid/documents/library'))) + d=UserProfiles(None, RecipeModel()) d.exec_() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 2554df93e6..38b70fc2bf 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1187,12 +1187,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.clean_custom() self.conn.commit() - def get_recipes(self): - return self.conn.get('SELECT id, script FROM feeds') - - def get_recipe(self, id): - return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False) - def get_books_for_category(self, category, id_): ans = set([]) @@ -3113,8 +3107,4 @@ books_series_link feeds s = self.conn.get('''SELECT book FROM books_plugin_data WHERE name=?''', (name,)) return [x[0] for x in s] - def get_custom_recipes(self): - for id, title, script in self.conn.get('SELECT id,title,script FROM feeds'): - yield id, title, script - diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index d1f22d379b..3fc9a2368a 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -582,4 +582,22 @@ class SchemaUpgrade(object): # statements self.conn.executescript(script) + def upgrade_version_19(self): + recipes = self.conn.get('SELECT id,title,script FROM feeds') + if recipes: + from calibre.web.feeds.recipes import custom_recipes, \ + custom_recipe_filename + bdir = os.path.dirname(custom_recipes.file_path) + for id_, title, script in recipes: + existing = frozenset(map(int, custom_recipes.iterkeys())) + if id_ in existing: + id_ = max(existing) + 1000 + id_ = str(id_) + fname = custom_recipe_filename(id_, title) + custom_recipes[id_] = (title, fname) + if isinstance(script, unicode): + script = script.encode('utf-8') + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) + diff --git a/src/calibre/web/feeds/recipes/__init__.py b/src/calibre/web/feeds/recipes/__init__.py index a72f500862..bfb46fa799 100644 --- a/src/calibre/web/feeds/recipes/__init__.py +++ b/src/calibre/web/feeds/recipes/__init__.py @@ -10,6 +10,7 @@ from calibre.web.feeds.news import BasicNewsRecipe, CustomIndexRecipe, \ from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ptempfile import PersistentTemporaryDirectory from calibre import __appname__, english_sort +from calibre.utils.config import JSONConfig BeautifulSoup, time, english_sort @@ -17,6 +18,14 @@ basic_recipes = (BasicNewsRecipe, AutomaticNewsRecipe, CustomIndexRecipe, CalibrePeriodical) _tdir = None _crep = 0 + +custom_recipes = JSONConfig('custom_recipes/index.json') + +def custom_recipe_filename(id_, title): + from calibre.utils.filenames import ascii_filename + return ascii_filename(title[:50]) + \ + ('_%s.recipe'%id_) + def compile_recipe(src): ''' Compile the code in src and return the first object that is a recipe or profile. diff --git a/src/calibre/web/feeds/recipes/collection.py b/src/calibre/web/feeds/recipes/collection.py index aa0051ca37..7082f780e6 100644 --- a/src/calibre/web/feeds/recipes/collection.py +++ b/src/calibre/web/feeds/recipes/collection.py @@ -88,19 +88,89 @@ def serialize_builtin_recipes(): def get_builtin_recipe_collection(): return etree.parse(P('builtin_recipes.xml')).getroot() -def get_custom_recipe_collection(db): - from calibre.web.feeds.recipes import compile_recipe +def get_custom_recipe_collection(*args): + from calibre.web.feeds.recipes import compile_recipe, \ + custom_recipes + bdir = os.path.dirname(custom_recipes.file_path) rmap = {} - for id, title, recipe in db.get_custom_recipes(): + for id_, x in custom_recipes.iteritems(): + title, fname = x + recipe = os.path.join(bdir, fname) try: + recipe = open(recipe, 'rb').read().decode('utf-8') recipe_class = compile_recipe(recipe) if recipe_class is not None: - rmap['custom:%d'%id] = recipe_class + rmap['custom:%s'%id_] = recipe_class except: + import traceback + traceback.print_exc() continue - return etree.fromstring(serialize_collection(rmap)) + +def update_custom_recipe(id_, title, script): + from calibre.web.feeds.recipes import custom_recipes, \ + custom_recipe_filename + id_ = str(int(id_)) + existing = custom_recipes.get(id_, None) + bdir = os.path.dirname(custom_recipes.file_path) + + if existing is None: + fname = custom_recipe_filename(id_, title) + else: + fname = existing[1] + if isinstance(script, unicode): + script = script.encode('utf-8') + + custom_recipes[id_] = (title, fname) + + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) + + +def add_custom_recipe(title, script): + from calibre.web.feeds.recipes import custom_recipes, \ + custom_recipe_filename + id_ = 1000 + keys = tuple(map(int, custom_recipes.iterkeys())) + if keys: + id_ = max(keys)+1 + id_ = str(id_) + bdir = os.path.dirname(custom_recipes.file_path) + + fname = custom_recipe_filename(id_, title) + if isinstance(script, unicode): + script = script.encode('utf-8') + + custom_recipes[id_] = (title, fname) + + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) + + +def remove_custom_recipe(id_): + from calibre.web.feeds.recipes import custom_recipes + id_ = str(int(id_)) + existing = custom_recipes.get(id_, None) + if existing is not None: + bdir = os.path.dirname(custom_recipes.file_path) + fname = existing[1] + del custom_recipes[id_] + try: + os.remove(os.path.join(bdir, fname)) + except: + pass + +def get_custom_recipe(id_): + from calibre.web.feeds.recipes import custom_recipes + id_ = str(int(id_)) + existing = custom_recipes.get(id_, None) + if existing is not None: + bdir = os.path.dirname(custom_recipes.file_path) + fname = existing[1] + with open(os.path.join(bdir, fname), 'rb') as f: + return f.read().decode('utf-8') + def get_builtin_recipe_titles(): return [r.get('title') for r in get_builtin_recipe_collection()] diff --git a/src/calibre/web/feeds/recipes/model.py b/src/calibre/web/feeds/recipes/model.py index 553fdcc3c3..19e73dd5f8 100644 --- a/src/calibre/web/feeds/recipes/model.py +++ b/src/calibre/web/feeds/recipes/model.py @@ -9,14 +9,15 @@ __docformat__ = 'restructuredtext en' import os, copy from PyQt4.Qt import QAbstractItemModel, QVariant, Qt, QColor, QFont, QIcon, \ - QModelIndex, QMetaObject, pyqtSlot, pyqtSignal + QModelIndex, pyqtSignal from calibre.utils.search_query_parser import SearchQueryParser from calibre.gui2 import NONE from calibre.utils.localization import get_language from calibre.web.feeds.recipes.collection import \ get_builtin_recipe_collection, get_custom_recipe_collection, \ - SchedulerConfig, download_builtin_recipe + SchedulerConfig, download_builtin_recipe, update_custom_recipe, \ + add_custom_recipe, remove_custom_recipe, get_custom_recipe from calibre.utils.pyparsing import ParseException class NewsTreeItem(object): @@ -122,26 +123,15 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser): LOCATIONS = ['all'] searched = pyqtSignal(object) - def __init__(self, db, *args): + def __init__(self, *args): QAbstractItemModel.__init__(self, *args) SearchQueryParser.__init__(self, locations=['all']) - self.db = db self.default_icon = QVariant(QIcon(I('news.png'))) self.custom_icon = QVariant(QIcon(I('user_profile.png'))) self.builtin_recipe_collection = get_builtin_recipe_collection() self.scheduler_config = SchedulerConfig() self.do_refresh() - @pyqtSlot() - def do_database_change(self): - self.db = self.newdb - self.newdb = None - self.do_refresh() - - def database_changed(self, db): - self.newdb = db - QMetaObject.invokeMethod(self, 'do_database_change', Qt.QueuedConnection) - def get_builtin_recipe(self, urn, download=True): if download: try: @@ -158,23 +148,24 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser): if recipe.get('id', False) == urn: if coll is self.builtin_recipe_collection: return self.get_builtin_recipe(urn[8:], download=download) - return self.db.get_feed(int(urn[len('custom:'):])) + return get_custom_recipe(int(urn[len('custom:'):])) def update_custom_recipe(self, urn, title, script): - self.db.update_feed(int(urn[len('custom:'):]), script, title) - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + id_ = int(urn[len('custom:'):]) + update_custom_recipe(id_, title, script) + self.custom_recipe_collection = get_custom_recipe_collection() def add_custom_recipe(self, title, script): - self.db.add_feed(title, script) - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + add_custom_recipe(title, script) + self.custom_recipe_collection = get_custom_recipe_collection() def remove_custom_recipes(self, urns): ids = [int(x[len('custom:'):]) for x in urns] - self.db.remove_feeds(ids) - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + for id_ in ids: remove_custom_recipe(id_) + self.custom_recipe_collection = get_custom_recipe_collection() def do_refresh(self, restrict_to_urns=set([])): - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + self.custom_recipe_collection = get_custom_recipe_collection() def factory(cls, parent, *args): args = list(args)