From dec2f6aa14d90855eced7566c153acf9069e14b8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Mar 2011 07:44:38 -0700 Subject: [PATCH 1/9] Fix #9286 (Paste cover fails in 0.7.48) --- src/calibre/library/database2.py | 3 +-- src/calibre/utils/magick/draw.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index e7d55da9df..2554df93e6 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -37,7 +37,7 @@ from calibre.utils.config import prefs, tweaks, from_json, to_json from calibre.utils.icu import sort_key from calibre.utils.search_query_parser import saved_searches, set_saved_searches from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format -from calibre.utils.magick.draw import minify_image, save_cover_data_to +from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.formatter_functions import load_user_template_functions @@ -951,7 +951,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if callable(getattr(data, 'read', None)): data = data.read() try: - data = minify_image(data, tweaks['maximum_cover_size']) save_cover_data_to(data, path) except (IOError, OSError): time.sleep(0.2) diff --git a/src/calibre/utils/magick/draw.py b/src/calibre/utils/magick/draw.py index 615aab9bd0..42659d70cc 100644 --- a/src/calibre/utils/magick/draw.py +++ b/src/calibre/utils/magick/draw.py @@ -47,7 +47,7 @@ def normalize_format_name(fmt): return fmt def save_cover_data_to(data, path, bgcolor='#ffffff', resize_to=None, - return_data=False, compression_quality=90): + return_data=False, compression_quality=90, minify_to=None): ''' Saves image in data to path, in the format specified by the path extension. Removes any transparency. If there is no transparency and no @@ -60,6 +60,9 @@ def save_cover_data_to(data, path, bgcolor='#ffffff', resize_to=None, compression (lossless). :param bgcolor: The color for transparent pixels. Must be specified in hex. :param resize_to: A tuple (width, height) or None for no resizing + :param minify_to: A tuple (width, height) to specify target size. The image + will be resized to fit into this target size. If None the value from the + tweak is used. ''' changed = False @@ -71,11 +74,18 @@ def save_cover_data_to(data, path, bgcolor='#ffffff', resize_to=None, if resize_to is not None: img.size = (resize_to[0], resize_to[1]) changed = True + owidth, oheight = img.size + nwidth, nheight = tweaks['maximum_cover_size'] if minify_to is None else minify_to + scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight) + if scaled: + img.size = (nwidth, nheight) + changed = True if img.has_transparent_pixels(): canvas = create_canvas(img.size[0], img.size[1], bgcolor) canvas.compose(img) img = canvas changed = True + if not changed: changed = fmt != orig_fmt From 28a43e15b7e34a1c78f28d37b8f7336555b9ca8d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Mar 2011 08:22:07 -0700 Subject: [PATCH 2/9] ... --- Changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog.yaml b/Changelog.yaml index 449826f874..c5eadc5e65 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -97,7 +97,7 @@ author: rylsfan - title: "Historia and Buctaras" - author: Silviu Coatara + author: Silviu Cotoara - title: "Buffalo News" author: ChappyOnIce @@ -174,7 +174,7 @@ author: Ricardo Jurado - title: "Various Romanian news sources" - author: Silviu Coatara + author: Silviu Cotoara - title: "Osnews.pl and SwiatCzytnikow" author: Tomasz Dlugosz From 41c25e037bca7095d926a773de884ab10af0c151 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Mar 2011 14:52:19 -0700 Subject: [PATCH 3/9] Catalog generation: Reuse cover from existing catalog, allows the use of a custom cover for catalogs --- src/calibre/library/catalog.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index f3640af4f0..ffa08eaed2 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -5103,6 +5103,19 @@ Author '{0}': recommendations.append(('book_producer',opts.output_profile, OptionRecommendation.HIGH)) + # If cover exists, use it + try: + search_text = 'title:"%s" author:%s' % ( + opts.catalog_title.replace('"', '\\"'), 'calibre') + matches = db.search(search_text, return_matches=True) + if matches: + cpath = db.cover(matches[0], index_is_id=True, as_path=True) + if cpath and os.path.exists(cpath): + recommendations.append(('cover', cpath, + OptionRecommendation.HIGH)) + except: + pass + # Run ebook-convert from calibre.ebooks.conversion.plumber import Plumber plumber = Plumber(os.path.join(catalog.catalogPath, From 64fd29fa3ef6b30b1df6e10f42c0f2404442d1af Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Mar 2011 14:54:19 -0700 Subject: [PATCH 4/9] Apple driver: Ignore invalid EPUBs when sending to iTunes --- src/calibre/devices/apple/driver.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 5ead675aab..76ecce3a8e 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -2512,7 +2512,12 @@ class ITUNES(DriverBase): # Refresh epub metadata with open(fpath,'r+b') as zfo: # Touch the OPF timestamp - zf_opf = ZipFile(fpath,'r') + try: + zf_opf = ZipFile(fpath,'r') + except: + raise UserFeedback("'%s' is not a valid EPUB" % metadata.title, + None, + level=UserFeedback.WARN) fnames = zf_opf.namelist() opf = [x for x in fnames if '.opf' in x][0] if opf: From 9eb94d425b9aa925fc9ea416f467fc3b8562e9f3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Mar 2011 14:57:49 -0700 Subject: [PATCH 5/9] Content server: Fix regression that caused various emtadata to be missing in the book details view. Fixes #8929 (Improve Content Server usability) --- src/calibre/library/server/browse.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index 585c1255a4..97bfc30f14 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -666,15 +666,13 @@ class BrowseServer(object): if add_category_links: added_key = False fm = mi.metadata_for_field(key) - if val and fm and fm['is_category'] and \ + if val and fm and fm['is_category'] and not fm['is_csp'] and\ key != 'formats' and fm['datatype'] not in ['rating']: categories = mi.get(key) if isinstance(categories, basestring): categories = [categories] dbtags = [] for category in categories: - if category not in ccache: - continue dbtag = None for tag in ccache[key]: if tag.name == category: From f836abbc2afe860b2d0a7a01ca901571bba08333 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Mar 2011 18:34:27 -0700 Subject: [PATCH 6/9] News download: More flexible news downlaod scheduling. You can now schedule by days of the week, days of the month and an interval, whcih can be as small as an hour for news sources that change rapidly --- src/calibre/gui2/dialogs/scheduler.py | 267 +++++++++++++++---- src/calibre/gui2/dialogs/scheduler.ui | 274 +++++++------------- src/calibre/web/feeds/recipes/collection.py | 56 +++- 3 files changed, 365 insertions(+), 232 deletions(-) diff --git a/src/calibre/gui2/dialogs/scheduler.py b/src/calibre/gui2/dialogs/scheduler.py index 460e1d9fdb..1f6a394556 100644 --- a/src/calibre/gui2/dialogs/scheduler.py +++ b/src/calibre/gui2/dialogs/scheduler.py @@ -8,9 +8,11 @@ Scheduler for automated recipe downloads ''' from datetime import timedelta +import calendar, textwrap -from PyQt4.Qt import QDialog, SIGNAL, Qt, QTime, QObject, QMenu, \ - QAction, QIcon, QMutex, QTimer, pyqtSignal +from PyQt4.Qt import QDialog, Qt, QTime, QObject, QMenu, QHBoxLayout, \ + QAction, QIcon, QMutex, QTimer, pyqtSignal, QWidget, QGridLayout, \ + QCheckBox, QTimeEdit, QLabel, QLineEdit, QDoubleSpinBox from calibre.gui2.dialogs.scheduler_ui import Ui_Dialog from calibre.gui2 import config as gconf, error_dialog @@ -18,9 +20,171 @@ from calibre.web.feeds.recipes.model import RecipeModel from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.date import utcnow from calibre.utils.network import internet_connected +from calibre.utils.ordered_dict import OrderedDict + +def convert_day_time_schedule(val): + day_of_week, hour, minute = val + if day_of_week == -1: + return (tuple(xrange(7)), hour, minute) + return ((day_of_week,), hour, minute) + +class Base(QWidget): + + def __init__(self, parent=None): + QWidget.__init__(self, parent) + self.l = QGridLayout() + self.setLayout(self.l) + self.setToolTip(textwrap.dedent(self.HELP)) + +class DaysOfWeek(Base): + + HELP = _('''\ + Download this periodical every week on the specified days after + the specified time. For example, if you choose: Monday after + 9:00 AM, then the periodical will be download every Monday as + soon after 9:00 AM as possible. + ''') + + def __init__(self, parent=None): + Base.__init__(self, parent) + self.days = [QCheckBox(calendar.day_abbr[d], self) for d in xrange(7)] + for i, cb in enumerate(self.days): + row = i % 2 + col = i // 2 + self.l.addWidget(cb, row, col, 1, 1) + + self.time = QTimeEdit(self) + self.time.setDisplayFormat('hh:mm AP') + self.hl = QHBoxLayout() + self.l1 = QLabel(_('&Download after:')) + self.l1.setBuddy(self.time) + self.hl.addWidget(self.l1) + self.hl.addWidget(self.time) + self.l.addLayout(self.hl, 1, 3, 1, 1) + self.initialize() + + def initialize(self, typ=None, val=None): + if typ is None: + typ = 'day/time' + val = (-1, 9, 0) + if typ == 'day/time': + val = convert_day_time_schedule(val) + + days_of_week, hour, minute = val + for i, d in enumerate(self.days): + d.setChecked(i in days_of_week) + + self.time.setTime(QTime(hour, minute)) + + @property + def schedule(self): + days_of_week = tuple([i for i, d in enumerate(self.days) if + d.isChecked()]) + t = self.time.time() + hour, minute = t.hour(), t.minute() + return 'days_of_week', (days_of_week, int(hour), int(minute)) + +class DaysOfMonth(Base): + + HELP = _('''\ + Download this periodical every month, on the specified days. + The download will happen as soon after the specified time as + possible on the specified days of each month. For example, + if you choose the 1st and the 15th after 9:00 AM, the + periodical will be downloaded on the 1st and 15th of every + month, as soon after 9:00 AM as possible. + ''') + + def __init__(self, parent=None): + Base.__init__(self, parent) + + self.l1 = QLabel(_('&Days of the month:')) + self.days = QLineEdit(self) + self.days.setToolTip(_('Comma separated list of days of the month.' + ' For example: 1, 15')) + self.l1.setBuddy(self.days) + + self.l2 = QLabel(_('Download &after:')) + self.time = QTimeEdit(self) + self.time.setDisplayFormat('hh:mm AP') + self.l2.setBuddy(self.time) + + self.l.addWidget(self.l1, 0, 0, 1, 1) + self.l.addWidget(self.days, 0, 1, 1, 1) + self.l.addWidget(self.l2, 1, 0, 1, 1) + self.l.addWidget(self.time, 1, 1, 1, 1) + + def initialize(self, typ=None, val=None): + if val is None: + val = ((1,), 9, 0) + days_of_month, hour, minute = val + self.days.setText(', '.join(map(str, map(int, days_of_month)))) + self.time.setTime(QTime(hour, minute)) + + @property + def schedule(self): + parts = [x.strip() for x in unicode(self.days.text()).split(',') if + x.strip()] + try: + days_of_month = tuple(map(int, parts)) + except: + days_of_month = (1,) + if not days_of_month: + days_of_month = (1,) + t = self.time.time() + hour, minute = t.hour(), t.minute() + return 'days_of_month', (days_of_month, int(hour), int(minute)) + +class EveryXDays(Base): + + HELP = _('''\ + Download this periodical every x days. For example, if you + choose 30 days, the periodical will be downloaded every 30 + days. Note that you can set periods of less than a day, like + 0.1 days to download a periodical more than once a day. + ''') + + def __init__(self, parent=None): + Base.__init__(self, parent) + self.l1 = QLabel(_('&Download every:')) + self.interval = QDoubleSpinBox(self) + self.interval.setMinimum(0.04) + self.interval.setSpecialValueText(_('every hour')) + self.interval.setMaximum(1000.0) + self.interval.setValue(31.0) + self.interval.setSuffix(' ' + _('days')) + self.interval.setSingleStep(1.0) + self.interval.setDecimals(2) + self.l1.setBuddy(self.interval) + self.l2 = QLabel(_('Note: You can set intervals of less than a day,' + ' by typing the value manually.')) + self.l2.setWordWrap(True) + + self.l.addWidget(self.l1, 0, 0, 1, 1) + self.l.addWidget(self.interval, 0, 1, 1, 1) + self.l.addWidget(self.l2, 1, 0, 1, -1) + + def initialize(self, typ=None, val=None): + if val is None: + val = 31.0 + self.interval.setValue(val) + + @property + def schedule(self): + schedule = self.interval.value() + return 'interval', schedule + class SchedulerDialog(QDialog, Ui_Dialog): + SCHEDULE_TYPES = OrderedDict([ + ('days_of_week', DaysOfWeek), + ('days_of_month', DaysOfMonth), + ('every_x_days', EveryXDays), + ]) + + download = pyqtSignal(object) + def __init__(self, recipe_model, parent=None): QDialog.__init__(self, parent) self.setupUi(self) @@ -30,6 +194,11 @@ class SchedulerDialog(QDialog, Ui_Dialog): _('%s news sources') % self.recipe_model.showing_count) + self.schedule_widgets = [] + for key in reversed(self.SCHEDULE_TYPES): + self.schedule_widgets.insert(0, self.SCHEDULE_TYPES[key](self)) + self.schedule_stack.insertWidget(0, self.schedule_widgets[0]) + self.search.initialize('scheduler_search_history') self.search.setMinimumContentsLength(15) self.search.search.connect(self.recipe_model.search) @@ -43,29 +212,43 @@ class SchedulerDialog(QDialog, Ui_Dialog): self.detail_box.setVisible(False) self.download_button.setVisible(False) self.recipes.currentChanged = self.current_changed - self.interval_button.setChecked(True) + for b, c in self.SCHEDULE_TYPES.iteritems(): + b = getattr(self, b) + b.toggled.connect(self.schedule_type_selected) + b.setToolTip(textwrap.dedent(c.HELP)) + self.days_of_week.setChecked(True) - self.connect(self.schedule, SIGNAL('stateChanged(int)'), - self.toggle_schedule_info) - self.connect(self.show_password, SIGNAL('stateChanged(int)'), - lambda state: self.password.setEchoMode(self.password.Normal if state == Qt.Checked else self.password.Password)) - self.connect(self.download_button, SIGNAL('clicked()'), - self.download_clicked) - self.connect(self.download_all_button, SIGNAL('clicked()'), + self.schedule.stateChanged[int].connect(self.toggle_schedule_info) + self.show_password.stateChanged[int].connect(self.set_pw_echo_mode) + self.download_button.clicked.connect(self.download_clicked) + self.download_all_button.clicked.connect( self.download_all_clicked) self.old_news.setValue(gconf['oldest_news']) + def set_pw_echo_mode(self, state): + self.password.setEchoMode(self.password.Normal + if state == Qt.Checked else self.password.Password) + + + def schedule_type_selected(self, *args): + for i, st in enumerate(self.SCHEDULE_TYPES): + if getattr(self, st).isChecked(): + self.schedule_stack.setCurrentIndex(i) + break + def keyPressEvent(self, ev): if ev.key() not in (Qt.Key_Enter, Qt.Key_Return): return QDialog.keyPressEvent(self, ev) def break_cycles(self): - self.disconnect(self.recipe_model, SIGNAL('searched(PyQt_PyObject)'), - self.search_done) - self.disconnect(self.recipe_model, SIGNAL('searched(PyQt_PyObject)'), - self.search.search_done) - self.search.search.disconnect() + try: + self.recipe_model.searched.disconnect(self.search_done) + self.recipe_model.searched.disconnect(self.search.search_done) + self.search.search.disconnect() + self.download.disconnect() + except: + pass self.recipe_model = None def search_done(self, *args): @@ -74,8 +257,9 @@ class SchedulerDialog(QDialog, Ui_Dialog): def toggle_schedule_info(self, *args): enabled = self.schedule.isChecked() - for x in ('daily_button', 'day', 'time', 'interval_button', 'interval'): + for x in self.SCHEDULE_TYPES: getattr(self, x).setEnabled(enabled) + self.schedule_stack.setEnabled(enabled) self.last_downloaded.setVisible(enabled) def current_changed(self, current, previous): @@ -97,14 +281,14 @@ class SchedulerDialog(QDialog, Ui_Dialog): return False return QDialog.accept(self) - def download_clicked(self): + def download_clicked(self, *args): self.commit() if self.commit() and self.current_urn: - self.emit(SIGNAL('download(PyQt_PyObject)'), self.current_urn) + self.download.emit(self.current_urn) - def download_all_clicked(self): + def download_all_clicked(self, *args): if self.commit() and self.commit(): - self.emit(SIGNAL('download(PyQt_PyObject)'), None) + self.download.emit(None) @property def current_urn(self): @@ -130,16 +314,8 @@ class SchedulerDialog(QDialog, Ui_Dialog): self.recipe_model.set_account_info(urn, un, pw) if self.schedule.isChecked(): - schedule_type = 'interval' if self.interval_button.isChecked() else 'day/time' - if schedule_type == 'interval': - schedule = self.interval.value() - if schedule < 0.1: - schedule = 1./24. - else: - day_of_week = self.day.currentIndex() - 1 - t = self.time.time() - hour, minute = t.hour(), t.minute() - schedule = (day_of_week, hour, minute) + schedule_type, schedule = \ + self.schedule_stack.currentWidget().schedule self.recipe_model.schedule_recipe(urn, schedule_type, schedule) else: self.recipe_model.un_schedule_recipe(urn) @@ -192,27 +368,27 @@ class SchedulerDialog(QDialog, Ui_Dialog): self.schedule.setChecked(scheduled) self.toggle_schedule_info() self.last_downloaded.setText(_('Last downloaded: never')) + ld_text = _('never') if scheduled: typ, sch, last_downloaded = schedule_info - if typ == 'interval': - self.interval_button.setChecked(True) - self.interval.setValue(sch) - elif typ == 'day/time': - self.daily_button.setChecked(True) - day, hour, minute = sch - self.day.setCurrentIndex(day+1) - self.time.setTime(QTime(hour, minute)) - d = utcnow() - last_downloaded def hm(x): return (x-x%3600)//3600, (x%3600 - (x%3600)%60)//60 hours, minutes = hm(d.seconds) tm = _('%d days, %d hours and %d minutes ago')%(d.days, hours, minutes) if d < timedelta(days=366): - self.last_downloaded.setText(_('Last downloaded')+': '+tm) - + ld_text = tm + else: + typ, sch = 'day/time', (-1, 9, 0) + sch_widget = {'day/time': 0, 'days_of_week': 0, 'days_of_month':1, + 'interval':2}[typ] + rb = getattr(self, list(self.SCHEDULE_TYPES)[sch_widget]) + rb.setChecked(True) + self.schedule_stack.setCurrentIndex(sch_widget) + self.schedule_stack.currentWidget().initialize(typ, sch) add_title_tag, custom_tags, keep_issues = customize_info self.add_title_tag.setChecked(add_title_tag) self.custom_tags.setText(u', '.join(custom_tags)) + self.last_downloaded.setText(_('Last downloaded:') + ' ' + ld_text) try: keep_issues = int(keep_issues) except: @@ -241,9 +417,9 @@ class Scheduler(QObject): self.news_icon = QIcon(I('news.png')) self.scheduler_action = QAction(QIcon(I('scheduler.png')), _('Schedule news download'), self) self.news_menu.addAction(self.scheduler_action) - self.connect(self.scheduler_action, SIGNAL('triggered(bool)'), self.show_dialog) + self.scheduler_action.triggered[bool].connect(self.show_dialog) self.cac = QAction(QIcon(I('user_profile.png')), _('Add a custom news source'), self) - self.connect(self.cac, SIGNAL('triggered(bool)'), self.customize_feeds) + self.cac.triggered[bool].connect(self.customize_feeds) self.news_menu.addAction(self.cac) self.news_menu.addSeparator() self.all_action = self.news_menu.addAction( @@ -252,7 +428,7 @@ class Scheduler(QObject): self.timer = QTimer(self) self.timer.start(int(self.INTERVAL * 60 * 1000)) - self.connect(self.timer, SIGNAL('timeout()'), self.check) + self.timer.timeout.connect(self.check) self.oldest = gconf['oldest_news'] QTimer.singleShot(5 * 1000, self.oldest_check) self.database_changed = self.recipe_model.database_changed @@ -276,8 +452,7 @@ class Scheduler(QObject): self.lock.lock() try: d = SchedulerDialog(self.recipe_model) - self.connect(d, SIGNAL('download(PyQt_PyObject)'), - self.download_clicked) + d.download.connect(self.download_clicked) d.exec_() gconf['oldest_news'] = self.oldest = d.old_news.value() d.break_cycles() diff --git a/src/calibre/gui2/dialogs/scheduler.ui b/src/calibre/gui2/dialogs/scheduler.ui index 26953bbe16..f295703b33 100644 --- a/src/calibre/gui2/dialogs/scheduler.ui +++ b/src/calibre/gui2/dialogs/scheduler.ui @@ -61,7 +61,7 @@ &Schedule - + @@ -76,141 +76,94 @@ - - - &Schedule for download: + + + Qt::Vertical + + + 20 + 40 + + + + + + + + QFrame::StyledPanel + + + QFrame::Raised + + + + + + &Schedule for download: + + + + + + + + + Days of week + + + + + + + Days of month + + + + + + + Every x days + + + + + + + + + + 16777215 + 75 + + + + + + + + + + + true + + + + - - - - - Every - - - - - - - - day - - - - - Monday - - - - - Tuesday - - - - - Wednesday - - - - - Thursday - - - - - Friday - - - - - Saturday - - - - - Sunday - - - - - - - - at - - - - - - - - - - Qt::Horizontal - - - - 40 - 20 - - - - - - - - - - - - Every - - - - - - - - 0 - 0 - - - - Interval at which to download this recipe. A value of zero means that the recipe will be downloaded every hour. - - - days - - - 1 - - - 0.000000000000000 - - - 365.100000000000023 - - - 1.000000000000000 - - - 1.000000000000000 - - - - - - - - - + + + Qt::Vertical - - true + + + 20 + 40 + - + @@ -343,6 +296,19 @@ + + + + Qt::Vertical + + + + 20 + 40 + + + + @@ -485,54 +451,6 @@ - - daily_button - toggled(bool) - day - setEnabled(bool) - - - 458 - 155 - - - 573 - 158 - - - - - daily_button - toggled(bool) - time - setEnabled(bool) - - - 458 - 155 - - - 684 - 157 - - - - - interval_button - toggled(bool) - interval - setEnabled(bool) - - - 458 - 212 - - - 752 - 215 - - - add_title_tag toggled(bool) diff --git a/src/calibre/web/feeds/recipes/collection.py b/src/calibre/web/feeds/recipes/collection.py index cd5a220dc3..aa0051ca37 100644 --- a/src/calibre/web/feeds/recipes/collection.py +++ b/src/calibre/web/feeds/recipes/collection.py @@ -231,6 +231,7 @@ class SchedulerConfig(object): if x.get('id', False) == recipe_id: typ, sch, last_downloaded = self.un_serialize_schedule(x) if typ == 'interval': + # Prevent downloads more frequent than once an hour actual_interval = now - last_downloaded nominal_interval = timedelta(days=sch) if abs(actual_interval - nominal_interval) < \ @@ -264,11 +265,16 @@ class SchedulerConfig(object): def serialize_schedule(self, typ, schedule): s = E.schedule({'type':typ}) if typ == 'interval': - if schedule < 0.1: - schedule = 1/24. + if schedule < 0.04: + schedule = 0.04 text = '%f'%schedule elif typ == 'day/time': text = '%d:%d:%d'%schedule + elif typ in ('days_of_week', 'days_of_month'): + dw = ','.join(map(str, map(int, schedule[0]))) + text = '%s:%d:%d'%(dw, schedule[1], schedule[2]) + else: + raise ValueError('Unknown schedule type: %r'%typ) s.text = text return s @@ -280,6 +286,11 @@ class SchedulerConfig(object): sch = float(sch) elif typ == 'day/time': sch = list(map(int, sch.split(':'))) + elif typ in ('days_of_week', 'days_of_month'): + parts = sch.split(':') + days = list(map(int, [x.strip() for x in + parts[0].split(',')])) + sch = [days, int(parts[1]), int(parts[2])] return typ, sch, parse_date(recipe.get('last_downloaded')) def recipe_needs_to_be_downloaded(self, recipe): @@ -287,19 +298,48 @@ class SchedulerConfig(object): typ, sch, ld = self.un_serialize_schedule(recipe) except: return False + + def is_time(now, hour, minute): + return now.hour > hour or \ + (now.hour == hour and now.minute >= minute) + + def is_weekday(day, now): + return day < 0 or day > 6 or \ + day == calendar.weekday(now.year, now.month, now.day) + + def was_downloaded_already_today(ld_local, now): + return ld_local.date() == now.date() + if typ == 'interval': return utcnow() - ld > timedelta(sch) elif typ == 'day/time': now = nowf() ld_local = ld.astimezone(tzlocal()) day, hour, minute = sch + return is_weekday(day, now) and \ + not was_downloaded_already_today(ld_local, now) and \ + is_time(now, hour, minute) + elif typ == 'days_of_week': + now = nowf() + ld_local = ld.astimezone(tzlocal()) + days, hour, minute = sch + have_day = False + for day in days: + if is_weekday(day, now): + have_day = True + break + return have_day and \ + not was_downloaded_already_today(ld_local, now) and \ + is_time(now, hour, minute) + elif typ == 'days_of_month': + now = nowf() + ld_local = ld.astimezone(tzlocal()) + days, hour, minute = sch + have_day = now.day in days + return have_day and \ + not was_downloaded_already_today(ld_local, now) and \ + is_time(now, hour, minute) - is_today = day < 0 or day > 6 or \ - day == calendar.weekday(now.year, now.month, now.day) - is_time = now.hour > hour or \ - (now.hour == hour and now.minute >= minute) - was_downloaded_already_today = ld_local.date() == now.date() - return is_today and not was_downloaded_already_today and is_time return False def set_account_info(self, urn, un, pw): From ca5813675de427f34eb903223758cd997cf3276f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Mar 2011 18:46:23 -0700 Subject: [PATCH 7/9] ... --- src/calibre/gui2/dialogs/scheduler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/scheduler.py b/src/calibre/gui2/dialogs/scheduler.py index 1f6a394556..67a13813df 100644 --- a/src/calibre/gui2/dialogs/scheduler.py +++ b/src/calibre/gui2/dialogs/scheduler.py @@ -21,6 +21,7 @@ from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.date import utcnow from calibre.utils.network import internet_connected from calibre.utils.ordered_dict import OrderedDict +from calibre import force_unicode def convert_day_time_schedule(val): day_of_week, hour, minute = val @@ -47,7 +48,8 @@ class DaysOfWeek(Base): def __init__(self, parent=None): Base.__init__(self, parent) - self.days = [QCheckBox(calendar.day_abbr[d], self) for d in xrange(7)] + self.days = [QCheckBox(force_unicode(calendar.day_abbr[d]), + self) for d in xrange(7)] for i, cb in enumerate(self.days): row = i % 2 col = i // 2 From 1f1128d5a4e8469ea4a947e0fae75d63ad7fea4b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 00:08:02 -0700 Subject: [PATCH 8/9] Custom recipes: Store custom recipes in the calibre config directory instead of the library database. This allows scheduling of custom recipes to work with multiple libraries. Note that you may have to re-schedule any existing custom recipes --- src/calibre/gui2/dialogs/scheduler.py | 10 +-- src/calibre/gui2/dialogs/user_profiles.py | 3 +- src/calibre/library/database2.py | 10 --- src/calibre/library/schema_upgrades.py | 18 +++++ src/calibre/web/feeds/recipes/__init__.py | 9 +++ src/calibre/web/feeds/recipes/collection.py | 80 +++++++++++++++++++-- src/calibre/web/feeds/recipes/model.py | 35 ++++----- 7 files changed, 122 insertions(+), 43 deletions(-) diff --git a/src/calibre/gui2/dialogs/scheduler.py b/src/calibre/gui2/dialogs/scheduler.py index 67a13813df..b2561342b8 100644 --- a/src/calibre/gui2/dialogs/scheduler.py +++ b/src/calibre/gui2/dialogs/scheduler.py @@ -411,7 +411,8 @@ class Scheduler(QObject): QObject.__init__(self, parent) self.internet_connection_failed = False self._parent = parent - self.recipe_model = RecipeModel(db) + self.recipe_model = RecipeModel() + self.db = db self.lock = QMutex(QMutex.Recursive) self.download_queue = set([]) @@ -433,7 +434,9 @@ class Scheduler(QObject): self.timer.timeout.connect(self.check) self.oldest = gconf['oldest_news'] QTimer.singleShot(5 * 1000, self.oldest_check) - self.database_changed = self.recipe_model.database_changed + + def database_changed(self, db): + self.db = db def oldest_check(self): if self.oldest > 0: @@ -549,7 +552,6 @@ class Scheduler(QObject): if __name__ == '__main__': from calibre.gui2 import is_ok_to_use_qt is_ok_to_use_qt() - from calibre.library.database2 import LibraryDatabase2 - d = SchedulerDialog(RecipeModel(LibraryDatabase2('/home/kovid/documents/library'))) + d = SchedulerDialog(RecipeModel()) d.exec_() diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index fe64deb430..290982caaf 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -366,8 +366,7 @@ class %(classname)s(%(base_class)s): if __name__ == '__main__': from calibre.gui2 import is_ok_to_use_qt is_ok_to_use_qt() - from calibre.library.database2 import LibraryDatabase2 from calibre.web.feeds.recipes.model import RecipeModel - d=UserProfiles(None, RecipeModel(LibraryDatabase2('/home/kovid/documents/library'))) + d=UserProfiles(None, RecipeModel()) d.exec_() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 2554df93e6..38b70fc2bf 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1187,12 +1187,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.clean_custom() self.conn.commit() - def get_recipes(self): - return self.conn.get('SELECT id, script FROM feeds') - - def get_recipe(self, id): - return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False) - def get_books_for_category(self, category, id_): ans = set([]) @@ -3113,8 +3107,4 @@ books_series_link feeds s = self.conn.get('''SELECT book FROM books_plugin_data WHERE name=?''', (name,)) return [x[0] for x in s] - def get_custom_recipes(self): - for id, title, script in self.conn.get('SELECT id,title,script FROM feeds'): - yield id, title, script - diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index d1f22d379b..3fc9a2368a 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -582,4 +582,22 @@ class SchemaUpgrade(object): # statements self.conn.executescript(script) + def upgrade_version_19(self): + recipes = self.conn.get('SELECT id,title,script FROM feeds') + if recipes: + from calibre.web.feeds.recipes import custom_recipes, \ + custom_recipe_filename + bdir = os.path.dirname(custom_recipes.file_path) + for id_, title, script in recipes: + existing = frozenset(map(int, custom_recipes.iterkeys())) + if id_ in existing: + id_ = max(existing) + 1000 + id_ = str(id_) + fname = custom_recipe_filename(id_, title) + custom_recipes[id_] = (title, fname) + if isinstance(script, unicode): + script = script.encode('utf-8') + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) + diff --git a/src/calibre/web/feeds/recipes/__init__.py b/src/calibre/web/feeds/recipes/__init__.py index a72f500862..bfb46fa799 100644 --- a/src/calibre/web/feeds/recipes/__init__.py +++ b/src/calibre/web/feeds/recipes/__init__.py @@ -10,6 +10,7 @@ from calibre.web.feeds.news import BasicNewsRecipe, CustomIndexRecipe, \ from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ptempfile import PersistentTemporaryDirectory from calibre import __appname__, english_sort +from calibre.utils.config import JSONConfig BeautifulSoup, time, english_sort @@ -17,6 +18,14 @@ basic_recipes = (BasicNewsRecipe, AutomaticNewsRecipe, CustomIndexRecipe, CalibrePeriodical) _tdir = None _crep = 0 + +custom_recipes = JSONConfig('custom_recipes/index.json') + +def custom_recipe_filename(id_, title): + from calibre.utils.filenames import ascii_filename + return ascii_filename(title[:50]) + \ + ('_%s.recipe'%id_) + def compile_recipe(src): ''' Compile the code in src and return the first object that is a recipe or profile. diff --git a/src/calibre/web/feeds/recipes/collection.py b/src/calibre/web/feeds/recipes/collection.py index aa0051ca37..7082f780e6 100644 --- a/src/calibre/web/feeds/recipes/collection.py +++ b/src/calibre/web/feeds/recipes/collection.py @@ -88,19 +88,89 @@ def serialize_builtin_recipes(): def get_builtin_recipe_collection(): return etree.parse(P('builtin_recipes.xml')).getroot() -def get_custom_recipe_collection(db): - from calibre.web.feeds.recipes import compile_recipe +def get_custom_recipe_collection(*args): + from calibre.web.feeds.recipes import compile_recipe, \ + custom_recipes + bdir = os.path.dirname(custom_recipes.file_path) rmap = {} - for id, title, recipe in db.get_custom_recipes(): + for id_, x in custom_recipes.iteritems(): + title, fname = x + recipe = os.path.join(bdir, fname) try: + recipe = open(recipe, 'rb').read().decode('utf-8') recipe_class = compile_recipe(recipe) if recipe_class is not None: - rmap['custom:%d'%id] = recipe_class + rmap['custom:%s'%id_] = recipe_class except: + import traceback + traceback.print_exc() continue - return etree.fromstring(serialize_collection(rmap)) + +def update_custom_recipe(id_, title, script): + from calibre.web.feeds.recipes import custom_recipes, \ + custom_recipe_filename + id_ = str(int(id_)) + existing = custom_recipes.get(id_, None) + bdir = os.path.dirname(custom_recipes.file_path) + + if existing is None: + fname = custom_recipe_filename(id_, title) + else: + fname = existing[1] + if isinstance(script, unicode): + script = script.encode('utf-8') + + custom_recipes[id_] = (title, fname) + + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) + + +def add_custom_recipe(title, script): + from calibre.web.feeds.recipes import custom_recipes, \ + custom_recipe_filename + id_ = 1000 + keys = tuple(map(int, custom_recipes.iterkeys())) + if keys: + id_ = max(keys)+1 + id_ = str(id_) + bdir = os.path.dirname(custom_recipes.file_path) + + fname = custom_recipe_filename(id_, title) + if isinstance(script, unicode): + script = script.encode('utf-8') + + custom_recipes[id_] = (title, fname) + + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) + + +def remove_custom_recipe(id_): + from calibre.web.feeds.recipes import custom_recipes + id_ = str(int(id_)) + existing = custom_recipes.get(id_, None) + if existing is not None: + bdir = os.path.dirname(custom_recipes.file_path) + fname = existing[1] + del custom_recipes[id_] + try: + os.remove(os.path.join(bdir, fname)) + except: + pass + +def get_custom_recipe(id_): + from calibre.web.feeds.recipes import custom_recipes + id_ = str(int(id_)) + existing = custom_recipes.get(id_, None) + if existing is not None: + bdir = os.path.dirname(custom_recipes.file_path) + fname = existing[1] + with open(os.path.join(bdir, fname), 'rb') as f: + return f.read().decode('utf-8') + def get_builtin_recipe_titles(): return [r.get('title') for r in get_builtin_recipe_collection()] diff --git a/src/calibre/web/feeds/recipes/model.py b/src/calibre/web/feeds/recipes/model.py index 553fdcc3c3..19e73dd5f8 100644 --- a/src/calibre/web/feeds/recipes/model.py +++ b/src/calibre/web/feeds/recipes/model.py @@ -9,14 +9,15 @@ __docformat__ = 'restructuredtext en' import os, copy from PyQt4.Qt import QAbstractItemModel, QVariant, Qt, QColor, QFont, QIcon, \ - QModelIndex, QMetaObject, pyqtSlot, pyqtSignal + QModelIndex, pyqtSignal from calibre.utils.search_query_parser import SearchQueryParser from calibre.gui2 import NONE from calibre.utils.localization import get_language from calibre.web.feeds.recipes.collection import \ get_builtin_recipe_collection, get_custom_recipe_collection, \ - SchedulerConfig, download_builtin_recipe + SchedulerConfig, download_builtin_recipe, update_custom_recipe, \ + add_custom_recipe, remove_custom_recipe, get_custom_recipe from calibre.utils.pyparsing import ParseException class NewsTreeItem(object): @@ -122,26 +123,15 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser): LOCATIONS = ['all'] searched = pyqtSignal(object) - def __init__(self, db, *args): + def __init__(self, *args): QAbstractItemModel.__init__(self, *args) SearchQueryParser.__init__(self, locations=['all']) - self.db = db self.default_icon = QVariant(QIcon(I('news.png'))) self.custom_icon = QVariant(QIcon(I('user_profile.png'))) self.builtin_recipe_collection = get_builtin_recipe_collection() self.scheduler_config = SchedulerConfig() self.do_refresh() - @pyqtSlot() - def do_database_change(self): - self.db = self.newdb - self.newdb = None - self.do_refresh() - - def database_changed(self, db): - self.newdb = db - QMetaObject.invokeMethod(self, 'do_database_change', Qt.QueuedConnection) - def get_builtin_recipe(self, urn, download=True): if download: try: @@ -158,23 +148,24 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser): if recipe.get('id', False) == urn: if coll is self.builtin_recipe_collection: return self.get_builtin_recipe(urn[8:], download=download) - return self.db.get_feed(int(urn[len('custom:'):])) + return get_custom_recipe(int(urn[len('custom:'):])) def update_custom_recipe(self, urn, title, script): - self.db.update_feed(int(urn[len('custom:'):]), script, title) - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + id_ = int(urn[len('custom:'):]) + update_custom_recipe(id_, title, script) + self.custom_recipe_collection = get_custom_recipe_collection() def add_custom_recipe(self, title, script): - self.db.add_feed(title, script) - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + add_custom_recipe(title, script) + self.custom_recipe_collection = get_custom_recipe_collection() def remove_custom_recipes(self, urns): ids = [int(x[len('custom:'):]) for x in urns] - self.db.remove_feeds(ids) - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + for id_ in ids: remove_custom_recipe(id_) + self.custom_recipe_collection = get_custom_recipe_collection() def do_refresh(self, restrict_to_urns=set([])): - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + self.custom_recipe_collection = get_custom_recipe_collection() def factory(cls, parent, *args): args = list(args) From aaefd462d92b4fdbd5dcdd4b1ac00602559edcfd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 00:15:13 -0700 Subject: [PATCH 9/9] ... --- src/calibre/gui2/dialogs/user_profiles.py | 9 +++++++-- src/calibre/gui2/dialogs/user_profiles.ui | 13 ++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index 290982caaf..f2388d2981 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -6,11 +6,11 @@ import time, os from PyQt4.Qt import SIGNAL, QUrl, QAbstractListModel, Qt, \ QVariant -from calibre.web.feeds.recipes import compile_recipe +from calibre.web.feeds.recipes import compile_recipe, custom_recipes from calibre.web.feeds.news import AutomaticNewsRecipe from calibre.gui2.dialogs.user_profiles_ui import Ui_Dialog from calibre.gui2 import error_dialog, question_dialog, open_url, \ - choose_files, ResizableDialog, NONE + choose_files, ResizableDialog, NONE, open_local_file from calibre.gui2.widgets import PythonHighlighter from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.icu import sort_key @@ -93,6 +93,7 @@ class UserProfiles(ResizableDialog, Ui_Dialog): self.connect(self.load_button, SIGNAL('clicked()'), self.load) self.connect(self.builtin_recipe_button, SIGNAL('clicked()'), self.add_builtin_recipe) self.connect(self.share_button, SIGNAL('clicked()'), self.share) + self.show_recipe_files_button.clicked.connect(self.show_recipe_files) self.connect(self.down_button, SIGNAL('clicked()'), self.down) self.connect(self.up_button, SIGNAL('clicked()'), self.up) self.connect(self.add_profile_button, SIGNAL('clicked(bool)'), @@ -102,6 +103,10 @@ class UserProfiles(ResizableDialog, Ui_Dialog): self.connect(self.toggle_mode_button, SIGNAL('clicked(bool)'), self.toggle_mode) self.clear() + def show_recipe_files(self, *args): + bdir = os.path.dirname(custom_recipes.file_path) + open_local_file(bdir) + def break_cycles(self): self.recipe_model = self._model.recipe_model = None self.available_profiles = None diff --git a/src/calibre/gui2/dialogs/user_profiles.ui b/src/calibre/gui2/dialogs/user_profiles.ui index 4ca47539d1..97e3d37db2 100644 --- a/src/calibre/gui2/dialogs/user_profiles.ui +++ b/src/calibre/gui2/dialogs/user_profiles.ui @@ -35,7 +35,7 @@ 0 0 730 - 600 + 601 @@ -102,6 +102,17 @@ + + + + S&how recipe files + + + + :/images/document_open.png:/images/document_open.png + + +