Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

News download: More flexible news downlaod scheduling. You can now schedule by days of the week, days of the month and an interval, whcih can be as small as an hour for news sources that change rapidly

Browse Source
This commit is contained in:
Kovid Goyal 2011-03-05 18:34:27 -07:00
parent 5ab5327b2e
commit f836abbc2a
3 changed files with 365 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

265
src/calibre/gui2/dialogs/scheduler.py
View File

@ -8,9 +8,11 @@ Scheduler for automated recipe downloads
'''
from datetime import timedelta
import calendar, textwrap
from PyQt4.Qt import QDialog, SIGNAL, Qt, QTime, QObject, QMenu, \
QAction, QIcon, QMutex, QTimer, pyqtSignal
from PyQt4.Qt import QDialog, Qt, QTime, QObject, QMenu, QHBoxLayout, \
QAction, QIcon, QMutex, QTimer, pyqtSignal, QWidget, QGridLayout, \
QCheckBox, QTimeEdit, QLabel, QLineEdit, QDoubleSpinBox
from calibre.gui2.dialogs.scheduler_ui import Ui_Dialog
from calibre.gui2 import config as gconf, error_dialog
@ -18,9 +20,171 @@ from calibre.web.feeds.recipes.model import RecipeModel
from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.date import utcnow
from calibre.utils.network import internet_connected
from calibre.utils.ordered_dict import OrderedDict
def convert_day_time_schedule(val):
day_of_week, hour, minute = val
if day_of_week == -1:
return (tuple(xrange(7)), hour, minute)
return ((day_of_week,), hour, minute)
class Base(QWidget):
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.l = QGridLayout()
self.setLayout(self.l)
self.setToolTip(textwrap.dedent(self.HELP))
class DaysOfWeek(Base):
HELP = _('''\
Download this periodical every week on the specified days after
the specified time. For example, if you choose: Monday after
9:00 AM, then the periodical will be download every Monday as
soon after 9:00 AM as possible.
''')
def __init__(self, parent=None):
Base.__init__(self, parent)
self.days = [QCheckBox(calendar.day_abbr[d], self) for d in xrange(7)]
for i, cb in enumerate(self.days):
row = i % 2
col = i // 2
self.l.addWidget(cb, row, col, 1, 1)
self.time = QTimeEdit(self)
self.time.setDisplayFormat('hh:mm AP')
self.hl = QHBoxLayout()
self.l1 = QLabel(_('&Download after:'))
self.l1.setBuddy(self.time)
self.hl.addWidget(self.l1)
self.hl.addWidget(self.time)
self.l.addLayout(self.hl, 1, 3, 1, 1)
self.initialize()
def initialize(self, typ=None, val=None):
if typ is None:
typ = 'day/time'
val = (-1, 9, 0)
if typ == 'day/time':
val = convert_day_time_schedule(val)
days_of_week, hour, minute = val
for i, d in enumerate(self.days):
d.setChecked(i in days_of_week)
self.time.setTime(QTime(hour, minute))
@property
def schedule(self):
days_of_week = tuple([i for i, d in enumerate(self.days) if
d.isChecked()])
t = self.time.time()
hour, minute = t.hour(), t.minute()
return 'days_of_week', (days_of_week, int(hour), int(minute))
class DaysOfMonth(Base):
HELP = _('''\
Download this periodical every month, on the specified days.
The download will happen as soon after the specified time as
possible on the specified days of each month. For example,
if you choose the 1st and the 15th after 9:00 AM, the
periodical will be downloaded on the 1st and 15th of every
month, as soon after 9:00 AM as possible.
''')
def __init__(self, parent=None):
Base.__init__(self, parent)
self.l1 = QLabel(_('&Days of the month:'))
self.days = QLineEdit(self)
self.days.setToolTip(_('Comma separated list of days of the month.'
' For example: 1, 15'))
self.l1.setBuddy(self.days)
self.l2 = QLabel(_('Download &after:'))
self.time = QTimeEdit(self)
self.time.setDisplayFormat('hh:mm AP')
self.l2.setBuddy(self.time)
self.l.addWidget(self.l1, 0, 0, 1, 1)
self.l.addWidget(self.days, 0, 1, 1, 1)
self.l.addWidget(self.l2, 1, 0, 1, 1)
self.l.addWidget(self.time, 1, 1, 1, 1)
def initialize(self, typ=None, val=None):
if val is None:
val = ((1,), 9, 0)
days_of_month, hour, minute = val
self.days.setText(', '.join(map(str, map(int, days_of_month))))
self.time.setTime(QTime(hour, minute))
@property
def schedule(self):
parts = [x.strip() for x in unicode(self.days.text()).split(',') if
x.strip()]
try:
days_of_month = tuple(map(int, parts))
except:
days_of_month = (1,)
if not days_of_month:
days_of_month = (1,)
t = self.time.time()
hour, minute = t.hour(), t.minute()
return 'days_of_month', (days_of_month, int(hour), int(minute))
class EveryXDays(Base):
HELP = _('''\
Download this periodical every x days. For example, if you
choose 30 days, the periodical will be downloaded every 30
days. Note that you can set periods of less than a day, like
0.1 days to download a periodical more than once a day.
''')
def __init__(self, parent=None):
Base.__init__(self, parent)
self.l1 = QLabel(_('&Download every:'))
self.interval = QDoubleSpinBox(self)
self.interval.setMinimum(0.04)
self.interval.setSpecialValueText(_('every hour'))
self.interval.setMaximum(1000.0)
self.interval.setValue(31.0)
self.interval.setSuffix(' ' + _('days'))
self.interval.setSingleStep(1.0)
self.interval.setDecimals(2)
self.l1.setBuddy(self.interval)
self.l2 = QLabel(_('Note: You can set intervals of less than a day,'
' by typing the value manually.'))
self.l2.setWordWrap(True)
self.l.addWidget(self.l1, 0, 0, 1, 1)
self.l.addWidget(self.interval, 0, 1, 1, 1)
self.l.addWidget(self.l2, 1, 0, 1, -1)
def initialize(self, typ=None, val=None):
if val is None:
val = 31.0
self.interval.setValue(val)
@property
def schedule(self):
schedule = self.interval.value()
return 'interval', schedule
class SchedulerDialog(QDialog, Ui_Dialog):
SCHEDULE_TYPES = OrderedDict([
('days_of_week', DaysOfWeek),
('days_of_month', DaysOfMonth),
('every_x_days', EveryXDays),
])
download = pyqtSignal(object)
def __init__(self, recipe_model, parent=None):
QDialog.__init__(self, parent)
self.setupUi(self)
@ -30,6 +194,11 @@ class SchedulerDialog(QDialog, Ui_Dialog):
_('%s news sources') %
self.recipe_model.showing_count)
self.schedule_widgets = []
for key in reversed(self.SCHEDULE_TYPES):
self.schedule_widgets.insert(0, self.SCHEDULE_TYPES[key](self))
self.schedule_stack.insertWidget(0, self.schedule_widgets[0])
self.search.initialize('scheduler_search_history')
self.search.setMinimumContentsLength(15)
self.search.search.connect(self.recipe_model.search)
@ -43,29 +212,43 @@ class SchedulerDialog(QDialog, Ui_Dialog):
self.detail_box.setVisible(False)
self.download_button.setVisible(False)
self.recipes.currentChanged = self.current_changed
self.interval_button.setChecked(True)
for b, c in self.SCHEDULE_TYPES.iteritems():
b = getattr(self, b)
b.toggled.connect(self.schedule_type_selected)
b.setToolTip(textwrap.dedent(c.HELP))
self.days_of_week.setChecked(True)
self.connect(self.schedule, SIGNAL('stateChanged(int)'),
self.toggle_schedule_info)
self.connect(self.show_password, SIGNAL('stateChanged(int)'),
lambda state: self.password.setEchoMode(self.password.Normal if state == Qt.Checked else self.password.Password))
self.connect(self.download_button, SIGNAL('clicked()'),
self.download_clicked)
self.connect(self.download_all_button, SIGNAL('clicked()'),
self.schedule.stateChanged[int].connect(self.toggle_schedule_info)
self.show_password.stateChanged[int].connect(self.set_pw_echo_mode)
self.download_button.clicked.connect(self.download_clicked)
self.download_all_button.clicked.connect(
self.download_all_clicked)
self.old_news.setValue(gconf['oldest_news'])
def set_pw_echo_mode(self, state):
self.password.setEchoMode(self.password.Normal
if state == Qt.Checked else self.password.Password)
def schedule_type_selected(self, *args):
for i, st in enumerate(self.SCHEDULE_TYPES):
if getattr(self, st).isChecked():
self.schedule_stack.setCurrentIndex(i)
break
def keyPressEvent(self, ev):
if ev.key() not in (Qt.Key_Enter, Qt.Key_Return):
return QDialog.keyPressEvent(self, ev)
def break_cycles(self):
self.disconnect(self.recipe_model, SIGNAL('searched(PyQt_PyObject)'),
self.search_done)
self.disconnect(self.recipe_model, SIGNAL('searched(PyQt_PyObject)'),
self.search.search_done)
try:
self.recipe_model.searched.disconnect(self.search_done)
self.recipe_model.searched.disconnect(self.search.search_done)
self.search.search.disconnect()
self.download.disconnect()
except:
pass
self.recipe_model = None
def search_done(self, *args):
@ -74,8 +257,9 @@ class SchedulerDialog(QDialog, Ui_Dialog):
def toggle_schedule_info(self, *args):
enabled = self.schedule.isChecked()
for x in ('daily_button', 'day', 'time', 'interval_button', 'interval'):
for x in self.SCHEDULE_TYPES:
getattr(self, x).setEnabled(enabled)
self.schedule_stack.setEnabled(enabled)
self.last_downloaded.setVisible(enabled)
def current_changed(self, current, previous):
@ -97,14 +281,14 @@ class SchedulerDialog(QDialog, Ui_Dialog):
return False
return QDialog.accept(self)
def download_clicked(self):
def download_clicked(self, *args):
self.commit()
if self.commit() and self.current_urn:
self.emit(SIGNAL('download(PyQt_PyObject)'), self.current_urn)
self.download.emit(self.current_urn)
def download_all_clicked(self):
def download_all_clicked(self, *args):
if self.commit() and self.commit():
self.emit(SIGNAL('download(PyQt_PyObject)'), None)
self.download.emit(None)
@property
def current_urn(self):
@ -130,16 +314,8 @@ class SchedulerDialog(QDialog, Ui_Dialog):
self.recipe_model.set_account_info(urn, un, pw)
if self.schedule.isChecked():
schedule_type = 'interval' if self.interval_button.isChecked() else 'day/time'
if schedule_type == 'interval':
schedule = self.interval.value()
if schedule < 0.1:
schedule = 1./24.
else:
day_of_week = self.day.currentIndex() - 1
t = self.time.time()
hour, minute = t.hour(), t.minute()
schedule = (day_of_week, hour, minute)
schedule_type, schedule = \
self.schedule_stack.currentWidget().schedule
self.recipe_model.schedule_recipe(urn, schedule_type, schedule)
else:
self.recipe_model.un_schedule_recipe(urn)
@ -192,27 +368,27 @@ class SchedulerDialog(QDialog, Ui_Dialog):
self.schedule.setChecked(scheduled)
self.toggle_schedule_info()
self.last_downloaded.setText(_('Last downloaded: never'))
ld_text = _('never')
if scheduled:
typ, sch, last_downloaded = schedule_info
if typ == 'interval':
self.interval_button.setChecked(True)
self.interval.setValue(sch)
elif typ == 'day/time':
self.daily_button.setChecked(True)
day, hour, minute = sch
self.day.setCurrentIndex(day+1)
self.time.setTime(QTime(hour, minute))
d = utcnow() - last_downloaded
def hm(x): return (x-x%3600)//3600, (x%3600 - (x%3600)%60)//60
hours, minutes = hm(d.seconds)
tm = _('%d days, %d hours and %d minutes ago')%(d.days, hours, minutes)
if d < timedelta(days=366):
self.last_downloaded.setText(_('Last downloaded')+': '+tm)
ld_text = tm
else:
typ, sch = 'day/time', (-1, 9, 0)
sch_widget = {'day/time': 0, 'days_of_week': 0, 'days_of_month':1,
'interval':2}[typ]
rb = getattr(self, list(self.SCHEDULE_TYPES)[sch_widget])
rb.setChecked(True)
self.schedule_stack.setCurrentIndex(sch_widget)
self.schedule_stack.currentWidget().initialize(typ, sch)
add_title_tag, custom_tags, keep_issues = customize_info
self.add_title_tag.setChecked(add_title_tag)
self.custom_tags.setText(u', '.join(custom_tags))
self.last_downloaded.setText(_('Last downloaded:') + ' ' + ld_text)
try:
keep_issues = int(keep_issues)
except:
@ -241,9 +417,9 @@ class Scheduler(QObject):
self.news_icon = QIcon(I('news.png'))
self.scheduler_action = QAction(QIcon(I('scheduler.png')), _('Schedule news download'), self)
self.news_menu.addAction(self.scheduler_action)
self.connect(self.scheduler_action, SIGNAL('triggered(bool)'), self.show_dialog)
self.scheduler_action.triggered[bool].connect(self.show_dialog)
self.cac = QAction(QIcon(I('user_profile.png')), _('Add a custom news source'), self)
self.connect(self.cac, SIGNAL('triggered(bool)'), self.customize_feeds)
self.cac.triggered[bool].connect(self.customize_feeds)
self.news_menu.addAction(self.cac)
self.news_menu.addSeparator()
self.all_action = self.news_menu.addAction(
@ -252,7 +428,7 @@ class Scheduler(QObject):
self.timer = QTimer(self)
self.timer.start(int(self.INTERVAL * 60 * 1000))
self.connect(self.timer, SIGNAL('timeout()'), self.check)
self.timer.timeout.connect(self.check)
self.oldest = gconf['oldest_news']
QTimer.singleShot(5 * 1000, self.oldest_check)
self.database_changed = self.recipe_model.database_changed
@ -276,8 +452,7 @@ class Scheduler(QObject):
self.lock.lock()
try:
d = SchedulerDialog(self.recipe_model)
self.connect(d, SIGNAL('download(PyQt_PyObject)'),
self.download_clicked)
d.download.connect(self.download_clicked)
d.exec_()
gconf['oldest_news'] = self.oldest = d.old_news.value()
d.break_cycles()

228
src/calibre/gui2/dialogs/scheduler.ui
View File

@ -61,7 +61,7 @@
<attribute name="title">
<string>&amp;Schedule</string>
</attribute>
<layout class="QVBoxLayout" name="verticalLayout_2">
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<widget class="QLabel" name="blurb">
<property name="text">
@ -75,6 +75,28 @@
</property>
</widget>
</item>
<item>
<spacer name="verticalSpacer_3">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QFrame" name="frame">
<property name="frameShape">
<enum>QFrame::StyledPanel</enum>
</property>
<property name="frameShadow">
<enum>QFrame::Raised</enum>
</property>
<layout class="QVBoxLayout" name="verticalLayout_2">
<item>
<widget class="QCheckBox" name="schedule">
<property name="text">
@ -82,126 +104,41 @@
</property>
</widget>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
<widget class="QRadioButton" name="daily_button">
<property name="text">
<string>Every </string>
</property>
</widget>
</item>
<item>
<widget class="QComboBox" name="day">
<item>
<property name="text">
<string>day</string>
</property>
</item>
<item>
<property name="text">
<string>Monday</string>
</property>
</item>
<item>
<property name="text">
<string>Tuesday</string>
</property>
</item>
<item>
<property name="text">
<string>Wednesday</string>
</property>
</item>
<item>
<property name="text">
<string>Thursday</string>
</property>
</item>
<item>
<property name="text">
<string>Friday</string>
</property>
</item>
<item>
<property name="text">
<string>Saturday</string>
</property>
</item>
<item>
<property name="text">
<string>Sunday</string>
</property>
</item>
</widget>
</item>
<item>
<widget class="QLabel" name="label_4">
<property name="text">
<string>at</string>
</property>
</widget>
</item>
<item>
<widget class="QTimeEdit" name="time"/>
</item>
<item>
<spacer name="horizontalSpacer">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QRadioButton" name="interval_button">
<widget class="QRadioButton" name="days_of_week">
<property name="text">
<string>Every </string>
<string>Days of week</string>
</property>
</widget>
</item>
<item>
<widget class="QDoubleSpinBox" name="interval">
<property name="sizePolicy">
<sizepolicy hsizetype="Expanding" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
<widget class="QRadioButton" name="days_of_month">
<property name="text">
<string>Days of month</string>
</property>
<property name="toolTip">
<string>Interval at which to download this recipe. A value of zero means that the recipe will be downloaded every hour.</string>
</property>
<property name="suffix">
<string> days</string>
</property>
<property name="decimals">
<number>1</number>
</property>
<property name="minimum">
<double>0.000000000000000</double>
</property>
<property name="maximum">
<double>365.100000000000023</double>
</property>
<property name="singleStep">
<double>1.000000000000000</double>
</property>
<property name="value">
<double>1.000000000000000</double>
</widget>
</item>
<item>
<widget class="QRadioButton" name="every_x_days">
<property name="text">
<string>Every x days</string>
</property>
</widget>
</item>
</layout>
</item>
<item>
<widget class="QStackedWidget" name="schedule_stack">
<property name="maximumSize">
<size>
<width>16777215</width>
<height>75</height>
</size>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="last_downloaded">
<property name="text">
@ -212,6 +149,22 @@
</property>
</widget>
</item>
</layout>
</widget>
</item>
<item>
<spacer name="verticalSpacer_4">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QGroupBox" name="account">
<property name="title">
@ -343,6 +296,19 @@
</widget>
</widget>
</item>
<item>
<spacer name="verticalSpacer_2">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QPushButton" name="download_button">
<property name="text">
@ -485,54 +451,6 @@
</hint>
</hints>
</connection>
<connection>
<sender>daily_button</sender>
<signal>toggled(bool)</signal>
<receiver>day</receiver>
<slot>setEnabled(bool)</slot>
<hints>
<hint type="sourcelabel">
<x>458</x>
<y>155</y>
</hint>
<hint type="destinationlabel">
<x>573</x>
<y>158</y>
</hint>
</hints>
</connection>
<connection>
<sender>daily_button</sender>
<signal>toggled(bool)</signal>
<receiver>time</receiver>
<slot>setEnabled(bool)</slot>
<hints>
<hint type="sourcelabel">
<x>458</x>
<y>155</y>
</hint>
<hint type="destinationlabel">
<x>684</x>
<y>157</y>
</hint>
</hints>
</connection>
<connection>
<sender>interval_button</sender>
<signal>toggled(bool)</signal>
<receiver>interval</receiver>
<slot>setEnabled(bool)</slot>
<hints>
<hint type="sourcelabel">
<x>458</x>
<y>212</y>
</hint>
<hint type="destinationlabel">
<x>752</x>
<y>215</y>
</hint>
</hints>
</connection>
<connection>
<sender>add_title_tag</sender>
<signal>toggled(bool)</signal>

56
src/calibre/web/feeds/recipes/collection.py
View File

@ -231,6 +231,7 @@ class SchedulerConfig(object):
if x.get('id', False) == recipe_id:
typ, sch, last_downloaded = self.un_serialize_schedule(x)
if typ == 'interval':
# Prevent downloads more frequent than once an hour
actual_interval = now - last_downloaded
nominal_interval = timedelta(days=sch)
if abs(actual_interval - nominal_interval) < \
@ -264,11 +265,16 @@ class SchedulerConfig(object):
def serialize_schedule(self, typ, schedule):
s = E.schedule({'type':typ})
if typ == 'interval':
if schedule < 0.1:
schedule = 1/24.
if schedule < 0.04:
schedule = 0.04
text = '%f'%schedule
elif typ == 'day/time':
text = '%d:%d:%d'%schedule
elif typ in ('days_of_week', 'days_of_month'):
dw = ','.join(map(str, map(int, schedule[0])))
text = '%s:%d:%d'%(dw, schedule[1], schedule[2])
else:
raise ValueError('Unknown schedule type: %r'%typ)
s.text = text
return s
@ -280,6 +286,11 @@ class SchedulerConfig(object):
sch = float(sch)
elif typ == 'day/time':
sch = list(map(int, sch.split(':')))
elif typ in ('days_of_week', 'days_of_month'):
parts = sch.split(':')
days = list(map(int, [x.strip() for x in
parts[0].split(',')]))
sch = [days, int(parts[1]), int(parts[2])]
return typ, sch, parse_date(recipe.get('last_downloaded'))
def recipe_needs_to_be_downloaded(self, recipe):
@ -287,19 +298,48 @@ class SchedulerConfig(object):
typ, sch, ld = self.un_serialize_schedule(recipe)
except:
return False
def is_time(now, hour, minute):
return now.hour > hour or \
(now.hour == hour and now.minute >= minute)
def is_weekday(day, now):
return day < 0 or day > 6 or \
day == calendar.weekday(now.year, now.month, now.day)
def was_downloaded_already_today(ld_local, now):
return ld_local.date() == now.date()
if typ == 'interval':
return utcnow() - ld > timedelta(sch)
elif typ == 'day/time':
now = nowf()
ld_local = ld.astimezone(tzlocal())
day, hour, minute = sch
return is_weekday(day, now) and \
not was_downloaded_already_today(ld_local, now) and \
is_time(now, hour, minute)
elif typ == 'days_of_week':
now = nowf()
ld_local = ld.astimezone(tzlocal())
days, hour, minute = sch
have_day = False
for day in days:
if is_weekday(day, now):
have_day = True
break
return have_day and \
not was_downloaded_already_today(ld_local, now) and \
is_time(now, hour, minute)
elif typ == 'days_of_month':
now = nowf()
ld_local = ld.astimezone(tzlocal())
days, hour, minute = sch
have_day = now.day in days
return have_day and \
not was_downloaded_already_today(ld_local, now) and \
is_time(now, hour, minute)
is_today = day < 0 or day > 6 or \
day == calendar.weekday(now.year, now.month, now.day)
is_time = now.hour > hour or \
(now.hour == hour and now.minute >= minute)
was_downloaded_already_today = ld_local.date() == now.date()
return is_today and not was_downloaded_already_today and is_time
return False
def set_account_info(self, urn, un, pw):