New recipe for Psychology Today by Krittika Goyal

This commit is contained in:
Kovid Goyal 2009-12-31 19:28:58 -07:00
parent 95cf14a3ad
commit bf090b2ba2
18 changed files with 319 additions and 158 deletions

View File

@ -0,0 +1,39 @@
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup
class PsychologyToday(BasicNewsRecipe):
title = u'Psychology Today'
language = 'en'
__author__ = 'Krittika Goyal'
oldest_article = 1 #days
max_articles_per_feed = 25
#encoding = 'latin1'
remove_stylesheets = True
#remove_tags_before = dict(name='h1', attrs={'class':'heading'})
#remove_tags_after = dict(name='td', attrs={'class':'newptool1'})
remove_tags = [
dict(name='iframe'),
dict(name='div', attrs={'class':['pt-box-title', 'pt-box-content', 'blog-entry-footer', 'item-list', 'article-sub-meta']}),
dict(name='div', attrs={'id':['block-td_search_160', 'block-cam_search_160']}),
#dict(name='ul', attrs={'class':'article-tools'}),
#dict(name='ul', attrs={'class':'articleTools'}),
]
feeds = [
('PSY TODAY',
'http://www.psychologytoday.com/articles/index.rss'),
]
def preprocess_html(self, soup):
story = soup.find(name='div', attrs={'id':'contentColumn'})
#td = heading.findParent(name='td')
#td.extract()
soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
body = soup.find(name='body')
body.insert(0, story)
for x in soup.findAll(name='p', text=lambda x:x and '--&gt;' in x):
p = x.findParent('p')
if p is not None:
p.extract()
return soup

View File

@ -47,11 +47,10 @@ class CYBOOKG3(USBMS):
SUPPORTS_SUB_DIRS = True
def upload_cover(self, path, filename, metadata):
coverdata = metadata.get('cover', None)
coverdata = getattr(metadata, 'thumbnail', None)
if coverdata:
coverdata = coverdata[2]
with open('%s_6090.t2b' % os.path.join(path, filename), 'wb') as t2bfile:
t2b.write_t2b(t2bfile, coverdata)
with open('%s_6090.t2b' % os.path.join(path, filename), 'wb') as t2bfile:
t2b.write_t2b(t2bfile, coverdata)
@classmethod
def can_handle(cls, device_info, debug=False):

View File

@ -267,15 +267,16 @@ class DevicePlugin(Plugin):
This method should raise a L{FreeSpaceError} if there is not enough
free space on the device. The text of the FreeSpaceError must contain the
word "card" if C{on_card} is not None otherwise it must contain the word "memory".
@param files: A list of paths and/or file-like objects.
@param names: A list of file names that the books should have
:files: A list of paths and/or file-like objects.
:names: A list of file names that the books should have
once uploaded to the device. len(names) == len(files)
@return: A list of 3-element tuples. The list is meant to be passed
:return: A list of 3-element tuples. The list is meant to be passed
to L{add_books_to_metadata}.
@param metadata: If not None, it is a list of dictionaries. Each dictionary
will have at least the key tags to allow the driver to choose book location
based on tags. len(metadata) == len(files). If your device does not support
hierarchical ebook folders, you can safely ignore this parameter.
:metadata: If not None, it is a list of :class:`MetaInformation` objects.
The idea is to use the metadata to determine where on the device to
put the book. len(metadata) == len(files). Apart from the regular
cover_data, there may also be a thumbnail attribute, which should
be used in preference.
'''
raise NotImplementedError()

View File

@ -11,10 +11,8 @@ Device driver for Ectaco Jetbook firmware >= JL04_v030e
import os
import re
import sys
from itertools import cycle
from calibre.devices.usbms.driver import USBMS
from calibre.utils.filenames import ascii_filename as sanitize
from calibre.ebooks.metadata import string_to_authors
class JETBOOK(USBMS):
@ -50,34 +48,14 @@ class JETBOOK(USBMS):
r'(?P<authors>.+)#(?P<title>.+)'
)
def upload_books(self, files, names, on_card=False, end_session=True,
metadata=None):
base_path = self._sanity_check(on_card, files)
paths = []
names = iter(names)
metadata = iter(metadata)
for i, infile in enumerate(files):
mdata, fname = metadata.next(), names.next()
path = os.path.dirname(self.create_upload_path(base_path, mdata, fname))
author = sanitize(mdata.get('authors','Unknown')).replace(' ', '_')
title = sanitize(mdata.get('title', 'Unknown')).replace(' ', '_')
fileext = os.path.splitext(os.path.basename(fname))[1]
fname = '%s#%s%s' % (author, title, fileext)
filepath = os.path.join(path, fname)
paths.append(filepath)
self.put_file(infile, filepath, replace_file=True)
self.report_progress((i+1) / float(len(files)), _('Transferring books to device...'))
self.report_progress(1.0, _('Transferring books to device...'))
return zip(paths, cycle([on_card]))
def filename_callback(self, fname, mi):
fileext = os.path.splitext(os.path.basename(fname))[1]
title = mi.title if mi.title else 'Unknown'
title = title.replace(' ', '_')
au = mi.format_authors()
if not au:
au = 'Unknown'
return '%s#%s%s' % (au, title, fileext)
@classmethod
def metadata_from_path(cls, path):

View File

@ -53,9 +53,9 @@ class NOOK(USBMS):
import Image, ImageDraw
coverdata = metadata.get('cover', None)
coverdata = getattr(metadata, 'thumbnail', None)
if coverdata:
cover = Image.open(cStringIO.StringIO(coverdata[2]))
cover = Image.open(cStringIO.StringIO(coverdata))
else:
coverdata = open(I('library.png'), 'rb').read()

View File

@ -842,49 +842,65 @@ class Device(DeviceConfig, DevicePlugin):
raise FreeSpaceError(_("There is insufficient free space on the storage card"))
return path
def filename_callback(self, default, mi):
'''
Callback to allow drivers to change the default file name
set by :method:`create_upload_path`.
'''
return default
def create_upload_path(self, path, mdata, fname):
path = os.path.abspath(path)
newpath = path
extra_components = []
if self.SUPPORTS_SUB_DIRS and self.settings().use_subdirs:
if 'tags' in mdata.keys():
for tag in mdata['tags']:
if tag.startswith(_('News')):
extra_components.append('news')
c = sanitize(mdata.get('title', ''))
if c:
extra_components.append(c)
c = sanitize(mdata.get('timestamp', ''))
if c:
extra_components.append(c)
break
elif tag.startswith('/'):
for c in tag.split('/'):
c = sanitize(c)
if not c: continue
extra_components.append(c)
break
special_tag = None
if mdata.tags:
for t in mdata.tags:
if t.startswith(_('News')) or t.startswith('/'):
special_tag = t
break
if not extra_components:
c = sanitize(mdata.get('authors', _('Unknown')))
if c:
extra_components.append(c)
c = sanitize(mdata.get('title', _('Unknown')))
if c:
extra_components.append(c)
newpath = os.path.join(newpath, c)
settings = self.settings()
template = settings.save_template
use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs
fname = sanitize(fname)
extra_components.append(fname)
extra_components = [str(x) for x in extra_components]
if special_tag is None:
from calibre.library.save_to_disk import get_components
extra_components = get_components(template, mdata, fname,
replace_whitespace=True)
else:
tag = special_tag
if tag.startswith(_('News')):
extra_components.append('News')
c = sanitize(mdata.title if mdata.title else '')
c = c.split('[')[0].strip()
if c:
extra_components.append(c)
else:
for c in tag.split('/'):
c = sanitize(c)
if not c: continue
extra_components.append(c)
if not use_subdirs:
extra_components = extra_components[:1]
if not extra_components:
fname = sanitize(self.filename_callback(fname, mdata))
extra_components.append(fname)
extra_components = [str(x) for x in extra_components]
def remove_trailing_periods(x):
ans = x
while ans.endswith('.'):
ans = ans[:-1]
ans = ans[:-1].strip()
if not ans:
ans = 'x'
return ans
extra_components = list(map(remove_trailing_periods, extra_components))
components = shorten_components_to(250 - len(path), extra_components)
filepath = os.path.join(path, *components)

View File

@ -6,12 +6,22 @@ __docformat__ = 'restructuredtext en'
from calibre.utils.config import Config, ConfigProxy
class DeviceConfig(object):
HELP_MESSAGE = _('Configure Device')
EXTRA_CUSTOMIZATION_MESSAGE = None
EXTRA_CUSTOMIZATION_DEFAULT = None
#: If None the default is used
SAVE_TEMPLATE = None
@classmethod
def _default_save_template(cls):
from calibre.library.save_to_disk import config
return cls.SAVE_TEMPLATE if cls.SAVE_TEMPLATE else \
config().parse().send_template
@classmethod
def _config(cls):
klass = cls if isinstance(cls, type) else cls.__class__
@ -22,6 +32,8 @@ class DeviceConfig(object):
help=_('Place files in sub directories if the device supports them'))
c.add_opt('read_metadata', default=True,
help=_('Read metadata from files on device'))
c.add_opt('save_template', default=cls._default_save_template(),
help=_('Template to control how books are saved'))
c.add_opt('extra_customization',
default=cls.EXTRA_CUSTOMIZATION_DEFAULT,
help=_('Extra customization'))
@ -52,6 +64,8 @@ class DeviceConfig(object):
if not ec:
ec = None
proxy['extra_customization'] = ec
st = unicode(config_widget.opt_save_template.text())
proxy['save_template'] = st
@classmethod
def settings(cls):

View File

@ -123,7 +123,8 @@ class USBMS(CLI, Device):
'''
:path: the full path were the associated book is located.
:filename: the name of the book file without the extension.
:metatdata: metadata belonging to the book. metadata.cover[2] for coverdata.
:metatdata: metadata belonging to the book. Use metadata.thumbnail
for cover
'''
pass

View File

@ -73,6 +73,8 @@ def _config():
'only take place when the Enter or Return key is pressed.')
c.add_opt('save_to_disk_template_history', default=[],
help='Previously used Save to Disk templates')
c.add_opt('send_to_device_template_history', default=[],
help='Previously used Send to Device templates')
c.add_opt('main_search_history', default=[],
help='Search history for the main GUI')
c.add_opt('viewer_search_history', default=[],

View File

@ -536,8 +536,7 @@ class DeviceGUI(object):
else:
_auto_ids = []
full_metadata = self.library_view.model().get_metadata(
ids, full_metadata=True, rows_are_ids=True)[-1]
full_metadata = self.library_view.model().metadata_for(ids)
files = [getattr(f, 'name', None) for f in files]
bad, remove_ids, jobnames = [], [], []
@ -707,19 +706,17 @@ class DeviceGUI(object):
if not files:
dynamic.set('news_to_be_synced', set([]))
return
metadata = self.library_view.model().get_metadata(ids,
rows_are_ids=True)
metadata = self.library_view.model().metadata_for(ids)
names = []
for mi in metadata:
prefix = ascii_filename(mi['title'])
prefix = ascii_filename(mi.title)
if not isinstance(prefix, unicode):
prefix = prefix.decode(preferred_encoding, 'replace')
prefix = ascii_filename(prefix)
names.append('%s_%d%s'%(prefix, id,
os.path.splitext(f.name)[1]))
cdata = mi['cover']
if cdata:
mi['cover'] = self.cover_to_thumbnail(cdata)
if mi.cover_data and mi.cover_data[1]:
mi.thumbnail = self.cover_to_thumbnail(mi.cover_data[1])
dynamic.set('news_to_be_synced', set([]))
if config['upload_news_to_device'] and files:
remove = ids if \
@ -751,29 +748,28 @@ class DeviceGUI(object):
else:
_auto_ids = []
metadata = self.library_view.model().get_metadata(ids, True)
metadata = self.library_view.model().metadata_for(ids)
ids = iter(ids)
for mi in metadata:
cdata = mi['cover']
if cdata:
mi['cover'] = self.cover_to_thumbnail(cdata)
metadata = iter(metadata)
if mi.cover_data and mi.cover_data[1]:
mi.thumbnail = self.cover_to_thumbnail(mi.cover_data[1])
imetadata = iter(metadata)
files = [getattr(f, 'name', None) for f in _files]
bad, good, gf, names, remove_ids = [], [], [], [], []
for f in files:
mi = metadata.next()
mi = imetadata.next()
id = ids.next()
if f is None:
bad.append(mi['title'])
bad.append(mi.title)
else:
remove_ids.append(id)
good.append(mi)
gf.append(f)
t = mi['title']
t = mi.title
if not t:
t = _('Unknown')
a = mi['authors']
a = mi.format_authors()
if not a:
a = _('Unknown')
prefix = ascii_filename(t+' - '+a)

View File

@ -46,6 +46,7 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
else:
self.extra_customization_label.setVisible(False)
self.opt_extra_customization.setVisible(False)
self.opt_save_template.setText(settings.save_template)
def up_column(self):

View File

@ -90,7 +90,7 @@
</property>
</widget>
</item>
<item row="3" column="0">
<item row="5" column="0">
<widget class="QLabel" name="extra_customization_label">
<property name="text">
<string>Extra customization</string>
@ -103,9 +103,22 @@
</property>
</widget>
</item>
<item row="4" column="0">
<item row="6" column="0">
<widget class="QLineEdit" name="opt_extra_customization"/>
</item>
<item row="3" column="0">
<widget class="QLabel" name="label">
<property name="text">
<string>Save &amp;template:</string>
</property>
<property name="buddy">
<cstring>opt_save_template</cstring>
</property>
</widget>
</item>
<item row="4" column="0">
<widget class="QLineEdit" name="opt_save_template"/>
</item>
</layout>
</widget>
<resources>

View File

@ -11,9 +11,7 @@ import textwrap
from PyQt4.Qt import QTabWidget
from calibre.gui2.dialogs.config.add_save_ui import Ui_TabWidget
from calibre.library.save_to_disk import config, FORMAT_ARG_DESCS, \
preprocess_template
from calibre.gui2 import error_dialog
from calibre.library.save_to_disk import config
from calibre.utils.config import prefs
from calibre.gui2.widgets import FilenamePattern
@ -22,8 +20,8 @@ class AddSave(QTabWidget, Ui_TabWidget):
def __init__(self, parent=None):
QTabWidget.__init__(self, parent)
self.setupUi(self)
while self.count() > 2:
self.removeTab(2)
while self.count() > 3:
self.removeTab(3)
c = config()
opts = c.parse()
for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf',
@ -41,36 +39,17 @@ class AddSave(QTabWidget, Ui_TabWidget):
g.setToolTip(help)
g.setWhatsThis(help)
help = '\n'.join(textwrap.wrap(c.get_option('template').help, 75))
self.opt_template.initialize('save_to_disk_template_history',
opts.template, help)
variables = sorted(FORMAT_ARG_DESCS.keys())
rows = []
for var in variables:
rows.append(u'<tr><td>%s</td><td>%s</td></tr>'%
(var, FORMAT_ARG_DESCS[var]))
table = u'<table>%s</table>'%(u'\n'.join(rows))
self.template_variables.setText(table)
self.opt_read_metadata_from_filename.setChecked(not prefs['read_file_metadata'])
self.filename_pattern = FilenamePattern(self)
self.metadata_box.layout().insertWidget(0, self.filename_pattern)
self.opt_swap_author_names.setChecked(prefs['swap_author_names'])
help = '\n'.join(textwrap.wrap(c.get_option('template').help, 75))
self.save_template.initialize('save_to_disk', opts.template, help)
self.send_template.initialize('send_to_device', opts.send_template, help)
def validate(self):
tmpl = preprocess_template(self.opt_template.text())
fa = {}
for x in FORMAT_ARG_DESCS.keys():
fa[x]=''
try:
tmpl.format(**fa)
except Exception, err:
error_dialog(self, _('Invalid template'),
'<p>'+_('The template %s is invalid:')%tmpl + \
'<br>'+str(err), show=True)
return False
return True
return self.save_template.validate() and self.send_template.validate()
def save_settings(self):
if not self.validate():
@ -79,12 +58,13 @@ class AddSave(QTabWidget, Ui_TabWidget):
for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf',
'replace_whitespace', 'to_lowercase'):
c.set(x, getattr(self, 'opt_'+x).isChecked())
for x in ('formats', 'template', 'timefmt'):
for x in ('formats', 'timefmt'):
val = unicode(getattr(self, 'opt_'+x).text()).strip()
if x == 'formats' and not val:
val = 'all'
c.set(x, val)
self.opt_template.save_history('save_to_disk_template_history')
self.save_template.save_settings(c, 'template')
self.send_template.save_settings(c, 'send_template')
prefs['read_file_metadata'] = not bool(self.opt_read_metadata_from_filename.isChecked())
pattern = self.filename_pattern.commit()
prefs['filename_pattern'] = pattern

View File

@ -141,38 +141,6 @@
<item row="6" column="1">
<widget class="QLineEdit" name="opt_formats"/>
</item>
<item row="7" column="0" colspan="2">
<widget class="QGroupBox" name="groupBox">
<property name="title">
<string>Save &amp;template</string>
</property>
<layout class="QGridLayout" name="gridLayout_2">
<item row="0" column="0">
<widget class="QLabel" name="label_4">
<property name="text">
<string>By adjusting the template below, you can control what folders the files are saved in and what filenames they are given. You can use the / character to indicate sub-folders. Available metadata variables are described below. If a particular book does not have some metadata, the variable will be replaced by the empty string.</string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="label_5">
<property name="text">
<string>Available variables:</string>
</property>
</widget>
</item>
<item row="3" column="0">
<widget class="QTextBrowser" name="template_variables"/>
</item>
<item row="1" column="0">
<widget class="HistoryBox" name="opt_template"/>
</item>
</layout>
</widget>
</item>
<item row="1" column="1">
<widget class="QCheckBox" name="opt_replace_whitespace">
<property name="text">
@ -187,14 +155,38 @@
</property>
</widget>
</item>
<item row="7" column="0" colspan="2">
<widget class="SaveTemplate" name="save_template" native="true"/>
</item>
</layout>
</widget>
<widget class="QWidget" name="tab_2">
<attribute name="title">
<string>Sending to &amp;device</string>
</attribute>
<layout class="QVBoxLayout" name="verticalLayout_2">
<item>
<widget class="QLabel" name="label_4">
<property name="text">
<string>Here you can control how calibre will save your books when you click the Send to Device button. This setting can be overriden for individual devices by customizing the device interface plugins in Preferences-&gt;Plugins</string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item>
<widget class="SaveTemplate" name="send_template" native="true"/>
</item>
</layout>
</widget>
</widget>
<customwidgets>
<customwidget>
<class>HistoryBox</class>
<extends>QComboBox</extends>
<header>calibre/gui2/dialogs/config/history.h</header>
<class>SaveTemplate</class>
<extends>QWidget</extends>
<header>calibre/gui2/dialogs/config/save_template.h</header>
<container>1</container>
</customwidget>
</customwidgets>
<resources/>

View File

@ -0,0 +1,58 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import with_statement
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
from PyQt4.Qt import QWidget
from calibre.gui2 import error_dialog
from calibre.gui2.dialogs.config.save_template_ui import Ui_Form
from calibre.library.save_to_disk import FORMAT_ARG_DESCS, \
preprocess_template
class SaveTemplate(QWidget, Ui_Form):
def __init__(self, *args):
QWidget.__init__(self, *args)
Ui_Form.__init__(self)
self.setupUi(self)
def initialize(self, name, default, help):
variables = sorted(FORMAT_ARG_DESCS.keys())
rows = []
for var in variables:
rows.append(u'<tr><td>%s</td><td>%s</td></tr>'%
(var, FORMAT_ARG_DESCS[var]))
table = u'<table>%s</table>'%(u'\n'.join(rows))
self.template_variables.setText(table)
self.opt_template.initialize(name+'_template_history',
default, help)
self.option_name = name
def validate(self):
tmpl = preprocess_template(self.opt_template.text())
fa = {}
for x in FORMAT_ARG_DESCS.keys():
fa[x]=''
try:
tmpl.format(**fa)
except Exception, err:
error_dialog(self, _('Invalid template'),
'<p>'+_('The template %s is invalid:')%tmpl + \
'<br>'+str(err), show=True)
return False
return True
def save_settings(self, config, name):
val = unicode(self.opt_template.text())
config.set(name, val)
self.opt_template.save_history(self.option_name+'_template_history')

View File

@ -0,0 +1,60 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>Form</class>
<widget class="QWidget" name="Form">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>400</width>
<height>300</height>
</rect>
</property>
<property name="windowTitle">
<string>Form</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<widget class="QGroupBox" name="groupBox">
<property name="title">
<string>Save &amp;template</string>
</property>
<layout class="QGridLayout" name="gridLayout_2">
<item row="0" column="0">
<widget class="QLabel" name="label_4">
<property name="text">
<string>By adjusting the template below, you can control what folders the files are saved in and what filenames they are given. You can use the / character to indicate sub-folders. Available metadata variables are described below. If a particular book does not have some metadata, the variable will be replaced by the empty string.</string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="label_5">
<property name="text">
<string>Available variables:</string>
</property>
</widget>
</item>
<item row="3" column="0">
<widget class="QTextBrowser" name="template_variables"/>
</item>
<item row="1" column="0">
<widget class="HistoryBox" name="opt_template"/>
</item>
</layout>
</widget>
</item>
</layout>
</widget>
<customwidgets>
<customwidget>
<class>HistoryBox</class>
<extends>QComboBox</extends>
<header>calibre/gui2/dialogs/config/history.h</header>
</customwidget>
</customwidgets>
<resources/>
<connections/>
</ui>

View File

@ -399,6 +399,8 @@ class BooksModel(QAbstractTableModel):
data[_('Author(s)')] = au
return data
def metadata_for(self, ids):
return [self.db.get_metadata(id, index_is_id=True) for id in ids]
def get_metadata(self, rows, rows_are_ids=False, full_metadata=False):
metadata, _full_metadata = [], []

View File

@ -18,6 +18,8 @@ from calibre.constants import preferred_encoding, filesystem_encoding
from calibre import strftime
DEFAULT_TEMPLATE = '{author_sort}/{title}/{title} - {authors}'
DEFAULT_SEND_TEMPLATE = '{author_sort}/{title} - {authors}'
FORMAT_ARG_DESCS = dict(
title=_('The title'),
authors=_('The authors'),
@ -62,6 +64,13 @@ def config(defaults=None):
'Default is "%s" which will save books into a per-author '
'subdirectory with filenames containing title and author. '
'Available controls are: {%s}')%(DEFAULT_TEMPLATE, ', '.join(FORMAT_ARGS)))
x('send_template', default=DEFAULT_SEND_TEMPLATE,
help=_('The template to control the filename and directory structure of files '
'sent to the device. '
'Default is "%s" which will save books into a per-author '
'directory with filenames containing title and author. '
'Available controls are: {%s}')%(DEFAULT_SEND_TEMPLATE, ', '.join(FORMAT_ARGS)))
x('asciiize', default=True,
help=_('Normally, calibre will convert all non English characters into English equivalents '
'for the file names. '