Merge from trunk

This commit is contained in:
Charles Haley 2013-04-14 10:07:03 +02:00
commit 2cf5b9e1da
11 changed files with 130 additions and 177 deletions

View File

@ -1,90 +0,0 @@
import re
from calibre.web.feeds.news import BasicNewsRecipe
class GiveMeSomethingToRead(BasicNewsRecipe):
title = u'Give Me Something To Read'
description = 'Curation / aggregation of articles on diverse topics'
language = 'en'
__author__ = 'barty on mobileread.com forum'
max_articles_per_feed = 100
no_stylesheets = False
timefmt = ' [%a, %d %b, %Y]'
oldest_article = 365
auto_cleanup = True
INDEX = 'http://givemesomethingtoread.com'
CATEGORIES = [
# comment out categories you don't want
# (user friendly name, system name, max number of articles to load)
('The Arts','arts',25),
('Science','science',30),
('Technology','technology',30),
('Politics','politics',20),
('Media','media',30),
('Crime','crime',15),
('Other articles','',10)
]
def parse_index(self):
self.cover_url = 'http://thegretchenshow.files.wordpress.com/2009/12/well-read-cat-small.jpg'
feeds = []
seen_urls = set([])
regex = re.compile( r'http://(www\.)?([^/:]+)', re.I)
for category in self.CATEGORIES:
(cat_name, tag, max_articles) = category
tagurl = '' if tag=='' else '/tagged/'+tag
self.log('Reading category:', cat_name)
articles = []
pageno = 1
while len(articles) < max_articles and pageno < 100:
page = "%s%s/page/%d" % (self.INDEX, tagurl, pageno) if pageno > 1 else self.INDEX + tagurl
pageno += 1
self.log('\tReading page:', page)
try:
soup = self.index_to_soup(page)
except:
break
headers = soup.findAll('h2')
if len(headers) == .0:
break
for header in headers:
atag = header.find('a')
url = atag['href']
# skip promotionals and duplicate
if url.startswith('http://givemesomethingtoread') or url.startswith('/') or url in seen_urls:
continue
seen_urls.add(url)
title = self.tag_to_string(header)
self.log('\tFound article:', title)
#self.log('\t', url)
desc = header.parent.find('blockquote')
desc = self.tag_to_string(desc) if desc else ''
m = regex.match( url)
if m:
desc = "[%s] %s" % (m.group(2), desc)
#self.log('\t', desc)
date = ''
p = header.parent.previousSibling
# navigate up to find h3, which contains the date
while p:
if hasattr(p,'name') and p.name == 'h3':
date = self.tag_to_string(p)
break
p = p.previousSibling
articles.append({'title':title,'url':url,'description':desc,'date':date})
if len(articles) >= max_articles:
break
if articles:
feeds.append((cat_name, articles))
return feeds

View File

@ -11,7 +11,8 @@ class PsychologyToday(BasicNewsRecipe):
language = 'en'
category = 'news'
encoding = 'UTF-8'
keep_only_tags = [dict(attrs={'class':['print-title', 'print-submitted', 'print-content', 'print-footer', 'print-source_url', 'print-links']})]
auto_cleanup = True
#keep_only_tags = [dict(attrs={'class':['print-title', 'print-submitted', 'print-content', 'print-footer', 'print-source_url', 'print-links']})]
no_javascript = True
no_stylesheets = True
@ -31,50 +32,32 @@ class PsychologyToday(BasicNewsRecipe):
self.timefmt = u' [%s]'%date
articles = []
for post in div.findAll('div', attrs={'class':'collections-node-feature-info'}):
for post in div.findAll('div', attrs={'class':'collections-node-feature collection-node-even'}):
title = self.tag_to_string(post.find('h2'))
author_item=post.find('div', attrs={'class':'collection-node-byline'})
author = re.sub(r'.*by\s',"",self.tag_to_string(author_item).strip())
title = title + u' (%s)'%author
article_page= self.index_to_soup('http://www.psychologytoday.com'+post.find('a', href=True)['href'])
print_page=article_page.find('li', attrs={'class':'print_html first'})
url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
url= 'http://www.psychologytoday.com'+post.find('a', href=True)['href']
#print_page=article_page.find('li', attrs={'class':'print_html first'})
#url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
desc = self.tag_to_string(post.find('div', attrs={'class':'collection-node-description'})).strip()
self.log('Found article:', title)
self.log('\t', url)
self.log('\t', desc)
articles.append({'title':title, 'url':url, 'date':'','description':desc})
for post in div.findAll('div', attrs={'class':'collections-node-feature collection-node-odd'}):
title = self.tag_to_string(post.find('h2'))
author_item=post.find('div', attrs={'class':'collection-node-byline'})
author = re.sub(r'.*by\s',"",self.tag_to_string(author_item).strip())
title = title + u' (%s)'%author
url= 'http://www.psychologytoday.com'+post.find('a', href=True)['href']
#print_page=article_page.find('li', attrs={'class':'print_html first'})
#url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
desc = self.tag_to_string(post.find('div', attrs={'class':'collection-node-description'})).strip()
self.log('Found article:', title)
self.log('\t', url)
self.log('\t', desc)
articles.append({'title':title, 'url':url, 'date':'','description':desc})
for post in div.findAll('div', attrs={'class':'collections-node-thumbnail-info'}):
title = self.tag_to_string(post.find('h2'))
author_item=post.find('div', attrs={'class':'collection-node-byline'})
article_page= self.index_to_soup('http://www.psychologytoday.com'+post.find('a', href=True)['href'])
print_page=article_page.find('li', attrs={'class':'print_html first'})
description = post.find('div', attrs={'class':'collection-node-description'})
author = re.sub(r'.*by\s',"",self.tag_to_string(description.nextSibling).strip())
desc = self.tag_to_string(description).strip()
url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
title = title + u' (%s)'%author
self.log('Found article:', title)
self.log('\t', url)
self.log('\t', desc)
articles.append({'title':title, 'url':url, 'date':'','description':desc})
for post in div.findAll('li', attrs={'class':['collection-item-list-odd','collection-item-list-even']}):
title = self.tag_to_string(post.find('h2'))
author_item=post.find('div', attrs={'class':'collection-node-byline'})
author = re.sub(r'.*by\s',"",self.tag_to_string(author_item).strip())
title = title + u' (%s)'%author
article_page= self.index_to_soup('http://www.psychologytoday.com'+post.find('a', href=True)['href'])
print_page=article_page.find('li', attrs={'class':'print_html first'})
if print_page is not None:
url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
desc = self.tag_to_string(post.find('div', attrs={'class':'collection-node-description'})).strip()
self.log('Found article:', title)
self.log('\t', url)
self.log('\t', desc)
articles.append({'title':title, 'url':url, 'date':'','description':desc})
return [('Current Issue', articles)]

View File

@ -50,6 +50,10 @@ class ScienceNewsIssue(BasicNewsRecipe):
dict(name='ul', attrs={'id':'toc'})
]
remove_tags= [ dict(name='a', attrs={'class':'enlarge print-no'}),
dict(name='a', attrs={'rel':'shadowbox'})
]
feeds = [(u"Science News Current Issues", u'http://www.sciencenews.org/view/feed/type/edition/name/issues.rss')]
match_regexps = [
@ -57,6 +61,12 @@ class ScienceNewsIssue(BasicNewsRecipe):
r'www.sciencenews.org/view/generic/id'
]
def image_url_processor(self, baseurl, url):
x = url.split('/')
if x[4] == u'scale':
url = u'http://www.sciencenews.org/view/download/id/' + x[6] + u'/name/' + x[-1]
return url
def get_cover_url(self):
cover_url = None
index = 'http://www.sciencenews.org/view/home'
@ -64,7 +74,6 @@ class ScienceNewsIssue(BasicNewsRecipe):
link_item = soup.find(name = 'img',alt = "issue")
if link_item:
cover_url = 'http://www.sciencenews.org' + link_item['src'] + '.jpg'
return cover_url
def preprocess_html(self, soup):

View File

@ -0,0 +1,11 @@
from calibre.web.feeds.news import BasicNewsRecipe
class AdvancedUserRecipe1365777047(BasicNewsRecipe):
title = u'The Feature'
__author__ = 'Jose Pinto'
language = 'en'
oldest_article = 30
max_articles_per_feed = 100
auto_cleanup = True
use_embedded_content = False
feeds = [(u'Latest', u'http://thefeature.net/rss/links')]

View File

@ -448,8 +448,15 @@
<xsl:template match = "rtf:field[@type='hyperlink']">
<xsl:element name ="a">
<xsl:attribute name = "href">
<xsl:if test = "not(contains(@link, '/'))">#</xsl:if>
<xsl:value-of select = "@link"/>
<xsl:choose>
<xsl:when test="@argument">
<xsl:value-of select="@argument"/>
</xsl:when>
<xsl:otherwise>
<xsl:if test = "not(contains(@link, '/'))">#</xsl:if>
<xsl:value-of select = "@link"/>
</xsl:otherwise>
</xsl:choose>
</xsl:attribute>
<xsl:apply-templates/>
</xsl:element>

View File

@ -175,7 +175,7 @@ def gui_polish(data):
if not data.pop('metadata'):
data.pop('opf')
if not data.pop('do_cover'):
data.pop('cover')
data.pop('cover', None)
file_map = {x:x for x in files}
opts = ALL_OPTS.copy()
opts.update(data)

View File

@ -716,10 +716,11 @@ def choose_save_file(window, name, title, filters=[], all_files=True):
ans = ans[0]
return ans
def choose_images(window, name, title, select_only_single_file=True):
def choose_images(window, name, title, select_only_single_file=True,
formats=('png', 'gif', 'jpg', 'jpeg', 'svg')):
mode = QFileDialog.ExistingFile if select_only_single_file else QFileDialog.ExistingFiles
fd = FileDialog(title=title, name=name,
filters=[('Images', ['png', 'gif', 'jpeg', 'jpg', 'svg'])],
filters=[('Images', list(formats))],
parent=window, add_all_files_filter=False, mode=mode,
)
fd.setParent(None)

View File

@ -17,14 +17,14 @@ from PyQt4.Qt import (QDialog, QGridLayout, QIcon, QCheckBox, QLabel, QFrame,
QSizePolicy, QTimer, QModelIndex, QTextEdit,
QInputDialog, QMenu)
from calibre.gui2 import error_dialog, Dispatcher, gprefs
from calibre.gui2 import error_dialog, Dispatcher, gprefs, question_dialog
from calibre.gui2.actions import InterfaceAction
from calibre.gui2.convert.metadata import create_opf_file
from calibre.gui2.dialogs.progress import ProgressDialog
from calibre.ptempfile import PersistentTemporaryDirectory
from calibre.utils.config_base import tweaks
class Polish(QDialog): # {{{
class Polish(QDialog): # {{{
def __init__(self, db, book_id_map, parent=None):
from calibre.ebooks.oeb.polish.main import HELP
@ -58,7 +58,7 @@ class Polish(QDialog): # {{{
' formats are not capable of supporting all the'
' metadata in calibre.</p><p>There is a separate option to'
' update the cover.</p>'),
'do_cover': _('<p>Update the covers in the ebook files to match the'
'do_cover': _('<p>Update the covers in the ebook files to match the'
' current cover in the calibre library.</p>'
'<p>If the ebook file does not have'
' an identifiable cover, a new cover is inserted.</p>'
@ -204,6 +204,15 @@ class Polish(QDialog): # {{{
ac[action] = saved_prefs[action] = bool(getattr(self, 'opt_'+action).isChecked())
if ac[action]:
something = True
if ac['jacket'] and not ac['metadata']:
if not question_dialog(self, _('Must update metadata'),
_('You have selected the option to add metadata as '
'a "book jacket". For this option to work, you '
'must also select the option to update metadata in'
' the book files. Do you want to select it?')):
return
ac['metadata'] = saved_prefs['metadata'] = True
self.opt_metadata.setChecked(True)
if not something:
return error_dialog(self, _('No actions selected'),
_('You must select at least one action, or click Cancel.'),
@ -275,7 +284,7 @@ class Polish(QDialog): # {{{
self.jobs.append((desc, data, book_id, base, is_orig))
# }}}
class Report(QDialog): # {{{
class Report(QDialog): # {{{
def __init__(self, parent):
QDialog.__init__(self, parent)
@ -427,7 +436,7 @@ class PolishAction(InterfaceAction):
supported = set(SUPPORTED)
for x in SUPPORTED:
supported.add('ORIGINAL_'+x)
ans = [(x, set( (db.formats(x, index_is_id=True) or '').split(',') )
ans = [(x, set((db.formats(x, index_is_id=True) or '').split(','))
.intersection(supported)) for x in book_ids]
ans = [x for x in ans if x[1]]
if not ans:
@ -476,8 +485,7 @@ class PolishAction(InterfaceAction):
db.save_original_format(book_id, fmt, notify=False)
with open(path, 'rb') as f:
db.add_format(book_id, fmt, f, index_is_id=True)
self.gui.status_bar.show_message(job.description + \
(' completed'), 2000)
self.gui.status_bar.show_message(job.description + (' completed'), 2000)
try:
shutil.rmtree(base)
parent = os.path.dirname(base)

View File

@ -927,9 +927,9 @@ class Cover(ImageView): # {{{
return sz
def select_cover(self, *args):
files = choose_images(self, 'change cover dialog',
_('Choose cover for ') +
self.dialog.title.current_val)
files = choose_images(
self, 'change cover dialog', _('Choose cover for ') + self.dialog.title.current_val,
formats=('png', 'gif', 'jpg', 'jpeg'))
if not files:
return
_file = files[0]

View File

@ -11,14 +11,14 @@ from base64 import b64encode
from PyQt4.Qt import (QWidget, QGridLayout, QListWidget, QSize, Qt, QUrl,
pyqtSlot, pyqtSignal, QVBoxLayout, QFrame, QLabel,
QLineEdit, QTimer, QPushButton, QIcon)
QLineEdit, QTimer, QPushButton, QIcon, QSplitter)
from PyQt4.QtWebKit import QWebView, QWebPage, QWebElement
from calibre.ebooks.oeb.display.webview import load_html
from calibre.gui2 import error_dialog, question_dialog
from calibre.gui2 import error_dialog, question_dialog, gprefs
from calibre.utils.logging import default_log
class Page(QWebPage): # {{{
class Page(QWebPage): # {{{
elem_clicked = pyqtSignal(object, object, object, object)
@ -67,7 +67,7 @@ class Page(QWebPage): # {{{
self.evaljs(self.js)
# }}}
class WebView(QWebView): # {{{
class WebView(QWebView): # {{{
elem_clicked = pyqtSignal(object, object, object, object)
@ -106,38 +106,46 @@ class ItemEdit(QWidget):
def __init__(self, parent):
QWidget.__init__(self, parent)
self.l = l = QGridLayout()
self.setLayout(l)
self.setLayout(QVBoxLayout())
self.la = la = QLabel('<b>'+_(
'Select a destination for the Table of Contents entry'))
l.addWidget(la, 0, 0, 1, 3)
self.layout().addWidget(la)
self.splitter = sp = QSplitter(self)
self.layout().addWidget(sp)
self.layout().setStretch(1, 10)
sp.setOpaqueResize(False)
sp.setChildrenCollapsible(False)
self.dest_list = dl = QListWidget(self)
dl.setMinimumWidth(250)
dl.currentItemChanged.connect(self.current_changed)
l.addWidget(dl, 1, 0, 2, 1)
sp.addWidget(dl)
w = self.w = QWidget(self)
l = w.l = QGridLayout()
w.setLayout(l)
self.view = WebView(self)
self.view.elem_clicked.connect(self.elem_clicked)
l.addWidget(self.view, 1, 1, 1, 3)
l.addWidget(self.view, 0, 0, 1, 3)
sp.addWidget(w)
self.search_text = s = QLineEdit(self)
s.setPlaceholderText(_('Search for text...'))
l.addWidget(s, 1, 0)
self.ns_button = b = QPushButton(QIcon(I('arrow-down.png')), _('Find &next'), self)
b.clicked.connect(self.find_next)
l.addWidget(b, 1, 1)
self.ps_button = b = QPushButton(QIcon(I('arrow-up.png')), _('Find &previous'), self)
l.addWidget(b, 1, 2)
b.clicked.connect(self.find_previous)
self.f = f = QFrame()
f.setFrameShape(f.StyledPanel)
f.setMinimumWidth(250)
l.addWidget(f, 1, 4, 2, 1)
self.search_text = s = QLineEdit(self)
s.setPlaceholderText(_('Search for text...'))
l.addWidget(s, 2, 1, 1, 1)
self.ns_button = b = QPushButton(QIcon(I('arrow-down.png')), _('Find &next'), self)
b.clicked.connect(self.find_next)
l.addWidget(b, 2, 2, 1, 1)
self.ps_button = b = QPushButton(QIcon(I('arrow-up.png')), _('Find &previous'), self)
l.addWidget(b, 2, 3, 1, 1)
b.clicked.connect(self.find_previous)
l.setRowStretch(1, 10)
l = f.l = QVBoxLayout()
f.setLayout(l)
sp.addWidget(f)
f.la = la = QLabel('<p>'+_(
'Here you can choose a destination for the Table of Contents\' entry'
@ -167,6 +175,10 @@ class ItemEdit(QWidget):
l.addStretch()
state = gprefs.get('toc_edit_splitter_state', None)
if state is not None:
sp.restoreState(state)
def keyPressEvent(self, ev):
if ev.key() in (Qt.Key_Return, Qt.Key_Enter) and self.search_text.hasFocus():
# Prevent pressing enter in the search box from triggering the dialog's accept() method
@ -236,6 +248,7 @@ class ItemEdit(QWidget):
if item is not None:
if where is None:
self.name.setText(item.data(0, Qt.DisplayRole).toString())
self.name.setCursorPosition(0)
toc = item.data(0, Qt.UserRole).toPyObject()
if toc.dest:
for i in xrange(self.dest_list.count()):
@ -272,7 +285,6 @@ class ItemEdit(QWidget):
loctext = _('Approximately %d%% from the top')%frac
return loctext
def elem_clicked(self, tag, frac, elem_id, loc):
self.current_frag = elem_id or loc
base = _('Location: A &lt;%s&gt; tag inside the file')%tag

View File

@ -14,7 +14,7 @@ from functools import partial
from PyQt4.Qt import (QPushButton, QFrame, QVariant, QMenu, QInputDialog,
QDialog, QVBoxLayout, QDialogButtonBox, QSize, QStackedWidget, QWidget,
QLabel, Qt, pyqtSignal, QIcon, QTreeWidget, QGridLayout, QTreeWidgetItem,
QToolButton, QItemSelectionModel, QCursor)
QToolButton, QItemSelectionModel, QCursor, QKeySequence)
from calibre.ebooks.oeb.polish.container import get_container, AZW3Container
from calibre.ebooks.oeb.polish.toc import (
@ -27,7 +27,7 @@ from calibre.utils.logging import GUILog
ICON_SIZE = 24
class XPathDialog(QDialog): # {{{
class XPathDialog(QDialog): # {{{
def __init__(self, parent):
QDialog.__init__(self, parent)
@ -118,7 +118,7 @@ class XPathDialog(QDialog): # {{{
return [w.xpath for w in self.widgets if w.xpath.strip()]
# }}}
class ItemView(QFrame): # {{{
class ItemView(QFrame): # {{{
add_new_item = pyqtSignal(object, object)
delete_item = pyqtSignal()
@ -207,7 +207,6 @@ class ItemView(QFrame): # {{{
)))
l.addWidget(b)
l.addStretch()
self.w1 = la = QLabel(_('<b>WARNING:</b> calibre only supports the '
'creation of linear ToCs in AZW3 files. In a '
@ -349,7 +348,9 @@ class ItemView(QFrame): # {{{
# }}}
class TreeWidget(QTreeWidget): # {{{
class TreeWidget(QTreeWidget): # {{{
edit_item = pyqtSignal()
def __init__(self, parent):
QTreeWidget.__init__(self, parent)
@ -510,25 +511,30 @@ class TreeWidget(QTreeWidget): # {{{
def show_context_menu(self, point):
item = self.currentItem()
def key(k):
sc = unicode(QKeySequence(k | Qt.CTRL).toString(QKeySequence.NativeText))
return ' [%s]'%sc
if item is not None:
m = QMenu()
ci = unicode(item.data(0, Qt.DisplayRole).toString())
p = item.parent() or self.invisibleRootItem()
idx = p.indexOfChild(item)
if idx > 0:
m.addAction(QIcon(I('arrow-up.png')), _('Move "%s" up')%ci, self.move_up)
m.addAction(QIcon(I('arrow-up.png')), (_('Move "%s" up')%ci)+key(Qt.Key_Up), self.move_up)
if idx + 1 < p.childCount():
m.addAction(QIcon(I('arrow-down.png')), _('Move "%s" down')%ci, self.move_down)
m.addAction(QIcon(I('arrow-down.png')), (_('Move "%s" down')%ci)+key(Qt.Key_Down), self.move_down)
m.addAction(QIcon(I('trash.png')), _('Remove all selected items'), self.del_items)
if item.parent() is not None:
m.addAction(QIcon(I('back.png')), _('Unindent "%s"')%ci, self.move_left)
m.addAction(QIcon(I('back.png')), (_('Unindent "%s"')%ci)+key(Qt.Key_Left), self.move_left)
if idx > 0:
m.addAction(QIcon(I('forward.png')), _('Indent "%s"')%ci, self.move_right)
m.addAction(QIcon(I('forward.png')), (_('Indent "%s"')%ci)+key(Qt.Key_Right), self.move_right)
m.addAction(QIcon(I('edit_input.png')), _('Change the location this entry points to'), self.edit_item)
m.addAction(_('Change all selected items to title case'), self.title_case)
m.exec_(QCursor.pos())
# }}}
class TOCView(QWidget): # {{{
class TOCView(QWidget): # {{{
add_new_item = pyqtSignal(object, object)
@ -537,6 +543,7 @@ class TOCView(QWidget): # {{{
l = self.l = QGridLayout()
self.setLayout(l)
self.tocw = t = TreeWidget(self)
self.tocw.edit_item.connect(self.edit_item)
l.addWidget(t, 0, 0, 7, 3)
self.up_button = b = QToolButton(self)
b.setIcon(QIcon(I('arrow-up.png')))
@ -595,6 +602,9 @@ class TOCView(QWidget): # {{{
l.setColumnStretch(2, 10)
def edit_item(self):
self.item_view.edit_item()
def event(self, e):
if e.type() == e.StatusTip:
txt = unicode(e.tip()) or self.default_msg
@ -742,12 +752,12 @@ class TOCView(QWidget): # {{{
else:
parent = item.parent() or self.root
idx = parent.indexOfChild(item)
if where == 'after': idx += 1
if where == 'after':
idx += 1
c = self.create_item(parent, child, idx=idx)
self.tocw.setCurrentItem(c, 0, QItemSelectionModel.ClearAndSelect)
self.tocw.scrollToItem(c)
def create_toc(self):
root = TOC()
@ -799,7 +809,7 @@ class TOCView(QWidget): # {{{
# }}}
class TOCEditor(QDialog): # {{{
class TOCEditor(QDialog): # {{{
explode_done = pyqtSignal(object)
writing_done = pyqtSignal(object)
@ -857,6 +867,7 @@ class TOCEditor(QDialog): # {{{
def accept(self):
if self.stacks.currentIndex() == 2:
self.toc_view.update_item(*self.item_edit.result)
gprefs['toc_edit_splitter_state'] = bytearray(self.item_edit.splitter.saveState())
self.stacks.setCurrentIndex(1)
elif self.stacks.currentIndex() == 1:
self.working = False
@ -883,6 +894,7 @@ class TOCEditor(QDialog): # {{{
if not self.bb.isEnabled():
return
if self.stacks.currentIndex() == 2:
gprefs['toc_edit_splitter_state'] = bytearray(self.item_edit.splitter.saveState())
self.stacks.setCurrentIndex(1)
else:
self.working = False
@ -938,5 +950,5 @@ if __name__ == '__main__':
d = TOCEditor(sys.argv[-1])
d.start()
d.exec_()
del d # Needed to prevent sigsegv in exit cleanup
del d # Needed to prevent sigsegv in exit cleanup