Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge from trunk

Browse Source
This commit is contained in:
Charles Haley 2010-12-02 19:08:12 +00:00
commit 71ca6e046c
8 changed files with 305 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
resources/recipes/nin.recipe
View File

@ -8,8 +8,8 @@ www.nin.co.rs
import re
from calibre import strftime
from calibre.web.feeds.news import BasicNewsRecipe
from contextlib import nested, closing
from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
from contextlib import closing
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre import entity_to_unicode
class Nin(BasicNewsRecipe):

63
src/calibre/ebooks/txt/markdownml.py Normal file
View File

@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
__license__ = 'GPL 3'
__copyright__ = '2009, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
'''
Transform OEB content into Markdown formatted plain text
'''
import re
from lxml import etree
from calibre.utils.html2text import html2text
class MarkdownMLizer(object):
def __init__(self, log):
self.log = log
def extract_content(self, oeb_book, opts):
self.log.info('Converting XHTML to Markdown formatted TXT...')
self.oeb_book = oeb_book
self.opts = opts
return self.mlize_spine()
def mlize_spine(self):
output = [u'']
for item in self.oeb_book.spine:
self.log.debug('Converting %s to Markdown formatted TXT...' % item.href)
html = unicode(etree.tostring(item.data, encoding=unicode))
if not self.opts.keep_links:
html = re.sub(r'<\s*a[^>]*>', '', html)
html = re.sub(r'<\s*/\s*a\s*>', '', html)
if not self.opts.keep_image_references:
html = re.sub(r'<\s*img[^>]*>', '', html)
html = re.sub(r'<\s*img\s*>', '', html)
text = html2text(html)
# Ensure the section ends with at least two new line characters.
# This is to prevent the last paragraph from a section being
# combined into the fist paragraph of the next.
end_chars = text[-4:]
# Convert all newlines to \n
end_chars = end_chars.replace('\r\n', '\n')
end_chars = end_chars.replace('\r', '\n')
end_chars = end_chars[-2:]
if not end_chars[1] == '\n':
text += '\n\n'
if end_chars[1] == '\n' and not end_chars[0] == '\n':
text += '\n'
output += text
output = u''.join(output)
return output

18
src/calibre/ebooks/txt/output.py
View File

@ -8,6 +8,7 @@ import os
from calibre.customize.conversion import OutputFormatPlugin, \
OptionRecommendation
from calibre.ebooks.txt.markdownml import MarkdownMLizer
from calibre.ebooks.txt.txtml import TXTMLizer
from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
@ -44,10 +45,27 @@ class TXTOutput(OutputFormatPlugin):
recommended_value=False, level=OptionRecommendation.LOW,
help=_('Force splitting on the max-line-length value when no space '
'is present. Also allows max-line-length to be below the minimum')),
OptionRecommendation(name='markdown_format',
recommended_value=False, level=OptionRecommendation.LOW,
help=_('Produce Markdown formatted text.')),
OptionRecommendation(name='keep_links',
recommended_value=False, level=OptionRecommendation.LOW,
help=_('Do not remove links within the document. This is only ' \
'useful when paired with the markdown-format option because' \
'links are always removed with plain text output.')),
OptionRecommendation(name='keep_image_references',
recommended_value=False, level=OptionRecommendation.LOW,
help=_('Do not remove image references within the document. This is only ' \
'useful when paired with the markdown-format option because' \
'image references are always removed with plain text output.')),
])
def convert(self, oeb_book, output_path, input_plugin, opts, log):
if opts.markdown_format:
writer = MarkdownMLizer(log)
else:
writer = TXTMLizer(log)
txt = writer.extract_content(oeb_book, opts)
log.debug('\tReplacing newlines with selected type...')

1
src/calibre/ebooks/txt/txtml.py
View File

@ -35,6 +35,7 @@ BLOCK_STYLES = [
SPACE_TAGS = [
'td',
'br',
]
class TXTMLizer(object):

41
src/calibre/gui2/actions/edit_metadata.py
View File

@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
import os
from functools import partial
from PyQt4.Qt import Qt, QMenu
from PyQt4.Qt import Qt, QMenu, QModelIndex
from calibre.gui2 import error_dialog, config
from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
@ -126,20 +126,35 @@ class EditMetadataAction(InterfaceAction):
if bulk or (bulk is None and len(rows) > 1):
return self.edit_bulk_metadata(checked)
def accepted(id):
self.gui.library_view.model().refresh_ids([id])
row_list = [r.row() for r in rows]
current_row = 0
changed = set([])
db = self.gui.library_view.model().db
for row in rows:
self.gui.iactions['View'].metadata_view_id = self.gui.library_view.model().db.id(row.row())
d = MetadataSingleDialog(self.gui, row.row(),
self.gui.library_view.model().db,
accepted_callback=accepted,
cancel_all=rows.index(row) < len(rows)-1)
d.view_format.connect(self.gui.iactions['View'].metadata_view_format)
d.exec_()
if d.cancel_all:
if len(row_list) == 1:
cr = row_list[0]
row_list = \
list(range(self.gui.library_view.model().rowCount(QModelIndex())))
current_row = row_list.index(cr)
while True:
prev = next_ = None
if current_row > 0:
prev = db.title(row_list[current_row-1])
if current_row < len(row_list) - 1:
next_ = db.title(row_list[current_row+1])
d = MetadataSingleDialog(self.gui, row_list[current_row], db,
prev=prev, next_=next_)
if d.exec_() != d.Accepted:
break
if rows:
changed.add(d.id)
if d.row_delta == 0:
break
current_row += d.row_delta
if changed:
self.gui.library_view.model().refresh_ids(list(changed))
current = self.gui.library_view.currentIndex()
m = self.gui.library_view.model()
if self.gui.cover_flow:

2
src/calibre/gui2/convert/txt_output.py
View File

@ -21,7 +21,7 @@ class PluginWidget(Widget, Ui_Form):
def __init__(self, parent, get_option, get_help, db=None, book_id=None):
Widget.__init__(self, parent,
['newline', 'max_line_length', 'force_max_line_length',
'inline_toc'])
'inline_toc', 'markdown_format', 'keep_links', 'keep_image_references'])
self.db, self.book_id = db, book_id
self.initialize_options(get_option, get_help, db, book_id)

25
src/calibre/gui2/convert/txt_output.ui
View File

@ -6,7 +6,7 @@
<rect>
<x>0</x>
<y>0</y>
<width>400</width>
<width>477</width>
<height>300</height>
</rect>
</property>
@ -27,7 +27,7 @@
<item row="0" column="1">
<widget class="QComboBox" name="opt_newline"/>
</item>
<item row="4" column="0">
<item row="7" column="0">
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
@ -67,6 +67,27 @@
</property>
</widget>
</item>
<item row="4" column="0">
<widget class="QCheckBox" name="opt_markdown_format">
<property name="text">
<string>Apply Markdown formatting to text</string>
</property>
</widget>
</item>
<item row="5" column="0">
<widget class="QCheckBox" name="opt_keep_links">
<property name="text">
<string>Do not remove links (&lt;a&gt; tags) before processing</string>
</property>
</widget>
</item>
<item row="6" column="0">
<widget class="QCheckBox" name="opt_keep_image_references">
<property name="text">
<string>Do not remove image references before processing</string>
</property>
</widget>
</item>
</layout>
</widget>
<resources/>

288
src/calibre/gui2/dialogs/metadata_single.py
View File

@ -7,9 +7,11 @@ add/remove formats
'''
import os, re, time, traceback, textwrap
from functools import partial
from PyQt4.Qt import SIGNAL, QObject, Qt, QTimer, QThread, QDate, \
QPixmap, QListWidgetItem, QDialog, pyqtSignal, QMessageBox
QPixmap, QListWidgetItem, QDialog, pyqtSignal, QMessageBox, QIcon, \
QPushButton
from calibre.gui2 import error_dialog, file_icon_provider, dynamic, \
choose_files, choose_images, ResizableDialog, \
@ -31,7 +33,7 @@ from calibre.gui2.preferences.social import SocialMetadata
from calibre.gui2.custom_column_widgets import populate_metadata_page
from calibre import strftime
class CoverFetcher(QThread):
class CoverFetcher(QThread): # {{{
def __init__(self, username, password, isbn, timeout, title, author):
self.username = username.strip() if username else username
@ -74,9 +76,9 @@ class CoverFetcher(QThread):
self.traceback = traceback.format_exc()
print self.traceback
# }}}
class Format(QListWidgetItem):
class Format(QListWidgetItem): # {{{
def __init__(self, parent, ext, size, path=None, timestamp=None):
self.path = path
@ -92,12 +94,60 @@ class Format(QListWidgetItem):
self.setToolTip(text)
self.setStatusTip(text)
# }}}
class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
COVER_FETCH_TIMEOUT = 240 # seconds
view_format = pyqtSignal(object)
# Cover processing {{{
def set_cover(self):
mi, ext = self.get_selected_format_metadata()
if mi is None:
return
cdata = None
if mi.cover and os.access(mi.cover, os.R_OK):
cdata = open(mi.cover).read()
elif mi.cover_data[1] is not None:
cdata = mi.cover_data[1]
if cdata is None:
error_dialog(self, _('Could not read cover'),
_('Could not read cover from %s format')%ext).exec_()
return
pix = QPixmap()
pix.loadFromData(cdata)
if pix.isNull():
error_dialog(self, _('Could not read cover'),
_('The cover in the %s format is invalid')%ext).exec_()
return
self.cover.setPixmap(pix)
self.update_cover_tooltip()
self.cover_changed = True
self.cpixmap = pix
self.cover_data = cdata
def trim_cover(self, *args):
from calibre.utils.magick import Image
cdata = self.cover_data
if not cdata:
return
im = Image()
im.load(cdata)
im.trim(10)
cdata = im.export('png')
pix = QPixmap()
pix.loadFromData(cdata)
self.cover.setPixmap(pix)
self.update_cover_tooltip()
self.cover_changed = True
self.cpixmap = pix
self.cover_data = cdata
def update_cover_tooltip(self):
p = self.cover.pixmap()
self.cover.setToolTip(_('Cover size: %dx%d pixels') %
@ -173,6 +223,76 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
self.cover_changed = True
self.cpixmap = pix
def cover_dropped(self, cover_data):
self.cover_changed = True
self.cover_data = cover_data
self.update_cover_tooltip()
def fetch_cover(self):
isbn = re.sub(r'[^0-9a-zA-Z]', '', unicode(self.isbn.text())).strip()
self.fetch_cover_button.setEnabled(False)
self.setCursor(Qt.WaitCursor)
title, author = map(unicode, (self.title.text(), self.authors.text()))
self.cover_fetcher = CoverFetcher(None, None, isbn,
self.timeout, title, author)
self.cover_fetcher.start()
self._hangcheck = QTimer(self)
self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck)
self.cf_start_time = time.time()
self.pi.start(_('Downloading cover...'))
self._hangcheck.start(100)
def hangcheck(self):
if not self.cover_fetcher.isFinished() and \
time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT:
return
self._hangcheck.stop()
try:
if self.cover_fetcher.isRunning():
self.cover_fetcher.terminate()
error_dialog(self, _('Cannot fetch cover'),
_('<b>Could not fetch cover.</b><br/>')+
_('The download timed out.')).exec_()
return
if self.cover_fetcher.needs_isbn:
error_dialog(self, _('Cannot fetch cover'),
_('Could not find cover for this book. Try '
'specifying the ISBN first.')).exec_()
return
if self.cover_fetcher.exception is not None:
err = self.cover_fetcher.exception
error_dialog(self, _('Cannot fetch cover'),
_('<b>Could not fetch cover.</b><br/>')+unicode(err)).exec_()
return
if self.cover_fetcher.errors and self.cover_fetcher.cover_data is None:
details = u'\n\n'.join([e[-1] + ': ' + e[1] for e in self.cover_fetcher.errors])
error_dialog(self, _('Cannot fetch cover'),
_('<b>Could not fetch cover.</b><br/>') +
_('For the error message from each cover source, '
'click Show details below.'), det_msg=details, show=True)
return
pix = QPixmap()
pix.loadFromData(self.cover_fetcher.cover_data)
if pix.isNull():
error_dialog(self, _('Bad cover'),
_('The cover is not a valid picture')).exec_()
else:
self.cover.setPixmap(pix)
self.update_cover_tooltip()
self.cover_changed = True
self.cpixmap = pix
self.cover_data = self.cover_fetcher.cover_data
finally:
self.fetch_cover_button.setEnabled(True)
self.unsetCursor()
self.pi.stop()
# }}}
# Formats processing {{{
def add_format(self, x):
files = choose_files(self, 'add formats dialog',
_("Choose formats for ") + unicode((self.title.text())),
@ -285,50 +405,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
self.comments.setPlainText(mi.comments)
def set_cover(self):
mi, ext = self.get_selected_format_metadata()
if mi is None:
return
cdata = None
if mi.cover and os.access(mi.cover, os.R_OK):
cdata = open(mi.cover).read()
elif mi.cover_data[1] is not None:
cdata = mi.cover_data[1]
if cdata is None:
error_dialog(self, _('Could not read cover'),
_('Could not read cover from %s format')%ext).exec_()
return
pix = QPixmap()
pix.loadFromData(cdata)
if pix.isNull():
error_dialog(self, _('Could not read cover'),
_('The cover in the %s format is invalid')%ext).exec_()
return
self.cover.setPixmap(pix)
self.update_cover_tooltip()
self.cover_changed = True
self.cpixmap = pix
self.cover_data = cdata
def trim_cover(self, *args):
from calibre.utils.magick import Image
cdata = self.cover_data
if not cdata:
return
im = Image()
im.load(cdata)
im.trim(10)
cdata = im.export('png')
pix = QPixmap()
pix.loadFromData(cdata)
self.cover.setPixmap(pix)
self.update_cover_tooltip()
self.cover_changed = True
self.cpixmap = pix
self.cover_data = cdata
def sync_formats(self):
old_extensions, new_extensions, paths = set(), set(), {}
for row in range(self.formats.count()):
@ -349,11 +425,14 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
if ext not in extensions:
self.db.remove_format(self.row, ext, notify=False)
def do_cancel_all(self):
self.cancel_all = True
self.reject()
def show_format(self, item, *args):
fmt = item.ext
self.view_format.emit(fmt)
def __init__(self, window, row, db, accepted_callback=None, cancel_all=False):
# }}}
def __init__(self, window, row, db, prev=None,
next_=None):
ResizableDialog.__init__(self, window)
self.bc_box.layout().setAlignment(self.cover, Qt.AlignCenter|Qt.AlignHCenter)
self.cancel_all = False
@ -365,16 +444,27 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
_(' The red color indicates that the current '
'author sort does not match the current author'))
if cancel_all:
self.__abort_button = self.button_box.addButton(self.button_box.Abort)
self.__abort_button.setToolTip(_('Abort the editing of all remaining books'))
self.connect(self.__abort_button, SIGNAL('clicked()'),
self.do_cancel_all)
self.row_delta = 0
if prev:
self.prev_button = QPushButton(QIcon(I('back.png')), _('Previous'),
self)
self.button_box.addButton(self.prev_button, self.button_box.ActionRole)
tip = _('Save changes and edit the metadata of %s')%prev
self.prev_button.setToolTip(tip)
self.prev_button.clicked.connect(partial(self.next_triggered,
-1))
if next_:
self.next_button = QPushButton(QIcon(I('forward.png')), _('Next'),
self)
self.button_box.addButton(self.next_button, self.button_box.ActionRole)
tip = _('Save changes and edit the metadata of %s')%next_
self.next_button.setToolTip(tip)
self.next_button.clicked.connect(partial(self.next_triggered, 1))
self.splitter.setStretchFactor(100, 1)
self.read_state()
self.db = db
self.pi = ProgressIndicator(self)
self.accepted_callback = accepted_callback
self.id = db.id(row)
self.row = row
self.cover_data = None
@ -423,6 +513,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
self.connect(self.reset_cover, SIGNAL('clicked()'), self.do_reset_cover)
self.connect(self.swap_button, SIGNAL('clicked()'), self.swap_title_author)
self.timeout = float(prefs['network_timeout'])
self.title.setText(db.title(row))
isbn = db.isbn(self.id, index_is_id=True)
if not isbn:
@ -491,6 +583,9 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
self.create_custom_column_editors()
self.generate_cover_button.clicked.connect(self.generate_cover)
self.original_author = unicode(self.authors.text()).strip()
self.original_title = unicode(self.title.text()).strip()
def create_custom_column_editors(self):
w = self.central_widget.widget(1)
layout = w.layout()
@ -543,10 +638,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
self.isbn.setStyleSheet('QLineEdit { background-color: rgba(255,0,0,20%) }')
self.isbn.setToolTip(_('This ISBN number is invalid'))
def show_format(self, item, *args):
fmt = item.ext
self.view_format.emit(fmt)
def deduce_author_sort(self):
au = unicode(self.authors.text())
au = re.sub(r'\s+et al\.$', '', au)
@ -559,9 +650,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
self.authors.setText(title)
self.author_sort.setText('')
def cover_dropped(self, cover_data):
self.cover_changed = True
self.cover_data = cover_data
def initialize_combos(self):
self.initalize_authors()
@ -637,67 +725,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
self.tags.setText(tag_string)
self.tags.update_tags_cache(self.db.all_tags())
def fetch_cover(self):
isbn = re.sub(r'[^0-9a-zA-Z]', '', unicode(self.isbn.text())).strip()
self.fetch_cover_button.setEnabled(False)
self.setCursor(Qt.WaitCursor)
title, author = map(unicode, (self.title.text(), self.authors.text()))
self.cover_fetcher = CoverFetcher(None, None, isbn,
self.timeout, title, author)
self.cover_fetcher.start()
self._hangcheck = QTimer(self)
self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck)
self.cf_start_time = time.time()
self.pi.start(_('Downloading cover...'))
self._hangcheck.start(100)
def hangcheck(self):
if not self.cover_fetcher.isFinished() and \
time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT:
return
self._hangcheck.stop()
try:
if self.cover_fetcher.isRunning():
self.cover_fetcher.terminate()
error_dialog(self, _('Cannot fetch cover'),
_('<b>Could not fetch cover.</b><br/>')+
_('The download timed out.')).exec_()
return
if self.cover_fetcher.needs_isbn:
error_dialog(self, _('Cannot fetch cover'),
_('Could not find cover for this book. Try '
'specifying the ISBN first.')).exec_()
return
if self.cover_fetcher.exception is not None:
err = self.cover_fetcher.exception
error_dialog(self, _('Cannot fetch cover'),
_('<b>Could not fetch cover.</b><br/>')+unicode(err)).exec_()
return
if self.cover_fetcher.errors and self.cover_fetcher.cover_data is None:
details = u'\n\n'.join([e[-1] + ': ' + e[1] for e in self.cover_fetcher.errors])
error_dialog(self, _('Cannot fetch cover'),
_('<b>Could not fetch cover.</b><br/>') +
_('For the error message from each cover source, '
'click Show details below.'), det_msg=details, show=True)
return
pix = QPixmap()
pix.loadFromData(self.cover_fetcher.cover_data)
if pix.isNull():
error_dialog(self, _('Bad cover'),
_('The cover is not a valid picture')).exec_()
else:
self.cover.setPixmap(pix)
self.update_cover_tooltip()
self.cover_changed = True
self.cpixmap = pix
self.cover_data = self.cover_fetcher.cover_data
finally:
self.fetch_cover_button.setEnabled(True)
self.unsetCursor()
self.pi.stop()
def fetch_metadata(self):
isbn = re.sub(r'[^0-9a-zA-Z]', '', unicode(self.isbn.text()))
@ -789,6 +816,10 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
unicode(self.tags.text()).split(',')],
notify=notify, commit=commit)
def next_triggered(self, row_delta, *args):
self.row_delta = row_delta
self.accept()
def accept(self):
cf = getattr(self, 'cover_fetcher', None)
if cf is not None and hasattr(cf, 'terminate'):
@ -798,9 +829,10 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
if self.formats_changed:
self.sync_formats()
title = unicode(self.title.text()).strip()
if title != self.original_title:
self.db.set_title(self.id, title, notify=False)
au = unicode(self.authors.text()).strip()
if au:
if au and au != self.original_author:
self.db.set_authors(self.id, string_to_authors(au), notify=False)
aus = unicode(self.author_sort.text()).strip()
if aus:
@ -850,8 +882,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
raise
self.save_state()
QDialog.accept(self)
if callable(self.accepted_callback):
self.accepted_callback(self.id)
def reject(self, *args):
cf = getattr(self, 'cover_fetcher', None)