merge from trunk

This commit is contained in:
ldolse 2011-02-02 12:17:35 +08:00
commit cbd5f19e0e
23 changed files with 193 additions and 82 deletions

View File

@ -585,7 +585,6 @@ application/vnd.osa.netdeploy
application/vnd.osgi.bundle
application/vnd.osgi.dp dp
application/vnd.otps.ct-kip+xml
application/vnd.palm oprc pdb pqa
application/vnd.paos.xml
application/vnd.pg.format str
application/vnd.pg.osasli ei6
@ -1082,7 +1081,6 @@ chemical/x-ncbi-asn1 asn
chemical/x-ncbi-asn1-ascii ent prt
chemical/x-ncbi-asn1-binary aso val
chemical/x-ncbi-asn1-spec asn
chemical/x-pdb ent pdb
chemical/x-rosdal ros
chemical/x-swissprot sw
chemical/x-vamas-iso14976 vms
@ -1379,3 +1377,5 @@ application/x-cbr cbr
application/x-cb7 cb7
application/x-koboreader-ebook kobo
image/wmf wmf
application/ereader pdb

View File

@ -35,7 +35,7 @@ class WallStreetJournal(BasicNewsRecipe):
remove_tags_before = dict(name='h1')
remove_tags = [
dict(id=["articleTabs_tab_article", "articleTabs_tab_comments", "articleTabs_tab_interactive","articleTabs_tab_video","articleTabs_tab_map","articleTabs_tab_slideshow"]),
dict(id=["articleTabs_tab_article", "articleTabs_tab_comments", "articleTabs_tab_interactive","articleTabs_tab_video","articleTabs_tab_map","articleTabs_tab_slideshow","articleTabs_tab_quotes","articleTabs_tab_document"]),
{'class':['footer_columns','network','insetCol3wide','interactive','video','slideshow','map','insettip','insetClose','more_in', "insetContent", 'articleTools_bottom', 'aTools', "tooltip", "adSummary", "nav-inline"]},
dict(rel='shortcut icon'),
]
@ -101,7 +101,7 @@ class WallStreetJournal(BasicNewsRecipe):
title = 'Front Section'
url = 'http://online.wsj.com' + a['href']
feeds = self.wsj_add_feed(feeds,title,url)
title = 'What''s News'
title = "What's News"
url = url.replace('pageone','whatsnews')
feeds = self.wsj_add_feed(feeds,title,url)
else:

View File

@ -10,7 +10,10 @@ class WallStreetJournal(BasicNewsRecipe):
title = 'Wall Street Journal (free)'
__author__ = 'Kovid Goyal, Sujata Raman, Joshua Oster-Morris, Starson17'
description = 'News and current affairs'
description = '''News and current affairs. This recipe only fetches complete
versions of the articles that are available free on the wsj.com website.
To get the rest of the articles, subscribe to the WSJ and use the other WSJ
recipe.'''
language = 'en'
cover_url = 'http://dealbreaker.com/images/thumbs/Wall%20Street%20Journal%20A1.JPG'
max_articles_per_feed = 1000
@ -151,6 +154,4 @@ class WallStreetJournal(BasicNewsRecipe):
return articles
def cleanup(self):
self.browser.open('http://online.wsj.com/logout?url=http://online.wsj.com')

View File

@ -22,13 +22,15 @@ Run an embedded python interpreter.
parser.add_option('-d', '--debug-device-driver', default=False, action='store_true',
help='Debug the specified device driver.')
parser.add_option('-g', '--gui', default=False, action='store_true',
help='Run the GUI',)
help='Run the GUI with debugging enabled. Debug output is '
'printed to stdout and stderr.')
parser.add_option('--gui-debug', default=None,
help='Run the GUI with a debug console, logging to the'
' specified path',)
' specified path. For internal use only, use the -g'
' option to run the GUI in debug mode',)
parser.add_option('--show-gui-debug', default=None,
help='Display the specified log file.',)
help='Display the specified log file. For internal use'
' only.',)
parser.add_option('-w', '--viewer', default=False, action='store_true',
help='Run the ebook viewer',)
parser.add_option('--paths', default=False, action='store_true',

View File

@ -35,6 +35,16 @@ class DevicePlugin(Plugin):
#: Height for thumbnails on the device
THUMBNAIL_HEIGHT = 68
#: Width for thumbnails on the device. Setting this will force thumbnails
#: to this size, not preserving aspect ratio. If it is not set, then
#: the aspect ratio will be preserved and the thumbnail will be no higher
#: than THUMBNAIL_HEIGHT
# THUMBNAIL_WIDTH = 68
#: Set this to True if the device supports updating cover thumbnails during
#: sync_booklists. Setting it to true will ask device.py to refresh the
#: cover thumbnails during book matching
WANTS_UPDATED_THUMBNAILS = False
#: Whether the metadata on books can be set via the GUI.
CAN_SET_METADATA = ['title', 'authors', 'collections']

View File

@ -8,5 +8,5 @@ CACHE_XML = 'Sony Reader/database/cache.xml'
CACHE_EXT = 'Sony Reader/database/cacheExt.xml'
MEDIA_THUMBNAIL = 'database/thumbnail'
CACHE_THUMBNAIL = 'Sony Reader/database/thumbnail'
CACHE_THUMBNAIL = 'Sony Reader/thumbnail'

View File

@ -81,12 +81,19 @@ class PRS505(USBMS):
_('Set this option to have separate book covers uploaded '
'every time you connect your device. Unset this option if '
'you have so many books on the reader that performance is '
'unacceptable.')
'unacceptable.'),
_('Preserve cover aspect ratio when building thumbnails') +
':::' +
_('Set this option if you want the cover thumbnails to have '
'the same aspect ratio (width to height) as the cover. '
'Unset it if you want the thumbnail to be the maximum size, '
'ignoring aspect ratio.')
]
EXTRA_CUSTOMIZATION_DEFAULT = [
', '.join(['series', 'tags']),
False,
False
False,
True
]
OPT_COLLECTIONS = 0
@ -96,7 +103,7 @@ class PRS505(USBMS):
plugboard = None
plugboard_func = None
THUMBNAIL_HEIGHT = 200
THUMBNAIL_HEIGHT = 217
MAX_PATH_LEN = 201 # 250 - (max(len(CACHE_THUMBNAIL), len(MEDIA_THUMBNAIL)) +
# len('main_thumbnail.jpg') + 1)
@ -138,6 +145,13 @@ class PRS505(USBMS):
if not write_cache(self._card_b_prefix):
self._card_b_prefix = None
self.booklist_class.rebuild_collections = self.rebuild_collections
# Set the thumbnail width to the theoretical max if the user has asked
# that we do not preserve aspect ratio
if not self.settings().extra_customization[3]:
self.THUMBNAIL_WIDTH = 168
# Set WANTS_UPDATED_THUMBNAILS if the user has asked that thumbnails be
# updated on every connect
self.WANTS_UPDATED_THUMBNAILS = self.settings().extra_customization[2]
def get_device_information(self, end_session=True):
return (self.gui_name, '', '', '')

View File

@ -46,7 +46,8 @@ HEURISTIC_OPTIONS = ['markup_chapter_headings',
'italicize_common_cases', 'fix_indents',
'html_unwrap_factor', 'unwrap_lines',
'delete_blank_paragraphs', 'format_scene_breaks',
'dehyphenate', 'renumber_headings']
'dehyphenate', 'renumber_headings',
'replace_scene_breaks']
def print_help(parser, log):
help = parser.format_help().encode(preferred_encoding, 'replace')
@ -143,7 +144,7 @@ def add_pipeline_options(parser, plumber):
' patterns. Disabled by default. Use %s to enable. '
' Individual actions can be disabled with the %s options.')
% ('--enable-heuristics', '--disable-*'),
['enable_heuristics', 'replace_scene_breaks'] + HEURISTIC_OPTIONS
['enable_heuristics'] + HEURISTIC_OPTIONS
),
'SEARCH AND REPLACE' : (

View File

@ -532,8 +532,9 @@ OptionRecommendation(name='format_scene_breaks',
'horizontal rules.')),
OptionRecommendation(name='replace_scene_breaks',
recommended_value=None, level=OptionRecommendation.LOW,
help=_('Replace scene breaks with the specified text.')),
recommended_value='', level=OptionRecommendation.LOW,
help=_('Replace scene breaks with the specified text. By default, the '
'text from the input document is used.')),
OptionRecommendation(name='dehyphenate',
recommended_value=True, level=OptionRecommendation.LOW,

View File

@ -499,10 +499,11 @@ class HeuristicProcessor(object):
hr_open = re.sub('45', str(divpercent), hr_open)
scene_break = hr_open+replacement_break+'</div>'
else:
scene_break = hr_open+'<hr style="height: 3px; background:#505050" /></div>'
scene_break = hr_open+'<hr style="height: 3px; background:#505050" /></div>'
elif re.match('^<img', replacement_break):
scene_break = self.scene_break_open+replacement_break+'</p>'
else:
from calibre.utils.html2text import html2text
replacement_break = html2text(replacement_break)
replacement_break = re.sub('\s', '&nbsp;', replacement_break)
scene_break = self.scene_break_open+replacement_break+'</p>'

View File

@ -422,6 +422,33 @@ class MetadataField(object):
elem = obj.create_metadata_element(self.name, is_dc=self.is_dc)
obj.set_text(elem, self.renderer(val))
class TitleSortField(MetadataField):
def __get__(self, obj, type=None):
c = self.__real_get__(obj, type)
if c is None:
matches = obj.title_path(obj.metadata)
if matches:
for match in matches:
ans = match.get('{%s}file-as'%obj.NAMESPACES['opf'], None)
if not ans:
ans = match.get('file-as', None)
if ans:
c = ans
if not c:
c = self.none_is
else:
c = c.strip()
return c
def __set__(self, obj, val):
MetadataField.__set__(self, obj, val)
matches = obj.title_path(obj.metadata)
if matches:
for match in matches:
for attr in list(match.attrib):
if attr.endswith('file-as'):
del match.attrib[attr]
def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)):
from calibre.utils.config import to_json
@ -490,6 +517,7 @@ class OPF(object): # {{{
rights = MetadataField('rights')
series = MetadataField('series', is_dc=False)
series_index = MetadataField('series_index', is_dc=False, formatter=float, none_is=1)
title_sort = TitleSortField('title_sort', is_dc=False)
rating = MetadataField('rating', is_dc=False, formatter=int)
pubdate = MetadataField('date', formatter=parse_date,
renderer=isoformat)
@ -776,30 +804,6 @@ class OPF(object): # {{{
return property(fget=fget, fset=fset)
@dynamic_property
def title_sort(self):
def fget(self):
matches = self.title_path(self.metadata)
if matches:
for match in matches:
ans = match.get('{%s}file-as'%self.NAMESPACES['opf'], None)
if not ans:
ans = match.get('file-as', None)
if ans:
return ans
def fset(self, val):
matches = self.title_path(self.metadata)
if matches:
for key in matches[0].attrib:
if key.endswith('file-as'):
matches[0].attrib.pop(key)
matches[0].set('{%s}file-as'%self.NAMESPACES['opf'], unicode(val))
return property(fget=fget, fset=fset)
@dynamic_property
def tags(self):
@ -1129,8 +1133,6 @@ class OPFCreator(Metadata):
metadata = M.metadata()
a = metadata.append
role = {}
if self.title_sort:
role = {'file-as':self.title_sort}
a(DC_ELEM('title', self.title if self.title else _('Unknown'),
opf_attrs=role))
for i, author in enumerate(self.authors):
@ -1165,6 +1167,8 @@ class OPFCreator(Metadata):
a(CAL_ELEM('calibre:series', self.series))
if self.series_index is not None:
a(CAL_ELEM('calibre:series_index', self.format_series_index()))
if self.title_sort:
a(CAL_ELEM('calibre:title_sort', self.title_sort))
if self.rating is not None:
a(CAL_ELEM('calibre:rating', str(self.rating)))
if self.timestamp is not None:
@ -1320,7 +1324,6 @@ def test_m2o():
mi.author_sort = 'author sort'
mi.pubdate = nowf()
mi.language = 'en'
mi.category = 'test'
mi.comments = 'what a fun book\n\n'
mi.publisher = 'publisher'
mi.isbn = 'boooo'
@ -1335,11 +1338,11 @@ def test_m2o():
opf = metadata_to_opf(mi)
print opf
newmi = MetaInformation(OPF(StringIO(opf)))
for attr in ('author_sort', 'title_sort', 'comments', 'category',
for attr in ('author_sort', 'title_sort', 'comments',
'publisher', 'series', 'series_index', 'rating',
'isbn', 'tags', 'cover_data', 'application_id',
'language', 'cover',
'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc',
'book_producer', 'timestamp',
'pubdate', 'rights', 'publication_type'):
o, n = getattr(mi, attr), getattr(newmi, attr)
if o != n and o.strip() != n.strip():
@ -1441,4 +1444,6 @@ def test_user_metadata():
print opf.render()
if __name__ == '__main__':
test_user_metadata()
#test_user_metadata()
#test_m2o()
test()

View File

@ -65,7 +65,7 @@ def to_metadata(browser, log, entry_):
mi = Metadata(title_, authors)
try:
raw = browser.open(id_url).read()
raw = browser.open_novisit(id_url).read()
feed = etree.fromstring(raw)
extra = entry(feed)[0]
except:
@ -129,7 +129,7 @@ class Worker(Thread):
for i in self.entries:
try:
ans = to_metadata(self.browser, self.log, i)
if ans is not None:
if isinstance(ans, Metadata):
self.result_queue.put(ans)
except:
self.log.exception(

View File

@ -103,6 +103,8 @@ class EXTHHeader(object):
pass
elif id == 108:
pass # Producer
elif id == 113:
pass # ASIN or UUID
#else:
# print 'unhandled metadata record', id, repr(content)

View File

@ -1547,6 +1547,31 @@ class MobiWriter(object):
rights = 'Unknown'
exth.write(pack('>II', EXTH_CODES['rights'], len(rights) + 8))
exth.write(rights)
nrecs += 1
# Write UUID as ASIN
uuid = None
from calibre.ebooks.oeb.base import OPF
for x in oeb.metadata['identifier']:
if x.get(OPF('scheme'), None).lower() == 'uuid' or unicode(x).startswith('urn:uuid:'):
uuid = unicode(x).split(':')[-1]
break
if uuid is None:
from uuid import uuid4
uuid = str(uuid4())
if isinstance(uuid, unicode):
uuid = uuid.encode('utf-8')
exth.write(pack('>II', 113, len(uuid) + 8))
exth.write(uuid)
nrecs += 1
# Write cdetype
if not self.opts.mobi_periodical:
data = 'EBOK'
exth.write(pack('>II', 501, len(data)+8))
exth.write(data)
nrecs += 1
# Add a publication date entry
if oeb.metadata['date'] != [] :

View File

@ -27,7 +27,8 @@ class HeuristicsWidget(Widget, Ui_Form):
'dehyphenate', 'renumber_headings']
)
self.db, self.book_id = db, book_id
self.rssb_defaults = ['', '<hr />', '* * *']
self.rssb_defaults = [u'', u'<hr />', u'* * *', u'• • •', u'✦ ✦ ✦',
u'✮ ✮ ✮', u'☆ ☆ ☆', u'❂ ❂ ❂', u'✣ ✣ ✣', u'❖ ❖ ❖', u'☼ ☼ ☼', u'✠ ✠ ✠']
self.initialize_options(get_option, get_help, db, book_id)
self.load_histories()
@ -40,11 +41,13 @@ class HeuristicsWidget(Widget, Ui_Form):
def restore_defaults(self, get_option):
Widget.restore_defaults(self, get_option)
self.save_histories()
rssb_hist = gprefs['replace_scene_breaks_history']
for x in self.rssb_defaults:
if x in rssb_hist:
del rssb_hist[rssb_hist.index(x)]
gprefs['replace_scene_breaks_history'] = self.rssb_defaults + gprefs['replace_scene_breaks_history']
self.load_histories()
def commit_options(self, save_defaults=False):
self.save_histories()
@ -69,6 +72,9 @@ class HeuristicsWidget(Widget, Ui_Form):
return True
def load_histories(self):
self.opt_replace_scene_breaks.clear()
self.opt_replace_scene_breaks.lineEdit().setText('')
val = unicode(self.opt_replace_scene_breaks.currentText())
rssb_hist = gprefs.get('replace_scene_breaks_history', self.rssb_defaults)
if val in rssb_hist:

View File

@ -164,7 +164,10 @@
</sizepolicy>
</property>
<property name="text">
<string>Replace soft scene breaks:</string>
<string>Replace soft scene &amp;breaks:</string>
</property>
<property name="buddy">
<cstring>opt_replace_scene_breaks</cstring>
</property>
</widget>
</item>

View File

@ -48,10 +48,10 @@
</property>
</widget>
</item>
<item row="5" column="0" colspan="3">
<item row="6" column="0" colspan="3">
<widget class="XPathEdit" name="opt_page_breaks_before" native="true"/>
</item>
<item row="6" column="0" colspan="3">
<item row="7" column="0" colspan="3">
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
@ -77,6 +77,16 @@
</property>
</spacer>
</item>
<item row="4" column="0" colspan="3">
<widget class="QLabel" name="label_2">
<property name="text">
<string>The header and footer removal options have been replaced by the Search &amp; Replace options. Click the Search &amp; Replace category in the bar to the left to use these options. Leave the replace field blank and enter your header/footer removal regexps into the search field.</string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
</layout>
</widget>
<customwidgets>

View File

@ -838,9 +838,10 @@ class DeviceMixin(object): # {{{
format_count[f] = 1
for f in self.device_manager.device.settings().format_map:
if f in format_count.keys():
formats.append((f, _('%i of %i Books') % (format_count[f], len(rows))), True if f in aval_out_formats else False)
formats.append((f, _('%i of %i Books') % (format_count[f],
len(rows)), True if f in aval_out_formats else False))
elif f in aval_out_formats:
formats.append((f, _('0 of %i Books') % len(rows)), True)
formats.append((f, _('0 of %i Books') % len(rows), True))
d = ChooseFormatDeviceDialog(self, _('Choose format to send to device'), formats)
if d.exec_() != QDialog.Accepted:
return
@ -871,6 +872,16 @@ class DeviceMixin(object): # {{{
self.send_by_mail(to, fmts, delete)
def cover_to_thumbnail(self, data):
if self.device_manager.device and \
hasattr(self.device_manager.device, 'THUMBNAIL_WIDTH'):
try:
return thumbnail(data,
self.device_manager.device.THUMBNAIL_WIDTH,
self.device_manager.device.THUMBNAIL_HEIGHT,
preserve_aspect_ratio=False)
except:
pass
return
ht = self.device_manager.device.THUMBNAIL_HEIGHT \
if self.device_manager else DevicePlugin.THUMBNAIL_HEIGHT
try:
@ -1272,6 +1283,8 @@ class DeviceMixin(object): # {{{
x = x.lower() if x else ''
return string_pat.sub('', x)
update_metadata = prefs['manage_device_metadata'] == 'on_connect'
# Force a reset if the caches are not initialized
if reset or not hasattr(self, 'db_book_title_cache'):
# Build a cache (map) of the library, so the search isn't On**2
@ -1284,8 +1297,13 @@ class DeviceMixin(object): # {{{
except:
return False
get_covers = False
if update_metadata and self.device_manager.is_device_connected:
if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
get_covers = True
for id in db.data.iterallids():
mi = db.get_metadata(id, index_is_id=True)
mi = db.get_metadata(id, index_is_id=True, get_cover=get_covers)
title = clean_string(mi.title)
if title not in db_book_title_cache:
db_book_title_cache[title] = \
@ -1311,7 +1329,6 @@ class DeviceMixin(object): # {{{
# the application_id to the db_id of the matching book. This value
# will be used by books_on_device to indicate matches.
update_metadata = prefs['manage_device_metadata'] == 'on_connect'
for booklist in booklists:
for book in booklist:
book.in_library = None
@ -1382,6 +1399,12 @@ class DeviceMixin(object): # {{{
if update_metadata:
if self.device_manager.is_device_connected:
if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
for blist in booklists:
for book in blist:
if book.cover and os.access(book.cover, os.R_OK):
book.thumbnail = \
self.cover_to_thumbnail(open(book.cover, 'rb').read())
plugboards = self.library_view.model().db.prefs.get('plugboards', {})
self.device_manager.sync_booklists(
Dispatcher(self.metadata_synced), booklists,

View File

@ -264,8 +264,9 @@ class EmailMixin(object): # {{{
if _auto_ids != []:
for id in _auto_ids:
if specific_format == None:
formats = [f.lower() for f in self.library_view.model().db.formats(id, index_is_id=True).split(',')]
formats = formats if formats != None else []
dbfmts = self.library_view.model().db.formats(id, index_is_id=True)
formats = [f.lower() for f in (dbfmts.split(',') if fmts else
[])]
if list(set(formats).intersection(available_input_formats())) != [] and list(set(fmts).intersection(available_output_formats())) != []:
auto.append(id)
else:

View File

@ -430,8 +430,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
authors = self.authors(id, index_is_id=True)
if not authors:
authors = _('Unknown')
author = ascii_filename(authors.split(',')[0][:self.PATH_LIMIT]).decode(filesystem_encoding, 'ignore')
title = ascii_filename(self.title(id, index_is_id=True)[:self.PATH_LIMIT]).decode(filesystem_encoding, 'ignore')
author = ascii_filename(authors.split(',')[0])[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
title = ascii_filename(self.title(id, index_is_id=True))[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
path = author + '/' + title + ' (%d)'%id
return path
@ -442,8 +442,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
authors = self.authors(id, index_is_id=True)
if not authors:
authors = _('Unknown')
author = ascii_filename(authors.split(',')[0][:self.PATH_LIMIT]).decode(filesystem_encoding, 'replace')
title = ascii_filename(self.title(id, index_is_id=True)[:self.PATH_LIMIT]).decode(filesystem_encoding, 'replace')
author = ascii_filename(authors.split(',')[0])[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
title = ascii_filename(self.title(id, index_is_id=True))[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
name = title + ' - ' + author
while name.endswith('.'):
name = name[:-1]

View File

@ -391,7 +391,7 @@ Take your pick:
* A tribute to the SONY Librie which was the first e-ink based e-book reader
* My wife chose it ;-)
|app| is pronounced as cal-i-ber *not* ca-libre. If you're wondering, |app| is the British/commonwealth spelling for caliber. Being Indian, that's the natural spelling for me.
|app| is pronounced as cal-i-ber *not* ca-li-bre. If you're wondering, |app| is the British/commonwealth spelling for caliber. Being Indian, that's the natural spelling for me.
Why does |app| show only some of my fonts on OS X?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

View File

@ -72,11 +72,17 @@ def save_cover_data_to(data, path, bgcolor='#ffffff', resize_to=None,
f.write(data)
return ret
def thumbnail(data, width=120, height=120, bgcolor='#ffffff', fmt='jpg'):
def thumbnail(data, width=120, height=120, bgcolor='#ffffff', fmt='jpg',
preserve_aspect_ratio=True):
img = Image()
img.load(data)
owidth, oheight = img.size
scaled, nwidth, nheight = fit_image(owidth, oheight, width, height)
if not preserve_aspect_ratio:
scaled = owidth > width or oheight > height
nwidth = width
nheight = height
else:
scaled, nwidth, nheight = fit_image(owidth, oheight, width, height)
if scaled:
img.size = (nwidth, nheight)
canvas = create_canvas(img.size[0], img.size[1], bgcolor)