From 81400b56c9fbea19c46d4af61fe7944e5bbe39bb Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Oct 2010 19:39:11 +0100 Subject: [PATCH 01/39] Make check_library delete work on linux and mac by checking the file type. --- src/calibre/gui2/dialogs/check_library.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/check_library.py b/src/calibre/gui2/dialogs/check_library.py index 741a42893d..f07a25c51a 100644 --- a/src/calibre/gui2/dialogs/check_library.py +++ b/src/calibre/gui2/dialogs/check_library.py @@ -11,7 +11,7 @@ from PyQt4.Qt import QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel, \ from calibre.gui2.dialogs.confirm_delete import confirm from calibre.library.check_library import CheckLibrary, CHECKS -from calibre.library.database2 import delete_file +from calibre.library.database2 import delete_file, delete_tree class Item(QTreeWidgetItem): pass @@ -147,7 +147,11 @@ class CheckLibraryDialog(QDialog): for it in items: if it.checkState(1): try: - delete_file(os.path.join(self.db.library_path ,unicode(it.text(1)))) + p = os.path.join(self.db.library_path ,unicode(it.text(1))) + if os.path.isdir(p): + delete_tree(p) + else: + delete_file(p) except: print 'failed to delete', os.path.join(self.db.library_path ,unicode(it.text(1))) self.run_the_check() From d1e7517fcec66484a07ddaa27b9aa88049db2091 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Oct 2010 19:43:35 +0100 Subject: [PATCH 02/39] Remove cancel button --- src/calibre/gui2/dialogs/check_library.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/calibre/gui2/dialogs/check_library.py b/src/calibre/gui2/dialogs/check_library.py index cf51999559..1cd11e7807 100644 --- a/src/calibre/gui2/dialogs/check_library.py +++ b/src/calibre/gui2/dialogs/check_library.py @@ -44,14 +44,10 @@ class CheckLibraryDialog(QDialog): self.delete = QPushButton('Delete &marked') self.delete.setDefault(False) self.delete.clicked.connect(self.delete_marked) - self.cancel = QPushButton('&Cancel') - self.cancel.setDefault(False) - self.cancel.clicked.connect(self.reject) self.bbox = QDialogButtonBox(self) - self.bbox.addButton(self.copy, QDialogButtonBox.ActionRole) self.bbox.addButton(self.check, QDialogButtonBox.ActionRole) self.bbox.addButton(self.delete, QDialogButtonBox.ActionRole) - self.bbox.addButton(self.cancel, QDialogButtonBox.RejectRole) + self.bbox.addButton(self.copy, QDialogButtonBox.ActionRole) self.bbox.addButton(self.ok, QDialogButtonBox.AcceptRole) h = QHBoxLayout() From 15b4bf7f8b598c4921cc151aea3f74f9248f06b4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 18 Oct 2010 13:38:08 -0600 Subject: [PATCH 03/39] /browse: Add an 'All book' top level category and Fix #7209 (New content server observations regarding restricted items) --- src/calibre/library/server/base.py | 1 + src/calibre/library/server/browse.py | 28 +++++++++++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/calibre/library/server/base.py b/src/calibre/library/server/base.py index 84e748a949..3a081fc427 100644 --- a/src/calibre/library/server/base.py +++ b/src/calibre/library/server/base.py @@ -148,6 +148,7 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache, cherrypy.engine.graceful() def set_search_restriction(self, restriction): + self.search_restriction_name = restriction if restriction: self.search_restriction = 'search:"%s"'%restriction else: diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index ea69ad77ef..ea86de4c1b 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -116,7 +116,10 @@ def render_rating(rating, container='span', prefix=None): # {{{ # }}} -def get_category_items(category, items, db, datatype): # {{{ +def get_category_items(category, items, restriction, datatype): # {{{ + + if category == 'search': + items = [x for x in items if x.name != restriction] def item(i): templ = (u'
' @@ -299,6 +302,7 @@ class BrowseServer(object): category_meta = self.db.field_metadata cats = [ (_('Newest'), 'newest', 'forward.png'), + (_('All books'), 'allbooks', 'book.png'), ] def getter(x): @@ -370,7 +374,8 @@ class BrowseServer(object): if len(items) <= self.opts.max_opds_ungrouped_items: script = 'false' - items = get_category_items(category, items, self.db, datatype) + items = get_category_items(category, items, + self.search_restriction_name, datatype) else: getter = lambda x: unicode(getattr(x, 'sort', x.name)) starts = set([]) @@ -440,7 +445,8 @@ class BrowseServer(object): entries.append(x) sort = self.browse_sort_categories(entries, sort) - entries = get_category_items(category, entries, self.db, datatype) + entries = get_category_items(category, entries, + self.search_restriction_name, datatype) return json.dumps(entries, ensure_ascii=False) @@ -451,6 +457,8 @@ class BrowseServer(object): ans = self.browse_toplevel() elif category == 'newest': raise cherrypy.InternalRedirect('/browse/matches/newest/dummy') + elif category == 'allbooks': + raise cherrypy.InternalRedirect('/browse/matches/allbooks/dummy') else: ans = self.browse_category(category, category_sort) @@ -478,16 +486,20 @@ class BrowseServer(object): raise cherrypy.HTTPError(404, 'invalid category id: %r'%cid) categories = self.categories_cache() - if category not in categories and category != 'newest': + if category not in categories and \ + category not in ('newest', 'allbooks'): raise cherrypy.HTTPError(404, 'category not found') fm = self.db.field_metadata try: category_name = fm[category]['name'] dt = fm[category]['datatype'] except: - if category != 'newest': + if category not in ('newest', 'allbooks'): raise - category_name = _('Newest') + category_name = { + 'newest' : _('Newest'), + 'allbooks' : _('All books'), + }[category] dt = None hide_sort = 'true' if dt == 'series' else 'false' @@ -498,8 +510,10 @@ class BrowseServer(object): except: raise cherrypy.HTTPError(404, 'Search: %r not understood'%which) elif category == 'newest': - ids = list(self.db.data.iterallids()) + ids = self.search_cache('') hide_sort = 'true' + elif category == 'allbooks': + ids = self.search_cache('') else: q = category if q == 'news': From f9006854a088217e0e057d0a583ad7a5d8733301 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 18 Oct 2010 13:40:15 -0600 Subject: [PATCH 04/39] Fix #7220 (Dismiss "Fetch Metadata" box when fetch fails) --- src/calibre/gui2/dialogs/fetch_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/fetch_metadata.py b/src/calibre/gui2/dialogs/fetch_metadata.py index eb6edce75d..6ee9cd9a96 100644 --- a/src/calibre/gui2/dialogs/fetch_metadata.py +++ b/src/calibre/gui2/dialogs/fetch_metadata.py @@ -190,7 +190,8 @@ class FetchMetadata(QDialog, Ui_FetchMetadata): if self.model.rowCount() < 1: info_dialog(self, _('No metadata found'), _('No metadata found, try adjusting the title and author ' - 'or the ISBN key.')).exec_() + 'and/or removing the ISBN.')).exec_() + self.reject() return self.matches.setModel(self.model) From 757b8fa4c056d190373635f5f78f281c5646da9e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 18 Oct 2010 13:50:39 -0600 Subject: [PATCH 05/39] Fix #7221 (You cannot delete a Series listing from List view) --- src/calibre/gui2/library/models.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 2946985342..0286acc782 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -783,18 +783,22 @@ class BooksModel(QAbstractTableModel): # {{{ self.db.set_rating(id, val) elif column == 'series': val = val.strip() - pat = re.compile(r'\[([.0-9]+)\]') - match = pat.search(val) - if match is not None: - self.db.set_series_index(id, float(match.group(1))) - val = pat.sub('', val).strip() - elif val: - if tweaks['series_index_auto_increment'] == 'next': - ni = self.db.get_next_series_num_for(val) - if ni != 1: - self.db.set_series_index(id, ni) - if val: + if not val: self.db.set_series(id, val) + self.db.set_series_index(id, 1.0) + else: + pat = re.compile(r'\[([.0-9]+)\]') + match = pat.search(val) + if match is not None: + self.db.set_series_index(id, float(match.group(1))) + val = pat.sub('', val).strip() + elif val: + if tweaks['series_index_auto_increment'] == 'next': + ni = self.db.get_next_series_num_for(val) + if ni != 1: + self.db.set_series_index(id, ni) + if val: + self.db.set_series(id, val) elif column == 'timestamp': if val.isNull() or not val.isValid(): return False From 1ebee86c83e5b3c89a2b3af5a8e62de6812b6504 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 18 Oct 2010 20:02:32 -0600 Subject: [PATCH 06/39] Support for the SONY periodical format. Now news downloaded with calibre will appear in the Periodicals section of your SONY and will have the special periodical navigation enabled. --- resources/recipes/atlantic.recipe | 8 +- src/calibre/customize/conversion.py | 5 + src/calibre/ebooks/epub/__init__.py | 16 ++- src/calibre/ebooks/epub/output.py | 13 +- src/calibre/ebooks/epub/periodical.py | 170 ++++++++++++++++++++++++++ src/calibre/ebooks/mobi/output.py | 9 +- src/calibre/web/feeds/news.py | 1 + 7 files changed, 210 insertions(+), 12 deletions(-) create mode 100644 src/calibre/ebooks/epub/periodical.py diff --git a/resources/recipes/atlantic.recipe b/resources/recipes/atlantic.recipe index a41a931e37..5ae0f7d993 100644 --- a/resources/recipes/atlantic.recipe +++ b/resources/recipes/atlantic.recipe @@ -71,7 +71,9 @@ class TheAtlantic(BasicNewsRecipe): for poem in soup.findAll('div', attrs={'class':'poem'}): title = self.tag_to_string(poem.find('h4')) desc = self.tag_to_string(poem.find(attrs={'class':'author'})) - url = 'http://www.theatlantic.com'+poem.find('a')['href'] + url = poem.find('a')['href'] + if url.startswith('/'): + url = 'http://www.theatlantic.com' + url self.log('\tFound article:', title, 'at', url) self.log('\t\t', desc) poems.append({'title':title, 'url':url, 'description':desc, @@ -83,7 +85,9 @@ class TheAtlantic(BasicNewsRecipe): if div is not None: self.log('Found section: Advice') title = self.tag_to_string(div.find('h4')) - url = 'http://www.theatlantic.com'+div.find('a')['href'] + url = div.find('a')['href'] + if url.startswith('/'): + url = 'http://www.theatlantic.com' + url desc = self.tag_to_string(div.find('p')) self.log('\tFound article:', title, 'at', url) self.log('\t\t', desc) diff --git a/src/calibre/customize/conversion.py b/src/calibre/customize/conversion.py index c36f83bd2f..ec83600a49 100644 --- a/src/calibre/customize/conversion.py +++ b/src/calibre/customize/conversion.py @@ -294,3 +294,8 @@ class OutputFormatPlugin(Plugin): ''' raise NotImplementedError + @property + def is_periodical(self): + return self.oeb.metadata.publication_type and \ + unicode(self.oeb.metadata.publication_type[0]).startswith('periodical:') + diff --git a/src/calibre/ebooks/epub/__init__.py b/src/calibre/ebooks/epub/__init__.py index f5de8421e0..53dd01d625 100644 --- a/src/calibre/ebooks/epub/__init__.py +++ b/src/calibre/ebooks/epub/__init__.py @@ -15,22 +15,30 @@ def rules(stylesheets): if r.type == r.STYLE_RULE: yield r -def initialize_container(path_to_container, opf_name='metadata.opf'): +def initialize_container(path_to_container, opf_name='metadata.opf', + extra_entries=[]): ''' Create an empty EPUB document, with a default skeleton. ''' - CONTAINER='''\ + rootfiles = '' + for path, mimetype, _ in extra_entries: + rootfiles += u''.format( + path, mimetype) + CONTAINER = u'''\ - + + {extra_entries} - '''%opf_name + '''.format(opf_name, extra_entries=rootfiles).encode('utf-8') zf = ZipFile(path_to_container, 'w') zf.writestr('mimetype', 'application/epub+zip', compression=ZIP_STORED) zf.writestr('META-INF/', '', 0700) zf.writestr('META-INF/container.xml', CONTAINER) + for path, _, data in extra_entries: + zf.writestr(path, data) return zf diff --git a/src/calibre/ebooks/epub/output.py b/src/calibre/ebooks/epub/output.py index 4146031cd2..38820010a8 100644 --- a/src/calibre/ebooks/epub/output.py +++ b/src/calibre/ebooks/epub/output.py @@ -106,6 +106,7 @@ class EPUBOutput(OutputFormatPlugin): recommendations = set([('pretty_print', True, OptionRecommendation.HIGH)]) + def workaround_webkit_quirks(self): # {{{ from calibre.ebooks.oeb.base import XPath for x in self.oeb.spine: @@ -183,6 +184,12 @@ class EPUBOutput(OutputFormatPlugin): with TemporaryDirectory('_epub_output') as tdir: from calibre.customize.ui import plugin_for_output_format + metadata_xml = None + extra_entries = [] + if self.is_periodical: + from calibre.ebooks.epub.periodical import sony_metadata + metadata_xml, atom_xml = sony_metadata(oeb) + extra_entries = [('atom.xml', 'application/atom+xml', atom_xml)] oeb_output = plugin_for_output_format('oeb') oeb_output.convert(oeb, tdir, input_plugin, opts, log) opf = [x for x in os.listdir(tdir) if x.endswith('.opf')][0] @@ -194,10 +201,14 @@ class EPUBOutput(OutputFormatPlugin): encryption = self.encrypt_fonts(encrypted_fonts, tdir, uuid) from calibre.ebooks.epub import initialize_container - epub = initialize_container(output_path, os.path.basename(opf)) + epub = initialize_container(output_path, os.path.basename(opf), + extra_entries=extra_entries) epub.add_dir(tdir) if encryption is not None: epub.writestr('META-INF/encryption.xml', encryption) + if metadata_xml is not None: + epub.writestr('META-INF/metadata.xml', + metadata_xml.encode('utf-8')) if opts.extract_to is not None: if os.path.exists(opts.extract_to): shutil.rmtree(opts.extract_to) diff --git a/src/calibre/ebooks/epub/periodical.py b/src/calibre/ebooks/epub/periodical.py new file mode 100644 index 0000000000..c68dc9e272 --- /dev/null +++ b/src/calibre/ebooks/epub/periodical.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from uuid import uuid4 + +from calibre.constants import __appname__, __version__ +from calibre import strftime, prepare_string_for_xml as xml + +SONY_METADATA = u'''\ + + + + {title} + {publisher} + {short_title} + {issue_date} + {language} + + + + + +''' + +SONY_ATOM = u'''\ + + + +{short_title} +{updated} +{id} +{entries} + +''' + +SONY_ATOM_SECTION = u'''\ + + {title} + + {id} + {updated} + {desc} + + newspaper/section + + +''' + +SONY_ATOM_ENTRY = u'''\ + + {title} + {author} + + {id} + {updated} + {desc} + + {word_count} + newspaper/article + + +''' + +def sony_metadata(oeb): + m = oeb.metadata + title = short_title = unicode(m.title[0]) + publisher = __appname__ + ' ' + __version__ + for k, n in m.title[0].attrib.items(): + if k.endswith('file-as'): + short_title = n + try: + date = unicode(m.date[0]).split('T')[0] + except: + date = strftime('%Y-%m-%d') + try: + language = unicode(m.language[0]).replace('_', '-') + except: + language = 'en' + short_title = xml(short_title, True) + + metadata = SONY_METADATA.format(title=xml(title), + short_title=short_title, + publisher=xml(publisher), issue_date=xml(date), + language=xml(language)) + + updated = strftime('%Y-%m-%dT%H:%M:%SZ') + + def cal_id(x): + for k, v in x.attrib.items(): + if k.endswith('scheme') and v == 'uuid': + return True + + try: + base_id = unicode(list(filter(cal_id, m.identifier))[0]) + except: + base_id = str(uuid4()) + + entries = [] + seen_titles = set([]) + for i, section in enumerate(oeb.toc): + if not section.href: + continue + secid = 'section%d'%i + sectitle = section.title + if not sectitle: + sectitle = _('Unknown') + d = 1 + bsectitle = sectitle + while sectitle in seen_titles: + sectitle = bsectitle + ' ' + str(d) + d += 1 + seen_titles.add(sectitle) + sectitle = xml(sectitle, True) + secdesc = section.description + if not secdesc: + secdesc = '' + secdesc = xml(secdesc) + entries.append(SONY_ATOM_SECTION.format(title=sectitle, + href=section.href, id=xml(base_id)+'/'+secid, + short_title=short_title, desc=secdesc, updated=updated)) + + for j, article in enumerate(section): + if not article.href: + continue + atitle = article.title + btitle = atitle + d = 1 + while atitle in seen_titles: + atitle = btitle + ' ' + str(d) + d += 1 + + auth = article.author if article.author else '' + desc = section.description + if not desc: + desc = '' + aid = 'article%d'%j + + entries.append(SONY_ATOM_ENTRY.format( + title=xml(atitle), + author=xml(auth), + updated=updated, + desc=desc, + short_title=short_title, + section_title=sectitle, + href=article.href, + word_count=str(1), + id=xml(base_id)+'/'+secid+'/'+aid + )) + + atom = SONY_ATOM.format(short_title=short_title, + entries='\n\n'.join(entries), updated=updated, + id=xml(base_id)).encode('utf-8') + + return metadata, atom + diff --git a/src/calibre/ebooks/mobi/output.py b/src/calibre/ebooks/mobi/output.py index 49da18ea7b..4159c6dd40 100644 --- a/src/calibre/ebooks/mobi/output.py +++ b/src/calibre/ebooks/mobi/output.py @@ -42,11 +42,10 @@ class MOBIOutput(OutputFormatPlugin): ]) def check_for_periodical(self): - if self.oeb.metadata.publication_type and \ - unicode(self.oeb.metadata.publication_type[0]).startswith('periodical:'): - self.periodicalize_toc() - self.check_for_masthead() - self.opts.mobi_periodical = True + if self.is_periodical: + self.periodicalize_toc() + self.check_for_masthead() + self.opts.mobi_periodical = True else: self.opts.mobi_periodical = False diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index f3d77061c3..f710b52204 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -1102,6 +1102,7 @@ class BasicNewsRecipe(Recipe): if self.output_profile.periodical_date_in_title: title += strftime(self.timefmt) mi = MetaInformation(title, [__appname__]) + mi.title_sort = self.short_title() mi.publisher = __appname__ mi.author_sort = __appname__ mi.publication_type = 'periodical:'+self.publication_type From 199d870b191ab863f21a4f55341501cbacd5c51b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 18 Oct 2010 21:25:58 -0600 Subject: [PATCH 07/39] Setting EPUB metadata: Fix date format. Fix language being overwritten by und when unspecified. Fix empty ISBN identifier being created --- .../recipes/theeconomictimes_india.recipe | 10 ++++---- src/calibre/ebooks/metadata/opf2.py | 25 +++++++++++-------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/resources/recipes/theeconomictimes_india.recipe b/resources/recipes/theeconomictimes_india.recipe index 5fef377f6e..59cd56b67e 100644 --- a/resources/recipes/theeconomictimes_india.recipe +++ b/resources/recipes/theeconomictimes_india.recipe @@ -19,18 +19,18 @@ class TheEconomicTimes(BasicNewsRecipe): simultaneous_downloads = 1 encoding = 'utf-8' language = 'en_IN' - publication_type = 'newspaper' + publication_type = 'newspaper' masthead_url = 'http://economictimes.indiatimes.com/photo/2676871.cms' - extra_css = """ body{font-family: Arial,Helvetica,sans-serif} + extra_css = """ body{font-family: Arial,Helvetica,sans-serif} .heading1{font-size: xx-large; font-weight: bold} """ - + conversion_options = { 'comment' : description , 'tags' : category , 'publisher' : publisher , 'language' : language } - + keep_only_tags = [dict(attrs={'class':['heading1','headingnext','Normal']})] remove_tags = [dict(name=['object','link','embed','iframe','base','table','meta'])] @@ -48,5 +48,5 @@ class TheEconomicTimes(BasicNewsRecipe): def preprocess_html(self, soup): for item in soup.findAll(style=True): - del item['style'] + del item['style'] return self.adeify_images(soup) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 5c2477c3dc..62d57f2251 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -382,11 +382,13 @@ class Guide(ResourceCollection): # {{{ class MetadataField(object): - def __init__(self, name, is_dc=True, formatter=None, none_is=None): + def __init__(self, name, is_dc=True, formatter=None, none_is=None, + renderer=lambda x: unicode(x)): self.name = name self.is_dc = is_dc self.formatter = formatter self.none_is = none_is + self.renderer = renderer def __real_get__(self, obj, type=None): ans = obj.get_metadata_element(self.name) @@ -418,7 +420,7 @@ class MetadataField(object): return if elem is None: elem = obj.create_metadata_element(self.name, is_dc=self.is_dc) - obj.set_text(elem, unicode(val)) + obj.set_text(elem, self.renderer(val)) def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)): @@ -489,10 +491,11 @@ class OPF(object): # {{{ series = MetadataField('series', is_dc=False) series_index = MetadataField('series_index', is_dc=False, formatter=float, none_is=1) rating = MetadataField('rating', is_dc=False, formatter=int) - pubdate = MetadataField('date', formatter=parse_date) + pubdate = MetadataField('date', formatter=parse_date, + renderer=isoformat) publication_type = MetadataField('publication_type', is_dc=False) timestamp = MetadataField('timestamp', is_dc=False, - formatter=parse_date) + formatter=parse_date, renderer=isoformat) def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True, @@ -826,11 +829,10 @@ class OPF(object): # {{{ def fset(self, val): matches = self.isbn_path(self.metadata) - if val is None: - if matches: - for x in matches: - x.getparent().remove(x) - return + if not val: + for x in matches: + x.getparent().remove(x) + return if not matches: attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'ISBN'} matches = [self.create_metadata_element('identifier', @@ -987,11 +989,14 @@ class OPF(object): # {{{ def smart_update(self, mi, replace_metadata=False): for attr in ('title', 'authors', 'author_sort', 'title_sort', 'publisher', 'series', 'series_index', 'rating', - 'isbn', 'language', 'tags', 'category', 'comments', + 'isbn', 'tags', 'category', 'comments', 'pubdate'): val = getattr(mi, attr, None) if val is not None and val != [] and val != (None, None): setattr(self, attr, val) + lang = getattr(mi, 'language', None) + if lang and lang != 'und': + self.language = lang temp = self.to_book_metadata() temp.smart_update(mi, replace_metadata=replace_metadata) self._user_metadata_ = temp.get_all_user_metadata(True) From b0ffb1fb6580f01fd931a9a721b50fbbe19439d2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 18 Oct 2010 22:31:53 -0600 Subject: [PATCH 08/39] ... --- src/calibre/ebooks/epub/periodical.py | 9 ++++++--- src/calibre/web/feeds/news.py | 3 +-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/epub/periodical.py b/src/calibre/ebooks/epub/periodical.py index c68dc9e272..ad75bb4706 100644 --- a/src/calibre/ebooks/epub/periodical.py +++ b/src/calibre/ebooks/epub/periodical.py @@ -80,9 +80,12 @@ def sony_metadata(oeb): m = oeb.metadata title = short_title = unicode(m.title[0]) publisher = __appname__ + ' ' + __version__ - for k, n in m.title[0].attrib.items(): - if k.endswith('file-as'): - short_title = n + try: + pt = unicode(oeb.metadata.publication_type[0]) + short_title = u''.join(pt.split(':')[2:]) + except: + pass + try: date = unicode(m.date[0]).split('T')[0] except: diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index f710b52204..cb6bf30bcf 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -1102,10 +1102,9 @@ class BasicNewsRecipe(Recipe): if self.output_profile.periodical_date_in_title: title += strftime(self.timefmt) mi = MetaInformation(title, [__appname__]) - mi.title_sort = self.short_title() mi.publisher = __appname__ mi.author_sort = __appname__ - mi.publication_type = 'periodical:'+self.publication_type + mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() mi.timestamp = nowf() mi.comments = self.description if not isinstance(mi.comments, unicode): From d463eff5eac01572d4932f6fcca95b04a346b044 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 00:03:55 -0600 Subject: [PATCH 09/39] ... --- src/calibre/ebooks/epub/periodical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/epub/periodical.py b/src/calibre/ebooks/epub/periodical.py index ad75bb4706..b46bea3719 100644 --- a/src/calibre/ebooks/epub/periodical.py +++ b/src/calibre/ebooks/epub/periodical.py @@ -82,7 +82,7 @@ def sony_metadata(oeb): publisher = __appname__ + ' ' + __version__ try: pt = unicode(oeb.metadata.publication_type[0]) - short_title = u''.join(pt.split(':')[2:]) + short_title = u':'.join(pt.split(':')[2:]) except: pass From f24f95cd1bebb962d0da39bcc395210a1c14fda6 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 19 Oct 2010 11:04:48 +0100 Subject: [PATCH 10/39] Make int and float custom fields work in save templates --- src/calibre/library/save_to_disk.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 94f9dbd229..c6cc12a978 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -131,15 +131,14 @@ class SafeFormat(TemplateFormatter): self.vformat(b['display']['composite_template'], [], kwargs) return self.composite_values[key] if key in kwargs: - return kwargs[key].replace('/', '_').replace('\\', '_') + val = kwargs[key] + return val.replace('/', '_').replace('\\', '_') return '' except: if DEBUG: traceback.print_exc() return key -safe_formatter = SafeFormat() - def get_components(template, mi, id, timefmt='%b %Y', length=250, sanitize_func=ascii_filename, replace_whitespace=False, to_lowercase=False): @@ -173,17 +172,22 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, custom_metadata = mi.get_all_user_metadata(make_copy=False) for key in custom_metadata: if key in format_args: + cm = custom_metadata[key] ## TODO: NEWMETA: should ratings be divided by 2? The standard rating isn't... - if custom_metadata[key]['datatype'] == 'series': + if cm['datatype'] == 'series': format_args[key] = tsfmt(format_args[key]) if key+'_index' in format_args: format_args[key+'_index'] = fmt_sidx(format_args[key+'_index']) - elif custom_metadata[key]['datatype'] == 'datetime': + elif cm['datatype'] == 'datetime': format_args[key] = strftime(timefmt, format_args[key].timetuple()) - elif custom_metadata[key]['datatype'] == 'bool': + elif cm['datatype'] == 'bool': format_args[key] = _('yes') if format_args[key] else _('no') - - components = safe_formatter.safe_format(template, format_args, + elif cm['datatype'] in ['int', 'float']: + if format_args[key] != 0: + format_args[key] = unicode(format_args[key]) + else: + format_args[key] = '' + components = SafeFormat().safe_format(template, format_args, 'G_C-EXCEPTION!', mi) components = [x.strip() for x in components.split('/') if x.strip()] components = [sanitize_func(x) for x in components if x] From d8fc285b70ab186d793493821d04c0cf7c65d2c6 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 19 Oct 2010 12:13:20 +0100 Subject: [PATCH 11/39] Make floats work in composite columns --- src/calibre/ebooks/metadata/book/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 3b96c98a7b..593e161df7 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -43,7 +43,7 @@ class SafeFormat(TemplateFormatter): b = self.book.get_user_metadata(key, False) if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0: v = '' - elif b and b['datatype'] == 'float' and b.get(key, 0.0) == 0.0: + elif b and b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0: v = '' else: ign, v = self.book.format_field(key.lower(), series_with_index=False) From 730bbb2d09980fe8a835ad360eced33a3d31ea10 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 08:45:14 -0600 Subject: [PATCH 12/39] EPUB Input: Handle EPUB files with multiple OPF files. Fixes #7229 (E-book Viewer crash: "IndexError:list index out of range") --- src/calibre/ebooks/epub/input.py | 34 ++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/calibre/ebooks/epub/input.py b/src/calibre/ebooks/epub/input.py index 30a3327b63..ec2004d81c 100644 --- a/src/calibre/ebooks/epub/input.py +++ b/src/calibre/ebooks/epub/input.py @@ -108,6 +108,27 @@ class EPUBInput(InputFormatPlugin): open('calibre_raster_cover.jpg', 'wb').write( renderer) + def find_opf(self): + def attr(n, attr): + for k, v in n.attrib.items(): + if k.endswith(attr): + return v + try: + with open('META-INF/container.xml') as f: + root = etree.fromstring(f.read()) + for r in root.xpath('//*[local-name()="rootfile"]'): + if attr(r, 'media-type') != "application/oebps-package+xml": + continue + path = attr(r, 'full-path') + if not path: + continue + path = os.path.join(os.getcwdu(), *path.split('/')) + if os.path.exists(path): + return path + except: + import traceback + traceback.print_exc() + def convert(self, stream, options, file_ext, log, accelerators): from calibre.utils.zipfile import ZipFile from calibre import walk @@ -116,12 +137,13 @@ class EPUBInput(InputFormatPlugin): zf = ZipFile(stream) zf.extractall(os.getcwd()) encfile = os.path.abspath(os.path.join('META-INF', 'encryption.xml')) - opf = None - for f in walk(u'.'): - if f.lower().endswith('.opf') and '__MACOSX' not in f and \ - not os.path.basename(f).startswith('.'): - opf = os.path.abspath(f) - break + opf = self.find_opf() + if opf is None: + for f in walk(u'.'): + if f.lower().endswith('.opf') and '__MACOSX' not in f and \ + not os.path.basename(f).startswith('.'): + opf = os.path.abspath(f) + break path = getattr(stream, 'name', 'stream') if opf is None: From 973afc073333c6cccbb10284cc688319dce68c38 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 11:34:46 -0600 Subject: [PATCH 13/39] Content server: Make /mobile a little prettier --- resources/content_server/mobile.css | 17 +++++++++++++++++ src/calibre/library/server/mobile.py | 21 +++++++++++---------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/resources/content_server/mobile.css b/resources/content_server/mobile.css index a887684841..28d12bb6db 100644 --- a/resources/content_server/mobile.css +++ b/resources/content_server/mobile.css @@ -1,5 +1,9 @@ /* CSS for the mobile version of the content server webpage */ +.body { + font-family: sans-serif; +} + .navigation table.buttons { width: 100%; } @@ -85,4 +89,17 @@ div.navigation { clear: both; } +.data-container { + display: inline-block; + vertical-align: middle; +} +.first-line { + font-size: larger; + font-weight: bold; +} + +.second-line { + margin-top: 0.75ex; + display: block; +} diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py index b9ca24a823..7c2f959131 100644 --- a/src/calibre/library/server/mobile.py +++ b/src/calibre/library/server/mobile.py @@ -112,7 +112,6 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS): CLASS('thumbnail')) data = TD() - last = None for fmt in book['formats'].split(','): a = ascii_filename(book['authors']) t = ascii_filename(book['title']) @@ -124,9 +123,11 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS): ), CLASS('button')) s.tail = u'' - last = s data.append(s) + div = DIV(CLASS('data-container')) + data.append(div) + series = u'[%s - %s]'%(book['series'], book['series_index']) \ if book['series'] else '' tags = u'Tags=[%s]'%book['tags'] if book['tags'] else '' @@ -137,13 +138,13 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS): if val: ctext += '%s=[%s] '%tuple(val.split(':#:')) - text = u'\u202f%s %s by %s - %s - %s %s %s' % (book['title'], series, - book['authors'], book['size'], book['timestamp'], tags, ctext) - - if last is None: - data.text = text - else: - last.tail += text + first = SPAN(u'\u202f%s %s by %s' % (book['title'], series, + book['authors']), CLASS('first-line')) + div.append(first) + second = SPAN(u'%s - %s %s %s' % ( book['size'], + book['timestamp'], + tags, ctext), CLASS('second-line')) + div.append(second) bookt.append(TR(thumbnail, data)) # }}} @@ -229,7 +230,7 @@ class MobileServer(object): no_tag_count=True) book['title'] = record[FM['title']] for x in ('timestamp', 'pubdate'): - book[x] = strftime('%Y/%m/%d %H:%M:%S', record[FM[x]]) + book[x] = strftime('%b, %Y', record[FM[x]]) book['id'] = record[FM['id']] books.append(book) for key in CKEYS: From 7dc8e70b711e5bf2a1fdbbdbe522b88ac31c2b9d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 14:33:19 -0600 Subject: [PATCH 14/39] /browse: Fix sorting on custom cols. Also specify sort order explicitly when sorting on boolean columns --- src/calibre/library/caches.py | 4 ++++ src/calibre/library/server/browse.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index c22f9e00b0..300ddbac0b 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -816,6 +816,10 @@ class SortKeyGenerator(object): if val is None: val = '' val = val.lower() + + elif dt == 'bool': + val = {True: 1, False: 2, None: 3}.get(val, 3) + yield val # }}} diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index ea86de4c1b..d8d67c3824 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' import operator, os, json from binascii import hexlify, unhexlify -from urllib import quote +from urllib import quote, unquote import cherrypy @@ -482,6 +482,8 @@ class BrowseServer(object): @Endpoint(sort_type='list') def browse_matches(self, category=None, cid=None, list_sort=None): + if list_sort: + list_sort = unquote(list_sort) if not cid: raise cherrypy.HTTPError(404, 'invalid category id: %r'%cid) categories = self.categories_cache() From 1dd72e682c9825f57331bfb511674d66184d821b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 15:12:18 -0600 Subject: [PATCH 15/39] USBMS drivers: Pass full filepath of ebook file to the upload_cover function --- src/calibre/devices/cybook/driver.py | 2 +- src/calibre/devices/hanvon/driver.py | 4 ++-- src/calibre/devices/misc.py | 2 +- src/calibre/devices/nook/driver.py | 2 +- src/calibre/devices/usbms/driver.py | 10 ++++++---- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/calibre/devices/cybook/driver.py b/src/calibre/devices/cybook/driver.py index d314646a87..7c436a7d0e 100644 --- a/src/calibre/devices/cybook/driver.py +++ b/src/calibre/devices/cybook/driver.py @@ -42,7 +42,7 @@ class CYBOOK(USBMS): DELETE_EXTS = ['.mbp', '.dat', '.bin', '_6090.t2b', '.thn'] SUPPORTS_SUB_DIRS = True - def upload_cover(self, path, filename, metadata): + def upload_cover(self, path, filename, metadata, filepath): coverdata = getattr(metadata, 'thumbnail', None) if coverdata and coverdata[2]: coverdata = coverdata[2] diff --git a/src/calibre/devices/hanvon/driver.py b/src/calibre/devices/hanvon/driver.py index 7933b9885d..1fe18afc58 100644 --- a/src/calibre/devices/hanvon/driver.py +++ b/src/calibre/devices/hanvon/driver.py @@ -77,7 +77,7 @@ class ALEX(N516): name = os.path.splitext(os.path.basename(file_abspath))[0] + '.png' return os.path.join(base, 'covers', name) - def upload_cover(self, path, filename, metadata): + def upload_cover(self, path, filename, metadata, filepath): from calibre.ebooks import calibre_cover from calibre.utils.magick.draw import thumbnail coverdata = getattr(metadata, 'thumbnail', None) @@ -129,7 +129,7 @@ class AZBOOKA(ALEX): def can_handle(self, device_info, debug=False): return not is_alex(device_info) - def upload_cover(self, path, filename, metadata): + def upload_cover(self, path, filename, metadata, filepath): pass class EB511(USBMS): diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index a1c9b790e4..bca4e8ec52 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -102,7 +102,7 @@ class PDNOVEL(USBMS): DELETE_EXTS = ['.jpg', '.jpeg', '.png'] - def upload_cover(self, path, filename, metadata): + def upload_cover(self, path, filename, metadata, filepath): coverdata = getattr(metadata, 'thumbnail', None) if coverdata and coverdata[2]: with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile: diff --git a/src/calibre/devices/nook/driver.py b/src/calibre/devices/nook/driver.py index f697ee5202..a809b2c08a 100644 --- a/src/calibre/devices/nook/driver.py +++ b/src/calibre/devices/nook/driver.py @@ -45,7 +45,7 @@ class NOOK(USBMS): DELETE_EXTS = ['.jpg'] SUPPORTS_SUB_DIRS = True - def upload_cover(self, path, filename, metadata): + def upload_cover(self, path, filename, metadata, filepath): try: from PIL import Image, ImageDraw Image, ImageDraw diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index a83a8eb0ea..2f26c4a353 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -186,7 +186,8 @@ class USBMS(CLI, Device): self.put_file(infile, filepath, replace_file=True) try: self.upload_cover(os.path.dirname(filepath), - os.path.splitext(os.path.basename(filepath))[0], mdata) + os.path.splitext(os.path.basename(filepath))[0], + mdata, filepath) except: # Failure to upload cover is not catastrophic import traceback traceback.print_exc() @@ -197,14 +198,15 @@ class USBMS(CLI, Device): debug_print('USBMS: finished uploading %d books'%(len(files))) return zip(paths, cycle([on_card])) - def upload_cover(self, path, filename, metadata): + def upload_cover(self, path, filename, metadata, filepath): ''' Upload book cover to the device. Default implementation does nothing. - :param path: the full path were the associated book is located. - :param filename: the name of the book file without the extension. + :param path: The full path to the directory where the associated book is located. + :param filename: The name of the book file without the extension. :param metadata: metadata belonging to the book. Use metadata.thumbnail for cover + :param filepath: The full path to the ebook file ''' pass From 5fb294486babed1e2471940a4ebee97cc199601f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 16:40:39 -0600 Subject: [PATCH 16/39] SONY driver: Add support for uploading covers and periodicals --- src/calibre/devices/prs505/__init__.py | 6 ++ src/calibre/devices/prs505/driver.py | 49 +++++++-- src/calibre/devices/prs505/sony_cache.py | 128 ++++++++++++++++++++++- 3 files changed, 173 insertions(+), 10 deletions(-) diff --git a/src/calibre/devices/prs505/__init__.py b/src/calibre/devices/prs505/__init__.py index 20f3b8d49b..48b7d98123 100644 --- a/src/calibre/devices/prs505/__init__.py +++ b/src/calibre/devices/prs505/__init__.py @@ -2,5 +2,11 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' MEDIA_XML = 'database/cache/media.xml' +MEDIA_EXT = 'database/cache/cacheExt.xml' CACHE_XML = 'Sony Reader/database/cache.xml' +CACHE_EXT = 'Sony Reader/database/cacheExt.xml' + +MEDIA_THUMBNAIL = 'database/thumbnail' +CACHE_THUMBNAIL = 'Sony Reader/database/thumbnail' + diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index bb62e4dc76..3bcf7715a2 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -9,10 +9,10 @@ Device driver for the SONY devices import os, time, re from calibre.devices.usbms.driver import USBMS, debug_print -from calibre.devices.prs505 import MEDIA_XML -from calibre.devices.prs505 import CACHE_XML +from calibre.devices.prs505 import MEDIA_XML, MEDIA_EXT, CACHE_XML, CACHE_EXT, \ + MEDIA_THUMBNAIL, CACHE_THUMBNAIL from calibre.devices.prs505.sony_cache import XMLCache -from calibre import __appname__ +from calibre import __appname__, prints from calibre.devices.usbms.books import CollectionsBookList class PRS505(USBMS): @@ -66,6 +66,8 @@ class PRS505(USBMS): plugboard = None plugboard_func = None + THUMBNAIL_HEIGHT = 200 + def windows_filter_pnp_id(self, pnp_id): return '_LAUNCHER' in pnp_id @@ -116,20 +118,21 @@ class PRS505(USBMS): return fname def initialize_XML_cache(self): - paths, prefixes = {}, {} - for prefix, path, source_id in [ - ('main', MEDIA_XML, 0), - ('card_a', CACHE_XML, 1), - ('card_b', CACHE_XML, 2) + paths, prefixes, ext_paths = {}, {}, {} + for prefix, path, ext_path, source_id in [ + ('main', MEDIA_XML, MEDIA_EXT, 0), + ('card_a', CACHE_XML, CACHE_EXT, 1), + ('card_b', CACHE_XML, CACHE_EXT, 2) ]: prefix = getattr(self, '_%s_prefix'%prefix) if prefix is not None and os.path.exists(prefix): paths[source_id] = os.path.join(prefix, *(path.split('/'))) + ext_paths[source_id] = os.path.join(prefix, *(ext_path.split('/'))) prefixes[source_id] = prefix d = os.path.dirname(paths[source_id]) if not os.path.exists(d): os.makedirs(d) - return XMLCache(paths, prefixes, self.settings().use_author_sort) + return XMLCache(paths, ext_paths, prefixes, self.settings().use_author_sort) def books(self, oncard=None, end_session=True): debug_print('PRS505: starting fetching books for card', oncard) @@ -174,3 +177,31 @@ class PRS505(USBMS): def set_plugboards(self, plugboards, pb_func): self.plugboards = plugboards self.plugboard_func = pb_func + + def upload_cover(self, path, filename, metadata, filepath): + if metadata.thumbnail and metadata.thumbnail[-1]: + path = path.replace('/', os.sep) + is_main = path.startswith(self._main_prefix) + thumbnail_dir = MEDIA_THUMBNAIL if is_main else CACHE_THUMBNAIL + prefix = None + if is_main: + prefix = self._main_prefix + else: + if self._card_a_prefix and \ + path.startswith(self._card_a_prefix): + prefix = self._card_a_prefix + elif self._card_b_prefix and \ + path.startswith(self._card_b_prefix): + prefix = self._card_b_prefix + if prefix is None: + prints('WARNING: Failed to find prefix for:', filepath) + return + thumbnail_dir = os.path.join(prefix, *thumbnail_dir.split('/')) + + relpath = os.path.relpath(filepath, prefix) + thumbnail_dir = os.path.join(thumbnail_dir, relpath) + if not os.path.exists(thumbnail_dir): + os.makedirs(thumbnail_dir) + with open(os.path.join(thumbnail_dir, 'main_thumbnail.jpg'), 'wb') as f: + f.write(metadata.thumbnail[-1]) + diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index ce24dcd03f..e272cd6372 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -9,6 +9,7 @@ import os, time from base64 import b64decode from uuid import uuid4 from lxml import etree +from datetime import date from calibre import prints, guess_type, isbytestring from calibre.devices.errors import DeviceError @@ -18,6 +19,20 @@ from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.metadata import authors_to_string, title_sort, \ authors_to_sort_string +''' +cahceExt.xml + +Periodical identifier sample from a PRS-650: + + + + + main_thumbnail.jpg + + + +''' + # Utility functions {{{ EMPTY_CARD_CACHE = '''\ @@ -25,6 +40,12 @@ EMPTY_CARD_CACHE = '''\ ''' +EMPTY_EXT_CACHE = '''\ + + + +''' + MIME_MAP = { "lrf" : "application/x-sony-bbeb", 'lrx' : 'application/x-sony-bbeb', @@ -63,7 +84,7 @@ def uuid(): class XMLCache(object): - def __init__(self, paths, prefixes, use_author_sort): + def __init__(self, paths, ext_paths, prefixes, use_author_sort): if DEBUG: debug_print('Building XMLCache...', paths) self.paths = paths @@ -85,6 +106,7 @@ class XMLCache(object): if os.access(path, os.R_OK): with open(path, 'rb') as f: raw = f.read() + self.roots[source_id] = etree.fromstring(xml_to_unicode( raw, strip_encoding_pats=True, assume_utf8=True, verbose=DEBUG)[0], @@ -93,6 +115,25 @@ class XMLCache(object): raise Exception(('The SONY database at %s is corrupted. Try ' ' disconnecting and reconnecting your reader.')%path) + self.ext_paths, self.ext_roots = {}, {} + for source_id, path in ext_paths.items(): + if not os.path.exists(path): + try: + with open(path, 'wb') as f: + f.write(EMPTY_EXT_CACHE) + except: + pass + if os.access(path, os.W_OK): + try: + with open(path, 'rb') as f: + self.ext_roots[source_id] = etree.fromstring( + xml_to_unicode(f.read(), + strip_encoding_pats=True, assume_utf8=True, + verbose=DEBUG)[0], parser=parser) + self.ext_paths[source_id] = path + except: + pass + # }}} recs = self.roots[0].xpath('//*[local-name()="records"]') @@ -352,12 +393,18 @@ class XMLCache(object): debug_print('Updating XML Cache:', i) root = self.record_roots[i] lpath_map = self.build_lpath_map(root) + ext_root = self.ext_roots[i] if i in self.ext_roots else None + ext_lpath_map = None + if ext_root is not None: + ext_lpath_map = self.build_lpath_map(ext_root) gtz_count = ltz_count = 0 use_tz_var = False for book in booklist: path = os.path.join(self.prefixes[i], *(book.lpath.split('/'))) record = lpath_map.get(book.lpath, None) + created = False if record is None: + created = True record = self.create_text_record(root, i, book.lpath) if plugboard is not None: newmi = book.deepcopy_metadata() @@ -373,6 +420,13 @@ class XMLCache(object): if book.device_collections is None: book.device_collections = [] book.device_collections = playlist_map.get(book.lpath, []) + + if created and ext_root is not None and \ + ext_lpath_map.get(book.lpath, None) is None: + ext_record = self.create_ext_text_record(ext_root, i, + book.lpath, book.thumbnail) + self.periodicalize_book(book, ext_record) + debug_print('Timezone votes: %d GMT, %d LTZ, use_tz_var=%s'% (gtz_count, ltz_count, use_tz_var)) self.update_playlists(i, root, booklist, collections_attributes) @@ -386,6 +440,47 @@ class XMLCache(object): self.fix_ids() debug_print('Finished update') + def is_sony_periodical(self, book): + if _('News') not in book.tags: + return False + if not book.lpath.lower().endswith('.epub'): + return False + if book.pubdate.date() < date(2010, 10, 17): + return False + return True + + def periodicalize_book(self, book, record): + if not self.is_sony_periodical(book): + return + record.set('conformsTo', + "http://xmlns.sony.net/e-book/prs/periodicals/1.0/newspaper/1.0") + + record.set('description', '') + + name = None + if '[' in book.title: + name = book.title.split('[')[0].strip() + if len(name) < 4: + name = None + if not name: + try: + name = [t for t in book.tags if t != _('News')][0] + except: + name = None + + if not name: + name = book.title + + record.set('periodicalName', name) + + try: + pubdate = strftime(book.pubdate.utctimetuple(), + zone=lambda x : x) + record.set('publicationDate', pubdate) + except: + pass + + def rebuild_collections(self, booklist, bl_index): if bl_index not in self.record_roots: return @@ -472,6 +567,25 @@ class XMLCache(object): root.append(ans) return ans + def create_ext_text_record(self, root, bl_id, lpath, thumbnail): + namespace = root.nsmap[None] + attrib = { 'path': lpath } + ans = root.makeelement('{%s}text'%namespace, attrib=attrib, + nsmap=root.nsmap) + ans.tail = '\n' + root[-1].tail = '\n' + '\t' + root.append(ans) + if thumbnail and thumbnail[-1]: + ans.text = '\n' + '\t\t' + t = root.makeelement('{%s}thumbnail'%namespace, + attrib={'width':str(thumbnail[0]), 'height':str(thumbnail[1])}, + nsmap=root.nsmap) + t.text = 'main_thumbnail.jpg' + ans.append(t) + t.tail = '\n\t' + return ans + + def update_text_record(self, record, book, path, bl_index, gtz_count, ltz_count, use_tz_var): ''' @@ -589,6 +703,18 @@ class XMLCache(object): '') with open(path, 'wb') as f: f.write(raw) + + for i, path in self.ext_paths.items(): + try: + raw = etree.tostring(self.ext_roots[i], encoding='UTF-8', + xml_declaration=True) + except: + continue + raw = raw.replace("", + '') + with open(path, 'wb') as f: + f.write(raw) + # }}} # Utility methods {{{ From 2bd9f63f29248eba346c22eeb691977591d24069 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 16:53:51 -0600 Subject: [PATCH 17/39] Linux device mounting: Mount the drive with the lowest kernel name as main memory --- src/calibre/devices/prs505/sony_cache.py | 6 +++--- src/calibre/devices/usbms/device.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index e272cd6372..15245d3cd5 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -97,8 +97,8 @@ class XMLCache(object): for source_id, path in paths.items(): if source_id == 0: if not os.path.exists(path): - raise DeviceError('The SONY XML cache media.xml does not exist. Try' - ' disconnecting and reconnecting your reader.') + raise DeviceError(('The SONY XML cache %r does not exist. Try' + ' disconnecting and reconnecting your reader.')%repr(path)) with open(path, 'rb') as f: raw = f.read() else: @@ -112,7 +112,7 @@ class XMLCache(object): verbose=DEBUG)[0], parser=parser) if self.roots[source_id] is None: - raise Exception(('The SONY database at %s is corrupted. Try ' + raise Exception(('The SONY database at %r is corrupted. Try ' ' disconnecting and reconnecting your reader.')%path) self.ext_paths, self.ext_roots = {}, {} diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 6f938cbcbd..aa4f0d06f4 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -523,7 +523,8 @@ class Device(DeviceConfig, DevicePlugin): devnodes.append(node) devnodes += list(repeat(None, 3)) - ans = tuple(['/dev/'+x if ok.get(x, False) else None for x in devnodes[:3]]) + ans = ['/dev/'+x if ok.get(x, False) else None for x in devnodes[:3]] + ans.sort(key=lambda x: x[5:] if x else 'zzzzz') return self.linux_swap_drives(ans) def linux_swap_drives(self, drives): From 8b0b1312e311cc4bc2eaa4d65bee0f6c96aa2b13 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 17:19:54 -0600 Subject: [PATCH 18/39] Linux device drivers: Fix udisks based ejecting for devices with multiple nodes --- src/calibre/devices/udisks.py | 20 ++++++-------- src/calibre/devices/usbms/device.py | 41 +++++++++++++++++------------ 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/calibre/devices/udisks.py b/src/calibre/devices/udisks.py index ba26c2b56c..d79b626f36 100644 --- a/src/calibre/devices/udisks.py +++ b/src/calibre/devices/udisks.py @@ -5,8 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import dbus -import os +import dbus, os def node_mountpoint(node): @@ -56,15 +55,6 @@ class UDisks(object): parent = device_node_path while parent[-1] in '0123456789': parent = parent[:-1] - devices = [str(x) for x in self.main.EnumerateDeviceFiles()] - for d in devices: - if d.startswith(parent) and d != parent: - try: - self.unmount(d) - except: - import traceback - print 'Failed to unmount:', d - traceback.print_exc() d = self.device(parent) d.DriveEject([]) @@ -76,13 +66,19 @@ def eject(node_path): u = UDisks() u.eject(node_path) +def umount(node_path): + u = UDisks() + u.unmount(node_path) + if __name__ == '__main__': import sys dev = sys.argv[1] print 'Testing with node', dev u = UDisks() print 'Mounted at:', u.mount(dev) - print 'Ejecting' + print 'Unmounting' + u.unmount(dev) + print 'Ejecting:' u.eject(dev) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index aa4f0d06f4..94744c521f 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -733,24 +733,31 @@ class Device(DeviceConfig, DevicePlugin): pass def eject_linux(self): - try: - from calibre.devices.udisks import eject - return eject(self._linux_main_device_node) - except: - pass - drives = self.find_device_nodes() + from calibre.devices.udisks import eject, umount + drives = [d for d in self.find_device_nodes() if d] + for d in drives: + try: + umount(d) + except: + pass + for d in drives: + try: + eject(d) + except Exception, e: + print 'Udisks eject call for:', d, 'failed:' + print '\t', str(e) + for drive in drives: - if drive: - cmd = 'calibre-mount-helper' - if getattr(sys, 'frozen_path', False): - cmd = os.path.join(sys.frozen_path, cmd) - cmd = [cmd, 'eject'] - mp = getattr(self, "_linux_mount_map", {}).get(drive, - 'dummy/')[:-1] - try: - subprocess.Popen(cmd + [drive, mp]).wait() - except: - pass + cmd = 'calibre-mount-helper' + if getattr(sys, 'frozen_path', False): + cmd = os.path.join(sys.frozen_path, cmd) + cmd = [cmd, 'eject'] + mp = getattr(self, "_linux_mount_map", {}).get(drive, + 'dummy/')[:-1] + try: + subprocess.Popen(cmd + [drive, mp]).wait() + except: + pass def eject(self): if islinux: From 4906e1fc420fc160e90fc4e329feaf2c2ee9eb7f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 17:21:15 -0600 Subject: [PATCH 19/39] ... --- src/calibre/devices/usbms/device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 94744c521f..f085ab8989 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -745,7 +745,7 @@ class Device(DeviceConfig, DevicePlugin): eject(d) except Exception, e: print 'Udisks eject call for:', d, 'failed:' - print '\t', str(e) + print '\t', e for drive in drives: cmd = 'calibre-mount-helper' From 5a8e4b2174e95527fa5b6a09c496a17015b4c222 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 18:26:37 -0600 Subject: [PATCH 20/39] ... --- resources/recipes/cacm.recipe | 74 +++++++++++++++++------------------ 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/resources/recipes/cacm.recipe b/resources/recipes/cacm.recipe index 1618bae742..e4af9d2024 100644 --- a/resources/recipes/cacm.recipe +++ b/resources/recipes/cacm.recipe @@ -1,37 +1,37 @@ -import datetime -from calibre.web.feeds.news import BasicNewsRecipe - -class AdvancedUserRecipe1286242553(BasicNewsRecipe): - title = u'CACM' - oldest_article = 7 - max_articles_per_feed = 100 - needs_subscription = True - feeds = [(u'CACM', u'http://cacm.acm.org/magazine.rss')] - language = 'en' - __author__ = 'jonmisurda' - no_stylesheets = True - remove_tags = [ - dict(name='div', attrs={'class':['FeatureBox', 'ArticleComments', 'SideColumn', \ - 'LeftColumn', 'RightColumn', 'SiteSearch', 'MainNavBar','more', 'SubMenu', 'inner']}) - ] - cover_url_pattern = 'http://cacm.acm.org/magazines/%d/%d' - - def get_browser(self): - br = BasicNewsRecipe.get_browser() - if self.username is not None and self.password is not None: - br.open('https://cacm.acm.org/login') - br.select_form(nr=1) - br['current_member[user]'] = self.username - br['current_member[passwd]'] = self.password - br.submit() - return br - - def get_cover_url(self): - now = datetime.datetime.now() - - cover_url = None - soup = self.index_to_soup(self.cover_url_pattern % (now.year, now.month)) - cover_item = soup.find('img',attrs={'alt':'magazine cover image'}) - if cover_item: - cover_url = cover_item['src'] - return cover_url +import datetime +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1286242553(BasicNewsRecipe): + title = u'CACM' + oldest_article = 7 + max_articles_per_feed = 100 + needs_subscription = True + feeds = [(u'CACM', u'http://cacm.acm.org/magazine.rss')] + language = 'en' + __author__ = 'jonmisurda' + no_stylesheets = True + remove_tags = [ + dict(name='div', attrs={'class':['FeatureBox', 'ArticleComments', 'SideColumn', \ + 'LeftColumn', 'RightColumn', 'SiteSearch', 'MainNavBar','more', 'SubMenu', 'inner']}) + ] + cover_url_pattern = 'http://cacm.acm.org/magazines/%d/%d' + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + if self.username is not None and self.password is not None: + br.open('https://cacm.acm.org/login') + br.select_form(nr=1) + br['current_member[user]'] = self.username + br['current_member[passwd]'] = self.password + br.submit() + return br + + def get_cover_url(self): + now = datetime.datetime.now() + + cover_url = None + soup = self.index_to_soup(self.cover_url_pattern % (now.year, now.month)) + cover_item = soup.find('img',attrs={'alt':'magazine cover image'}) + if cover_item: + cover_url = cover_item['src'] + return cover_url From 1c2ff97a5d8b64edffc80d42a0e64ea152a94842 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 19 Oct 2010 19:06:31 -0600 Subject: [PATCH 21/39] ... --- src/calibre/devices/usbms/device.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index f085ab8989..f826167d16 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -740,12 +740,17 @@ class Device(DeviceConfig, DevicePlugin): umount(d) except: pass + failures = False for d in drives: try: eject(d) except Exception, e: print 'Udisks eject call for:', d, 'failed:' print '\t', e + failures = True + + if not failures: + return for drive in drives: cmd = 'calibre-mount-helper' From abc1f7525c8a2af18a714ee57944c720750b7ec7 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 20 Oct 2010 12:28:55 +0100 Subject: [PATCH 22/39] Fix restore not to die when conflicting custom columns are encountered. --- src/calibre/library/restore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index 16aba3aebd..748d60b0b2 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -170,8 +170,8 @@ class Restore(Thread): label = cfm['label'] if label in m and args != m[label]: if label not in self.conflicting_custom_cols: - self.conflicting_custom_cols[label] = set([m[label]]) - self.conflicting_custom_cols[label].add(args) + self.conflicting_custom_cols[label] = [] + self.conflicting_custom_cols[label].append(args) m[cfm['label']] = args db = RestoreDatabase(self.library_path) From 8f371fdd4f1279375e5cfa5a79be73dc4c0b40b0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 20 Oct 2010 22:12:50 -0600 Subject: [PATCH 23/39] /browse: Force AJAX requests to never be cached as the user has no way of requesting a refresh for them. --- resources/content_server/browse/browse.js | 3 +++ src/calibre/library/server/browse.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/resources/content_server/browse/browse.js b/resources/content_server/browse/browse.js index 5e3cee14c0..e7e359c163 100644 --- a/resources/content_server/browse/browse.js +++ b/resources/content_server/browse/browse.js @@ -156,6 +156,7 @@ function category() { if (href) { $.ajax({ url:href, + cache: false, data:{'sort':cookie(sort_cookie_name)}, success: function(data) { this.children(".loaded").html(data); @@ -212,6 +213,7 @@ function load_page(elem) { url: href, context: elem, dataType: "json", + cache : false, type: 'POST', timeout: 600000, //milliseconds (10 minutes) data: {'ids': ids}, @@ -263,6 +265,7 @@ function show_details(a_dom) { $.ajax({ url: book.find('.details-href').attr('title'), context: bd, + cache: false, dataType: "json", timeout: 600000, //milliseconds (10 minutes) error: function(xhr, stat, err) { diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index d8d67c3824..463fcd6fde 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -168,6 +168,9 @@ class Endpoint(object): # {{{ sort_val = cookie[eself.sort_cookie_name].value kwargs[eself.sort_kwarg] = sort_val + # Remove AJAX caching disabling jquery workaround arg + kwargs.pop('_', None) + ans = func(self, *args, **kwargs) cherrypy.response.headers['Content-Type'] = eself.mimetype updated = self.db.last_modified() From 5fc0d687307bb921c35076bcd92bbcc87d25906a Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 21 Oct 2010 09:54:21 +0100 Subject: [PATCH 24/39] Improve conflicting custom column error reporting when restoring a database --- src/calibre/library/restore.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index 748d60b0b2..bc2c740279 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -71,9 +71,17 @@ class Restore(Thread): if self.conflicting_custom_cols: ans += '\n\n' - ans += 'The following custom columns were not fully restored:\n' + ans += 'The following custom columns have conflicting definitions ' \ + 'and were not fully restored:\n' for x in self.conflicting_custom_cols: ans += '\t#'+x+'\n' + ans += '\tused:\t%s, %s, %s, %s\n'%(self.custom_columns[x][1], + self.custom_columns[x][2], + self.custom_columns[x][3], + self.custom_columns[x][5]) + for coldef in self.conflicting_custom_cols[x]: + ans += '\tother:\t%s, %s, %s, %s\n'%(coldef[1], coldef[2], + coldef[3], coldef[5]) if self.mismatched_dirs: ans += '\n\n' @@ -152,7 +160,7 @@ class Restore(Thread): def create_cc_metadata(self): self.books.sort(key=itemgetter('timestamp')) - m = {} + self.custom_columns = {} fields = ('label', 'name', 'datatype', 'is_multiple', 'is_editable', 'display') for b in self.books: @@ -168,16 +176,17 @@ class Restore(Thread): if len(args) == len(fields): # TODO: Do series type columns need special handling? label = cfm['label'] - if label in m and args != m[label]: + if label in self.custom_columns and args != self.custom_columns[label]: if label not in self.conflicting_custom_cols: self.conflicting_custom_cols[label] = [] - self.conflicting_custom_cols[label].append(args) - m[cfm['label']] = args + if self.custom_columns[label] not in self.conflicting_custom_cols[label]: + self.conflicting_custom_cols[label].append(self.custom_columns[label]) + self.custom_columns[label] = args db = RestoreDatabase(self.library_path) - self.progress_callback(None, len(m)) - if len(m): - for i,args in enumerate(m.values()): + self.progress_callback(None, len(self.custom_columns)) + if len(self.custom_columns): + for i,args in enumerate(self.custom_columns.values()): db.create_custom_column(*args) self.progress_callback(_('creating custom column ')+args[0], i+1) db.conn.close() From 506cd50dd1774b3961150b3a97d132a7c1cd0c1e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 21 Oct 2010 07:49:13 -0700 Subject: [PATCH 25/39] Fix #7228 (The Economic Times of India - News Headers are missing) --- resources/recipes/theeconomictimes_india.recipe | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/resources/recipes/theeconomictimes_india.recipe b/resources/recipes/theeconomictimes_india.recipe index 59cd56b67e..92d2a64a70 100644 --- a/resources/recipes/theeconomictimes_india.recipe +++ b/resources/recipes/theeconomictimes_india.recipe @@ -21,8 +21,9 @@ class TheEconomicTimes(BasicNewsRecipe): language = 'en_IN' publication_type = 'newspaper' masthead_url = 'http://economictimes.indiatimes.com/photo/2676871.cms' - extra_css = """ body{font-family: Arial,Helvetica,sans-serif} - .heading1{font-size: xx-large; font-weight: bold} """ + extra_css = """ + body{font-family: Arial,Helvetica,sans-serif} + """ conversion_options = { 'comment' : description @@ -31,8 +32,9 @@ class TheEconomicTimes(BasicNewsRecipe): , 'language' : language } - keep_only_tags = [dict(attrs={'class':['heading1','headingnext','Normal']})] + keep_only_tags = [dict(attrs={'class':'printdiv'})] remove_tags = [dict(name=['object','link','embed','iframe','base','table','meta'])] + remove_attributes = ['name'] feeds = [(u'All articles', u'http://economictimes.indiatimes.com/rssfeedsdefault.cms')] From 6dce871b053c803f4a0283ad1929fb8e375ce82c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 21 Oct 2010 07:50:22 -0700 Subject: [PATCH 26/39] Fix #7187 (New Scientist recipe update) --- resources/recipes/new_scientist.recipe | 29 +++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/resources/recipes/new_scientist.recipe b/resources/recipes/new_scientist.recipe index 2e864565ff..02bbbe4d42 100644 --- a/resources/recipes/new_scientist.recipe +++ b/resources/recipes/new_scientist.recipe @@ -8,11 +8,11 @@ import re from calibre.web.feeds.news import BasicNewsRecipe class NewScientist(BasicNewsRecipe): - title = 'New Scientist - Online News' + title = 'New Scientist - Online News w. subscription' __author__ = 'Darko Miletic' description = 'Science news and science articles from New Scientist.' language = 'en' - publisher = 'New Scientist' + publisher = 'Reed Business Information Ltd.' category = 'science news, science articles, science jobs, drugs, cancer, depression, computer software' oldest_article = 7 max_articles_per_feed = 100 @@ -21,7 +21,12 @@ class NewScientist(BasicNewsRecipe): cover_url = 'http://www.newscientist.com/currentcover.jpg' masthead_url = 'http://www.newscientist.com/img/misc/ns_logo.jpg' encoding = 'utf-8' - extra_css = ' body{font-family: Arial,sans-serif} img{margin-bottom: 0.8em} ' + needs_subscription = 'optional' + extra_css = """ + body{font-family: Arial,sans-serif} + img{margin-bottom: 0.8em} + .quotebx{font-size: x-large; font-weight: bold; margin-right: 2em; margin-left: 2em} + """ conversion_options = { 'comment' : description @@ -33,15 +38,27 @@ class NewScientist(BasicNewsRecipe): keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})] + def get_browser(self): + br = BasicNewsRecipe.get_browser() + br.open('http://www.newscientist.com/') + if self.username is not None and self.password is not None: + br.open('https://www.newscientist.com/user/login?redirectURL=') + br.select_form(nr=2) + br['loginId' ] = self.username + br['password'] = self.password + br.submit() + return br + remove_tags = [ dict(name='div' , attrs={'class':['hldBd','adline','pnl','infotext' ]}) ,dict(name='div' , attrs={'id' :['compnl','artIssueInfo','artTools','comments','blgsocial','sharebtns']}) ,dict(name='p' , attrs={'class':['marker','infotext' ]}) ,dict(name='meta' , attrs={'name' :'description' }) - ,dict(name='a' , attrs={'rel' :'tag' }) + ,dict(name='a' , attrs={'rel' :'tag' }) + ,dict(name=['link','base','meta','iframe','object','embed']) ] remove_tags_after = dict(attrs={'class':['nbpcopy','comments']}) - remove_attributes = ['height','width'] + remove_attributes = ['height','width','lang'] feeds = [ (u'Latest Headlines' , u'http://feeds.newscientist.com/science-news' ) @@ -62,6 +79,8 @@ class NewScientist(BasicNewsRecipe): return url + '?full=true&print=true' def preprocess_html(self, soup): + for item in soup.findAll(['quote','quotetext']): + item.name='p' for tg in soup.findAll('a'): if tg.string == 'Home': tg.parent.extract() From eafc6e72c11864f6b0d084938d940624cc704144 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 21 Oct 2010 07:51:31 -0700 Subject: [PATCH 27/39] Fix #7180 (Fox news website changed) --- resources/recipes/foxnews.recipe | 39 +++++++++----------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/resources/recipes/foxnews.recipe b/resources/recipes/foxnews.recipe index e7e76390b5..916bd28ad2 100644 --- a/resources/recipes/foxnews.recipe +++ b/resources/recipes/foxnews.recipe @@ -4,7 +4,6 @@ __copyright__ = '2010, Darko Miletic ' foxnews.com ''' -import re from calibre.web.feeds.news import BasicNewsRecipe class FoxNews(BasicNewsRecipe): @@ -21,11 +20,10 @@ class FoxNews(BasicNewsRecipe): language = 'en' publication_type = 'newsportal' remove_empty_feeds = True - extra_css = ' body{font-family: Arial,sans-serif } img{margin-bottom: 0.4em} .caption{font-size: x-small} ' - - preprocess_regexps = [ - (re.compile(r'.*?', re.DOTALL|re.IGNORECASE),lambda match: '') - ] + extra_css = """ + body{font-family: Arial,sans-serif } + .caption{font-size: x-small} + """ conversion_options = { 'comment' : description @@ -34,27 +32,15 @@ class FoxNews(BasicNewsRecipe): , 'language' : language } - remove_attributes = ['xmlns'] - - keep_only_tags = [ - dict(name='div', attrs={'id' :['story','browse-story-content']}) - ,dict(name='div', attrs={'class':['posts articles','slideshow']}) - ,dict(name='h4' , attrs={'class':'storyDate'}) - ,dict(name='h1' , attrs={'xmlns:functx':'http://www.functx.com'}) - ,dict(name='div', attrs={'class':'authInfo'}) - ,dict(name='div', attrs={'id':'articleCont'}) - ] + remove_attributes = ['xmlns','lang'] remove_tags = [ - dict(name='div', attrs={'class':['share-links','quigo quigo2','share-text','storyControls','socShare','btm-links']}) - ,dict(name='div', attrs={'id' :['otherMedia','loomia_display','img-all-path','story-vcmId','story-url','pane-browse-story-comments','story_related']}) - ,dict(name='ul' , attrs={'class':['tools','tools alt','tools alt2','tabs']}) - ,dict(name='a' , attrs={'class':'join-discussion'}) - ,dict(name='ul' , attrs={'class':['tools','tools alt','tools alt2']}) - ,dict(name='p' , attrs={'class':'see_fullarchive'}) - ,dict(name=['object','embed','link','script']) + dict(name=['object','embed','link','script','iframe','meta','base']) + ,dict(attrs={'class':['user-control','url-description','ad-context']}) ] + remove_tags_before=dict(name='h1') + remove_tags_after =dict(attrs={'class':'url-description'}) feeds = [ (u'Latest Headlines', u'http://feeds.foxnews.com/foxnews/latest' ) @@ -67,8 +53,5 @@ class FoxNews(BasicNewsRecipe): ,(u'Entertainment' , u'http://feeds.foxnews.com/foxnews/entertainment' ) ] - def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - return self.adeify_images(soup) - + def print_version(self, url): + return url + 'print' From 0559826083bf6f653aba88e6bbbedb67d1288064 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 21 Oct 2010 07:53:53 -0700 Subject: [PATCH 28/39] Fix #7147 (Auto merge books not respecting article sort tweak) --- src/calibre/library/database2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index bbfef47977..b21299c335 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -748,10 +748,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return False def find_identical_books(self, mi): - fuzzy_title_patterns = [(re.compile(pat), repl) for pat, repl in + fuzzy_title_patterns = [(re.compile(pat, re.IGNORECASE), repl) for pat, repl in [ (r'[\[\](){}<>\'";,:#]', ''), - (r'^(the|a|an) ', ''), + (tweaks.get('title_sort_articles', r'^(a|the|an)\s+'), ''), (r'[-._]', ' '), (r'\s+', ' ') ] From 2a2f1fd988e6e35ef4c71d5b9c2343c1c0570ba7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 21 Oct 2010 07:56:16 -0700 Subject: [PATCH 29/39] Fix #7252 (New recipe for The Economic Collapse blog) --- resources/images/news/theecocolapse.png | Bin 0 -> 1264 bytes resources/recipes/theecocolapse.recipe | 46 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 resources/images/news/theecocolapse.png create mode 100644 resources/recipes/theecocolapse.recipe diff --git a/resources/images/news/theecocolapse.png b/resources/images/news/theecocolapse.png new file mode 100644 index 0000000000000000000000000000000000000000..1c45ec14bf41c20c9a86311f4af9ccd47de799df GIT binary patch literal 1264 zcmeAS@N?(olHy`uVBq!ia0vp^0w65F1|AWFkDz^H9*yf;%kG#rN>m48Y1xyIaN$&n8)EXe}nRBoyVD>IS z-#0E7udq$NamUy=eV^+KH%`xcPal5RbAIN}x!)5mr1bXgt$JF-%Ohi%WF%|!zx&R;|JQGI^}YApDl0ZK{r-0G%fG*Q+fTX#rfrV9f6p&ajN4bS?jC#klu<6-#|8AM~r)OlO zJ^Pf}w(|JVSGhY%vkMC=DoVC2f4={JRrP$kP%TcUwfAy44t(Ya)nYJ`>^q&purp@q zr1Q)vo1Ye$ac|fp)UU5!z4!9v=4So4JG;M|&tIW_;cBYUT#JVvS6nT!bY&FbS`?wd za3tyFm+kp7)4fEySBe~$DhUC z86ByfK?^TrWJnz^_m9wc^M;}3#{>@-rMq|S?mu{Ndiu*ae9gttq|&K${No;tFf9fSPa!7;t1@O3^UH^>Z(n?8@HPOWB%ilC%MHXo?TE}{PDrTgbQhl zGiUcEvN#H`Fgh%sJ^y+ZJKMHxlTN;U+sf)WNyYP##p? zyz*Y2y64V0Dw(sEg=(!8YfxEz-DtAr^vjtITerF-)dE4XvQ)22-o2Q(eZ|l3?six( z&Ec}5jLe+5rB;U@&Xb${_R31*^s@U+SFg(3hlShM&6&5aBr|Q<>eYd9cLV>F|NS-9 zCok8wUamXv%5?p^cR7HbW&HDJ=hmqndnzyMysx+8IJeT=>Ei}qiJ@BJ8c~vxSdwa$ zT$Bo=7>o=I4RnFX$S}mf+{)C%%Fsd=$S^Rt;`i$jiiX_$l+3hB6b+VECWclfW)KYt Sz52mG4Gf;HelF{r5}E+n8aB)T literal 0 HcmV?d00001 diff --git a/resources/recipes/theecocolapse.recipe b/resources/recipes/theecocolapse.recipe new file mode 100644 index 0000000000..6743ca68b5 --- /dev/null +++ b/resources/recipes/theecocolapse.recipe @@ -0,0 +1,46 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Darko Miletic ' +''' +theeconomiccollapseblog.com +''' + +from calibre.web.feeds.news import BasicNewsRecipe +class TheEconomicCollapse(BasicNewsRecipe): + title = 'The Economic Collapse' + __author__ = 'Darko Miletic' + description = 'Are You Prepared For The Coming Economic Collapse And The Next Great Depression?' + publisher = 'The Economic Collapse' + category = 'news, politics, USA, economy' + oldest_article = 2 + max_articles_per_feed = 200 + no_stylesheets = True + encoding = 'utf8' + use_embedded_content = False + language = 'en' + remove_empty_feeds = True + extra_css = """ + body{font-family: Tahoma,Arial,sans-serif } + img{margin-bottom: 0.4em} + """ + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + } + + remove_tags = [ + dict(attrs={'class':'sociable'}) + ,dict(name=['iframe','object','embed','meta','link','base']) + ] + remove_attributes=['lang','onclick','width','height'] + keep_only_tags=[dict(attrs={'class':['post-headline','post-bodycopy clearfix','']})] + + feeds = [(u'Posts', u'http://theeconomiccollapseblog.com/feed')] + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + return self.adeify_images(soup) + From 91f8c368c19e7d47019694457977911e4eba28e7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 21 Oct 2010 07:59:42 -0700 Subject: [PATCH 30/39] Fix #7249 (updated "el_pais" recipe, corrects disappearing titles) --- resources/recipes/el_pais.recipe | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/resources/recipes/el_pais.recipe b/resources/recipes/el_pais.recipe index 1e2164b2af..2e358060b8 100644 --- a/resources/recipes/el_pais.recipe +++ b/resources/recipes/el_pais.recipe @@ -2,7 +2,7 @@ __license__ = 'GPL v3' __author__ = 'Jordi Balcells, based on an earlier version by Lorenzo Vigentini & Kovid Goyal' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -description = 'Main daily newspaper from Spain - v1.03 (03, September 2010)' +description = 'Main daily newspaper from Spain - v1.04 (19, October 2010)' __docformat__ = 'restructuredtext en' ''' @@ -32,19 +32,16 @@ class ElPais(BasicNewsRecipe): remove_javascript = True no_stylesheets = True - keep_only_tags = [ dict(name='div', attrs={'class':['cabecera_noticia','cabecera_noticia_reportaje','cabecera_noticia_opinion','contenido_noticia','caja_despiece','presentacion']})] - - extra_css = ''' - p{style:normal size:12 serif} + keep_only_tags = [ dict(name='div', attrs={'class':['cabecera_noticia_reportaje estirar','cabecera_noticia_opinion estirar','cabecera_noticia estirar','contenido_noticia','caja_despiece']})] - ''' + extra_css = ' p{text-align: justify; font-size: 100%} body{ text-align: left; font-family: serif; font-size: 100% } h1{ font-family: sans-serif; font-size:200%; font-weight: bolder; text-align: justify; } h2{ font-family: sans-serif; font-size:150%; font-weight: 500; text-align: justify } h3{ font-family: sans-serif; font-size:125%; font-weight: 500; text-align: justify } img{margin-bottom: 0.4em} ' remove_tags = [ dict(name='div', attrs={'class':['zona_superior','pie_enlaces_inferiores','contorno_f','ampliar']}), - dict(name='div', attrs={'class':['limpiar','mod_apoyo','borde_sup','votos','info_complementa','info_relacionada','buscador_m','nav_ant_sig']}), + dict(name='div', attrs={'class':['limpiar','mod_apoyo','borde_sup','votos estirar','info_complementa','info_relacionada','buscador_m','nav_ant_sig']}), dict(name='div', attrs={'id':['suscribirse suscrito','google_noticia','utilidades','coment','foros_not','pie','lomas','calendar']}), dict(name='p', attrs={'class':'nav_meses'}), - dict(attrs={'class':['enlaces_m','miniaturas_m']}) + dict(attrs={'class':['enlaces_m','miniaturas_m','nav_miniaturas_m']}) ] feeds = [ From 77284f75284ae7e5f8eb0823cd1f406316e0280b Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 21 Oct 2010 21:09:47 +0100 Subject: [PATCH 31/39] Fix exceptions when referencing invalid fields XXX_index --- src/calibre/ebooks/metadata/book/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 593e161df7..9286226a3e 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -501,13 +501,15 @@ class Metadata(object): if key.startswith('#') and key.endswith('_index'): tkey = key[:-6] # strip the _index cmeta = self.get_user_metadata(tkey, make_copy=False) - if cmeta['datatype'] == 'series': + if cmeta and cmeta['datatype'] == 'series': if self.get(tkey): res = self.get_extra(tkey) return (unicode(cmeta['name']+'_index'), self.format_series_index(res), res, cmeta) else: return (unicode(cmeta['name']+'_index'), '', '', cmeta) + else: + return (key, key, None, None) if key in self.custom_field_keys(): res = self.get(key, None) From 6375d4cc451ec5a3d28a078e4170134321c2bb86 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 21 Oct 2010 21:14:04 +0100 Subject: [PATCH 32/39] Fix the fix --- src/calibre/ebooks/metadata/book/base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 9286226a3e..125cd542b8 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -508,8 +508,6 @@ class Metadata(object): self.format_series_index(res), res, cmeta) else: return (unicode(cmeta['name']+'_index'), '', '', cmeta) - else: - return (key, key, None, None) if key in self.custom_field_keys(): res = self.get(key, None) From eccb70cede67d608164b51f9393958bea79fcfc2 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 22 Oct 2010 10:00:24 +0100 Subject: [PATCH 33/39] Fix bulk edit of bool columns when the tristate tweak is set to no --- src/calibre/gui2/custom_column_widgets.py | 33 ++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index 3be4c19d17..3103d7c459 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -429,7 +429,38 @@ class BulkBase(Base): self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify) class BulkBool(BulkBase, Bool): - pass + + def get_initial_value(self, book_ids): + value = None + for book_id in book_ids: + val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True) + if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None: + val = False + if value is not None and value != val: + return None + value = val + return value + + def setup_ui(self, parent): + self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), + QComboBox(parent)] + w = self.widgets[1] + items = [_('Yes'), _('No'), _('Undefined')] + icons = [I('ok.png'), I('list_remove.png'), I('blank.png')] + for icon, text in zip(icons, items): + w.addItem(QIcon(icon), text) + + def setter(self, val): + val = {None: 2, False: 1, True: 0}[val] + self.widgets[1].setCurrentIndex(val) + + def commit(self, book_ids, notify=False): + val = self.gui_val + val = self.normalize_ui_val(val) + if val != self.initial_val: + if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None: + val = False + self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify) class BulkInt(BulkBase, Int): pass From 8ad31bbf087b7503e94cd5b142a2c8af514ca9d7 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 22 Oct 2010 12:31:56 +0100 Subject: [PATCH 34/39] Change bulk edit and the template function titlecase to use utils.titlecase instead of str.title() --- src/calibre/gui2/dialogs/metadata_bulk.py | 7 ++++--- src/calibre/utils/formatter.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index e27f4b5eab..de62f20de0 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -16,6 +16,7 @@ from calibre.gui2.custom_column_widgets import populate_metadata_page from calibre.gui2 import error_dialog from calibre.gui2.progress_indicator import ProgressIndicator from calibre.utils.config import dynamic +from calibre.utils.titlecase import titlecase class MyBlockingBusy(QDialog): @@ -115,7 +116,7 @@ class MyBlockingBusy(QDialog): aum = [a.strip().replace('|', ',') for a in aum.split(',')] new_title = authors_to_string(aum) if do_title_case: - new_title = new_title.title() + new_title = titlecase(new_title) self.db.set_title(id, new_title, notify=False) title_set = True if title: @@ -123,7 +124,7 @@ class MyBlockingBusy(QDialog): self.db.set_authors(id, new_authors, notify=False) if do_title_case and not title_set: title = self.db.title(id, index_is_id=True) - self.db.set_title(id, title.title(), notify=False) + self.db.set_title(id, titlecase(title), notify=False) if au: self.db.set_authors(id, string_to_authors(au), notify=False) elif self.current_phase == 2: @@ -179,7 +180,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): s_r_functions = { '' : lambda x: x, _('Lower Case') : lambda x: x.lower(), _('Upper Case') : lambda x: x.upper(), - _('Title Case') : lambda x: x.title(), + _('Title Case') : lambda x: titlecase(x), } s_r_match_modes = [ _('Character match'), diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index 76c086cc58..336ac2390b 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -7,6 +7,7 @@ Created on 23 Sep 2010 import re, string, traceback from calibre.constants import DEBUG +from calibre.utils.titlecase import titlecase class TemplateFormatter(string.Formatter): ''' @@ -81,7 +82,7 @@ class TemplateFormatter(string.Formatter): functions = { 'uppercase' : (0, lambda s,x: x.upper()), 'lowercase' : (0, lambda s,x: x.lower()), - 'titlecase' : (0, lambda s,x: x.title()), + 'titlecase' : (0, lambda s,x: titlecase(x)), 'capitalize' : (0, lambda s,x: x.capitalize()), 'contains' : (3, _contains), 'ifempty' : (1, _ifempty), From abd8ca4cb9a179e71f7512175e88b0decea027d9 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 22 Oct 2010 13:55:17 +0100 Subject: [PATCH 35/39] Add the possibility of sorting collections by an arbitrary metadata field --- resources/default_tweaks.py | 18 ++++++++++++++ src/calibre/devices/usbms/books.py | 40 ++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 86921886ad..dbb1154172 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -137,6 +137,24 @@ auto_connect_to_folder = '' sony_collection_renaming_rules={} +# Specify how sony collections are sorted. This tweak is only applicable if +# metadata management is set to automatic. You can indicate which metadata is to +# be used to sort on a collection-by-collection basis. The format of the tweak +# is a list of metadata fields from which collections are made, followed by the +# name of the metadata field containing the sort value. +# Example: The following indicates that collections built from pubdate and tags +# are to be sorted by the value in the custom column '#mydate', that collections +# built from 'series' are to be sorted by 'series_index', and that all other +# collections are to be sorted by title. If a collection metadata field is not +# named, then if it is a series- based collection it is sorted by series order, +# otherwise it is sorted by title order. +# [(['pubdate', 'tags'],'#mydate'), (['series'],'series_index'), (['*'], 'title')] +# Note that the bracketing and parentheses are required. The syntax is +# [ ( [list of fields], sort field ) , ( [ list of fields ] , sort field ) ] +# Default: empty (no rules), so no collection attributes are named. +sony_collection_sorting_rules = [] + + # Create search terms to apply a query across several built-in search terms. # Syntax: {'new term':['existing term 1', 'term 2', ...], 'new':['old'...] ...} # Example: create the term 'myseries' that when used as myseries:foo would diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index 462d78b233..f54bfbddf9 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -99,6 +99,15 @@ class CollectionsBookList(BookList): def supports_collections(self): return True + def in_category_sort_rules(self, attr): + sorts = tweaks['sony_collection_sorting_rules'] + for attrs,sortattr in sorts: + if attr in attrs or '*' in attrs: + print 'in category sort:', attr, sortattr + return sortattr + print 'in category sort:', attr, 'None' + return None + def compute_category_name(self, attr, category, field_meta): renames = tweaks['sony_collection_renaming_rules'] attr_name = renames.get(attr, None) @@ -116,6 +125,7 @@ class CollectionsBookList(BookList): from calibre.devices.usbms.driver import debug_print debug_print('Starting get_collections:', prefs['manage_device_metadata']) debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules']) + debug_print('Sorting rules:', tweaks['sony_collection_sorting_rules']) # Complexity: we can use renaming rules only when using automatic # management. Otherwise we don't always have the metadata to make the @@ -171,6 +181,7 @@ class CollectionsBookList(BookList): else: val = [val] + sort_attr = self.in_category_sort_rules(attr) for category in val: is_series = False if doing_dc: @@ -199,22 +210,41 @@ class CollectionsBookList(BookList): if cat_name not in collections: collections[cat_name] = {} - if is_series: + if use_renaming_rules and sort_attr: + sort_val = book.get(sort_attr, None) + collections[cat_name][lpath] = \ + (book, sort_val, book.get('title_sort', 'zzzz')) + elif is_series: if doing_dc: collections[cat_name][lpath] = \ - (book, book.get('series_index', sys.maxint)) + (book, book.get('series_index', sys.maxint), '') else: collections[cat_name][lpath] = \ - (book, book.get(attr+'_index', sys.maxint)) + (book, book.get(attr+'_index', sys.maxint), '') else: if lpath not in collections[cat_name]: collections[cat_name][lpath] = \ - (book, book.get('title_sort', 'zzzz')) + (book, book.get('title_sort', 'zzzz'), '') # Sort collections result = {} + + def none_cmp(xx, yy): + x = xx[1] + y = yy[1] + if x is None and y is None: + return cmp(xx[2], yy[2]) + if x is None: + return 1 + if y is None: + return -1 + c = cmp(x, y) + if c != 0: + return c + return cmp(xx[2], yy[2]) + for category, lpaths in collections.items(): books = lpaths.values() - books.sort(cmp=lambda x,y:cmp(x[1], y[1])) + books.sort(cmp=none_cmp) result[category] = [x[0] for x in books] return result From e568a55d5944c483751baa65b11c73c12e244aa6 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 22 Oct 2010 13:58:19 +0100 Subject: [PATCH 36/39] Improve sony_collection_renaming_rules documentation. --- resources/default_tweaks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index dbb1154172..270b7e0b06 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -106,7 +106,8 @@ title_sort_articles=r'^(A|The|An)\s+' auto_connect_to_folder = '' -# Specify renaming rules for sony collections. Collections on Sonys are named +# Specify renaming rules for sony collections. This tweak is only applicable if +# metadata management is set to automatic. Collections on Sonys are named # depending upon whether the field is standard or custom. A collection derived # from a standard field is named for the value in that field. For example, if # the standard 'series' column contains the name 'Darkover', then the series From 122fb530873c8559fe5434b3479f935473f63458 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 22 Oct 2010 14:01:11 +0100 Subject: [PATCH 37/39] Remove print statements --- src/calibre/devices/usbms/books.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index f54bfbddf9..5063daa29f 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -103,9 +103,7 @@ class CollectionsBookList(BookList): sorts = tweaks['sony_collection_sorting_rules'] for attrs,sortattr in sorts: if attr in attrs or '*' in attrs: - print 'in category sort:', attr, sortattr return sortattr - print 'in category sort:', attr, 'None' return None def compute_category_name(self, attr, category, field_meta): From fddf3abf92b84eabc0a11b9bceda826e0a24be37 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 22 Oct 2010 18:20:26 +0100 Subject: [PATCH 38/39] Fix broken autonumbering for standard series columns --- src/calibre/gui2/dialogs/metadata_bulk.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index de62f20de0..32350c36b7 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -51,6 +51,7 @@ class MyBlockingBusy(QDialog): self.start() self.args = args + self.series_start_value = None self.db = db self.ids = ids self.error = None @@ -148,8 +149,10 @@ class MyBlockingBusy(QDialog): if do_series: if do_series_restart: - next = series_start_value - series_start_value += 1 + if self.series_start_value is None: + self.series_start_value = series_start_value + next = self.series_start_value + self.series_start_value += 1 else: next = self.db.get_next_series_num_for(series) self.db.set_series(id, series, notify=False, commit=False) From 69481d62d6ad31546d83f89fa63172765bc11d9c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 22 Oct 2010 14:40:43 -0700 Subject: [PATCH 39/39] Increase image size for comics in Kindle DX profile for better conversion of comics to PDF --- src/calibre/customize/profiles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/customize/profiles.py b/src/calibre/customize/profiles.py index 4fa53b1cdb..27f0805f86 100644 --- a/src/calibre/customize/profiles.py +++ b/src/calibre/customize/profiles.py @@ -583,7 +583,8 @@ class KindleDXOutput(OutputProfile): # Screen size is a best guess screen_size = (744, 1022) dpi = 150.0 - comic_screen_size = (741, 1022) + comic_screen_size = (771, 1116) + #comic_screen_size = (741, 1022) supports_mobi_indexing = True periodical_date_in_title = False mobi_ems_per_blockquote = 2.0