From 034e289ef397a76e5062864e700cc4ac7983cfa3 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Jun 2011 17:57:25 +0100 Subject: [PATCH 01/11] Author link first try --- src/calibre/ebooks/metadata/book/__init__.py | 2 ++ src/calibre/ebooks/metadata/book/base.py | 1 + src/calibre/ebooks/metadata/opf2.py | 7 +++-- src/calibre/gui2/book_details.py | 10 +++++++ .../gui2/dialogs/edit_authors_dialog.py | 22 +++++++++----- src/calibre/gui2/library/models.py | 8 ++++- src/calibre/gui2/tag_view.py | 4 ++- src/calibre/library/database2.py | 30 +++++++++++++------ src/calibre/library/schema_upgrades.py | 10 +++++++ src/calibre/library/sqlite.py | 6 ++-- 10 files changed, 77 insertions(+), 23 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py index fae858aabd..50e7b916ee 100644 --- a/src/calibre/ebooks/metadata/book/__init__.py +++ b/src/calibre/ebooks/metadata/book/__init__.py @@ -86,6 +86,8 @@ CALIBRE_METADATA_FIELDS = frozenset([ # a dict of user category names, where the value is a list of item names # from the book that are in that category 'user_categories', + # a dict of author to an associated hyperlink + 'author_link_map', ] ) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 382cb6c5a2..c28c65f7c9 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -34,6 +34,7 @@ NULL_VALUES = { 'authors' : [_('Unknown')], 'title' : _('Unknown'), 'user_categories' : {}, + 'author_link_map' : {}, 'language' : 'und' } diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 80fb84633b..c1cd2a739f 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -538,7 +538,8 @@ class OPF(object): # {{{ user_categories = MetadataField('user_categories', is_dc=False, formatter=json.loads, renderer=dump_user_categories) - + author_link_map = MetadataField('author_link_map', is_dc=False, + formatter=json.loads) def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True, populate_spine=True): @@ -1039,7 +1040,7 @@ class OPF(object): # {{{ for attr in ('title', 'authors', 'author_sort', 'title_sort', 'publisher', 'series', 'series_index', 'rating', 'isbn', 'tags', 'category', 'comments', - 'pubdate', 'user_categories'): + 'pubdate', 'user_categories', 'author_link_map'): val = getattr(mi, attr, None) if val is not None and val != [] and val != (None, None): setattr(self, attr, val) @@ -1336,6 +1337,8 @@ def metadata_to_opf(mi, as_string=True): for tag in mi.tags: factory(DC('subject'), tag) meta = lambda n, c: factory('meta', name='calibre:'+n, content=c) + if getattr(mi, 'author_link_map', None) is not None: + meta('author_link_map', json.dumps(mi.author_link_map)) if mi.series: meta('series', mi.series) if mi.series_index is not None: diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index f94e179166..ef21773ae4 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -121,6 +121,16 @@ def render_data(mi, use_roman_numbers=True, all_fields=False): if links: ans.append((field, u'%s%s'%( _('Ids')+':', links))) + elif field == 'authors' and not isdevice: + authors = [] + for aut in mi.authors: + if mi.author_link_map[aut]: + authors.append(u'%s' % + (mi.author_link_map[aut], aut)) + else: + authors.append(aut) + ans.append((field, u'%s%s'%(name, + u' & '.join(authors)))) else: val = mi.format_field(field)[-1] if val is None: diff --git a/src/calibre/gui2/dialogs/edit_authors_dialog.py b/src/calibre/gui2/dialogs/edit_authors_dialog.py index a791551d27..1087c3cb82 100644 --- a/src/calibre/gui2/dialogs/edit_authors_dialog.py +++ b/src/calibre/gui2/dialogs/edit_authors_dialog.py @@ -33,7 +33,7 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog): # Set up the column headings self.table.setSelectionMode(QAbstractItemView.SingleSelection) - self.table.setColumnCount(2) + self.table.setColumnCount(3) self.down_arrow_icon = QIcon(I('arrow-down.png')) self.up_arrow_icon = QIcon(I('arrow-up.png')) self.blank_icon = QIcon(I('blank.png')) @@ -43,26 +43,33 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog): self.aus_col = QTableWidgetItem(_('Author sort')) self.table.setHorizontalHeaderItem(1, self.aus_col) self.aus_col.setIcon(self.up_arrow_icon) + self.aul_col = QTableWidgetItem(_('Link')) + self.table.setHorizontalHeaderItem(2, self.aul_col) + self.aus_col.setIcon(self.blank_icon) # Add the data self.authors = {} auts = db.get_authors_with_ids() self.table.setRowCount(len(auts)) select_item = None - for row, (id, author, sort) in enumerate(auts): + for row, (id, author, sort, link) in enumerate(auts): author = author.replace('|', ',') - self.authors[id] = (author, sort) + self.authors[id] = (author, sort, link) aut = tableItem(author) aut.setData(Qt.UserRole, id) sort = tableItem(sort) + link = tableItem(link) self.table.setItem(row, 0, aut) self.table.setItem(row, 1, sort) + self.table.setItem(row, 2, link) if id == id_to_select: if select_sort: select_item = sort else: select_item = aut self.table.resizeColumnsToContents() + if self.table.columnWidth(2) < 200: + self.table.setColumnWidth(2, 200) # set up the cellChanged signal only after the table is filled self.table.cellChanged.connect(self.cell_changed) @@ -236,9 +243,10 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog): id = self.table.item(row, 0).data(Qt.UserRole).toInt()[0] aut = unicode(self.table.item(row, 0).text()).strip() sort = unicode(self.table.item(row, 1).text()).strip() - orig_aut,orig_sort = self.authors[id] - if orig_aut != aut or orig_sort != sort: - self.result.append((id, orig_aut, aut, sort)) + link = unicode(self.table.item(row, 2).text()).strip() + orig_aut,orig_sort,orig_link = self.authors[id] + if orig_aut != aut or orig_sort != sort or orig_link != link: + self.result.append((id, orig_aut, aut, sort, link)) def do_recalc_author_sort(self): self.table.cellChanged.disconnect() @@ -276,6 +284,6 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog): c.setText(author_to_author_sort(aut)) item = c else: - item = self.table.item(row, 1) + item = self.table.item(row, col) self.table.setCurrentItem(item) self.table.scrollToItem(item) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 40d6e2b6cf..d4aeada313 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -90,6 +90,7 @@ class BooksModel(QAbstractTableModel): # {{{ self.ids_to_highlight_set = set() self.current_highlighted_idx = None self.highlight_only = False + self.current_row = -1 self.colors = frozenset([unicode(c) for c in QColor.colorNames()]) self.read_config() @@ -172,6 +173,7 @@ class BooksModel(QAbstractTableModel): # {{{ self.color_cache = defaultdict(dict) for row in rows: if row == current_row: + self.current_row = row self.new_bookdisplay_data.emit( self.get_book_display_info(row)) self.dataChanged.emit(self.index(row, 0), self.index(row, @@ -329,6 +331,8 @@ class BooksModel(QAbstractTableModel): # {{{ def refresh(self, reset=True): self.db.refresh(field=None) self.resort(reset=reset) + if self.current_row >= 0: + self.new_bookdisplay_data.emit(self.get_book_display_info(self.current_row)) def reset(self): self.color_cache = defaultdict(dict) @@ -368,12 +372,14 @@ class BooksModel(QAbstractTableModel): # {{{ def current_changed(self, current, previous, emit_signal=True): if current.isValid(): - idx = current.row() + self.current_row = idx = current.row() data = self.get_book_display_info(idx) if emit_signal: self.new_bookdisplay_data.emit(data) else: return data + else: + self.current_row = -1 def get_book_info(self, index): if isinstance(index, int): diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 3b8c27866c..21309a1592 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -2081,12 +2081,14 @@ class TagBrowserMixin(object): # {{{ editor = EditAuthorsDialog(parent, db, id, select_sort) d = editor.exec_() if d: - for (id, old_author, new_author, new_sort) in editor.result: + for (id, old_author, new_author, new_sort, new_link) in editor.result: if old_author != new_author: # The id might change if the new author already exists id = db.rename_author(id, new_author) db.set_sort_field_for_author(id, unicode(new_sort), commit=False, notify=False) + db.set_link_field_for_author(id, unicode(new_link), + commit=False, notify=False) db.commit() self.library_view.model().refresh() self.tags_view.recount() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 4c61438e35..9602f8ef1d 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -367,7 +367,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): 'uuid', 'has_cover', ('au_map', 'authors', 'author', - 'aum_sortconcat(link.id, authors.name, authors.sort)'), + 'aum_sortconcat(link.id, authors.name, authors.sort, authors.link)'), 'last_modified', '(SELECT identifiers_concat(type, val) FROM identifiers WHERE identifiers.book=books.id) identifiers', ] @@ -894,13 +894,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): aut_list = [] aum = [] aus = {} - for (author, author_sort) in aut_list: - aum.append(author.replace('|', ',')) - aus[author] = author_sort.replace('|', ',') + aul = {} + for (author, author_sort, link) in aut_list: + aut = author.replace('|', ',') + aum.append(aut) + aus[aut] = author_sort.replace('|', ',') + aul[aut] = link mi.title = row[fm['title']] mi.authors = aum mi.author_sort = row[fm['author_sort']] mi.author_sort_map = aus + mi.author_link_map = aul mi.comments = row[fm['comments']] mi.publisher = row[fm['publisher']] mi.timestamp = row[fm['timestamp']] @@ -2002,13 +2006,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def authors_with_sort_strings(self, id, index_is_id=False): id = id if index_is_id else self.id(id) aut_strings = self.conn.get(''' - SELECT authors.id, authors.name, authors.sort + SELECT authors.id, authors.name, authors.sort, authors.link FROM authors, books_authors_link as bl WHERE bl.book=? and authors.id=bl.author ORDER BY bl.id''', (id,)) result = [] - for (id_, author, sort,) in aut_strings: - result.append((id_, author.replace('|', ','), sort)) + for (id_, author, sort, link) in aut_strings: + result.append((id_, author.replace('|', ','), sort, link)) return result # Given a book, return the author_sort string for authors of the book @@ -2048,7 +2052,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): aum = self.authors_with_sort_strings(id_, index_is_id=True) self.data.set(id_, self.FIELD_MAP['au_map'], - ':#:'.join([':::'.join((au.replace(',', '|'), aus)) for (_, au, aus) in aum]), + ':#:'.join([':::'.join((au.replace(',', '|'), aus, aul)) + for (_, au, aus, aul) in aum]), row_is_id=True) def _set_authors(self, id, authors, allow_case_change=False): @@ -2399,7 +2404,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.commit() def get_authors_with_ids(self): - result = self.conn.get('SELECT id,name,sort FROM authors') + result = self.conn.get('SELECT id,name,sort,link FROM authors') if not result: return [] return result @@ -2410,6 +2415,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): (author,), all=False) return result + def set_link_field_for_author(self, aid, link, commit=True, notify=False): + if not link: + link = '' + self.conn.execute('UPDATE authors SET link=? WHERE id=?', (link.strip(), aid)) + if commit: + self.conn.commit() + def set_sort_field_for_author(self, old_id, new_sort, commit=True, notify=False): self.conn.execute('UPDATE authors SET sort=? WHERE id=?', \ (new_sort.strip(), old_id)) diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index 3fc9a2368a..3c64785178 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -600,4 +600,14 @@ class SchemaUpgrade(object): with open(os.path.join(bdir, fname), 'wb') as f: f.write(script) + def upgrade_version_20(self): + ''' + Add a link column to the authors table. + ''' + + script = ''' + ALTER TABLE authors ADD COLUMN link TEXT NON NULL DEFAULT ""; + ''' + self.conn.executescript(script) + diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 96874d2c27..a2a85806f5 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -144,9 +144,9 @@ class AumSortedConcatenate(object): def __init__(self): self.ans = {} - def step(self, ndx, author, sort): + def step(self, ndx, author, sort, link): if author is not None: - self.ans[ndx] = author + ':::' + sort + self.ans[ndx] = ':::'.join((author, sort, link)) def finalize(self): keys = self.ans.keys() @@ -229,7 +229,7 @@ class DBThread(Thread): load_c_extensions(self.conn) self.conn.row_factory = sqlite.Row if self.row_factory else lambda cursor, row : list(row) self.conn.create_aggregate('concat', 1, Concatenate) - self.conn.create_aggregate('aum_sortconcat', 3, AumSortedConcatenate) + self.conn.create_aggregate('aum_sortconcat', 4, AumSortedConcatenate) self.conn.create_collation('PYNOCASE', partial(pynocase, encoding=encoding)) self.conn.create_function('title_sort', 1, title_sort) From 448196eff35a4336b1e93ac46c955bb8ca3023da Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 28 Jun 2011 10:11:54 -0600 Subject: [PATCH 02/11] Add proxy info to FAQ --- src/calibre/manual/faq.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 97551b403f..c67d44b7d5 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -558,11 +558,16 @@ Most readers do not support this. You should complain to the manufacturer about Another alternative is to create a catalog in ebook form containing a listing of all the books in your calibre library, with their metadata. Click the arrow next to the convert button to access the catalog creation tool. And before you ask, no you cannot have the catalog "link directly to" books on your reader. +How do I get |app| to use my HTTP proxy? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, |app| uses whatever proxy settings are set in your OS. Sometimes these are incorrect, for example, on windows if you don't use Internet Explorer then the proxy settings may not be up to date. You can tell |app| to use a particular proxy server by setting the http_proxy environment variable. The format of the variable is: http://username:password@servername you should ask your network admin to give you the correct value for this variable. Note that |app| only supports HTTP proxies not SOCKS proxies. You can see the current proxies used by |app| in Preferences->Miscellaneous. + I want some feature added to |app|. What can I do? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You have two choices: 1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development `_. - 2. `Open a ticket `_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it. + 2. `Open a bug requesting the feature `_ . Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it. Why doesn't |app| have an automatic update? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From c417377ba7135682aaf9bdd13d3140da7e6985f3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 28 Jun 2011 10:17:22 -0600 Subject: [PATCH 03/11] ... --- recipes/ming_pao.recipe | 18 +++++++++--------- recipes/ming_pao_toronto.recipe | 18 +++++++++--------- recipes/ming_pao_vancouver.recipe | 18 +++++++++--------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/recipes/ming_pao.recipe b/recipes/ming_pao.recipe index 3566fca667..947d85692f 100644 --- a/recipes/ming_pao.recipe +++ b/recipes/ming_pao.recipe @@ -179,17 +179,17 @@ class MPRecipe(BasicNewsRecipe): def get_dtlocal(self): dt_utc = datetime.datetime.utcnow() if __Region__ == 'Hong Kong': - # convert UTC to local hk time - at HKT 4.30am, all news are available - dt_local = dt_utc + datetime.timedelta(8.0/24) - datetime.timedelta(4.5/24) - # dt_local = dt_utc.astimezone(pytz.timezone('Asia/Hong_Kong')) - datetime.timedelta(4.5/24) + # convert UTC to local hk time - at HKT 5.30am, all news are available + dt_local = dt_utc + datetime.timedelta(8.0/24) - datetime.timedelta(5.5/24) + # dt_local = dt_utc.astimezone(pytz.timezone('Asia/Hong_Kong')) - datetime.timedelta(5.5/24) elif __Region__ == 'Vancouver': - # convert UTC to local Vancouver time - at PST time 4.30am, all news are available - dt_local = dt_utc + datetime.timedelta(-8.0/24) - datetime.timedelta(4.5/24) - #dt_local = dt_utc.astimezone(pytz.timezone('America/Vancouver')) - datetime.timedelta(4.5/24) + # convert UTC to local Vancouver time - at PST time 5.30am, all news are available + dt_local = dt_utc + datetime.timedelta(-8.0/24) - datetime.timedelta(5.5/24) + #dt_local = dt_utc.astimezone(pytz.timezone('America/Vancouver')) - datetime.timedelta(5.5/24) elif __Region__ == 'Toronto': - # convert UTC to local Toronto time - at EST time 4.30am, all news are available - dt_local = dt_utc + datetime.timedelta(-5.0/24) - datetime.timedelta(4.5/24) - #dt_local = dt_utc.astimezone(pytz.timezone('America/Toronto')) - datetime.timedelta(4.5/24) + # convert UTC to local Toronto time - at EST time 8.30am, all news are available + dt_local = dt_utc + datetime.timedelta(-5.0/24) - datetime.timedelta(8.5/24) + #dt_local = dt_utc.astimezone(pytz.timezone('America/Toronto')) - datetime.timedelta(8.5/24) return dt_local def get_fetchdate(self): diff --git a/recipes/ming_pao_toronto.recipe b/recipes/ming_pao_toronto.recipe index 677a8272b0..9f3d7f510c 100644 --- a/recipes/ming_pao_toronto.recipe +++ b/recipes/ming_pao_toronto.recipe @@ -179,17 +179,17 @@ class MPRecipe(BasicNewsRecipe): def get_dtlocal(self): dt_utc = datetime.datetime.utcnow() if __Region__ == 'Hong Kong': - # convert UTC to local hk time - at HKT 4.30am, all news are available - dt_local = dt_utc + datetime.timedelta(8.0/24) - datetime.timedelta(4.5/24) - # dt_local = dt_utc.astimezone(pytz.timezone('Asia/Hong_Kong')) - datetime.timedelta(4.5/24) + # convert UTC to local hk time - at HKT 5.30am, all news are available + dt_local = dt_utc + datetime.timedelta(8.0/24) - datetime.timedelta(5.5/24) + # dt_local = dt_utc.astimezone(pytz.timezone('Asia/Hong_Kong')) - datetime.timedelta(5.5/24) elif __Region__ == 'Vancouver': - # convert UTC to local Vancouver time - at PST time 4.30am, all news are available - dt_local = dt_utc + datetime.timedelta(-8.0/24) - datetime.timedelta(4.5/24) - #dt_local = dt_utc.astimezone(pytz.timezone('America/Vancouver')) - datetime.timedelta(4.5/24) + # convert UTC to local Vancouver time - at PST time 5.30am, all news are available + dt_local = dt_utc + datetime.timedelta(-8.0/24) - datetime.timedelta(5.5/24) + #dt_local = dt_utc.astimezone(pytz.timezone('America/Vancouver')) - datetime.timedelta(5.5/24) elif __Region__ == 'Toronto': - # convert UTC to local Toronto time - at EST time 4.30am, all news are available - dt_local = dt_utc + datetime.timedelta(-5.0/24) - datetime.timedelta(4.5/24) - #dt_local = dt_utc.astimezone(pytz.timezone('America/Toronto')) - datetime.timedelta(4.5/24) + # convert UTC to local Toronto time - at EST time 8.30am, all news are available + dt_local = dt_utc + datetime.timedelta(-5.0/24) - datetime.timedelta(8.5/24) + #dt_local = dt_utc.astimezone(pytz.timezone('America/Toronto')) - datetime.timedelta(8.5/24) return dt_local def get_fetchdate(self): diff --git a/recipes/ming_pao_vancouver.recipe b/recipes/ming_pao_vancouver.recipe index 3312c8f7b8..3b13211d01 100644 --- a/recipes/ming_pao_vancouver.recipe +++ b/recipes/ming_pao_vancouver.recipe @@ -179,17 +179,17 @@ class MPRecipe(BasicNewsRecipe): def get_dtlocal(self): dt_utc = datetime.datetime.utcnow() if __Region__ == 'Hong Kong': - # convert UTC to local hk time - at HKT 4.30am, all news are available - dt_local = dt_utc + datetime.timedelta(8.0/24) - datetime.timedelta(4.5/24) - # dt_local = dt_utc.astimezone(pytz.timezone('Asia/Hong_Kong')) - datetime.timedelta(4.5/24) + # convert UTC to local hk time - at HKT 5.30am, all news are available + dt_local = dt_utc + datetime.timedelta(8.0/24) - datetime.timedelta(5.5/24) + # dt_local = dt_utc.astimezone(pytz.timezone('Asia/Hong_Kong')) - datetime.timedelta(5.5/24) elif __Region__ == 'Vancouver': - # convert UTC to local Vancouver time - at PST time 4.30am, all news are available - dt_local = dt_utc + datetime.timedelta(-8.0/24) - datetime.timedelta(4.5/24) - #dt_local = dt_utc.astimezone(pytz.timezone('America/Vancouver')) - datetime.timedelta(4.5/24) + # convert UTC to local Vancouver time - at PST time 5.30am, all news are available + dt_local = dt_utc + datetime.timedelta(-8.0/24) - datetime.timedelta(5.5/24) + #dt_local = dt_utc.astimezone(pytz.timezone('America/Vancouver')) - datetime.timedelta(5.5/24) elif __Region__ == 'Toronto': - # convert UTC to local Toronto time - at EST time 4.30am, all news are available - dt_local = dt_utc + datetime.timedelta(-5.0/24) - datetime.timedelta(4.5/24) - #dt_local = dt_utc.astimezone(pytz.timezone('America/Toronto')) - datetime.timedelta(4.5/24) + # convert UTC to local Toronto time - at EST time 8.30am, all news are available + dt_local = dt_utc + datetime.timedelta(-5.0/24) - datetime.timedelta(8.5/24) + #dt_local = dt_utc.astimezone(pytz.timezone('America/Toronto')) - datetime.timedelta(8.5/24) return dt_local def get_fetchdate(self): From fc02b538f5558a54aeb7a608ecbd2b17772265e1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 28 Jun 2011 11:05:44 -0600 Subject: [PATCH 04/11] ... --- src/calibre/library/database2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index c8fd660e1a..ef709cd85e 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1245,6 +1245,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) shutil.copyfileobj(f, ret) ret.seek(0) + # Various bits of code try to use the name as the default + # title when reading metadata, so set it ret.name = f.name else: ret = f.read() From d3a93c500b50f7a36aec753de4ea4e324b122b7e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 28 Jun 2011 12:06:24 -0600 Subject: [PATCH 05/11] Fix #802288 (ISBN ID not recognized when not lower case) --- src/calibre/gui2/metadata/basic_widgets.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index 2d6c79d0e3..227a2257bc 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -1092,11 +1092,12 @@ class IdentifiersEdit(QLineEdit): # {{{ for x in parts: c = x.split(':') if len(c) > 1: - if c[0] == 'isbn': + itype = c[0].lower() + if itype == 'isbn': v = check_isbn(c[1]) if v is not None: c[1] = v - ans[c[0]] = c[1] + ans[itype] = c[1] return ans def fset(self, val): if not val: @@ -1112,7 +1113,7 @@ class IdentifiersEdit(QLineEdit): # {{{ if v is not None: val[k] = v ids = sorted(val.iteritems(), key=keygen) - txt = ', '.join(['%s:%s'%(k, v) for k, v in ids]) + txt = ', '.join(['%s:%s'%(k.lower(), v) for k, v in ids]) self.setText(txt.strip()) self.setCursorPosition(0) return property(fget=fget, fset=fset) From aceb4655f22bf916da388afdaacb25598492072c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 28 Jun 2011 19:43:14 +0100 Subject: [PATCH 06/11] Commit to merge from trunk --- resources/metadata_sqlite.sql | 3 ++- src/calibre/ebooks/metadata/opf2.py | 11 +++++++-- src/calibre/gui2/__init__.py | 8 +++++++ src/calibre/gui2/book_details.py | 16 +++++++++++-- src/calibre/gui2/dialogs/restore_library.py | 2 +- src/calibre/gui2/preferences/look_feel.py | 1 + src/calibre/gui2/preferences/look_feel.ui | 26 +++++++++++++++++++-- src/calibre/gui2/tag_browser/view.py | 2 +- src/calibre/library/restore.py | 12 ++++++++++ src/calibre/library/schema_upgrades.py | 2 +- 10 files changed, 73 insertions(+), 10 deletions(-) diff --git a/resources/metadata_sqlite.sql b/resources/metadata_sqlite.sql index aa29d4b8de..83f55c2762 100644 --- a/resources/metadata_sqlite.sql +++ b/resources/metadata_sqlite.sql @@ -1,6 +1,7 @@ CREATE TABLE authors ( id INTEGER PRIMARY KEY, name TEXT NOT NULL COLLATE NOCASE, sort TEXT COLLATE NOCASE, + link TEXT NOT NULL DEFAULT "", UNIQUE(name) ); CREATE TABLE books ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -545,4 +546,4 @@ CREATE TRIGGER series_update_trg BEGIN UPDATE series SET sort=NEW.name WHERE id=NEW.id; END; -pragma user_version=20; +pragma user_version=21; diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index c1cd2a739f..2ec4ed83ea 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -481,6 +481,13 @@ def dump_user_categories(cats): return json.dumps(object_to_unicode(cats), ensure_ascii=False, skipkeys=True) +def dump_author_links(links): + if not links: + links = {} + from calibre.ebooks.metadata.book.json_codec import object_to_unicode + return json.dumps(object_to_unicode(links), ensure_ascii=False, + skipkeys=True) + class OPF(object): # {{{ MIMETYPE = 'application/oebps-package+xml' @@ -539,7 +546,7 @@ class OPF(object): # {{{ formatter=json.loads, renderer=dump_user_categories) author_link_map = MetadataField('author_link_map', is_dc=False, - formatter=json.loads) + formatter=json.loads, renderer=dump_author_links) def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True, populate_spine=True): @@ -1338,7 +1345,7 @@ def metadata_to_opf(mi, as_string=True): factory(DC('subject'), tag) meta = lambda n, c: factory('meta', name='calibre:'+n, content=c) if getattr(mi, 'author_link_map', None) is not None: - meta('author_link_map', json.dumps(mi.author_link_map)) + meta('author_link_map', dump_author_links(mi.author_link_map)) if mi.series: meta('series', mi.series) if mi.series_index is not None: diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 8dbc72ab98..88c5653ee7 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -181,6 +181,14 @@ def _config(): # {{{ help=_('Show the average rating per item indication in the tag browser')) c.add_opt('disable_animations', default=False, help=_('Disable UI animations')) + c.add_opt('default_author_link', + default='http://en.wikipedia.org/w/index.php?search={author}', + help='

' + + _('Enter a template to be used to create a link for' + 'an author in the books information dialog. This template will ' + 'be used when no link has been provided for the author using ' + 'Manage Authors. You can use the values {author} and ' + '{author_sort}, and any template function.') + '

') # This option is no longer used. It remains for compatibility with upgrades # so the value can be migrated diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index ef21773ae4..1927b1448e 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -5,6 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import urllib2 from PyQt4.Qt import (QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, QPropertyAnimation, QEasingCurve, QApplication, QFontInfo, @@ -23,6 +24,7 @@ from calibre.library.comments import comments_to_html from calibre.gui2 import (config, open_local_file, open_url, pixmap_to_data, gprefs) from calibre.utils.icu import sort_key +from calibre.utils.formatter import EvalFormatter def render_html(mi, css, vertical, widget, all_fields=False): # {{{ table = render_data(mi, all_fields=all_fields, @@ -123,10 +125,20 @@ def render_data(mi, use_roman_numbers=True, all_fields=False): _('Ids')+':', links))) elif field == 'authors' and not isdevice: authors = [] + formatter = EvalFormatter() for aut in mi.authors: if mi.author_link_map[aut]: - authors.append(u'%s' % - (mi.author_link_map[aut], aut)) + link = mi.author_link_map[aut] + elif config.get('default_author_link'): + vals = {'author': aut} + try: + vals['author_sort'] = mi.author_sort_map[aut] + except: + vals['author_sort'] = aut + link = formatter.safe_format( + config.get('default_author_link'), vals, '', vals) + if link: + authors.append(u'%s'%(urllib2.quote(link), aut)) else: authors.append(aut) ans.append((field, u'%s%s'%(name, diff --git a/src/calibre/gui2/dialogs/restore_library.py b/src/calibre/gui2/dialogs/restore_library.py index a57d6c86c1..60b224d1cd 100644 --- a/src/calibre/gui2/dialogs/restore_library.py +++ b/src/calibre/gui2/dialogs/restore_library.py @@ -54,7 +54,7 @@ class DBRestore(QDialog): def reject(self): self.rejected = True self.restorer.progress_callback = lambda x, y: x - QDialog.rejecet(self) + QDialog.reject(self) def update(self): if self.restorer.is_alive(): diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index a2850679f1..841193373b 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -138,6 +138,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): (_('Partitioned'), 'partition')] r('tags_browser_partition_method', gprefs, choices=choices) r('tags_browser_collapse_at', gprefs) + r('default_author_link', config) choices = set([k for k in db.field_metadata.all_field_keys() if db.field_metadata[k]['is_category'] and diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index cc9133a36f..8dadfe3424 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -192,7 +192,7 @@ Book Details - + Select displayed metadata @@ -243,6 +243,28 @@ + + + + + + Default author link template: + + +

Enter a template that will be used to create a link for +an author in the books information dialog. Used when no link has been +provided for the author in Manage Authors.

 + + + opt_default_author_link + + + + + + + + @@ -253,7 +275,7 @@ - + Note that <b>comments</b> will always be displayed at the end, regardless of the position you assign here. diff --git a/src/calibre/gui2/tag_browser/view.py b/src/calibre/gui2/tag_browser/view.py index 1fad4eb9a3..c878630234 100644 --- a/src/calibre/gui2/tag_browser/view.py +++ b/src/calibre/gui2/tag_browser/view.py @@ -136,7 +136,7 @@ class TagsView(QTreeView): # {{{ return expanded_categories, state_map def reread_collapse_parameters(self): - self._model.reread_collapse_parameters(self.get_state()[1]) + self._model.reread_collapse_model(self.get_state()) def set_database(self, db, tag_match, sort_by): self._model.set_database(db) diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index 8bd7174849..4fab2edbd6 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -53,6 +53,7 @@ class Restore(Thread): self.mismatched_dirs = [] self.successes = 0 self.tb = None + self.authors_links = {} @property def errors_occurred(self): @@ -160,6 +161,12 @@ class Restore(Thread): else: self.mismatched_dirs.append(dirpath) + alm = mi.get('author_link_map', {}) + for author, link in alm.iteritems(): + existing_link, timestamp = self.authors_links.get(author, (None, None)) + if existing_link is None or existing_link != link and timestamp < mi.timestamp: + self.authors_links[author] = (link, mi.timestamp) + def create_cc_metadata(self): self.books.sort(key=itemgetter('timestamp')) self.custom_columns = {} @@ -206,6 +213,11 @@ class Restore(Thread): self.failed_restores.append((book, traceback.format_exc())) self.progress_callback(book['mi'].title, i+1) + for author in self.authors_links.iterkeys(): + link, ign = self.authors_links[author] + db.conn.execute('UPDATE authors SET link=? WHERE name=?', + (link, author.replace(',', '|'))) + db.conn.commit() db.conn.close() def restore_book(self, book, db): diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index 3c64785178..2907e43098 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -606,7 +606,7 @@ class SchemaUpgrade(object): ''' script = ''' - ALTER TABLE authors ADD COLUMN link TEXT NON NULL DEFAULT ""; + ALTER TABLE authors ADD COLUMN link TEXT NOT NULL DEFAULT ""; ''' self.conn.executescript(script) From 251f5ca7fd2996ac44b906221c151579d578eb4c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 28 Jun 2011 19:50:43 +0100 Subject: [PATCH 07/11] Ready to push --- src/calibre/gui2/preferences/look_feel.ui | 7 +------ src/calibre/gui2/tag_browser/view.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index 8dadfe3424..f8dc9dc782 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -250,11 +250,6 @@ Default author link template: - -

Enter a template that will be used to create a link for -an author in the books information dialog. Used when no link has been -provided for the author in Manage Authors.

 - opt_default_author_link @@ -275,7 +270,7 @@ provided for the author in Manage Authors.

 - + Note that <b>comments</b> will always be displayed at the end, regardless of the position you assign here. diff --git a/src/calibre/gui2/tag_browser/view.py b/src/calibre/gui2/tag_browser/view.py index e9059d78aa..586d01ff87 100644 --- a/src/calibre/gui2/tag_browser/view.py +++ b/src/calibre/gui2/tag_browser/view.py @@ -136,7 +136,7 @@ class TagsView(QTreeView): # {{{ return expanded_categories, state_map def reread_collapse_parameters(self): - self._model.reread_collapse_model(self.get_state()) + self._model.reread_collapse_model(self.get_state()[1]) def set_database(self, db, tag_match, sort_by): self._model.set_database(db) From 1a1b75411c93845ff1296bc19ea1d7db6fe26524 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 28 Jun 2011 13:00:53 -0600 Subject: [PATCH 08/11] Fix #802100 (Error upon pulling HBR feed (Change in HBR format)) --- recipes/hbr.recipe | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/recipes/hbr.recipe b/recipes/hbr.recipe index cd7dcd2061..1152a48784 100644 --- a/recipes/hbr.recipe +++ b/recipes/hbr.recipe @@ -1,5 +1,6 @@ from calibre.web.feeds.news import BasicNewsRecipe import re +from datetime import date, timedelta class HBR(BasicNewsRecipe): @@ -12,13 +13,14 @@ class HBR(BasicNewsRecipe): no_stylesheets = True LOGIN_URL = 'http://hbr.org/login?request_url=/' - INDEX = 'http://hbr.org/current' + INDEX = 'http://hbr.org/archive-toc/BR' keep_only_tags = [dict(name='div', id='pageContainer')] remove_tags = [dict(id=['mastheadContainer', 'magazineHeadline', 'articleToolbarTopRD', 'pageRightSubColumn', 'pageRightColumn', 'todayOnHBRListWidget', 'mostWidget', 'keepUpWithHBR', 'mailingListTout', 'partnerCenter', 'pageFooter', + 'superNavHeadContainer', 'hbrDisqus', 'articleToolbarTop', 'articleToolbarBottom', 'articleToolbarRD']), dict(name='iframe')] extra_css = ''' @@ -55,9 +57,14 @@ class HBR(BasicNewsRecipe): def hbr_get_toc(self): - soup = self.index_to_soup(self.INDEX) - url = soup.find('a', text=lambda t:'Full Table of Contents' in t).parent.get('href') - return self.index_to_soup('http://hbr.org'+url) + today = date.today() + future = today + timedelta(days=30) + for x in [x.strftime('%y%m') for x in (future, today)]: + url = self.INDEX + x + soup = self.index_to_soup(url) + if not soup.find(text='Issue Not Found'): + return soup + raise Exception('Could not find current issue') def hbr_parse_section(self, container, feeds): current_section = None From 283f9b57a797ff68661fb1362c2d114a4ee28f70 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 28 Jun 2011 13:39:54 -0600 Subject: [PATCH 09/11] ... --- src/calibre/db/tables.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 7240b3ec6e..735d2f69a0 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -105,11 +105,13 @@ class ManyToManyTable(ManyToOneTable): class AuthorsTable(ManyToManyTable): def read_id_maps(self, db): + self.alink_map = {} for row in db.conn.execute( - 'SELECT id, name, sort FROM authors'): + 'SELECT id, name, sort, link FROM authors'): self.id_map[row[0]] = row[1] self.extra_map[row[0]] = (row[2] if row[2] else author_to_author_sort(row[1])) + self.alink_map[row[0]] = row[3] class FormatsTable(ManyToManyTable): From 7da2a2561774614e782e1538eb60b35d64a30e37 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 28 Jun 2011 21:08:37 +0100 Subject: [PATCH 10/11] Fixes for authors link stuff. --- src/calibre/gui2/book_details.py | 9 +++++---- src/calibre/gui2/tag_browser/ui.py | 4 +++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 2d7b3b1479..5d396e2e96 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -130,15 +130,16 @@ def render_data(mi, use_roman_numbers=True, all_fields=False): if mi.author_link_map[aut]: link = mi.author_link_map[aut] elif gprefs.get('default_author_link'): - vals = {'author': aut} + vals = {'author': aut.replace(' ', '+')} try: - vals['author_sort'] = mi.author_sort_map[aut] + vals['author_sort'] = mi.author_sort_map[aut].replace(' ', '+') except: - vals['author_sort'] = aut + vals['author_sort'] = aut.replace(' ', '+') link = formatter.safe_format( gprefs.get('default_author_link'), vals, '', vals) if link: - authors.append(u'%s'%(urllib2.quote(link), aut)) + link = prepare_string_for_xml(link) + authors.append(u'%s'%(link, aut)) else: authors.append(aut) ans.append((field, u'%s%s'%(name, diff --git a/src/calibre/gui2/tag_browser/ui.py b/src/calibre/gui2/tag_browser/ui.py index 0b01e20154..509f923be1 100644 --- a/src/calibre/gui2/tag_browser/ui.py +++ b/src/calibre/gui2/tag_browser/ui.py @@ -270,12 +270,14 @@ class TagBrowserMixin(object): # {{{ editor = EditAuthorsDialog(parent, db, id, select_sort) d = editor.exec_() if d: - for (id, old_author, new_author, new_sort) in editor.result: + for (id, old_author, new_author, new_sort, new_link) in editor.result: if old_author != new_author: # The id might change if the new author already exists id = db.rename_author(id, new_author) db.set_sort_field_for_author(id, unicode(new_sort), commit=False, notify=False) + db.set_link_field_for_author(id, unicode(new_link), + commit=False, notify=False) db.commit() self.library_view.model().refresh() self.tags_view.recount() From f07d04dc332096dcc8ad56873df2aa369e561f82 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 28 Jun 2011 14:51:50 -0600 Subject: [PATCH 11/11] Fix #803147 (Updated recipe for El Universal from Venezuela) --- recipes/eluniversal_ve.recipe | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/recipes/eluniversal_ve.recipe b/recipes/eluniversal_ve.recipe index 28667cd39b..d7c2c4710b 100644 --- a/recipes/eluniversal_ve.recipe +++ b/recipes/eluniversal_ve.recipe @@ -1,5 +1,5 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Darko Miletic ' +__copyright__ = '2010-2011, Darko Miletic ' ''' www.eluniversal.com ''' @@ -15,12 +15,20 @@ class ElUniversal(BasicNewsRecipe): max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False + remove_empty_feeds = True encoding = 'cp1252' publisher = 'El Universal' category = 'news, Caracas, Venezuela, world' language = 'es_VE' + publication_type = 'newspaper' cover_url = strftime('http://static.eluniversal.com/%Y/%m/%d/portada.jpg') - + extra_css = """ + .txt60{font-family: Tahoma,Geneva,sans-serif; font-size: small} + .txt29{font-family: Tahoma,Geneva,sans-serif; font-size: small; color: gray} + .txt38{font-family: Georgia,"Times New Roman",Times,serif; font-size: xx-large} + .txt35{font-family: Georgia,"Times New Roman",Times,serif; font-size: large} + body{font-family: Verdana,Arial,Helvetica,sans-serif} + """ conversion_options = { 'comments' : description ,'tags' : category @@ -28,10 +36,11 @@ class ElUniversal(BasicNewsRecipe): ,'publisher' : publisher } - keep_only_tags = [dict(name='div', attrs={'class':'Nota'})] + remove_tags_before=dict(attrs={'class':'header-print MB10'}) + remove_tags_after= dict(attrs={'id':'SizeText'}) remove_tags = [ - dict(name=['object','link','script','iframe']) - ,dict(name='div',attrs={'class':'Herramientas'}) + dict(name=['object','link','script','iframe','meta']) + ,dict(attrs={'class':'header-print MB10'}) ] feeds = [