diff --git a/resources/recipes/nytimes.recipe b/resources/recipes/nytimes.recipe index 33758e8c47..eba717027e 100644 --- a/resources/recipes/nytimes.recipe +++ b/resources/recipes/nytimes.recipe @@ -391,10 +391,14 @@ class NYTimes(BasicNewsRecipe): return ans def preprocess_html(self, soup): - # Skip ad pages before actual article + # Skip ad pages served before actual article skip_tag = soup.find(True, {'name':'skip'}) if skip_tag is not None: - soup = self.index_to_soup(skip_tag.parent['href']) + self.log.error("Found forwarding link: %s" % skip_tag.parent['href']) + url = 'http://www.nytimes.com' + re.sub(r'\?.*', '', skip_tag.parent['href']) + url += '?pagewanted=all' + self.log.error("Skipping ad to article at '%s'" % url) + soup = self.index_to_soup(url) return self.strip_anchors(soup) def postprocess_html(self,soup, True): diff --git a/resources/recipes/nytimes_sub.recipe b/resources/recipes/nytimes_sub.recipe index 79c0d49223..c08b06572d 100644 --- a/resources/recipes/nytimes_sub.recipe +++ b/resources/recipes/nytimes_sub.recipe @@ -280,18 +280,14 @@ class NYTimes(BasicNewsRecipe): return ans def preprocess_html(self, soup): - ''' - refresh = soup.find('meta', {'http-equiv':'refresh'}) - if refresh is None: - return soup - content = refresh.get('content').partition('=')[2] - raw = self.browser.open('http://www.nytimes.com'+content).read() - return BeautifulSoup(raw.decode('cp1252', 'replace')) - ''' - # Skip ad pages before actual article + # Skip ad pages served before actual article skip_tag = soup.find(True, {'name':'skip'}) if skip_tag is not None: - soup = self.index_to_soup(skip_tag.parent['href']) + self.log.error("Found forwarding link: %s" % skip_tag.parent['href']) + url = 'http://www.nytimes.com' + re.sub(r'\?.*', '', skip_tag.parent['href']) + url += '?pagewanted=all' + self.log.error("Skipping ad to article at '%s'" % url) + soup = self.index_to_soup(url) return self.strip_anchors(soup) def postprocess_html(self,soup, True): diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index 413db1cc0b..6108aa329d 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -416,9 +416,9 @@ class HTMLInput(InputFormatPlugin): link = unquote(link).replace('/', os.sep) if not link.strip(): return link_ - if base and not os.path.isabs(link): - link = os.path.join(base, link) try: + if base and not os.path.isabs(link): + link = os.path.join(base, link) link = os.path.abspath(link) except: return link_ diff --git a/src/calibre/ebooks/metadata/html.py b/src/calibre/ebooks/metadata/html.py index d5aa9b8bef..45b592c709 100644 --- a/src/calibre/ebooks/metadata/html.py +++ b/src/calibre/ebooks/metadata/html.py @@ -11,7 +11,7 @@ import re from calibre.ebooks.metadata import MetaInformation from calibre.ebooks.chardet import xml_to_unicode - +from calibre import entity_to_unicode def get_metadata(stream): src = stream.read() @@ -43,6 +43,10 @@ def get_metadata_(src, encoding=None): if match: author = match.group(2).replace(',', ';') + ent_pat = re.compile(r'&(\S+)?;') + title = ent_pat.sub(entity_to_unicode, title) + if author: + author = ent_pat.sub(entity_to_unicode, author) mi = MetaInformation(title, [author] if author else None) # Publisher diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py index 4b4ce63a8f..d5fe3b6181 100644 --- a/src/calibre/gui2/dialogs/tag_list_editor.py +++ b/src/calibre/gui2/dialogs/tag_list_editor.py @@ -1,7 +1,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' from PyQt4.QtCore import SIGNAL, Qt -from PyQt4.QtGui import QDialog +from PyQt4.QtGui import QDialog, QListWidgetItem from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor from calibre.gui2 import question_dialog, error_dialog @@ -9,7 +9,7 @@ from calibre.gui2 import question_dialog, error_dialog class TagListEditor(QDialog, Ui_TagListEditor): def tag_cmp(self, x, y): - return cmp(x.lower(), y.lower()) + return cmp(x.text().lower(), y.text().lower()) def __init__(self, window, db, tag_to_match): QDialog.__init__(self, window) @@ -19,11 +19,11 @@ class TagListEditor(QDialog, Ui_TagListEditor): self.to_rename = {} self.to_delete = [] self.db = db - all_tags = [tag for tag in self.db.all_tags()] - all_tags = list(set(all_tags)) - all_tags.sort(cmp=self.tag_cmp) - for tag in all_tags: - self.available_tags.addItem(tag) + all_tags = db.get_tags_and_ids() + for tag in sorted(all_tags.keys()): + item = QListWidgetItem(tag) + item.setData(all_tags[tag]) + self.available_tags.addItem(item) items = self.available_tags.findItems(tag_to_match, Qt.MatchExactly) if len(items) == 1: diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index c47c821913..73f7f3839d 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -121,7 +121,8 @@ class GuiRunner(QObject): def start_gui(self): from calibre.gui2.ui import Main - main = Main(self.library_path, self.db, self.listener, self.opts, self.actions) + main = Main(self.opts) + main.initialize(self.library_path, self.db, self.listener, self.actions) add_filesystem_book = partial(main.add_filesystem_book, allow_device=False) sys.excepthook = main.unhandled_exception if len(self.args) > 1: diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 88f82e78c6..11db157ed4 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -91,7 +91,7 @@ class TagsView(QTreeView): # {{{ return try: if action == 'manage_tags': - self.tag_list_edit.emit(category); + self.tag_list_edit.emit(category) return if action == 'manage_categories': self.user_category_edit.emit(category) @@ -122,12 +122,12 @@ class TagsView(QTreeView): # {{{ if item.type == TagTreeItem.CATEGORY: category = unicode(item.name.toString()) self.context_menu = QMenu(self) - self.context_menu.addAction(_('Hide column %s') % category, + self.context_menu.addAction(_('Hide %s') % category, partial(self.context_menu_handler, action='hide', category=category)) if self.hidden_categories: self.context_menu.addSeparator() - m = self.context_menu.addMenu(_('Show column')) + m = self.context_menu.addMenu(_('Show category')) for col in self.hidden_categories: m.addAction(col, partial(self.context_menu_handler, action='show', category=col)) @@ -154,7 +154,7 @@ class TagsView(QTreeView): # {{{ category=tag_name)) self.context_menu.popup(self.mapToGlobal(point)) - return True; + return True def clear(self): self.model().clear_state() @@ -315,7 +315,7 @@ class TagsModel(QAbstractItemModel): # {{{ data = self.get_node_tree(config['sort_by_popularity']) self.root_item = TagTreeItem() for i, r in enumerate(self.row_map): - if self.categories[i]in self.hidden_categories: + if self.hidden_categories and self.categories[i] in self.hidden_categories: continue if self.db.field_metadata[r]['kind'] != 'user': tt = _('The lookup/search name is "{0}"').format(r) @@ -351,9 +351,9 @@ class TagsModel(QAbstractItemModel): # {{{ def refresh(self): data = self.get_node_tree(config['sort_by_popularity']) # get category data - row_index = -1; + row_index = -1 for i, r in enumerate(self.row_map): - if self.categories[i] in self.hidden_categories: + if self.hidden_categories and self.categories[i] in self.hidden_categories: continue row_index += 1 category = self.root_item.children[row_index] @@ -485,9 +485,9 @@ class TagsModel(QAbstractItemModel): # {{{ def tokens(self): ans = [] tags_seen = set() - row_index = -1; + row_index = -1 for i, key in enumerate(self.row_map): - if self.categories[i] in self.hidden_categories: + if self.hidden_categories and self.categories[i] in self.hidden_categories: continue row_index += 1 if key.endswith(':'): # User category, so skip it. The tag will be marked in its real category @@ -498,7 +498,7 @@ class TagsModel(QAbstractItemModel): # {{{ if tag.state > 0: prefix = ' not ' if tag.state == 2 else '' category = key if key != 'news' else 'tag' - if tag.name[0] == u'\u2605': # char is a star. Assume rating + if tag.name and tag.name[0] == u'\u2605': # char is a star. Assume rating ans.append('%s%s:%s'%(prefix, category, len(tag.name))) else: if category == 'tags': diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index d7869a57a8..773f44acd2 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -129,13 +129,18 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): pixmap_to_data(pixmap)) self.last_time = datetime.datetime.now() - def __init__(self, library_path, db, listener, opts, actions, parent=None): + + def __init__(self, opts, parent=None): + MainWindow.__init__(self, opts, parent) + self.opts = opts + + def initialize(self, library_path, db, listener, actions): + opts = self.opts self.last_time = datetime.datetime.now() self.preferences_action, self.quit_action = actions self.library_path = library_path self.spare_servers = [] self.must_restart_before_config = False - MainWindow.__init__(self, opts, parent) # Initialize fontconfig in a separate thread as this can be a lengthy # process if run for the first time on this machine from calibre.utils.fonts import fontconfig diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 4e2a18e395..1eae6866a4 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -648,6 +648,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.execute(st%dict(ltable='publishers', table='publishers', ltable_col='publisher')) self.conn.execute(st%dict(ltable='tags', table='tags', ltable_col='tag')) self.conn.execute(st%dict(ltable='series', table='series', ltable_col='series')) + for id_, tag in self.conn.get('SELECT id, name FROM tags', all=True): + if not tag.strip(): + self.conn.execute('DELETE FROM books_tags_link WHERE tag=?', + (id_,)) + self.conn.execute('DELETE FROM tags WHERE id=?', (id_,)) self.clean_custom() self.conn.commit() @@ -980,8 +985,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if notify: self.notify('metadata', [id]) + def get_tags_and_ids(self): + result = self.conn.get('SELECT * FROM tags') + if not result: + return {} + r = {} + for k,v in result: + r[v] = k + return r + def rename_tag(self, old, new): self.conn.execute('UPDATE tags SET name=? WHERE name=?', (new, old)) + self.conn.commit() def get_tags(self, id): result = self.conn.get( diff --git a/src/calibre/utils/magick_draw.py b/src/calibre/utils/magick_draw.py index 0288107b45..5625da0869 100644 --- a/src/calibre/utils/magick_draw.py +++ b/src/calibre/utils/magick_draw.py @@ -51,6 +51,8 @@ class FontMetrics(object): def get_font_metrics(image, d_wand, text): + if isinstance(text, unicode): + text = text.encode('utf-8') ret = p.MagickQueryFontMetrics(image, d_wand, text) return FontMetrics(ret)