From 1a361f694f4c4a82e1e0a3d42afc8cb4e4144fc8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 17 Jan 2012 18:12:45 +0530 Subject: [PATCH 01/13] Tweakers.net by Roedi06 --- recipes/tweakers_net.recipe | 66 +++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 recipes/tweakers_net.recipe diff --git a/recipes/tweakers_net.recipe b/recipes/tweakers_net.recipe new file mode 100644 index 0000000000..f9bbe27ec9 --- /dev/null +++ b/recipes/tweakers_net.recipe @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__docformat__ = 'restructuredtext en' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class Tweakers(BasicNewsRecipe): + title = u'Tweakers.net - with Reactions' + __author__ = 'Roedi06' + language = 'nl' + oldest_article = 7 + max_articles_per_feed = 100 + cover_url = 'http://img51.imageshack.us/img51/7470/tweakersnetebook.gif' + + keep_only_tags = [dict(name='div', attrs={'class':'columnwrapper news'}), + {'id':'reacties'}, + ] + + remove_tags = [dict(name='div', attrs={'id' : ['utracker']}), + {'id' : ['channelNav']}, + {'id' : ['contentArea']}, + {'class' : ['breadCrumb']}, + {'class' : ['nextPrevious ellipsis']}, + {'class' : ['advertorial']}, + {'class' : ['sidebar']}, + {'class' : ['filterBox']}, + {'id' : ['toggleButtonTxt']}, + {'id' : ['socialButtons']}, + {'class' : ['button']}, + {'class' : ['textadTop']}, + {'class' : ['commentLink']}, + {'title' : ['Reageer op deze reactie']}, + {'class' : ['pageIndex']}, + {'class' : ['reactieHeader collapsed']}, + ] + no_stylesheets=True + + preprocess_regexps = [ + (re.compile(r'', re.IGNORECASE | re.DOTALL), lambda match : ''), + (re.compile(r'

', re.IGNORECASE | re.DOTALL), lambda match : ''), + (re.compile(r'

', re.IGNORECASE | re.DOTALL), lambda match : ''), + (re.compile(r''), lambda h1: ''), + (re.compile(r''), lambda h2: ''), + (re.compile(r'', re.IGNORECASE | re.DOTALL), lambda match : ''), + (re.compile(r'', re.IGNORECASE | re.DOTALL), lambda match : ''), + (re.compile(r'
.*?
'), lambda h1: ''), + ] + + extra_css = '.reactieHeader { color: #333333; font-size: 6px; border-bottom:solid 2px #333333; border-top:solid 1px #333333; } \ + .reactieContent { font-family:"Times New Roman",Georgia,Serif; color: #000000; font-size: 8px; } \ + .quote { font-family:"Times New Roman",Georgia,Serif; padding-left:2px; border-left:solid 3px #666666; color: #666666; }' + + + feeds = [(u'Tweakers.net', u'http://feeds.feedburner.com/tweakers/nieuws')] + + def print_version(self, url): + return url + '?max=200' + From d6b82e364881527d37b08fb7613462dba8f7692d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 17 Jan 2012 22:37:44 +0530 Subject: [PATCH 02/13] Fix HTML 5 parser choking on comments --- src/calibre/ebooks/oeb/parse_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/oeb/parse_utils.py b/src/calibre/ebooks/oeb/parse_utils.py index 712427d457..f0b48afb39 100644 --- a/src/calibre/ebooks/oeb/parse_utils.py +++ b/src/calibre/ebooks/oeb/parse_utils.py @@ -103,7 +103,7 @@ def html5_parse(data, max_nesting_depth=100): xmlns_declaration = '{%s}'%XMLNS_NS non_html5_namespaces = {} seen_namespaces = set() - for elem in tuple(data.iter()): + for elem in tuple(data.iter(tag=etree.Element)): elem.attrib.pop('xmlns', None) namespaces = {} for x in tuple(elem.attrib): From b36b552ec6bc90f055475b8557465feb9171ea6d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 17 Jan 2012 22:40:22 +0530 Subject: [PATCH 03/13] Fix reading metadata from CHM files with non-ascii titles --- src/calibre/ebooks/chm/metadata.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/chm/metadata.py b/src/calibre/ebooks/chm/metadata.py index 26b09c7676..ea67947231 100644 --- a/src/calibre/ebooks/chm/metadata.py +++ b/src/calibre/ebooks/chm/metadata.py @@ -6,13 +6,14 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re +import re, codecs from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.metadata import string_to_authors, MetaInformation from calibre.utils.logging import default_log from calibre.ptempfile import TemporaryFile +from calibre import force_unicode def _clean(s): return s.replace(u'\u00a0', u' ') @@ -138,6 +139,13 @@ def get_metadata_from_reader(rdr): resolve_entities=True)[0]) title = rdr.title + try: + x = rdr.GetEncoding() + codecs.lookup(x) + enc = x + except: + enc = 'cp1252' + title = force_unicode(title, enc) authors = _get_authors(home) mi = MetaInformation(title, authors) publisher = _get_publisher(home) From ca2ccdcff7b06fa510328264c7520951d885445a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 17 Jan 2012 22:59:46 +0530 Subject: [PATCH 04/13] CHM Input: Do not choke on CHM files with non ascii internal filenames on windows. Fixes #917696 (Several problems in handling CHM file) --- src/calibre/ebooks/chm/input.py | 3 +++ src/calibre/ebooks/chm/reader.py | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/chm/input.py b/src/calibre/ebooks/chm/input.py index 9aa8272ee9..f36685bd91 100644 --- a/src/calibre/ebooks/chm/input.py +++ b/src/calibre/ebooks/chm/input.py @@ -11,6 +11,7 @@ from calibre.customize.conversion import InputFormatPlugin from calibre.ptempfile import TemporaryDirectory from calibre.utils.localization import get_lang from calibre.utils.filenames import ascii_filename +from calibre.constants import filesystem_encoding class CHMInput(InputFormatPlugin): @@ -36,6 +37,8 @@ class CHMInput(InputFormatPlugin): log.debug('Processing CHM...') with TemporaryDirectory('_chm2oeb') as tdir: + if not isinstance(tdir, unicode): + tdir = tdir.decode(filesystem_encoding) html_input = plugin_for_input_format('html') for opt in html_input.options: setattr(options, opt.option.name, opt.recommended_value) diff --git a/src/calibre/ebooks/chm/reader.py b/src/calibre/ebooks/chm/reader.py index 05ec388a9b..ad362895d6 100644 --- a/src/calibre/ebooks/chm/reader.py +++ b/src/calibre/ebooks/chm/reader.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ,' \ ' and Alex Bramley .' -import os, re +import os, re, codecs from calibre import guess_type as guess_mimetype from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString @@ -99,8 +99,17 @@ class CHMReader(CHMFile): def ExtractFiles(self, output_dir=os.getcwdu(), debug_dump=False): html_files = set([]) + try: + x = self.GetEncoding() + codecs.lookup(x) + enc = x + except: + enc = None for path in self.Contents(): - lpath = os.path.join(output_dir, path) + fpath = path + if not isinstance(path, unicode) and enc: + fpath = path.decode(enc) + lpath = os.path.join(output_dir, fpath) self._ensure_dir(lpath) try: data = self.GetFile(path) @@ -123,6 +132,7 @@ class CHMReader(CHMFile): self.log.warn('%r filename too long, skipping'%path) continue raise + if debug_dump: import shutil shutil.copytree(output_dir, os.path.join(debug_dump, 'debug_dump')) From 2f4941613ca4f9b5ccbf620d80f15423d2ec4e97 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 17 Jan 2012 23:55:30 +0530 Subject: [PATCH 05/13] Nicer implementation of the mutable classes --- src/calibre/db/lazy.py | 94 ++++++++++++++++++++++++++ src/calibre/library/database2.py | 109 +------------------------------ 2 files changed, 97 insertions(+), 106 deletions(-) create mode 100644 src/calibre/db/lazy.py diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py new file mode 100644 index 0000000000..b53cf2b54a --- /dev/null +++ b/src/calibre/db/lazy.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import weakref +from functools import wraps +from collections import MutableMapping, MutableSequence + +def resolved(f): + @wraps(f) + def wrapper(self, *args, **kwargs): + if getattr(self, '_must_resolve', True): + self._must_resolve = False + self._resolve() + return f(self, *args, **kwargs) + return wrapper + +class MutableBaseMixin(object): # {{{ + @resolved + def __str__(self): + return str(self.values) + + @resolved + def __repr__(self): + return repr(self.values) + + @resolved + def __unicode__(self): + return unicode(self.values) + + @resolved + def __len__(self): + return len(self.values) + + @resolved + def __iter__(self): + return iter(self.values) + + @resolved + def __contains__(self, key): + return key in self.values + + @resolved + def __getitem__(self, fmt): + return self.values[fmt] + + @resolved + def __setitem__(self, key, val): + self.values[key] = val + + @resolved + def __delitem__(self, key): + del self.values[key] + +# }}} + +class FormatMetadata(MutableBaseMixin, MutableMapping): # {{{ + + def __init__(self, db, id_, formats): + self.dbwref = weakref.ref(db) + self.id_ = id_ + self.formats = formats + self.values = {} + + def _resolve(self): + db = self.dbwref() + for f in self.formats: + try: + self.values[f] = db.format_metadata(self.id_, f) + except: + pass + +class FormatsList(MutableBaseMixin, MutableSequence): + + def __init__(self, formats, format_metadata): + self.formats = formats + self.format_metadata = format_metadata + self.values = [] + + def _resolve(self): + self.values = [f for f in self.formats if f in self.format_metadata] + + def insert(self, idx, val): + self._resolve() + self.values.insert(idx, val) + +# }}} + + diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 513f0bc690..caa8134579 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -7,8 +7,8 @@ __docformat__ = 'restructuredtext en' The database used to store ebook metadata ''' import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \ - json, uuid, hashlib, copy, weakref -from collections import defaultdict, MutableMapping, MutableSequence + json, uuid, hashlib, copy +from collections import defaultdict import threading, random from itertools import repeat from math import ceil @@ -40,6 +40,7 @@ from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.formatter_functions import load_user_template_functions from calibre.db.errors import NoSuchFormat +from calibre.db.lazy import FormatMetadata, FormatsList from calibre.utils.localization import (canonicalize_lang, calibre_langcode_to_name) @@ -81,110 +82,6 @@ class Tag(object): def __repr__(self): return str(self) -class MutablePrintMixin(object): # {{{ - def __str__(self): - self._resolve() - return str(self.values) - - def __repr__(self): - self._resolve() - return repr(self.values) - - def __unicode__(self): - self._resolve() - return unicode(self.values) -# }}} - -class FormatMetadata(MutableMapping, MutablePrintMixin): # {{{ - - def __init__(self, db, id_, formats): - self.dbwref = weakref.ref(db) - self.id_ = id_ - self.formats = formats - self._must_do = True - self.values = {} - - def _resolve(self): - if self._must_do: - self._must_do = False - db = self.dbwref() - for f in self.formats: - try: - self.values[f] = db.format_metadata(self.id_, f) - except: - pass - - def __contains__(self, key): - self._resolve() - return key in self.values - - def __getitem__(self, fmt): - self._resolve() - return self.values[fmt] - - def __setitem__(self, key, val): - self._resolve() - self.values[key] = val - - def __delitem__(self, key): - self._resolve() - self.values.__delitem__(key) - - def __len__(self): - self._resolve() - return len(self.values) - - def __iter__(self): - self._resolve() - return self.values.__iter__() - - -class FormatsList(MutableSequence, MutablePrintMixin): - - def __init__(self, formats, format_metadata): - self.formats = formats - self.format_metadata = format_metadata - self._must_do = True - self.values = [] - - def _resolve(self): - if self._must_do: - self._must_do = False - for f in self.formats: - try: - if f in self.format_metadata: - self.values.append(f) - except: - pass - - def __getitem__(self, dex): - self._resolve() - return self.values[dex] - - def __setitem__(self, key, dex): - self._resolve() - self.values[key] = dex - - def __delitem__(self, dex): - self._resolve() - self.values.__delitem__(dex) - - def __len__(self): - self._resolve() - return len(self.values) - - def __iter__(self): - self._resolve() - return self.values.__iter__() - - def insert(self, idx, val): - self._resolve() - self.values.insert(idx, val) - -# }}} - - - class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ''' An ebook metadata database that stores references to ebook files on disk. From f5ba57a0fe65fe89fed0fd4e7a94bf41ac344136 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jan 2012 10:49:34 +0530 Subject: [PATCH 06/13] ... --- src/calibre/db/lazy.py | 44 ++++++++++++++++---------------- src/calibre/ebooks/chm/reader.py | 4 +-- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index b53cf2b54a..8bbbfe40d2 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -21,73 +21,73 @@ def resolved(f): return wrapper class MutableBaseMixin(object): # {{{ + @resolved def __str__(self): - return str(self.values) + return str(self._values) @resolved def __repr__(self): - return repr(self.values) + return repr(self._values) @resolved def __unicode__(self): - return unicode(self.values) + return unicode(self._values) @resolved def __len__(self): - return len(self.values) + return len(self._values) @resolved def __iter__(self): - return iter(self.values) + return iter(self._values) @resolved def __contains__(self, key): - return key in self.values + return key in self._values @resolved def __getitem__(self, fmt): - return self.values[fmt] + return self._values[fmt] @resolved def __setitem__(self, key, val): - self.values[key] = val + self._values[key] = val @resolved def __delitem__(self, key): - del self.values[key] + del self._values[key] # }}} class FormatMetadata(MutableBaseMixin, MutableMapping): # {{{ def __init__(self, db, id_, formats): - self.dbwref = weakref.ref(db) - self.id_ = id_ - self.formats = formats - self.values = {} + self._dbwref = weakref.ref(db) + self._id = id_ + self._formats = formats def _resolve(self): - db = self.dbwref() - for f in self.formats: + db = self._dbwref() + self._values = {} + for f in self._formats: try: - self.values[f] = db.format_metadata(self.id_, f) + self._values[f] = db.format_metadata(self._id, f) except: pass class FormatsList(MutableBaseMixin, MutableSequence): def __init__(self, formats, format_metadata): - self.formats = formats - self.format_metadata = format_metadata - self.values = [] + self._formats = formats + self._format_metadata = format_metadata def _resolve(self): - self.values = [f for f in self.formats if f in self.format_metadata] + self._values = [f for f in self._formats if f in self._format_metadata] + @resolved def insert(self, idx, val): - self._resolve() - self.values.insert(idx, val) + self._values.insert(idx, val) # }}} diff --git a/src/calibre/ebooks/chm/reader.py b/src/calibre/ebooks/chm/reader.py index ad362895d6..fc7d865265 100644 --- a/src/calibre/ebooks/chm/reader.py +++ b/src/calibre/ebooks/chm/reader.py @@ -104,10 +104,10 @@ class CHMReader(CHMFile): codecs.lookup(x) enc = x except: - enc = None + enc = 'cp1252' for path in self.Contents(): fpath = path - if not isinstance(path, unicode) and enc: + if not isinstance(path, unicode): fpath = path.decode(enc) lpath = os.path.join(output_dir, fpath) self._ensure_dir(lpath) From d38d2824890e12074479203a6f28a3092bec0495 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jan 2012 10:54:18 +0530 Subject: [PATCH 07/13] Make subscription optional for ESPN --- recipes/espn.recipe | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/recipes/espn.recipe b/recipes/espn.recipe index 34c772f767..03c95d0001 100644 --- a/recipes/espn.recipe +++ b/recipes/espn.recipe @@ -20,7 +20,7 @@ class ESPN(BasicNewsRecipe): use_embedded_content = False remove_javascript = True - needs_subscription = True + needs_subscription = 'optional' encoding= 'ISO-8859-1' remove_tags_before = dict(name='font', attrs={'class':'date'}) @@ -75,32 +75,30 @@ class ESPN(BasicNewsRecipe): return soup - - def get_browser(self): br = BasicNewsRecipe.get_browser() - br.set_handle_refresh(False) - url = ('https://r.espn.go.com/members/v3_1/login') - raw = br.open(url).read() - raw = re.sub(r'(?s)
.*?id="regsigninbtn".*?
', '', raw) - with TemporaryFile(suffix='.htm') as fname: - with open(fname, 'wb') as f: - f.write(raw) - br.open_local_file(fname) + if self.username and self.password: + br.set_handle_refresh(False) + url = ('https://r.espn.go.com/members/v3_1/login') + raw = br.open(url).read() + raw = re.sub(r'(?s)
.*?id="regsigninbtn".*?
', '', raw) + with TemporaryFile(suffix='.htm') as fname: + with open(fname, 'wb') as f: + f.write(raw) + br.open_local_file(fname) - br.form = br.forms().next() - br.form.find_control(name='username', type='text').value = self.username - br.form['password'] = self.password - br.submit().read() - br.open('http://espn.go.com').read() - br.set_handle_refresh(True) + br.form = br.forms().next() + br.form.find_control(name='username', type='text').value = self.username + br.form['password'] = self.password + br.submit().read() + br.open('http://espn.go.com').read() + br.set_handle_refresh(True) return br def get_article_url(self, article): return article.get('guid', None) def print_version(self, url): - if 'eticket' in url: return url.partition('&')[0].replace('story?', 'print?') match = re.search(r'story\?(id=\d+)', url) From a2e76a5993da9a9b0aa0471c8b0929d5a6f6f9c4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jan 2012 11:09:50 +0530 Subject: [PATCH 08/13] ... --- src/calibre/db/cache.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index e79d496cd9..10fe0bb014 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -14,6 +14,7 @@ from functools import wraps, partial from calibre.db.locking import create_locks, RecordLock from calibre.db.fields import create_field from calibre.db.tables import VirtualTable +from calibre.db.lazy import FormatMetadata, FormatsList from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.date import now @@ -127,14 +128,8 @@ class Cache(object): if not formats: good_formats = None else: - good_formats = [] - for f in formats: - try: - mi.format_metadata[f] = self._format_metadata(book_id, f) - except: - pass - else: - good_formats.append(f) + mi.format_metadata = FormatMetadata(self, id, formats) + good_formats = FormatsList(formats, mi.format_metadata) mi.formats = good_formats mi.has_cover = _('Yes') if self._field_for('cover', book_id, default_value=False) else '' From 64503d07fe1b9d20eececa563942d29ac7f6b55b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jan 2012 11:17:58 +0530 Subject: [PATCH 09/13] ... --- src/calibre/db/lazy.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index 8bbbfe40d2..d68af171c6 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -11,6 +11,13 @@ import weakref from functools import wraps from collections import MutableMapping, MutableSequence +''' +Avoid doing stats on all files in a book when getting metadata for that book. +Speeds up calibre startup with large libraries/libraries on a network share, +with a composite custom column. +''' + +# Lazy format metadata retrieval {{{ def resolved(f): @wraps(f) def wrapper(self, *args, **kwargs): @@ -20,7 +27,7 @@ def resolved(f): return f(self, *args, **kwargs) return wrapper -class MutableBaseMixin(object): # {{{ +class MutableBase(object): @resolved def __str__(self): @@ -58,9 +65,8 @@ class MutableBaseMixin(object): # {{{ def __delitem__(self, key): del self._values[key] -# }}} -class FormatMetadata(MutableBaseMixin, MutableMapping): # {{{ +class FormatMetadata(MutableBase, MutableMapping): def __init__(self, db, id_, formats): self._dbwref = weakref.ref(db) @@ -76,7 +82,7 @@ class FormatMetadata(MutableBaseMixin, MutableMapping): # {{{ except: pass -class FormatsList(MutableBaseMixin, MutableSequence): +class FormatsList(MutableBase, MutableSequence): def __init__(self, formats, format_metadata): self._formats = formats @@ -91,4 +97,3 @@ class FormatsList(MutableBaseMixin, MutableSequence): # }}} - From 10e56ca5b521abc87019011947fb427a0cd205a4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jan 2012 11:53:42 +0530 Subject: [PATCH 10/13] Fix #918027 (The Reference Mode icon in ebook reader does not match documentation) --- src/calibre/manual/images/bookmark.png | Bin 1649 -> 1494 bytes src/calibre/manual/images/pref_button.png | Bin 1334 -> 1709 bytes src/calibre/manual/images/ref_mode_button.png | Bin 733 -> 2320 bytes 3 files changed, 0 insertions(+), 0 deletions(-) diff --git a/src/calibre/manual/images/bookmark.png b/src/calibre/manual/images/bookmark.png index c6671a2541d2d6223718f470d00b8c9567bcf099..4ba83fb49cfb4f3fc0465263c0f58f50ac7da7c1 100644 GIT binary patch delta 1491 zcmV;^1uXjU4Au)UiBL{Q4GJ0x0000DNk~Le0000u0000X2m$~A00TY$8UO$Q19L)2 zR0s$N+u={(ks%j<1$jwCK~z|U?U>JR8$}ey-i0A|^JetfnfKn#Tb!Nw4gL>e|24pF^%C@dpe6QXO+=bDA34Lgp^1pP z28NSc;qnWia(&{_v!;5Yq7ZLM_g-s{uTj+5q`YpsJI@O{78 zXtY}`L-0gG9tiX>P^C!|q-?v8&1Nzg!!Q7Vh&n;L(QI^rR-}V;HhuTz%@gmRLx!=W zl!-(lpU(q-K($(RUAN`?j%Aw3M-#z!95Zi1BfPjeFotEn@H@uZjr?WXo zq?8JRKx=K9=H%pLy(|!wg@oeG#)PiZ8wB4 z3_0gQ2qwg%KOXIyau}$$?2R26?>|OBL_l3M>#a9`=Rf}ffVOL6Mkbf_O5UM>hCyVS z<^WCr*rXVF=dAbB&&4};`Wh>xIOmCkZ2JD8fCga@M^9gEo;{GTtf@;M{qV`BmD$IkaA&H>ts0hf;UjNK06n?pSwN)-}c`?SAX_}T}xutU71@{kHF1hJk zCb=#1b+?L$HcuZA0T2w^e*5}$0WfpzTGVRw`T!B7QYmL7T`svpP^K?joW5{z(Bps| zedgr~xxTpxV#IY^078tvJZK#~dh*H@qh6nXzj342UMVG|94+MBXFCUe4A+Y|4Afip z3gd;rt8IxG+~^!Tb4Hvvp&vYG7K!8S|Ehf!_N1t6tXLo`IlitF=rn)1R?7 zl41`<06Z=Cu%F9L3V z?y__0RHxBs&(8MFLc~-m)okpXp<=!*l>I?#)f$&ZDwXQH5F&6cq?A%7kTC#2M20AY zFbqRVX^)SGK@cu2ZF$2~D#fKytJZeOx>jplQ*Af=IHOw7&A)f;scDISJ@k?#XzzVBzVS;klxhEWtbj^ktt>6 z3bedbb~0((wmr{VSXfwEdfN2sLJGv#6#)PkV{45{@yTP$N=H$+SX@k+k_(QAYVq$* zrD7aE{&03C2tqSq77Ak%6BD*=JK40mRGvCDc`%@E$#rthH{ad@rlk~0tB5*(ZNJHo zF~;It#2Aw@VcJQ*RRciFF-dE!6f%Yji#(lXN5+%3X=4fztM!%AuTP^;J0s5URJu1% zue_3<8WRbUMluKkAe2UuGo`f#0t6I>kkY8EtS%E>p|x5Sf^*IV7iKESIhU4ySSpqd479qkW{+il|8Y)*T7@bKRTM-~pj4zt6CxT( zBU7-wCzW~&$pW&PjpQxf>f1o(%Pl+Z{v7#t{>b8{DN@-16 zE2RNQ69qvC08$zZ5rA=y$QWmgF&;~B&SMYY{6(s~fi)9DWI_neNlLAh)>><=NRv{U zh&1Uq@e@GzUV;n|8M2rZSy!AR0KRY`x~_b@ZNF09Thg0d^hs4S?+vh0GOur8hi zuLGhWcu?>l;>ClC=t0(lAbJ>eM!kqExGN&=DDw%r+uJ?e)!o%yU)hx(5%D}!^>#Zm z{bi3cdzg1flFZ2e{}=IIMnooEf4}wy9(M4<1^^yGY=^!N8sdli-wid}^nOd#hXYaU z{1R(F&TO|j_r>+#MNg%}DjYQ(Dt=9^{!m9V(c~xc`o$sn=l>vbGziJUrEjTdX5{fN zkDR#>RL+|ACFAQ46v8xqsx_6TUo?dp`FMEtC70BP~fC!a;@)dDR)Ac%0~JHhZ1<*84R>q0UhhubLD&VzI*{S%4(zeDKOZ%le6ntaxj zPQW*T9FoDOOj11dacTdh%>O!s+TqN_s^qF}|K7AOM$?}u);>)65|D!kAP6D35tXL0 zp(l9b6^y(TyD85@`>N93e+!t1SIKow{-Ju;IlHNSQw5`{IAbd3O0{P+r?3rx0fA4D zuR=2DF{@|e`s?}Tn<}j3G-}N$i)!kJyqP*21i6-3{l0EpHpNL@eq2Q}x-_eTsxm_= zsHh;Kd=YsAaF84>hU&{f0-7Zv1aZ-5aGgC zbo+92{Dso=r*$-@f)YszAH&CREw~QE0Ui+e7(N9Ez#uOmKtWHy97A~8qu{cpOB#9n zcG~!JYIS1H+3NG}cV>G3Ve4k82zLW27Mh6MBo7ALFrcRw3xa0_Fvocm;Sx%E9bt;IyxK#favsv z#;rQl|47`cf6PQA*ZQ3U0#FDF1~ecb1REe3lr0#)r=&Wk;wRq?qF_SyX&6oTNC(n@zh0K0n=+=W8iB}FWo{=PsUTV^xL;VRex4zdGp zuV}Nfe`MQxYah5Kd;=;}g#`TH1tHimQ^{S=wMjGkrV|HJM~0A1$vY}CU=Q?1R{@;I z*j&i7rjSBZJaWAWHct8Atu6;nV z4QgHwNi?R z_J`<)_r6c#y|>nR@A^mQy!Xy~@!pGgMG>ICvK~z|U?U;LTRMj2FfA{X)y}Nnr8uH#GtQ*A8BsgeLuxt%2 zN(%_VI-LP)F{oH!3Syaxf7oFx{!yJ#TXlTKqAgIA5p+OJDj<+Bffyw^ab-=&W8c~A z^KS0Vz2}^JPybjNGeXMEB#e&ZZ+?HA*FE=hf9H3;=geh)++TFTcWvx<4FK>X#5eZC zf}oTpB@6(*guJ&S)a6T43W7{Djb8RyZn_{&^~St0$!g({$`siwzJ%N#5mb%xMt0fU zQqI6!SqMZlN~tJgdw%Y(7S>vLV_Xn(EWF9c9Y6cEXCR1G4Xe~*n($rI0MXGC?hOm` zsvI}hI1e6w@AO9!lmYcTIb;t9qAE~U}d~xq#Qc& zS=}|I7|9miG(8aKypdp37~e85l<*7$W@H(9d=V9Yx)9CTD zJ&{EEWLNLkzDVa+gT4OYOjn0{Koq4>NkWRO5_GW>tRzw?o@I`HdTvS(3Q_xl(LK$l zt=X3TQ1tj&k9Rng>2lT8m^hYYSxqGbqf2^@#ViQIGS`h$f=CKtSz+Fyx|(dWagMXR zqB#G5nwh1sR2l%P$_pyX3jIT|Iailu^6^`qYvHXq*$e{a=iA z*r}>jR+QX)LrveHZ_UajtCrpZ04&R5Lj2)>Nb8xi+cvDtx8;n*faloz*RE0|v9sGF zj!L{~!rj5>xzteB@Z#?GhNAJ=PDd)84o7133$9;LKOZ3k0NK_&jyM0kX|F9e>+U;m zPYNU9;n-Ey+zNYf^U>C}Pd2pM%Vp6SYmSg>z)qi{CpY!L{x^9|( zXUooV_w@G;1^~chG%i^*-(q4v?dt9u@a1G@73Sxqgpr})n5wE~GcOAXjx$U|pf?i0 zFh#{BX+h}j^{E6cykRa@vZmSD=N-=gO!f*&aUp~n|3e1b@962 ztQznKON$Glp@61MD3|&pK_lT103;KC@q_RG$ILM^?Zui#REz*XWdMMN1O5+MKKA;8 zn>YM+*YkgsCDA|Fm&Nm!B>?@3Nh$sCSlhqeKA29YU)#Mu5{;+RVkQ9zMl!-owprV^A=MV2p10D6}LKvmUO_Z;YUUr;@x(MSjxTV}JFR}_?gsd5Hx z%LD6XIm&c?9_Wt*$8nEt-Q*~@XHEoJk}SNbe$j&EOMfxD@@jEpgdn71tP!&1!F%Uc zU#p)0^xr2arRiks-rucx^v^p1U~#=`+m=lX)ih1zdGiWTP38D zlY-C{34}00NS5UT2M($j&2r9v+`8%hNXQ=!`Xk|xZ?M0tXvVfJ52_fox3`BvL4*)O zNY#-I8PgSsolib9=)G_dQ%YAZyDb_Sf^qv^62&>QD~pQ?0)gPhhqh-L|G4@0^RBC% zHVBOnf|arAjH_&Lk@Ad+Bc&yU-ob%yn*QJ2cJdG^!35*6YBG_qmlR!pJS-lKjvq#+ zcR5zxz9JG~!|H~D{Cof?FDsu;h!x9!Y2Y{j z2n76x4;^l9ZeFmUuBN(rYyctT<(FO^6~%&rg3W)}$QcX(kWM8tC*1TvBopyWA}hPg$J>8iQ!ngBhGoNkpT%N%?u8e8KJU-#Z%M>sR|En82qB{*!us`%vEc{+XzKrx zgHlQRCc#l9C1{{?zsvU;I>e479O002ovPDHLkV1g0H BOxged delta 1330 zcmV-21pHfRut4}H~uAAz>vL&B$meJHerMVG+V?snT{m!`F|Ei*ea*N17Lw6xon zcJaaUbaL;x_x$GEGylvT^yk$ke|V^dA8vX8U?;asmgQi3=yv~Yp72Ja(Uda7WSW5x zN(R9d@70^(sKso}ot+yFN3Qv<2ZMoLu@|!_Hz((XmCsEhK_C!pyx6j2$y3WpN_{@x zFNY3`qKFZ+1p?7XL@@BTqFoD%7Nn=UCPaY7*@>r5pQ9ANQdVY8F^58-e~lk)bUIx2 zG~1lnIayhmHk(Zkqm)kcEE3m%QhMOPPx*O|mX?+Z1_Qz<6bgAf*^7!66c#=<{DCIk z3W*609j>20FTbR?7ywjNMHt)D(ijGfOg}kGK+xUYqpGl|s3;zfQvi)pgiv;N7S9W! zD4I;B2Pa6N2Z3OFMn*pN~4j8&-a%nD}8b@kboc->$`rVZSLIpUhkEfni{Ls z^7?D9dOVquB>P+cHX2ggnVCjGFk5C`zTA@Oc035g{Rz&TJwI<=fBv;=zPh?PzyC&c z)h?IId1qFZ;~gE5bXU4j5G*sTP0g2_E<47P6!`lu%g>uzUtd>STie>&`o!WzL(>BQ zA)L!)@4VS5D~j0rPf^k0aK|l8)25QZZnqULeKH!2cs%YkYu*?cKnQU5UcM<7e~V3&U@T4S>1Zp2*WTK zW0qwx#_jE)rl#f<_d-ky0RR*fV@lLNLY%e<7adIga!DTLXb$Ny*YXk<)l* zDl1#{{(J9sb#+OS1OTyEj1Xe6SXx>x4-K2-ooW0K49o1^y$1jc27@F?qS(9l+pm*H z0066Aeu)rLQBkd`s-h_S_I@*KR!&k00N}-y&)aRb+PeC;*S?jNd8xYc-eb!VqO6NC{-Dw z*OUW4h~7WfX^4gVbwL3Udbx$a=g$4xo0`e|*9hDsd8^am1h2O<95G|T%r5%H19W19 zP-+H2&~B?tCX*ydx7+P@>y-mjMrs5Ad?$83=%q_qc87801hp`H^z4Es1pvIza(+Bn zx%(@reBQQFRvg8i=k|6$>z%CZN*|y=Q<)Pr2s%F#Ta93{m&GOMWNK&Pp$Oj0bu@Ut6Pt+kiZw~ z@?YOkzIV@6W2KU=>r%=%j-x0_l0*n0gpg8dnkJO6B0egM<(O*nV|t=dHQb6FsZIroS;lHF7Zt zGO#KurT(E#EUD!ROjP{Iv%h}|PoIufS38d5hhfCIv~8Nt^KIL#@$t;AUDC4B>GTaZ z+(0R%lnNoB{_9Sd?!v6jxr(HG*8`z&I!*M=yFPdiekFhNih30QM!k`{@BODfH|GQN z0Ahdypaw;Qc8--U@g*orkkEoUJ^aF0ILxIz>vm&SSDGv z1;HAuy(*6b0RYfyQ9L)tFMdqpd-ooG=%H(Gz8Mh#AW0HJh-D-!{~6gmIi!xR9KBxa z%L70wmM=N<xtx)qFyF#cDj8Ca{il;;Tvza>~%fNsWHzIn%0M;Ao5O9{!0 zkLwc?Bm4GE-hO*wYATz}B4S@&DMcw#?{6GE740F~5VNQC?&;FZOXptPZ`RkH<8`U6 z2>{@G^gGXs7nWdoCr-{DOvK42VP>ZE;DgLG^Z9%_od$r7RR{o*-pcCR52y8j=4VjL z5!8?mYuRb5`-k)$2jZtSibxOT@PhkdFo6Na*XV8iu{S>FQ#y42{o5*)OeWJ$wvDD# z5-vR1n)}IW+hP;b)F`2rC)6M`MUCy;zNR3bUApi4rezJ(8cZNb*vS4h)M}~TZX-RK zaP9%PuB+?1u77m*D<#2sF*^HbuX%zhFU|aBqpDj)%5%s}s+W+Zs8zAtQ)cfC{bet? zX}`$lH)##7iRVUsQO`Z^rcP7~zgrpqlLC-Q|5XEu4Jw^AK__z_lbY)#P9YMwEFe%i^L1b|E?w2Eo zzmNcc5QY;E5JaYNrs<4nHg7f?)!1K5f@&PBinyZ)BfP{=^bW*+6sC_I+nNA@Zk!SU zlyi#62w7{lJDpA#h9CXdz7g_=DdQUBOw$c6NL*WMErs31*ssRDrjik%7V2Z1|7(7F zh9t?>1UxQf{F(Dp+nuGH13)H|?zryS+FGa6Nj5mb@{b5jF->P0XBxNlwPvH*>nuio zH4f@Y*inFzW?lVAyHxaU+P_I|a3t=if7?mF5^5`{a3^aH#dIzXLrvE$gFCI3uIrq0 z+qV08LHH;c`z~F2Ok=uWF=H{8tIdX0KtofAd`K$Vm6L6;Cz|R+MdaKpaG>vm^7{)rp=v}Ju3={Jh>qwKd;Z>cQX&8oLgo_x*tKnyZ zz|CKf2!l^yvoV%<9wh52kh3#w$7!`%?RNY8SrFu;{8OY1DMzXy!Hd3||QW&F4bLq1JkfVHgHQXfzs)M#FKO{#~GyBH2Tby9Fsnsw34*o%r2` zPj>scySzZUYwcZ=l?$sGTuQIZGnv9KpU_et)$>xJ@ZOoT{iX;2QmGUo5<+132T(tT zG$P0Wab)H`D1Os*-Da~sIZ?JQt0rzTEtmb+8piTjySiEj04ZgmP~e;sLX@&G`z`1_ zAyPM~@_mUaxvf^SzPhtqHq9&Q(v(uZ<7kASrh!s~kSOS3Dy3Uyu~1m5)@rp{k|a?S z*|yzpoh2Rtd%K5w!ys(b&uJ9*?4Ihk4wpfol&Ze?7o%Pu+O~Z(lK~+>N+l&oDV2)j zXz{hzgJPj{>#ZkFoeIM+2!eihs+1B!#PJTtIYR-)$Hr2r!8^YN!FvzuQW?nx<(QV~kRIVK5FL(DL~NQ4=C=w^=r;kB{ryw=>%ot{W^axs!T$#VOOiwiA*B>T2q|PDgp@)EC51?YP*Nm9N+A*<)%w%(*?*xCLMS3KiiA=| q5h`%qLDBce*%w5 zL_t(oh3%Kki_<_D#@}~lYyRIwlitaxk=!)RYn_!h*#EXZBc=pfGgL+d?2v)ZD z)DPH;$X*0L(TgG#wN)3tSEL8ic_$vG4Iwj0nl`K;`y9e#n7qGv-lv_k`0C|zSSztL zBdx>MVObKf`*!L>zqF8`#vJ-8FLWIoEa1q;`+5K*)Ra` zeZSZ1=|YGY4pP`LYNb-KZ^E3yp~5QW)dZxl(_kUbah!g?pH$IQsZ?q-8Yv4BodJOB zx{zhd>Mx%&E5phgrNbGpScxQzHQIcz@JYvUeBV!6U=}Tx%gttU9A}8=f4Z(|(W)3A zcV=K~CSxoU7S4BE1$0<*twf; zlOAY2`V0V?mskJvH;C5dQ@c6mY$!y0@cNf~uL}UR{oAyyVlp#7X)CM#;_` Date: Wed, 18 Jan 2012 13:35:00 +0530 Subject: [PATCH 11/13] ... --- src/calibre/db/lazy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/db/lazy.py b/src/calibre/db/lazy.py index d68af171c6..be9334c056 100644 --- a/src/calibre/db/lazy.py +++ b/src/calibre/db/lazy.py @@ -22,8 +22,8 @@ def resolved(f): @wraps(f) def wrapper(self, *args, **kwargs): if getattr(self, '_must_resolve', True): - self._must_resolve = False self._resolve() + self._must_resolve = False return f(self, *args, **kwargs) return wrapper From 66421de3105efdf95f44386fc050776502edaf6b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jan 2012 15:35:50 +0530 Subject: [PATCH 12/13] Cache the format filename info in memory --- src/calibre/library/database2.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index caa8134579..07bfba53b3 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -170,6 +170,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except: traceback.print_exc() self.field_metadata = FieldMetadata() + self.format_filename_cache = defaultdict(dict) self._library_id_ = None # Create the lock to be used to guard access to the metadata writer # queues. This must be an RLock, not a Lock @@ -310,6 +311,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if not self.is_second_db: load_user_template_functions(self.prefs.get('user_template_functions', [])) + # Load the format filename cache + self.format_filename_cache = defaultdict(dict) + for book_id, fmt, name in self.conn.get( + 'SELECT book,format,name FROM data'): + self.format_filename_cache[book_id][fmt.upper()] = name + self.conn.executescript(''' DROP TRIGGER IF EXISTS author_insert_trg; CREATE TEMP TRIGGER author_insert_trg @@ -599,7 +606,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): fname = self.construct_file_name(id) changed = False for format in formats: - name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False) + name = self.format_filename_cache[id].get(format.upper(), None) if name and name != fname: changed = True break @@ -1139,12 +1146,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def format_files(self, index, index_is_id=False): id = index if index_is_id else self.id(index) - try: - formats = self.conn.get('SELECT name,format FROM data WHERE book=?', (id,)) - formats = map(lambda x:(x[0], x[1]), formats) - return formats - except: - return [] + return [(v, k) for k, v in self.format_filename_cache[id].iteritems()] def formats(self, index, index_is_id=False, verify_formats=True): ''' Return available formats as a comma separated list or None if there are no available formats ''' @@ -1230,7 +1232,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ''' id = index if index_is_id else self.id(index) try: - name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False) + name = self.format_filename_cache[id][format.upper()] except: return None if name: @@ -1327,11 +1329,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def add_format(self, index, format, stream, index_is_id=False, path=None, notify=True, replace=True): id = index if index_is_id else self.id(index) - if format: - self.format_metadata_cache[id].pop(format.upper(), None) + if not format: format = '' + self.format_metadata_cache[id].pop(format.upper(), None) + name = self.format_filename_cache[id].get(format.upper(), None) if path is None: path = os.path.join(self.library_path, self.path(id, index_is_id=True)) - name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False) if name and not replace: return False name = self.construct_file_name(id) @@ -1349,6 +1351,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.execute('INSERT OR REPLACE INTO data (book,format,uncompressed_size,name) VALUES (?,?,?,?)', (id, format.upper(), size, name)) self.conn.commit() + self.format_filename_cache[id][format.upper()] = name self.refresh_ids([id]) if notify: self.notify('metadata', [id]) @@ -1396,9 +1399,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def remove_format(self, index, format, index_is_id=False, notify=True, commit=True, db_only=False): id = index if index_is_id else self.id(index) - if format: - self.format_metadata_cache[id].pop(format.upper(), None) - name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False) + if not format: format = '' + self.format_metadata_cache[id].pop(format.upper(), None) + name = self.format_filename_cache[id].pop(format.upper(), None) if name: if not db_only: try: From a19fdb42cda865d126e1b999711ffe61d1f0beef Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jan 2012 15:49:56 +0530 Subject: [PATCH 13/13] ... --- src/calibre/library/database2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 07bfba53b3..00ca0e39a2 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -315,7 +315,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.format_filename_cache = defaultdict(dict) for book_id, fmt, name in self.conn.get( 'SELECT book,format,name FROM data'): - self.format_filename_cache[book_id][fmt.upper()] = name + self.format_filename_cache[book_id][fmt.upper() if fmt else ''] = name self.conn.executescript(''' DROP TRIGGER IF EXISTS author_insert_trg;