From 1f2daebce60aa01438ddbe35b02746552a399114 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 18 Jan 2013 19:33:45 +0530 Subject: [PATCH 01/43] CHM Input: Support hierarchical table of contents. Do not generate an inline table of contents when a metadata table of contents is present. Also correctly decode the text in the table of contents --- src/calibre/ebooks/chm/reader.py | 2 +- .../ebooks/conversion/plugins/chm_input.py | 206 +++++++----------- src/calibre/utils/chm/chm.py | 16 +- 3 files changed, 93 insertions(+), 131 deletions(-) diff --git a/src/calibre/ebooks/chm/reader.py b/src/calibre/ebooks/chm/reader.py index 28d7f1e7c7..3e5c0d24d9 100644 --- a/src/calibre/ebooks/chm/reader.py +++ b/src/calibre/ebooks/chm/reader.py @@ -100,7 +100,7 @@ class CHMReader(CHMFile): def ExtractFiles(self, output_dir=os.getcwdu(), debug_dump=False): html_files = set([]) try: - x = self.GetEncoding() + x = self.get_encoding() codecs.lookup(x) enc = x except: diff --git a/src/calibre/ebooks/conversion/plugins/chm_input.py b/src/calibre/ebooks/conversion/plugins/chm_input.py index 05f7a32aa4..a846682432 100644 --- a/src/calibre/ebooks/conversion/plugins/chm_input.py +++ b/src/calibre/ebooks/conversion/plugins/chm_input.py @@ -7,8 +7,6 @@ import os from calibre.customize.conversion import InputFormatPlugin from calibre.ptempfile import TemporaryDirectory -from calibre.utils.localization import get_lang -from calibre.utils.filenames import ascii_filename from calibre.constants import filesystem_encoding class CHMInput(InputFormatPlugin): @@ -57,6 +55,7 @@ class CHMInput(InputFormatPlugin): mainpath = os.path.join(tdir, mainname) metadata = get_metadata_from_reader(self._chm_reader) + encoding = self._chm_reader.get_encoding() or options.input_encoding or 'cp1252' self._chm_reader.CloseCHM() # print tdir, mainpath # from calibre import ipython @@ -64,15 +63,31 @@ class CHMInput(InputFormatPlugin): options.debug_pipeline = None options.input_encoding = 'utf-8' - # try a custom conversion: - #oeb = self._create_oebbook(mainpath, tdir, options, log, metadata) - # try using html converter: - htmlpath = self._create_html_root(mainpath, log) + htmlpath, toc = self._create_html_root(mainpath, log, encoding) oeb = self._create_oebbook_html(htmlpath, tdir, options, log, metadata) options.debug_pipeline = odi - #log.debug('DEBUG: Not removing tempdir %s' % tdir) + if toc.count() > 1: + oeb.toc = self.parse_html_toc(oeb.spine[0]) + oeb.manifest.remove(oeb.spine[0]) + oeb.auto_generated_toc = False return oeb + def parse_html_toc(self, item): + from calibre.ebooks.oeb.base import TOC, XPath + dx = XPath('./h:div') + ax = XPath('./h:a[1]') + + def do_node(parent, div): + for child in dx(div): + a = ax(child)[0] + c = parent.add(a.text, a.attrib['href']) + do_node(c, child) + + toc = TOC() + root = XPath('//h:div[1]')(item.data)[0] + do_node(toc, root) + return toc + def _create_oebbook_html(self, htmlpath, basedir, opts, log, mi): # use HTMLInput plugin to generate book from calibre.customize.builtins import HTMLInput @@ -81,78 +96,22 @@ class CHMInput(InputFormatPlugin): oeb = htmlinput.create_oebbook(htmlpath, basedir, opts, log, mi) return oeb - - def _create_oebbook(self, hhcpath, basedir, opts, log, mi): - import uuid - from lxml import html - from calibre.ebooks.conversion.plumber import create_oebbook - from calibre.ebooks.oeb.base import DirContainer - oeb = create_oebbook(log, None, opts, - encoding=opts.input_encoding, populate=False) - self.oeb = oeb - - metadata = oeb.metadata - if mi.title: - metadata.add('title', mi.title) - if mi.authors: - for a in mi.authors: - metadata.add('creator', a, attrib={'role':'aut'}) - if mi.publisher: - metadata.add('publisher', mi.publisher) - if mi.isbn: - metadata.add('identifier', mi.isbn, attrib={'scheme':'ISBN'}) - if not metadata.language: - oeb.logger.warn(u'Language not specified') - metadata.add('language', get_lang().replace('_', '-')) - if not metadata.creator: - oeb.logger.warn('Creator not specified') - metadata.add('creator', _('Unknown')) - if not metadata.title: - oeb.logger.warn('Title not specified') - metadata.add('title', _('Unknown')) - - bookid = str(uuid.uuid4()) - metadata.add('identifier', bookid, id='uuid_id', scheme='uuid') - for ident in metadata.identifier: - if 'id' in ident.attrib: - self.oeb.uid = metadata.identifier[0] - break - - hhcdata = self._read_file(hhcpath) - hhcroot = html.fromstring(hhcdata) - chapters = self._process_nodes(hhcroot) - #print "=============================" - #print "Printing hhcroot" - #print etree.tostring(hhcroot, pretty_print=True) - #print "=============================" - log.debug('Found %d section nodes' % len(chapters)) - - if len(chapters) > 0: - path0 = chapters[0][1] - subpath = os.path.dirname(path0) - htmlpath = os.path.join(basedir, subpath) - - oeb.container = DirContainer(htmlpath, log) - for chapter in chapters: - title = chapter[0] - basename = os.path.basename(chapter[1]) - self._add_item(oeb, title, basename) - - oeb.container = DirContainer(htmlpath, oeb.log) - return oeb - - def _create_html_root(self, hhcpath, log): + def _create_html_root(self, hhcpath, log, encoding): from lxml import html from urllib import unquote as _unquote from calibre.ebooks.oeb.base import urlquote + from calibre.ebooks.chardet import xml_to_unicode hhcdata = self._read_file(hhcpath) + hhcdata = hhcdata.decode(encoding) + hhcdata = xml_to_unicode(hhcdata, verbose=True, + strip_encoding_pats=True, resolve_entities=True)[0] hhcroot = html.fromstring(hhcdata) - chapters = self._process_nodes(hhcroot) + toc = self._process_nodes(hhcroot) #print "=============================" #print "Printing hhcroot" #print etree.tostring(hhcroot, pretty_print=True) #print "=============================" - log.debug('Found %d section nodes' % len(chapters)) + log.debug('Found %d section nodes' % toc.count()) htmlpath = os.path.splitext(hhcpath)[0] + ".html" base = os.path.dirname(os.path.abspath(htmlpath)) @@ -168,37 +127,40 @@ class CHMInput(InputFormatPlugin): x = y return x + def donode(item, parent, base, subpath): + for child in item: + title = child.title + if not title: continue + raw = unquote_path(child.href or '') + rsrcname = os.path.basename(raw) + rsrcpath = os.path.join(subpath, rsrcname) + if (not os.path.exists(os.path.join(base, rsrcpath)) and + os.path.exists(os.path.join(base, raw))): + rsrcpath = raw + + if '%' not in rsrcpath: + rsrcpath = urlquote(rsrcpath) + if not raw: + rsrcpath = '' + c = DIV(A(title, href=rsrcpath)) + donode(child, c, base, subpath) + parent.append(c) + with open(htmlpath, 'wb') as f: - if chapters: - f.write('\n') - path0 = chapters[0][1] + if toc.count() > 1: + from lxml.html.builder import HTML, BODY, DIV, A + path0 = toc[0].href path0 = unquote_path(path0) subpath = os.path.dirname(path0) base = os.path.dirname(f.name) - - for chapter in chapters: - title = chapter[0] - raw = unquote_path(chapter[1]) - rsrcname = os.path.basename(raw) - rsrcpath = os.path.join(subpath, rsrcname) - if (not os.path.exists(os.path.join(base, rsrcpath)) and - os.path.exists(os.path.join(base, raw))): - rsrcpath = raw - - # title should already be url encoded - if '%' not in rsrcpath: - rsrcpath = urlquote(rsrcpath) - url = "
" + title + " \n" - if isinstance(url, unicode): - url = url.encode('utf-8') - f.write(url) - - f.write("") + root = DIV() + donode(toc, root, base, subpath) + raw = html.tostring(HTML(BODY(root)), encoding='utf-8', + pretty_print=True) + f.write(raw) else: f.write(hhcdata) - return htmlpath - + return htmlpath, toc def _read_file(self, name): f = open(name, 'rb') @@ -206,41 +168,27 @@ class CHMInput(InputFormatPlugin): f.close() return data - def _visit_node(self, node, chapters, depth): - # check that node is a normal node (not a comment, DOCTYPE, etc.) - # (normal nodes have string tags) - if isinstance(node.tag, basestring): - from calibre.ebooks.chm.reader import match_string - - chapter_path = None - if match_string(node.tag, 'object') and match_string(node.attrib['type'], 'text/sitemap'): - chapter_title = None - for child in node: - if match_string(child.tag,'param') and match_string(child.attrib['name'], 'name'): - chapter_title = child.attrib['value'] - if match_string(child.tag,'param') and match_string(child.attrib['name'],'local'): - chapter_path = child.attrib['value'] - if chapter_title is not None and chapter_path is not None: - chapter = [chapter_title, chapter_path, depth] - chapters.append(chapter) - if node.tag=="UL": - depth = depth + 1 - if node.tag=="/UL": - depth = depth - 1 + def add_node(self, node, toc, ancestor_map): + from calibre.ebooks.chm.reader import match_string + if match_string(node.attrib['type'], 'text/sitemap'): + p = node.xpath('ancestor::ul[1]/ancestor::li[1]/object[1]') + parent = p[0] if p else None + toc = ancestor_map.get(parent, toc) + title = href = u'' + for param in node.xpath('./param'): + if match_string(param.attrib['name'], 'name'): + title = param.attrib['value'] + elif match_string(param.attrib['name'], 'local'): + href = param.attrib['value'] + child = toc.add(title or _('Unknown'), href) + ancestor_map[node] = child def _process_nodes(self, root): - chapters = [] - depth = 0 - for node in root.iter(): - self._visit_node(node, chapters, depth) - return chapters + from calibre.ebooks.oeb.base import TOC + toc = TOC() + ancestor_map = {} + for node in root.xpath('//object'): + self.add_node(node, toc, ancestor_map) + return toc - def _add_item(self, oeb, title, path): - bname = os.path.basename(path) - id, href = oeb.manifest.generate(id='html', - href=ascii_filename(bname)) - item = oeb.manifest.add(id, href, 'text/html') - item.html_input_href = bname - oeb.spine.add(item, True) - oeb.toc.add(title, item.href) diff --git a/src/calibre/utils/chm/chm.py b/src/calibre/utils/chm/chm.py index 3b3f2c39d9..02fe19e44e 100644 --- a/src/calibre/utils/chm/chm.py +++ b/src/calibre/utils/chm/chm.py @@ -28,6 +28,7 @@ import array import string import sys +import codecs import calibre.utils.chm.chmlib as chmlib from calibre.constants import plugins @@ -184,7 +185,7 @@ locale_table = { 0x0420 : ('iso8859_6', "Urdu", "Arabic"), 0x0443 : ('iso8859_9', "Uzbek_Latin", "Turkish"), 0x0843 : ('cp1251', "Uzbek_Cyrillic", "Cyrillic"), - 0x042a : (None, "Vietnamese", "Vietnamese") + 0x042a : ('cp1258', "Vietnamese", "Vietnamese") } class CHMFile: @@ -434,6 +435,19 @@ class CHMFile: else: return None + def get_encoding(self): + ans = self.GetEncoding() + if ans is None: + lcid = self.GetLCID() + if lcid is not None: + ans = lcid[0] + if ans: + try: + codecs.lookup(ans) + except: + ans = None + return ans + def GetDWORD(self, buff, idx=0): '''Internal method. Reads a double word (4 bytes) from a buffer. From 805d8c622372bbf7adaf639bdb447203f93bb59d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 18 Jan 2013 20:00:23 +0530 Subject: [PATCH 02/43] E-book viewer: Allow entries in the Table of Contents that do not point anywhere, instead of just ignoring them. --- src/calibre/ebooks/metadata/toc.py | 11 +++++------ src/calibre/gui2/viewer/toc.py | 14 ++++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/calibre/ebooks/metadata/toc.py b/src/calibre/ebooks/metadata/toc.py index 0f22603a8b..25eb154b74 100644 --- a/src/calibre/ebooks/metadata/toc.py +++ b/src/calibre/ebooks/metadata/toc.py @@ -194,12 +194,11 @@ class TOC(list): content = content_path(np) if content and text: content = content[0] - src = get_attr(content, attr='src') - if src: - purl = urlparse(content.get('src')) - href, fragment = unquote(purl[2]), unquote(purl[5]) - nd = dest.add_item(href, fragment, text) - nd.play_order = play_order + # if get_attr(content, attr='src'): + purl = urlparse(content.get('src')) + href, fragment = unquote(purl[2]), unquote(purl[5]) + nd = dest.add_item(href, fragment, text) + nd.play_order = play_order for c in np_path(np): process_navpoint(c, nd) diff --git a/src/calibre/gui2/viewer/toc.py b/src/calibre/gui2/viewer/toc.py index b0e97bea65..094191308e 100644 --- a/src/calibre/gui2/viewer/toc.py +++ b/src/calibre/gui2/viewer/toc.py @@ -56,7 +56,7 @@ class TOCItem(QStandardItem): self.title = text self.parent = parent QStandardItem.__init__(self, text if text else '') - self.abspath = toc.abspath + self.abspath = toc.abspath if toc.href else None self.fragment = toc.fragment all_items.append(self) self.bold_font = QFont(self.font()) @@ -70,11 +70,13 @@ class TOCItem(QStandardItem): if si == self.abspath: spos = i break - try: - am = getattr(spine[i], 'anchor_map', {}) - except UnboundLocalError: - # Spine was empty? - am = {} + am = {} + if self.abspath is not None: + try: + am = getattr(spine[i], 'anchor_map', {}) + except UnboundLocalError: + # Spine was empty? + pass frag = self.fragment if (self.fragment and self.fragment in am) else None self.starts_at = spos self.start_anchor = frag From 67e691c2ff886caf4e98368d8702172d1f39e41d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 18 Jan 2013 21:19:55 +0530 Subject: [PATCH 03/43] Conversion: Preserve ToC entries that point nowhere instead of causing them to point to a non-existant file --- src/calibre/ebooks/oeb/reader.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index 0461491d2f..68db089073 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -373,16 +373,12 @@ class OEBReader(object): if not title: self._toc_from_navpoint(item, toc, child) continue - if not href: - gc = xpath(child, 'ncx:navPoint') - if not gc: - # This node is useless - continue - href = 'missing.html' - - href = item.abshref(urlnormalize(href[0])) + if (not href or not href[0]) and not xpath(child, 'ncx:navPoint'): + # This node is useless + continue + href = item.abshref(urlnormalize(href[0])) if href and href[0] else '' path, _ = urldefrag(href) - if path not in self.oeb.manifest.hrefs: + if href and path not in self.oeb.manifest.hrefs: self.logger.warn('TOC reference %r not found' % href) gc = xpath(child, 'ncx:navPoint') if not gc: From 70a22e0327cc02d07590f5156f19de5030741046 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 18 Jan 2013 21:32:56 +0530 Subject: [PATCH 04/43] Do not choke when reading metadata from MOBI files with incorrectly encoded metadata fields --- src/calibre/ebooks/mobi/reader/headers.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/calibre/ebooks/mobi/reader/headers.py b/src/calibre/ebooks/mobi/reader/headers.py index 5a3de7e705..690b4b488e 100644 --- a/src/calibre/ebooks/mobi/reader/headers.py +++ b/src/calibre/ebooks/mobi/reader/headers.py @@ -13,6 +13,7 @@ from calibre.utils.date import parse_date from calibre.ebooks.mobi import MobiError from calibre.ebooks.metadata import MetaInformation, check_isbn from calibre.ebooks.mobi.langcodes import main_language, sub_language, mobi2iana +from calibre.utils.cleantext import clean_ascii_chars from calibre.utils.localization import canonicalize_lang NULL_INDEX = 0xffffffff @@ -31,6 +32,8 @@ class EXTHHeader(object): # {{{ self.kf8_header = None self.uuid = self.cdetype = None + self.decode = lambda x : clean_ascii_chars(x.decode(codec, 'replace')) + while left > 0: left -= 1 idx, size = struct.unpack('>LL', raw[pos:pos + 8]) @@ -66,7 +69,7 @@ class EXTHHeader(object): # {{{ # title contains non ASCII chars or non filename safe chars # they are messed up in the PDB header try: - title = content.decode(codec) + title = self.decode(content) except: pass elif idx == 524: # Lang code @@ -80,31 +83,30 @@ class EXTHHeader(object): # {{{ #else: # print 'unknown record', idx, repr(content) if title: - self.mi.title = replace_entities(title) + self.mi.title = replace_entities(clean_ascii_chars(title)) def process_metadata(self, idx, content, codec): if idx == 100: if self.mi.is_null('authors'): self.mi.authors = [] - au = content.decode(codec, 'ignore').strip() + au = self.decode(content).strip() self.mi.authors.append(au) if self.mi.is_null('author_sort') and re.match(r'\S+?\s*,\s+\S+', au.strip()): self.mi.author_sort = au.strip() elif idx == 101: - self.mi.publisher = content.decode(codec, 'ignore').strip() + self.mi.publisher = self.decode(content).strip() if self.mi.publisher in {'Unknown', _('Unknown')}: self.mi.publisher = None elif idx == 103: - self.mi.comments = content.decode(codec, 'ignore') + self.mi.comments = self.decode(content).strip() elif idx == 104: - raw = check_isbn(content.decode(codec, 'ignore').strip().replace('-', '')) + raw = check_isbn(self.decode(content).strip().replace('-', '')) if raw: self.mi.isbn = raw elif idx == 105: if not self.mi.tags: self.mi.tags = [] - self.mi.tags.extend([x.strip() for x in content.decode(codec, - 'ignore').split(';')]) + self.mi.tags.extend([x.strip() for x in self.decode(content).split(';')]) self.mi.tags = list(set(self.mi.tags)) elif idx == 106: try: @@ -112,7 +114,7 @@ class EXTHHeader(object): # {{{ except: pass elif idx == 108: - self.mi.book_producer = content.decode(codec, 'ignore').strip() + self.mi.book_producer = self.decode(content).strip() elif idx == 112: # dc:source set in some EBSP amazon samples try: content = content.decode(codec).strip() From f0518e333660abed3436e0f4dbdabd4f3bfd1cbd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 18 Jan 2013 21:54:57 +0530 Subject: [PATCH 05/43] Have the metadata download dialog remember its last used size. Fixes #1101150 ([Feature Request] automatically adjust the size of the meta data choosing screen) --- src/calibre/gui2/metadata/single_download.py | 23 ++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py index ee8cd84261..72b0fd9b1c 100644 --- a/src/calibre/gui2/metadata/single_download.py +++ b/src/calibre/gui2/metadata/single_download.py @@ -31,7 +31,7 @@ from calibre.utils.logging import GUILog as Log from calibre.ebooks.metadata.sources.identify import urls_from_identifiers from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.opf2 import OPF -from calibre.gui2 import error_dialog, NONE, rating_font +from calibre.gui2 import error_dialog, NONE, rating_font, gprefs from calibre.utils.date import (utcnow, fromordinal, format_date, UNDEFINED_DATE, as_utc) from calibre.library.comments import comments_to_html @@ -264,6 +264,15 @@ class ResultsView(QTableView): # {{{ sm = self.selectionModel() sm.select(idx, sm.ClearAndSelect|sm.Rows) + def resize_delegate(self): + self.rt_delegate.max_width = int(self.width()/2.1) + self.resizeColumnsToContents() + + def resizeEvent(self, ev): + ret = super(ResultsView, self).resizeEvent(ev) + self.resize_delegate() + return ret + def currentChanged(self, current, previous): ret = QTableView.currentChanged(self, current, previous) self.show_details(current) @@ -385,7 +394,7 @@ class IdentifyWorker(Thread): # {{{ def sample_results(self): m1 = Metadata('The Great Gatsby', ['Francis Scott Fitzgerald']) - m2 = Metadata('The Great Gatsby', ['F. Scott Fitzgerald']) + m2 = Metadata('The Great Gatsby - An extra long title to test resizing', ['F. Scott Fitzgerald']) m1.has_cached_cover_url = True m2.has_cached_cover_url = False m1.comments = 'Some comments '*10 @@ -963,12 +972,16 @@ class FullFetch(QDialog): # {{{ self.covers_widget.chosen.connect(self.ok_clicked) self.stack.addWidget(self.covers_widget) + self.resize(850, 600) + geom = gprefs.get('metadata_single_gui_geom', None) + if geom is not None and geom: + self.restoreGeometry(geom) + # Workaround for Qt 4.8.0 bug that causes the frame of the window to go # off the top of the screen if a max height is not set for the # QWebView. Seems to only happen on windows, but keep it for all # platforms just in case. - self.identify_widget.comments_view.setMaximumHeight(500) - self.resize(850, 600) + self.identify_widget.comments_view.setMaximumHeight(self.height()-100) self.finished.connect(self.cleanup) @@ -995,12 +1008,14 @@ class FullFetch(QDialog): # {{{ self.covers_widget.reset_covers() def accept(self): + gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry()) if self.stack.currentIndex() == 1: return QDialog.accept(self) # Prevent the usual dialog accept mechanisms from working pass def reject(self): + gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry()) self.identify_widget.cancel() self.covers_widget.cancel() return QDialog.reject(self) From 8402205d11827cbda6f67d987e91e4d9a8e38e25 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 19 Jan 2013 11:35:14 +0530 Subject: [PATCH 06/43] Fix datetime tests and read many-many fields in id order --- src/calibre/db/tables.py | 2 +- src/calibre/db/tests/reading.py | 38 ++++++++++++++++----------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index c5d7ee216c..58768c9ff5 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -151,7 +151,7 @@ class ManyToManyTable(ManyToOneTable): def read_maps(self, db): for row in db.conn.execute( - 'SELECT book, {0} FROM {1}'.format( + 'SELECT book, {0} FROM {1} ORDER BY id'.format( self.metadata['link_column'], self.link_table)): if row[1] not in self.col_book_map: self.col_book_map[row[1]] = [] diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index d1ff81440c..b722d30793 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -10,7 +10,7 @@ __docformat__ = 'restructuredtext en' import shutil, unittest, tempfile, datetime from cStringIO import StringIO -from calibre.utils.date import local_tz +from calibre.utils.date import utc_tz from calibre.db.tests.base import BaseTest class ReadingTest(BaseTest): @@ -37,12 +37,12 @@ class ReadingTest(BaseTest): 'tags': (), 'formats':(), 'identifiers': {}, - 'timestamp': datetime.datetime(2011, 9, 7, 13, 54, 41, - tzinfo=local_tz), - 'pubdate': datetime.datetime(2011, 9, 7, 13, 54, 41, - tzinfo=local_tz), - 'last_modified': datetime.datetime(2011, 9, 7, 13, 54, 41, - tzinfo=local_tz), + 'timestamp': datetime.datetime(2011, 9, 7, 19, 54, 41, + tzinfo=utc_tz), + 'pubdate': datetime.datetime(2011, 9, 7, 19, 54, 41, + tzinfo=utc_tz), + 'last_modified': datetime.datetime(2011, 9, 7, 19, 54, 41, + tzinfo=utc_tz), 'publisher': None, 'languages': (), 'comments': None, @@ -69,17 +69,17 @@ class ReadingTest(BaseTest): 'formats': (), 'rating': 4.0, 'identifiers': {'test':'one'}, - 'timestamp': datetime.datetime(2011, 9, 5, 15, 6, - tzinfo=local_tz), - 'pubdate': datetime.datetime(2011, 9, 5, 15, 6, - tzinfo=local_tz), + 'timestamp': datetime.datetime(2011, 9, 5, 21, 6, + tzinfo=utc_tz), + 'pubdate': datetime.datetime(2011, 9, 5, 21, 6, + tzinfo=utc_tz), 'publisher': 'Publisher One', 'languages': ('eng',), 'comments': '

Comments One

', '#enum':'One', '#authors':('Custom One', 'Custom Two'), - '#date':datetime.datetime(2011, 9, 5, 0, 0, - tzinfo=local_tz), + '#date':datetime.datetime(2011, 9, 5, 6, 0, + tzinfo=utc_tz), '#rating':2.0, '#series':'My Series One', '#series_index': 1.0, @@ -98,17 +98,17 @@ class ReadingTest(BaseTest): 'tags': ('Tag One',), 'formats':(), 'identifiers': {'test':'two'}, - 'timestamp': datetime.datetime(2011, 9, 6, 0, 0, - tzinfo=local_tz), - 'pubdate': datetime.datetime(2011, 8, 5, 0, 0, - tzinfo=local_tz), + 'timestamp': datetime.datetime(2011, 9, 6, 6, 0, + tzinfo=utc_tz), + 'pubdate': datetime.datetime(2011, 8, 5, 6, 0, + tzinfo=utc_tz), 'publisher': 'Publisher Two', 'languages': ('deu',), 'comments': '

Comments Two

', '#enum':'Two', '#authors':('My Author Two',), - '#date':datetime.datetime(2011, 9, 1, 0, 0, - tzinfo=local_tz), + '#date':datetime.datetime(2011, 9, 1, 6, 0, + tzinfo=utc_tz), '#rating':4.0, '#series':'My Series Two', '#series_index': 3.0, From 5b08c1ed60e754ddda0beb4ca529a0d8c09d97f5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 19 Jan 2013 11:45:23 +0530 Subject: [PATCH 07/43] Fix get_metadata() test --- src/calibre/db/tests/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index 3264465050..8e72721c4e 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -7,8 +7,8 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' - import unittest, os, shutil +from future_builtins import map class BaseTest(unittest.TestCase): @@ -39,7 +39,10 @@ class BaseTest(unittest.TestCase): 'ondevice_col', 'last_modified'}.union(allfk1) for attr in all_keys: if attr == 'user_metadata': continue + if attr == 'format_metadata': continue # TODO: Not implemented yet attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr) + if attr == 'formats': + attr1, attr2 = map(lambda x:tuple(x) if x else (), (attr1, attr2)) self.assertEqual(attr1, attr2, '%s not the same: %r != %r'%(attr, attr1, attr2)) if attr.startswith('#'): From 3e4d847eeefe9d921ab84146d77fd145e6343234 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 19 Jan 2013 21:28:50 +0530 Subject: [PATCH 08/43] Fix Barrons login form parsing --- recipes/barrons.recipe | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/barrons.recipe b/recipes/barrons.recipe index 9d79aed728..41ed7e26ec 100644 --- a/recipes/barrons.recipe +++ b/recipes/barrons.recipe @@ -64,8 +64,8 @@ class Barrons(BasicNewsRecipe): br = BasicNewsRecipe.get_browser() if self.username is not None and self.password is not None: br.open('http://commerce.barrons.com/auth/login') - br.select_form(name='login_form') - br['user'] = self.username + br.select_form(nr=0) + br['username'] = self.username br['password'] = self.password br.submit() return br From 419f3b63947815ba32b3ab2d770ec046694bf8e1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 19 Jan 2013 22:37:27 +0530 Subject: [PATCH 09/43] Add language dependent sorting for series like columns --- src/calibre/db/cache.py | 22 +++++++++---- src/calibre/db/fields.py | 33 +++++++++++++++---- src/calibre/db/tests/metadata.db | Bin 230400 -> 230400 bytes src/calibre/db/tests/reading.py | 53 ++++++++++++++++--------------- 4 files changed, 68 insertions(+), 40 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 10fe0bb014..a631f9ea46 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -269,11 +269,11 @@ class Cache(object): return () @read_api - def all_book_ids(self): + def all_book_ids(self, type=frozenset): ''' Frozen set of all known book ids. ''' - return frozenset(self.fields['uuid']) + return type(self.fields['uuid']) @read_api def all_field_ids(self, name): @@ -316,6 +316,10 @@ class Cache(object): self.format_metadata_cache[book_id][fmt] = ans return ans + @read_api + def pref(self, name): + return self.backend.prefs[name] + @api def get_metadata(self, book_id, get_cover=False, get_user_categories=True, cover_as_data=False): @@ -378,17 +382,21 @@ class Cache(object): all_book_ids = frozenset(self._all_book_ids() if ids_to_sort is None else ids_to_sort) get_metadata = partial(self._get_metadata, get_user_categories=False) + def get_lang(book_id): + ans = self._field_for('languages', book_id) + return ans[0] if ans else None fm = {'title':'sort', 'authors':'author_sort'} def sort_key(field): 'Handle series type fields' - ans = self.fields[fm.get(field, field)].sort_keys_for_books(get_metadata, - all_book_ids) idx = field + '_index' - if idx in self.fields: - idx_ans = self.fields[idx].sort_keys_for_books(get_metadata, - all_book_ids) + is_series = idx in self.fields + ans = self.fields[fm.get(field, field)].sort_keys_for_books( + get_metadata, get_lang, all_book_ids,) + if is_series: + idx_ans = self.fields[idx].sort_keys_for_books( + get_metadata, get_lang, all_book_ids) ans = {k:(v, idx_ans[k]) for k, v in ans.iteritems()} return ans diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index e154900031..3808052549 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -11,6 +11,8 @@ __docformat__ = 'restructuredtext en' from threading import Lock from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY +from calibre.ebooks.metadata import title_sort +from calibre.utils.config_base import tweaks from calibre.utils.icu import sort_key from calibre.utils.date import UNDEFINED_DATE from calibre.utils.localization import calibre_langcode_to_name @@ -72,7 +74,7 @@ class Field(object): ''' return iter(()) - def sort_keys_for_books(self, get_metadata, all_book_ids): + def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): ''' Return a mapping of book_id -> sort_key. The sort key is suitable for use in sorting the list of all books by this field, via the python cmp @@ -96,7 +98,7 @@ class OneToOneField(Field): def __iter__(self): return self.table.book_col_map.iterkeys() - def sort_keys_for_books(self, get_metadata, all_book_ids): + def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): return {id_ : self._sort_key(self.table.book_col_map.get(id_, self._default_sort_key)) for id_ in all_book_ids} @@ -133,7 +135,7 @@ class CompositeField(OneToOneField): ans = mi.get('#'+self.metadata['label']) return ans - def sort_keys_for_books(self, get_metadata, all_book_ids): + def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): return {id_ : sort_key(self.get_value_with_cache(id_, get_metadata)) for id_ in all_book_ids} @@ -170,7 +172,7 @@ class OnDeviceField(OneToOneField): def __iter__(self): return iter(()) - def sort_keys_for_books(self, get_metadata, all_book_ids): + def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): return {id_ : self.for_book(id_) for id_ in all_book_ids} @@ -196,7 +198,7 @@ class ManyToOneField(Field): def __iter__(self): return self.table.id_map.iterkeys() - def sort_keys_for_books(self, get_metadata, all_book_ids): + def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): ans = {id_ : self.table.book_col_map.get(id_, None) for id_ in all_book_ids} sk_map = {cid : (self._default_sort_key if cid is None else @@ -227,7 +229,7 @@ class ManyToManyField(Field): def __iter__(self): return self.table.id_map.iterkeys() - def sort_keys_for_books(self, get_metadata, all_book_ids): + def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): ans = {id_ : self.table.book_col_map.get(id_, ()) for id_ in all_book_ids} all_cids = set() @@ -248,7 +250,7 @@ class IdentifiersField(ManyToManyField): ids = default_value return ids - def sort_keys_for_books(self, get_metadata, all_book_ids): + def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): 'Sort by identifier keys' ans = {id_ : self.table.book_col_map.get(id_, ()) for id_ in all_book_ids} @@ -274,6 +276,21 @@ class FormatsField(ManyToManyField): def format_fname(self, book_id, fmt): return self.table.fname_map[book_id][fmt.upper()] +class SeriesField(ManyToOneField): + + def sort_key_for_series(self, book_id, get_lang, series_sort_order): + sid = self.table.book_col_map.get(book_id, None) + if sid is None: + return self._default_sort_key + return self._sort_key(title_sort(self.table.id_map[sid], + order=series_sort_order, + lang=get_lang(book_id))) + + def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + sso = tweaks['title_series_sorting'] + return {book_id:self.sort_key_for_series(book_id, get_lang, sso) for book_id + in all_book_ids} + def create_field(name, table): cls = { ONE_ONE : OneToOneField, @@ -290,5 +307,7 @@ def create_field(name, table): cls = IdentifiersField elif table.metadata['datatype'] == 'composite': cls = CompositeField + elif table.metadata['datatype'] == 'series': + cls = SeriesField return cls(name, table) diff --git a/src/calibre/db/tests/metadata.db b/src/calibre/db/tests/metadata.db index 63a096e2f44d9ecce9f32da2f408d56ebe6949ce..da54c61ad5c83bde8917933e2d73f2e6ae370273 100644 GIT binary patch delta 991 zcmaKrTWC{B9L8tPcbdjnn?zHr_Oi#;P+N_?rlK253%XjWHnv^pO=8jW3085} zq!qmElM>-CvJVvm@nzQ%aUN8#DEK0*^}6e=X?+k8S`>?*4|Ps-7eVyEcjjZh;Wu-> z;lz(-$B$+o&zsZ17#sgAeqzarj1K0SC>ail?D^HMib_}IO0BYXRgHU9m3w(@ZPiM* zJ5gmF{#KTJZ?&GqTf9x$D_$Wdi*t!Kam6ABrZOg@_vy%-#T6q`k~5dNnmb2&Gq*7Q zPMN2aDr=QrltapCZb{@`dt(_Ga0X4Rba1eohCCRP7&ex;Jd&T79I56$dKyD<-h98x z%I#sjKjMq-^aXTZWPi9v4|Z+I36R%=9P{s8JsE7>$(T5J5XBsoH5Cclh=W87i{~1r zGsDC15-m6n9iF7^Mj2c=DCb`x{c};uw}tfHps!Dl`h$L{MOep4f(tTw(sn#1tuGgD zP6Hk|5+gTDcp&-mW`V;r!7zbaNxS(v5`T})E6az@9fyQhejLVq~<0xlGcSE#RV@~!SB-Q2CSh!Ho+}^Yr|pid*XE%R!;Q?kilzc z*b7aR?Z93x{^~><=l4at8*$KwI+W1TcKmGY{y@aH8k4p`Sm|057LuzQtw!>Y&g_Dg zS@h}iXtEB~O0_$UHAT|h(B^g&n$p%s;dTs&b0Ksnv@`$<9k0g%D%pw-gY^$B3cx{s zHcS4Chzz`qFpI2SY{Y-8zB+VX3PM$BGJs>^r(KB1i(u?AjkiFf!cLqJ;V7EYlTT>W zj|y>dKQ<^-*e|JtQFv)6giJBjE=w@3>FF}fqG?5RCWNy8>njpJ3}T$qRWBSOcOQDW zvB@)vt;Q44U56n~<;~bmGn+6=MB}jWOZ9+i{0`|SCEqE^`W1Nv52U>>bx-Q9)E%i& Usg%@h+TDVDN<}55FOCF00>i)|-v9sr delta 1190 zcmbVLU2GIp6ux)9*>%CPZK0J)f0(5PTWE3r+GPu-)&6RU{f|;w+JzmqtJ@!Ux3#u3 z6cd4HjF2H5OiYBtgb+h?vyuBC5426h2NQu(ph0QHgoN-S#J+&B-f0?$FUEK?U*_EN zopa{gIo};WVjn+ZKbpO&kumnG#L_xe;IlNc;yddTL1AC(Dt7vdoj%Q3TIMb-EAehA z_4!;Lzu`_zJmpHtJl-<5XUkTX)8p|bAEl;#iAVS&neM%h%oMK3x>k;rW_gvdCzLHk zS)!0!{En+x7D;nFHlyb8SWy*Cspj7^W@KG_m=oJjrulqD+Dc8X4e@ZV9*YNh2TZTu zxILBU2*-N#sNDLW1VsXUp}{~%H&dNO8YM!=pza_txu0f(DDtf1DsyG->F=wm2FNRjOo#?z0wuxkB0Tw=$Nr)(!<-0+R1#|$Ie|AR(qn**nIJ{ zQ91R2;`bUqOnFHSqAgpV;c|z0;Ahaxa1YHm3muaI(_Yr!`0 zO&wamXXxcIRMF=(C=p{ja0vW5P3*yo^wpbaqc3_97Wbo49<;s!8lBsT6mmxJq4>HP zt(@PWL?sI8+%B}xVjXNW-HO#BuNC{`3>dpf>E+0$GhNt8pH@k0ya8V7eg_V*|1H>X zJ_uE{4ofY%y_BefCj3np;^Ku)bW5!~x`4w&m`|^JdI1G# zC_@allp#tRN4DdJ^jw#ik+>!?Epb)iio~46WqS8@`H#8sr>&oF_IHoXZxsq tx4b0B$NqUqwk}sPJDsD`JqXca4OWQXN07#;p&C{?wHH=#^dJV5zX2&wWI6x< diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index b722d30793..d77d3ac6eb 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -63,7 +63,7 @@ class ReadingTest(BaseTest): 'sort': 'One', 'authors': ('Author One',), 'author_sort': 'One, Author', - 'series' : 'Series One', + 'series' : 'A Series One', 'series_index': 1.0, 'tags':('Tag Two', 'Tag One'), 'formats': (), @@ -92,7 +92,7 @@ class ReadingTest(BaseTest): 'sort': 'Title Two', 'authors': ('Author Two', 'Author One'), 'author_sort': 'Two, Author & One, Author', - 'series' : 'Series One', + 'series' : 'A Series One', 'series_index': 2.0, 'rating': 6.0, 'tags': ('Tag One',), @@ -130,30 +130,31 @@ class ReadingTest(BaseTest): 'Test sorting' cache = self.init_cache(self.library_path) for field, order in { - 'title' : [2, 1, 3], - 'authors': [2, 1, 3], - 'series' : [3, 2, 1], - 'tags' : [3, 1, 2], - 'rating' : [3, 2, 1], - # 'identifiers': [3, 2, 1], There is no stable sort since 1 and - # 2 have the same identifier keys - # TODO: Add an empty book to the db and ensure that empty - # fields sort the same as they do in db2 - 'timestamp': [2, 1, 3], - 'pubdate' : [1, 2, 3], - 'publisher': [3, 2, 1], - 'last_modified': [2, 1, 3], - 'languages': [3, 2, 1], - 'comments': [3, 2, 1], - '#enum' : [3, 2, 1], - '#authors' : [3, 2, 1], - '#date': [3, 1, 2], - '#rating':[3, 2, 1], - '#series':[3, 2, 1], - '#tags':[3, 2, 1], - '#yesno':[3, 1, 2], - '#comments':[3, 2, 1], - }.iteritems(): + 'title' : [2, 1, 3], + 'authors': [2, 1, 3], + 'series' : [3, 1, 2], + 'tags' : [3, 1, 2], + 'rating' : [3, 2, 1], + # 'identifiers': [3, 2, 1], There is no stable sort since 1 and + # 2 have the same identifier keys + # 'last_modified': [3, 2, 1], There is no stable sort as two + # records have the exact same value + 'timestamp': [2, 1, 3], + 'pubdate' : [1, 2, 3], + 'publisher': [3, 2, 1], + 'languages': [3, 2, 1], + 'comments': [3, 2, 1], + '#enum' : [3, 2, 1], + '#authors' : [3, 2, 1], + '#date': [3, 1, 2], + '#rating':[3, 2, 1], + '#series':[3, 2, 1], + '#tags':[3, 2, 1], + '#yesno':[3, 1, 2], + '#comments':[3, 2, 1], + # TODO: Add an empty book to the db and ensure that empty + # fields sort the same as they do in db2 + }.iteritems(): x = list(reversed(order)) self.assertEqual(order, cache.multisort([(field, True)], ids_to_sort=x), From 2d72a307593ea937bcd115784363cae0e89df560 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 20 Jan 2013 13:48:34 +0530 Subject: [PATCH 10/43] ... --- recipes/conowego_pl.recipe | 0 recipes/{ => icons}/spiders_web_pl.png | Bin recipes/linux_journal.recipe | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 recipes/conowego_pl.recipe rename recipes/{ => icons}/spiders_web_pl.png (100%) mode change 100755 => 100644 recipes/linux_journal.recipe diff --git a/recipes/conowego_pl.recipe b/recipes/conowego_pl.recipe old mode 100755 new mode 100644 diff --git a/recipes/spiders_web_pl.png b/recipes/icons/spiders_web_pl.png similarity index 100% rename from recipes/spiders_web_pl.png rename to recipes/icons/spiders_web_pl.png diff --git a/recipes/linux_journal.recipe b/recipes/linux_journal.recipe old mode 100755 new mode 100644 From aff8f66fa1c8d99af9d7ae476641f047e022d7f0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 20 Jan 2013 14:31:16 +0530 Subject: [PATCH 11/43] Fix Michelle Malkin --- recipes/michellemalkin.recipe | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/recipes/michellemalkin.recipe b/recipes/michellemalkin.recipe index e933ed8f1c..0b1f0ebdaa 100644 --- a/recipes/michellemalkin.recipe +++ b/recipes/michellemalkin.recipe @@ -18,6 +18,8 @@ class MichelleMalkin(BasicNewsRecipe): remove_javascript = True no_stylesheets = True + auto_cleanup = True + use_embedded_content = False conversion_options = { @@ -29,16 +31,16 @@ class MichelleMalkin(BasicNewsRecipe): } - keep_only_tags = [ - dict(name='div', attrs={'class':'article'}) - ] + #keep_only_tags = [ + #dict(name='div', attrs={'class':'article'}) + #] - remove_tags = [ - dict(name=['iframe', 'embed', 'object']), - dict(name='div', attrs={'id':['comments', 'commentForm']}), - dict(name='div', attrs={'class':['postCategories', 'comments', 'blogInfo', 'postInfo']}) + #remove_tags = [ + #dict(name=['iframe', 'embed', 'object']), + #dict(name='div', attrs={'id':['comments', 'commentForm']}), + #dict(name='div', attrs={'class':['postCategories', 'comments', 'blogInfo', 'postInfo']}) - ] + #] feeds = [(u'http://feeds.feedburner.com/michellemalkin/posts')] From 63b164241a5846ca40aa20f361c20a1b29333068 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 20 Jan 2013 14:34:26 +0530 Subject: [PATCH 12/43] Start work on implementing search in the new backend. Searching for date columns working. --- src/calibre/db/cache.py | 6 + src/calibre/db/fields.py | 46 ++++++ src/calibre/db/search.py | 284 ++++++++++++++++++++++++++++++++ src/calibre/db/tests/reading.py | 20 +++ 4 files changed, 356 insertions(+) create mode 100644 src/calibre/db/search.py diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index a631f9ea46..88a2196a61 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -13,6 +13,7 @@ from functools import wraps, partial from calibre.db.locking import create_locks, RecordLock from calibre.db.fields import create_field +from calibre.db.search import Search from calibre.db.tables import VirtualTable from calibre.db.lazy import FormatMetadata, FormatsList from calibre.ebooks.metadata.book.base import Metadata @@ -50,6 +51,7 @@ class Cache(object): self.record_lock = RecordLock(self.read_lock) self.format_metadata_cache = defaultdict(dict) self.formatter_template_cache = {} + self._search_api = Search(self.field_metadata.get_search_terms()) # Implement locking for all simple read/write API methods # An unlocked version of the method is stored with the name starting @@ -409,6 +411,10 @@ class Cache(object): else: return sorted(all_book_ids, key=partial(SortKey, fields, sort_keys)) + @read_api + def search(self, query, restriction): + return self._search_api(self, query, restriction) + # }}} class SortKey(object): diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 3808052549..43e89cdc6f 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -9,6 +9,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' from threading import Lock +from collections import defaultdict from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY from calibre.ebooks.metadata import title_sort @@ -83,6 +84,15 @@ class Field(object): ''' raise NotImplementedError() + def iter_searchable_values(self, get_metadata, candidates, default_value=None): + ''' + Return a generator that yields items of the form (value, set of books + ids that have this value). Here, value is a searchable value. For + OneToOneField the set of books ids will contain only a single id, but for + other fields it will generally have more than one id. Returned books_ids + are restricted to the set of ids in candidates. + ''' + raise NotImplementedError() class OneToOneField(Field): @@ -102,6 +112,11 @@ class OneToOneField(Field): return {id_ : self._sort_key(self.table.book_col_map.get(id_, self._default_sort_key)) for id_ in all_book_ids} + def iter_searchable_values(self, get_metadata, candidates, default_value=None): + cbm = self.table.book_col_map + for book_id in candidates: + yield cbm.get(book_id, default_value), {book_id} + class CompositeField(OneToOneField): def __init__(self, *args, **kwargs): @@ -139,6 +154,9 @@ class CompositeField(OneToOneField): return {id_ : sort_key(self.get_value_with_cache(id_, get_metadata)) for id_ in all_book_ids} + def iter_searchable_values(self, get_metadata, candidates, default_value=None): + for book_id in candidates: + yield self.get_value_with_cache(book_id, get_metadata), {book_id} class OnDeviceField(OneToOneField): @@ -176,6 +194,10 @@ class OnDeviceField(OneToOneField): return {id_ : self.for_book(id_) for id_ in all_book_ids} + def iter_searchable_values(self, get_metadata, candidates, default_value=None): + for book_id in candidates: + yield self.for_book(book_id, default_value=default_value), {book_id} + class ManyToOneField(Field): def for_book(self, book_id, default_value=None): @@ -206,6 +228,13 @@ class ManyToOneField(Field): for cid in ans.itervalues()} return {id_ : sk_map[cid] for id_, cid in ans.iteritems()} + def iter_searchable_values(self, get_metadata, candidates, default_value=None): + cbm = self.table.col_book_map + for item_id, val in self.table.id_map.iteritems(): + book_ids = set(cbm.get(item_id, ())).intersection(candidates) + if book_ids: + yield val, book_ids + class ManyToManyField(Field): def __init__(self, *args, **kwargs): @@ -241,6 +270,12 @@ class ManyToManyField(Field): (self._default_sort_key,)) for id_, cids in ans.iteritems()} + def iter_searchable_values(self, get_metadata, candidates, default_value=None): + cbm = self.table.col_book_map + for item_id, val in self.table.id_map.iteritems(): + book_ids = set(cbm.get(item_id, ())).intersection(candidates) + if book_ids: + yield val, book_ids class IdentifiersField(ManyToManyField): @@ -276,6 +311,17 @@ class FormatsField(ManyToManyField): def format_fname(self, book_id, fmt): return self.table.fname_map[book_id][fmt.upper()] + def iter_searchable_values(self, get_metadata, candidates, default_value=None): + val_map = defaultdict(set) + cbm = self.table.book_col_map + for book_id in candidates: + vals = cbm.get(book_id, ()) + for val in vals: + val_map[val].add(book_id) + + for val, book_ids in val_map.iteritems(): + yield val, book_ids + class SeriesField(ManyToOneField): def sort_key_for_series(self, book_id, get_lang, series_sort_order): diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py new file mode 100644 index 0000000000..d304deeb9a --- /dev/null +++ b/src/calibre/db/search.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import re +from functools import partial +from datetime import timedelta + +from calibre.utils.config_base import prefs +from calibre.utils.date import parse_date, UNDEFINED_DATE, now +from calibre.utils.search_query_parser import SearchQueryParser, ParseException + +# TODO: Thread safety of saved searches + +class DateSearch(object): # {{{ + + def __init__(self): + self.operators = { + '=' : (1, self.eq), + '!=' : (2, self.ne), + '>' : (1, self.gt), + '>=' : (2, self.ge), + '<' : (1, self.lt), + '<=' : (2, self.le), + } + self.local_today = { '_today', 'today', icu_lower(_('today')) } + self.local_yesterday = { '_yesterday', 'yesterday', icu_lower(_('yesterday')) } + self.local_thismonth = { '_thismonth', 'thismonth', icu_lower(_('thismonth')) } + self.daysago_pat = re.compile(r'(%s|daysago|_daysago)$'%_('daysago')) + + def eq(self, dbdate, query, field_count): + if dbdate.year == query.year: + if field_count == 1: + return True + if dbdate.month == query.month: + if field_count == 2: + return True + return dbdate.day == query.day + return False + + def ne(self, *args): + return not self.eq(*args) + + def gt(self, dbdate, query, field_count): + if dbdate.year > query.year: + return True + if field_count > 1 and dbdate.year == query.year: + if dbdate.month > query.month: + return True + return (field_count == 3 and dbdate.month == query.month and + dbdate.day > query.day) + return False + + def le(self, *args): + return not self.gt(*args) + + def lt(self, dbdate, query, field_count): + if dbdate.year < query.year: + return True + if field_count > 1 and dbdate.year == query.year: + if dbdate.month < query.month: + return True + return (field_count == 3 and dbdate.month == query.month and + dbdate.day < query.day) + return False + + def ge(self, *args): + return not self.lt(*args) + + def __call__(self, query, field_iter): + matches = set() + if len(query) < 2: + return matches + + if query == 'false': + for v, book_ids in field_iter(): + if isinstance(v, (str, unicode)): + v = parse_date(v) + if v is None or v <= UNDEFINED_DATE: + matches |= book_ids + return matches + + if query == 'true': + for v, book_ids in field_iter(): + if isinstance(v, (str, unicode)): + v = parse_date(v) + if v is not None and v > UNDEFINED_DATE: + matches |= book_ids + return matches + + relop = None + for k, op in self.operators.iteritems(): + if query.startswith(k): + p, relop = op + query = query[p:] + if relop is None: + relop = self.operators['='][-1] + + if query in self.local_today: + qd = now() + field_count = 3 + elif query in self.local_yesterday: + qd = now() - timedelta(1) + field_count = 3 + elif query in self.local_thismonth: + qd = now() + field_count = 2 + else: + m = self.daysago_pat.search(query) + if m is not None: + num = query[:-len(m.group(1))] + try: + qd = now() - timedelta(int(num)) + except: + raise ParseException(query, len(query), 'Number conversion error') + field_count = 3 + else: + try: + qd = parse_date(query, as_utc=False) + except: + raise ParseException(query, len(query), 'Date conversion error') + if '-' in query: + field_count = query.count('-') + 1 + else: + field_count = query.count('/') + 1 + + for v, book_ids in field_iter(): + if isinstance(v, (str, unicode)): + v = parse_date(v) + if v is not None and relop(v, qd, field_count): + matches |= book_ids + + return matches +# }}} + +class Parser(SearchQueryParser): + + def __init__(self, dbcache, all_book_ids, gst, date_search, + limit_search_columns, limit_search_columns_to, locations): + self.dbcache, self.all_book_ids = dbcache, all_book_ids + self.all_search_locations = frozenset(locations) + self.grouped_search_terms = gst + self.date_search = date_search + self.limit_search_columns, self.limit_search_columns_to = ( + limit_search_columns, limit_search_columns_to) + super(Parser, self).__init__(locations, optimize=True) + + @property + def field_metadata(self): + return self.dbcache.field_metadata + + def universal_set(self): + return self.all_book_ids + + def field_iter(self, name, candidates): + get_metadata = partial(self.dbcache._get_metadata, get_user_categories=False) + return self.dbcache.fields[name].iter_searchable_values(get_metadata, + candidates) + + def get_matches(self, location, query, candidates=None, + allow_recursion=True): + # If candidates is not None, it must not be modified. Changing its + # value will break query optimization in the search parser + matches = set() + + if candidates is None: + candidates = self.all_book_ids + if not candidates or not query or not query.strip(): + return matches + if location not in self.all_search_locations: + return matches + + if (len(location) > 2 and location.startswith('@') and + location[1:] in self.grouped_search_terms): + location = location[1:] + + # get metadata key associated with the search term. Eliminates + # dealing with plurals and other aliases + # original_location = location + location = self.field_metadata.search_term_to_field_key( + icu_lower(location.strip())) + # grouped search terms + if isinstance(location, list): + if allow_recursion: + if query.lower() == 'false': + invert = True + query = 'true' + else: + invert = False + for loc in location: + c = candidates.copy() + m = self.get_matches(loc, query, + candidates=c, allow_recursion=False) + matches |= m + c -= m + if len(c) == 0: + break + if invert: + matches = self.all_book_ids - matches + return matches + raise ParseException(query, len(query), 'Recursive query group detected') + + # If the user has asked to restrict searching over all field, apply + # that restriction + if (location == 'all' and self.limit_search_columns and + self.limit_search_columns_to): + terms = set() + for l in self.limit_search_columns_to: + l = icu_lower(l.strip()) + if l and l != 'all' and l in self.all_search_locations: + terms.add(l) + if terms: + c = candidates.copy() + for l in terms: + try: + m = self.get_matches(l, query, + candidates=c, allow_recursion=allow_recursion) + matches |= m + c -= m + if len(c) == 0: + break + except: + pass + return matches + + if location in self.field_metadata: + fm = self.field_metadata[location] + # take care of dates special case + if (fm['datatype'] == 'datetime' or + (fm['datatype'] == 'composite' and + fm['display'].get('composite_sort', '') == 'date')): + if location == 'date': + location = 'timestamp' + return self.date_search( + icu_lower(query), partial(self.field_iter, location, candidates)) + + return matches + + +class Search(object): + + def __init__(self, all_search_locations): + self.all_search_locations = all_search_locations + self.date_search = DateSearch() + + def change_locations(self, newlocs): + self.all_search_locations = newlocs + + def __call__(self, dbcache, query, search_restriction): + ''' + Return the set of ids of all records that match the specified + query and restriction + ''' + q = '' + if not query or not query.strip(): + q = search_restriction + else: + q = query + if search_restriction: + q = u'(%s) and (%s)' % (search_restriction, query) + + all_book_ids = dbcache.all_book_ids(type=set) + if not q: + return all_book_ids + + # We construct a new parser instance per search as pyparsing is not + # thread safe. On my desktop, constructing a SearchQueryParser instance + # takes 0.000975 seconds and restoring it from a pickle takes + # 0.000974 seconds. + sqp = Parser( + dbcache, all_book_ids, dbcache.pref('grouped_search_terms'), + self.date_search, prefs[ 'limit_search_columns' ], + prefs[ 'limit_search_columns_to' ], self.all_search_locations) + try: + ret = sqp.parse(query) + finally: + sqp.dbcache = None + return ret + diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index d77d3ac6eb..22d1bba37e 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -191,6 +191,26 @@ class ReadingTest(BaseTest): # }}} + def test_searching(self): # {{{ + 'Test searching returns the same data for both backends' + from calibre.library.database2 import LibraryDatabase2 + old = LibraryDatabase2(self.library_path) + oldvals = {query:set(old.search_getting_ids(query, '')) for query in ( + 'date:9/6/2011', 'date:true', 'date:false', 'pubdate:9/2011', + '#date:true', 'date:<100daysago', 'date:>9/6/2011', + '#date:>9/1/2011', '#date:=2011', + )} + old = None + + cache = self.init_cache(self.library_path) + for query, ans in oldvals.iteritems(): + nr = cache.search(query, '') + self.assertEqual(ans, nr, + 'Old result: %r != New result: %r for search: %s'%( + ans, nr, query)) + + # }}} + def tests(): return unittest.TestLoader().loadTestsFromTestCase(ReadingTest) From 556582f1bac11bfd6ef1119770debf884425d453 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 20 Jan 2013 16:33:28 +0530 Subject: [PATCH 13/43] Implement numeric field searches --- src/calibre/db/fields.py | 14 ++++ src/calibre/db/search.py | 118 +++++++++++++++++++++++++++++-- src/calibre/db/tables.py | 6 +- src/calibre/db/tests/metadata.db | Bin 230400 -> 235520 bytes src/calibre/db/tests/reading.py | 12 +++- 5 files changed, 141 insertions(+), 9 deletions(-) diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 43e89cdc6f..194cb33011 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -20,6 +20,8 @@ from calibre.utils.localization import calibre_langcode_to_name class Field(object): + is_many = False + def __init__(self, name, table): self.name, self.table = name, table self.has_text_data = self.metadata['datatype'] in ('text', 'comments', @@ -200,6 +202,8 @@ class OnDeviceField(OneToOneField): class ManyToOneField(Field): + is_many = True + def for_book(self, book_id, default_value=None): ids = self.table.book_col_map.get(book_id, None) if ids is not None: @@ -237,6 +241,8 @@ class ManyToOneField(Field): class ManyToManyField(Field): + is_many = True + def __init__(self, *args, **kwargs): Field.__init__(self, *args, **kwargs) self.alphabetical_sort = self.name != 'authors' @@ -277,6 +283,14 @@ class ManyToManyField(Field): if book_ids: yield val, book_ids + def iter_counts(self, candidates): + val_map = defaultdict(set) + cbm = self.table.book_col_map + for book_id in candidates: + val_map[len(cbm.get(book_id, ()))].add(book_id) + for count, book_ids in val_map.iteritems(): + yield count, book_ids + class IdentifiersField(ManyToManyField): def for_book(self, book_id, default_value=None): diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index d304deeb9a..fe9cec79c8 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -138,14 +138,101 @@ class DateSearch(object): # {{{ return matches # }}} +class NumericSearch(object): # {{{ + + def __init__(self): + self.operators = { + '=':( 1, lambda r, q: r == q ), + '>':( 1, lambda r, q: r is not None and r > q ), + '<':( 1, lambda r, q: r is not None and r < q ), + '!=':( 2, lambda r, q: r != q ), + '>=':( 2, lambda r, q: r is not None and r >= q ), + '<=':( 2, lambda r, q: r is not None and r <= q ) + } + + def __call__(self, query, field_iter, location, datatype, candidates, is_many=False): + matches = set() + if not query: + return matches + + q = '' + cast = adjust = lambda x: x + dt = datatype + + if is_many and query in {'true', 'false'}: + valcheck = lambda x: True + if datatype == 'rating': + valcheck = lambda x: x is not None and x > 0 + found = set() + for val, book_ids in field_iter(): + if valcheck(val): + found |= book_ids + return found if query == 'true' else candidates - found + + if query == 'false': + if location == 'cover': + relop = lambda x,y: not bool(x) + else: + relop = lambda x,y: x is None + elif query == 'true': + if location == 'cover': + relop = lambda x,y: bool(x) + else: + relop = lambda x,y: x is not None + else: + relop = None + for k, op in self.operators.iteritems(): + if query.startswith(k): + p, relop = op + query = query[p:] + if relop is None: + p, relop = self.operators['='] + + cast = int + if dt == 'rating': + cast = lambda x: 0 if x is None else int(x) + adjust = lambda x: x/2 + elif dt in ('float', 'composite'): + cast = float + + mult = 1.0 + if len(query) > 1: + mult = query[-1].lower() + mult = {'k': 1024.,'m': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) + if mult != 1.0: + query = query[:-1] + else: + mult = 1.0 + + try: + q = cast(query) * mult + except: + raise ParseException(query, len(query), + 'Non-numeric value in query: %r'%query) + + for val, book_ids in field_iter(): + if val is None: + continue + try: + v = cast(val) + except: + v = None + if v: + v = adjust(v) + if relop(v, q): + matches |= book_ids + return matches + +# }}} + class Parser(SearchQueryParser): - def __init__(self, dbcache, all_book_ids, gst, date_search, + def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, limit_search_columns, limit_search_columns_to, locations): self.dbcache, self.all_book_ids = dbcache, all_book_ids self.all_search_locations = frozenset(locations) self.grouped_search_terms = gst - self.date_search = date_search + self.date_search, self.num_search = date_search, num_search self.limit_search_columns, self.limit_search_columns_to = ( limit_search_columns, limit_search_columns_to) super(Parser, self).__init__(locations, optimize=True) @@ -230,15 +317,33 @@ class Parser(SearchQueryParser): if location in self.field_metadata: fm = self.field_metadata[location] + dt = fm['datatype'] + # take care of dates special case - if (fm['datatype'] == 'datetime' or - (fm['datatype'] == 'composite' and - fm['display'].get('composite_sort', '') == 'date')): + if (dt == 'datetime' or ( + dt == 'composite' and + fm['display'].get('composite_sort', '') == 'date')): if location == 'date': location = 'timestamp' return self.date_search( icu_lower(query), partial(self.field_iter, location, candidates)) + # take care of numbers special case + if (dt in ('rating', 'int', 'float') or + (dt == 'composite' and + fm['display'].get('composite_sort', '') == 'number')): + field = self.dbcache.fields[location] + return self.num_search( + icu_lower(query), partial(self.field_iter, location, candidates), + location, dt, candidates, is_many=field.is_many) + + # take care of the 'count' operator for is_multiples + if (fm['is_multiple'] and + len(query) > 1 and query[0] == '#' and query[1] in '=<>!'): + return self.num_search(icu_lower(query[1:]), partial( + self.dbcache.fields[location].iter_counts, candidates), + location, dt, candidates) + return matches @@ -247,6 +352,7 @@ class Search(object): def __init__(self, all_search_locations): self.all_search_locations = all_search_locations self.date_search = DateSearch() + self.num_search = NumericSearch() def change_locations(self, newlocs): self.all_search_locations = newlocs @@ -274,7 +380,7 @@ class Search(object): # 0.000974 seconds. sqp = Parser( dbcache, all_book_ids, dbcache.pref('grouped_search_terms'), - self.date_search, prefs[ 'limit_search_columns' ], + self.date_search, self.num_search, prefs[ 'limit_search_columns' ], prefs[ 'limit_search_columns_to' ], self.all_search_locations) try: ret = sqp.parse(query) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 58768c9ff5..234a7fe4a8 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -148,11 +148,11 @@ class ManyToManyTable(ManyToOneTable): ''' table_type = MANY_MANY + selectq = 'SELECT book, {0} FROM {1}' def read_maps(self, db): for row in db.conn.execute( - 'SELECT book, {0} FROM {1} ORDER BY id'.format( - self.metadata['link_column'], self.link_table)): + self.selectq.format(self.metadata['link_column'], self.link_table)): if row[1] not in self.col_book_map: self.col_book_map[row[1]] = [] self.col_book_map[row[1]].append(row[0]) @@ -168,6 +168,8 @@ class ManyToManyTable(ManyToOneTable): class AuthorsTable(ManyToManyTable): + selectq = 'SELECT book, {0} FROM {1} ORDER BY id' + def read_id_maps(self, db): self.alink_map = {} self.asort_map = {} diff --git a/src/calibre/db/tests/metadata.db b/src/calibre/db/tests/metadata.db index da54c61ad5c83bde8917933e2d73f2e6ae370273..4bd6dfe4f97d037ed7c343a87e6cc929d153b5d9 100644 GIT binary patch delta 2440 zcmdT_dr(x@8NcWJ?%mxB3$kX-f+#HD10&$x<)Kg=*8y3RphA#L6EVvI3!?jAb{B&@ z3LBD4lGah?)|^Qv)7a@hjZ-_ycDU^{ol)sjTaB?XF|AF-SCwfVXqz^vIt4dY(obmhO^2(A5_sYt$%H-`i zDHdeV5!IZTOX)~b$=pgGwr8N2wDrW|k&v$~66^_ueTqVpug&iuBz0_o>`cl1g&z!<5-^3E_#$a5yud+MnAK^YMYfnl-38{c7Fon;T<7YeP)~Pe^O#ee4n1a2 zw9ZczMg^S1kI{ooc$og3PEacu7Tyy^Rhu=oZe41M;BYtyX(FT}81ctz^_Q^{NPm8~ zC)65<`Z^-fkUySZ<;<^-cC`hnoQwKhMJ4Wz!TiDGyS;9tr(%zQ5Aipg#1FA2nK96h z6d8^g$kZeoik9IvK1<8`0W{!5) z*0RBHXfogqQ~wZbofPKg)JO-Hs*^*e60arYy6pG@w;iIe%og+S3bgxT z0e`e@yY4feh=MY&H4+KNeEw*_7ms$u;{G^~_2=N&ui7Dn^57->qzk2>v3mUYLb5}B z^F#|S;1f>q8K@Ud**V42r!sVk&&u|sJT(b*>P8wt@*g*3kznj&c|%h6Ov56I5(tru zx+K#0YH| z%LBOASSQ)e6CNKR_sj$@K2OG`81iIU0-E zfh@WtOHq7ZN4UZxvY#uwQjVM;b|}JCcI#!DO%!(d1j)22&O*-9&SjjNNp_?OOSDI7 z@CT5y+TBMnKqR9_Se_S)*@75kswm7~jYrttCfFsTi`ao}C}w4Ak#8!Pv0m4J@~<36 zYq-u8L=Ky*!BQS{ID!i2gZhn4_9KsNUJny{_HlfN{7$>O5h7*X^(bVUtMMpvHsTxF z;SH$cl_m)9vic!-wdyDEOTyL+qK3U*i*jwS4nxGm_2IqO+4|%&8Q+3?8pW#NQF6d+6{t(z5h=n6uvPzll?YO_wm=sF6 zdU7I%X`LLrr5$-`1}FLqadCHb&BVge?w;0QS8RJAs*jO>4~K#Pr-e&amd-*J%wkq3 z?AFtGNc*xKeMFuGbZg-byh>Po1D3G$y;#Z)Jq4@!*0DM4U@dIyQVkYs-;1Jyuv`zE zhF0T;K5ZQJkF3FqZR}_@ve+}t$k)ER3qita8{pE)x^dDl`j9DYh-f?eu%FD-!+|!K zw0#MDOvVQQkzCS54|tQL2DZSXcSij+`<*jI*jwkNu&J6F;h>O zl=HKOk6uXQF^HLX7R_wOZ}PdI$%W(&Qm9ZX2(>hqF6CXjN*rlQY-Y>dD71a8!IqF$=XV9uYA$Jp5qh%0quTcX~aU0qCG zY-A? xk}Z(zq|It0DTx;rhD^e;=U&NG?Z+mWryr6j>W_DYJN0>xqZRk!9i!0x^e>HH*n$86 delta 1544 zcmaJ>eQZ-z6o2>rUc0_p)^!4#gGoC;zHx=EYy%viIK)X7G6@2s>()nC9PQ|bjIV41 zf(mFL4|jtxK_Vtfh`~MV)riQ3$v_Z>OeAyiVT=rcfDr`t58l23N@6_8yEo_D^Z7gH zyvBX0jc=rG9prdLlBA3GKO*iAw~d`BgyVi0VG>+9^@vkCxvlOa-C=VPy^vnArMnBA zZckBOe$j;d@gC2F!u)(~f<5scV!hXHwZKkyXepUPbjQdj)pukqA;j7;SaIsg?jG1> zSt5kUW+ZQlmTC7u(6ajvD19%~486C+Bnx4wBLCY4kJ=>m)5~y}4R_RRGTc$K%zHV# z1?{@mqL@--5sDJIrBadc8;*;N4naE}7(}4=1TvB_YGqrdRqB**8ha7P91Ntl=wWIj zjZ&xPuzjv~*5@MA3MQc&f8q!}027)$IEBw}P&ix-oi#-Aa1%FmN8S}^w|Tl?dJRUu z{iPz0=_e4i5UWIP)5qy3I+HG?>*>2f@DFV)OEF0$TY}n^Lwo3?0&Q<=c2>+6h~^YI zJ^5puR;Qqxs2T~XO-;l7+K#rK?DJVfmfQoX!r|Je*B4Q}u}Cl)^TpIyB(7>F4rjOo zp!tqKBpk0({obhRi&WQmV`^ksboJVyURf$}Xq(%!ixyU7_+;s6M@U@}^~S>9D%BfZ z8LC$O3o8ax$ABwN5Nehk)j zwvJ}Q{7Bc1w)STpKL%&McPP3lnk3rKqeq}|YZF7+cs|Lc+>n%i3XC(6lPq!pH=gh@9LPKHZ zTZbpv;BsW}$))(tP&v~aM^^K6q_d7Sn99<^NaIcO&_J4}!c216ry+RQL7y<0Uxs~# z$&YN;dN|q01)`|cwW84T5t#X=1$d3HD-mRJ5#g$#d6vCD13BzKF>Ktk7~c}MZ!xCw z>GRP}Qj;4JjMk_TFIvvXVqANf)@S5of@5zKpcgiFpqSPyPb_26`yMPyhe` diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 22d1bba37e..35f4a7333d 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -65,7 +65,7 @@ class ReadingTest(BaseTest): 'author_sort': 'One, Author', 'series' : 'A Series One', 'series_index': 1.0, - 'tags':('Tag Two', 'Tag One'), + 'tags':('Tag One', 'Tag Two'), 'formats': (), 'rating': 4.0, 'identifiers': {'test':'one'}, @@ -196,9 +196,19 @@ class ReadingTest(BaseTest): from calibre.library.database2 import LibraryDatabase2 old = LibraryDatabase2(self.library_path) oldvals = {query:set(old.search_getting_ids(query, '')) for query in ( + # Date tests 'date:9/6/2011', 'date:true', 'date:false', 'pubdate:9/2011', '#date:true', 'date:<100daysago', 'date:>9/6/2011', '#date:>9/1/2011', '#date:=2011', + + # Number tests + 'rating:3', 'rating:>2', 'rating:=2', 'rating:true', + 'rating:false', 'rating:>4', 'tags:#<2', 'tags:#>7', + 'cover:false', 'cover:true', '#float:>11', '#float:<1k', + '#float:10.01', + + # TODO: Tests for searching the size column and + # cover:true|false )} old = None From 9d29d7ab3470da23828cb96fccdf6faa281c7096 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 20 Jan 2013 17:37:39 +0530 Subject: [PATCH 14/43] ... --- src/calibre/db/tests/reading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 35f4a7333d..7c1ff45968 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -205,7 +205,7 @@ class ReadingTest(BaseTest): 'rating:3', 'rating:>2', 'rating:=2', 'rating:true', 'rating:false', 'rating:>4', 'tags:#<2', 'tags:#>7', 'cover:false', 'cover:true', '#float:>11', '#float:<1k', - '#float:10.01', + '#float:10.01', 'series_index:1', 'series_index:<3', # TODO: Tests for searching the size column and # cover:true|false From 62805ec26444891cbdc212bad949698903adcf79 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 20 Jan 2013 19:33:43 +0530 Subject: [PATCH 15/43] ... --- src/calibre/db/fields.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 194cb33011..34a12c9491 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -89,10 +89,8 @@ class Field(object): def iter_searchable_values(self, get_metadata, candidates, default_value=None): ''' Return a generator that yields items of the form (value, set of books - ids that have this value). Here, value is a searchable value. For - OneToOneField the set of books ids will contain only a single id, but for - other fields it will generally have more than one id. Returned books_ids - are restricted to the set of ids in candidates. + ids that have this value). Here, value is a searchable value. Returned + books_ids are restricted to the set of ids in candidates. ''' raise NotImplementedError() @@ -116,8 +114,17 @@ class OneToOneField(Field): def iter_searchable_values(self, get_metadata, candidates, default_value=None): cbm = self.table.book_col_map - for book_id in candidates: - yield cbm.get(book_id, default_value), {book_id} + if (self.name in {'id', 'uuid', 'title'} or + self.metadata['datatype'] == 'datetime'): + # Values are likely to be unique + for book_id in candidates: + yield cbm.get(book_id, default_value), {book_id} + else: + val_map = defaultdict(set) + for book_id in candidates: + val_map[cbm.get(book_id, default_value)].add(book_id) + for val, book_ids in val_map.iteritems(): + yield val, book_ids class CompositeField(OneToOneField): @@ -157,8 +164,11 @@ class CompositeField(OneToOneField): all_book_ids} def iter_searchable_values(self, get_metadata, candidates, default_value=None): + val_map = defaultdict(set) for book_id in candidates: - yield self.get_value_with_cache(book_id, get_metadata), {book_id} + val_map[self.get_value_with_cache(book_id, get_metadata)].add(book_id) + for val, book_ids in val_map.iteritems(): + yield val, book_ids class OnDeviceField(OneToOneField): @@ -197,8 +207,11 @@ class OnDeviceField(OneToOneField): all_book_ids} def iter_searchable_values(self, get_metadata, candidates, default_value=None): + val_map = defaultdict(set) for book_id in candidates: - yield self.for_book(book_id, default_value=default_value), {book_id} + val_map[self.for_book(book_id, default_value=default_value)].add(book_id) + for val, book_ids in val_map.iteritems(): + yield val, book_ids class ManyToOneField(Field): From 3a299104fae93ee7807d71d5b2935f5de3d444e5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 20 Jan 2013 19:43:12 +0530 Subject: [PATCH 16/43] ... --- src/calibre/db/tests/reading.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 7c1ff45968..4792f498f8 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -205,7 +205,8 @@ class ReadingTest(BaseTest): 'rating:3', 'rating:>2', 'rating:=2', 'rating:true', 'rating:false', 'rating:>4', 'tags:#<2', 'tags:#>7', 'cover:false', 'cover:true', '#float:>11', '#float:<1k', - '#float:10.01', 'series_index:1', 'series_index:<3', + '#float:10.01', 'series_index:1', 'series_index:<3', 'id:1', + 'id:>2', # TODO: Tests for searching the size column and # cover:true|false From f4a8a2f2677b8d6fbbf15bed7b5737d4bc7667fa Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 20 Jan 2013 15:44:27 +0100 Subject: [PATCH 17/43] Fix resending metadata on every connect if a book's cover is smaller than what the device wants. --- src/calibre/devices/smart_device_app/driver.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index b4842876b1..f59d75f5d0 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -886,10 +886,12 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self._debug('extension path lengths', self.exts_path_lengths) self.THUMBNAIL_HEIGHT = result.get('coverHeight', self.DEFAULT_THUMBNAIL_HEIGHT) + self._debug('cover height', self.THUMBNAIL_HEIGHT) if 'coverWidth' in result: # Setting this field forces the aspect ratio self.THUMBNAIL_WIDTH = result.get('coverWidth', (self.DEFAULT_THUMBNAIL_HEIGHT/3) * 4) + self._debug('cover width', self.THUMBNAIL_WIDTH) elif hasattr(self, 'THUMBNAIL_WIDTH'): delattr(self, 'THUMBNAIL_WIDTH') @@ -1023,14 +1025,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if '_series_sort_' in result: del result['_series_sort_'] book = self.json_codec.raw_to_book(result, SDBook, self.PREFIX) - - # If the thumbnail is the wrong size, zero the last mod date - # so the metadata will be resent - thumbnail = book.get('thumbnail', None) - if thumbnail and not (thumbnail[0] == self.THUMBNAIL_HEIGHT or - thumbnail[1] == self.THUMBNAIL_HEIGHT): - book.set('last_modified', UNDEFINED_DATE) - bl.add_book(book, replace_metadata=True) if '_new_book_' in result: book.set('_new_book_', True) @@ -1086,7 +1080,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if count: for i,book in enumerate(books_to_send): - self._debug('sending metadata for book', book.lpath) + self._debug('sending metadata for book', book.lpath, book.title) self._set_known_metadata(book) opcode, result = self._call_client( 'SEND_BOOK_METADATA', From e0df1634ab6f12ee0961602efa683b45e8e2177d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 20 Jan 2013 22:25:49 +0530 Subject: [PATCH 18/43] Boolean searches --- src/calibre/db/search.py | 71 ++++++++++++++++++++++++++++++++- src/calibre/db/tests/reading.py | 4 ++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index fe9cec79c8..5360c9ef38 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -17,6 +17,22 @@ from calibre.utils.search_query_parser import SearchQueryParser, ParseException # TODO: Thread safety of saved searches +def force_to_bool(val): + if isinstance(val, (str, unicode)): + try: + val = icu_lower(val) + if not val: + val = None + elif val in [_('yes'), _('checked'), 'true', 'yes']: + val = True + elif val in [_('no'), _('unchecked'), 'false', 'no']: + val = False + else: + val = bool(int(val)) + except: + val = None + return val + class DateSearch(object): # {{{ def __init__(self): @@ -225,14 +241,57 @@ class NumericSearch(object): # {{{ # }}} +class BoolenSearch(object): # {{{ + + def __init__(self): + self.local_no = icu_lower(_('no')) + self.local_yes = icu_lower(_('yes')) + self.local_unchecked = icu_lower(_('unchecked')) + self.local_checked = icu_lower(_('checked')) + self.local_empty = icu_lower(_('empty')) + self.local_blank = icu_lower(_('blank')) + self.local_bool_values = { + self.local_no, self.local_unchecked, '_no', 'false', 'no', + self.local_yes, self.local_checked, '_yes', 'true', 'yes', + self.local_empty, self.local_blank, '_empty', 'empty'} + + def __call__(self, query, field_iter, bools_are_tristate): + matches = set() + if query not in self.local_bool_values: + raise ParseException(_('Invalid boolean query "{0}"').format(query)) + for val, book_ids in field_iter(): + val = force_to_bool(val) + if not bools_are_tristate: + if val is None or not val: # item is None or set to false + if query in { self.local_no, self.local_unchecked, 'no', '_no', 'false' }: + matches |= book_ids + else: # item is explicitly set to true + if query in { self.local_yes, self.local_checked, 'yes', '_yes', 'true' }: + matches |= book_ids + else: + if val is None: + if query in { self.local_empty, self.local_blank, 'empty', '_empty', 'false' }: + matches |= book_ids + elif not val: # is not None and false + if query in { self.local_no, self.local_unchecked, 'no', '_no', 'true' }: + matches |= book_ids + else: # item is not None and true + if query in { self.local_yes, self.local_checked, 'yes', '_yes', 'true' }: + matches |= book_ids + return matches + +# }}} + class Parser(SearchQueryParser): def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, - limit_search_columns, limit_search_columns_to, locations): + bool_search, limit_search_columns, limit_search_columns_to, + locations): self.dbcache, self.all_book_ids = dbcache, all_book_ids self.all_search_locations = frozenset(locations) self.grouped_search_terms = gst self.date_search, self.num_search = date_search, num_search + self.bool_search = bool_search self.limit_search_columns, self.limit_search_columns_to = ( limit_search_columns, limit_search_columns_to) super(Parser, self).__init__(locations, optimize=True) @@ -344,6 +403,12 @@ class Parser(SearchQueryParser): self.dbcache.fields[location].iter_counts, candidates), location, dt, candidates) + # take care of boolean special case + if dt == 'bool': + return self.bool_search(icu_lower(query), + partial(self.field_iter, location, candidates), + self.dbcache.pref('bools_are_tristate')) + return matches @@ -353,6 +418,7 @@ class Search(object): self.all_search_locations = all_search_locations self.date_search = DateSearch() self.num_search = NumericSearch() + self.bool_search = BoolenSearch() def change_locations(self, newlocs): self.all_search_locations = newlocs @@ -380,7 +446,8 @@ class Search(object): # 0.000974 seconds. sqp = Parser( dbcache, all_book_ids, dbcache.pref('grouped_search_terms'), - self.date_search, self.num_search, prefs[ 'limit_search_columns' ], + self.date_search, self.num_search, self.bool_search, + prefs[ 'limit_search_columns' ], prefs[ 'limit_search_columns_to' ], self.all_search_locations) try: ret = sqp.parse(query) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 4792f498f8..6069de8026 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -208,6 +208,10 @@ class ReadingTest(BaseTest): '#float:10.01', 'series_index:1', 'series_index:<3', 'id:1', 'id:>2', + # Bool tests + '#yesno:true', '#yesno:false', '#yesno:yes', '#yesno:no', + '#yesno:empty', + # TODO: Tests for searching the size column and # cover:true|false )} From d915e49815bcfac404e93d69ac6b996cbc6b5710 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 20 Jan 2013 23:27:30 +0530 Subject: [PATCH 19/43] Move _match to db.search module --- src/calibre/db/search.py | 45 +++++++++++++++ src/calibre/gui2/library/models.py | 9 +-- .../gui2/store/config/chooser/models.py | 11 ++-- .../gui2/store/stores/mobileread/models.py | 19 ++++--- src/calibre/library/caches.py | 55 +++---------------- 5 files changed, 74 insertions(+), 65 deletions(-) diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 5360c9ef38..de95cf69dc 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -13,9 +13,15 @@ from datetime import timedelta from calibre.utils.config_base import prefs from calibre.utils.date import parse_date, UNDEFINED_DATE, now +from calibre.utils.icu import primary_find from calibre.utils.search_query_parser import SearchQueryParser, ParseException # TODO: Thread safety of saved searches +CONTAINS_MATCH = 0 +EQUALS_MATCH = 1 +REGEXP_MATCH = 2 + +# Utils {{{ def force_to_bool(val): if isinstance(val, (str, unicode)): @@ -33,6 +39,45 @@ def force_to_bool(val): val = None return val +def _match(query, value, matchkind, use_primary_find_in_search=True): + if query.startswith('..'): + query = query[1:] + sq = query[1:] + internal_match_ok = True + else: + internal_match_ok = False + for t in value: + try: ### ignore regexp exceptions, required because search-ahead tries before typing is finished + t = icu_lower(t) + if (matchkind == EQUALS_MATCH): + if internal_match_ok: + if query == t: + return True + comps = [c.strip() for c in t.split('.') if c.strip()] + for comp in comps: + if sq == comp: + return True + elif query[0] == '.': + if t.startswith(query[1:]): + ql = len(query) - 1 + if (len(t) == ql) or (t[ql:ql+1] == '.'): + return True + elif query == t: + return True + elif matchkind == REGEXP_MATCH: + if re.search(query, t, re.I|re.UNICODE): + return True + elif matchkind == CONTAINS_MATCH: + if use_primary_find_in_search: + if primary_find(query, t)[0] != -1: + return True + elif query in t: + return True + except re.error: + pass + return False +# }}} + class DateSearch(object): # {{{ def __init__(self): diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 891b775448..bbd8566a37 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -16,12 +16,12 @@ from calibre.utils.pyparsing import ParseException from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors from calibre.ebooks.metadata.book.base import SafeFormat from calibre.ptempfile import PersistentTemporaryFile -from calibre.utils.config import tweaks, device_prefs +from calibre.utils.config import tweaks, device_prefs, prefs from calibre.utils.date import dt_factory, qt_to_dt, as_local_time from calibre.utils.icu import sort_key from calibre.utils.search_query_parser import SearchQueryParser -from calibre.library.caches import (_match, CONTAINS_MATCH, EQUALS_MATCH, - REGEXP_MATCH, MetadataBackup, force_to_bool) +from calibre.db.search import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH +from calibre.library.caches import (MetadataBackup, force_to_bool) from calibre.library.save_to_disk import find_plugboard from calibre import strftime, isbytestring from calibre.constants import filesystem_encoding, DEBUG @@ -1037,6 +1037,7 @@ class OnDeviceSearch(SearchQueryParser): # {{{ } for x in ('author', 'format'): q[x+'s'] = q[x] + upf = prefs['use_primary_find_in_search'] for index, row in enumerate(self.model.db): for locvalue in locations: accessor = q[locvalue] @@ -1063,7 +1064,7 @@ class OnDeviceSearch(SearchQueryParser): # {{{ vals = accessor(row).split(',') else: vals = [accessor(row)] - if _match(query, vals, m): + if _match(query, vals, m, use_primary_find_in_search=upf): matches.add(index) break except ValueError: # Unicode errors diff --git a/src/calibre/gui2/store/config/chooser/models.py b/src/calibre/gui2/store/config/chooser/models.py index 24f6bdfc25..036b45bcaf 100644 --- a/src/calibre/gui2/store/config/chooser/models.py +++ b/src/calibre/gui2/store/config/chooser/models.py @@ -10,8 +10,8 @@ from PyQt4.Qt import (Qt, QAbstractItemModel, QIcon, QVariant, QModelIndex, QSiz from calibre.gui2 import NONE from calibre.customize.ui import is_disabled, disable_plugin, enable_plugin -from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ - REGEXP_MATCH +from calibre.db.search import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH +from calibre.utils.config_base import prefs from calibre.utils.icu import sort_key from calibre.utils.search_query_parser import SearchQueryParser @@ -60,13 +60,13 @@ class Matches(QAbstractItemModel): index = self.createIndex(i, 0) data = QVariant(True) self.setData(index, data, Qt.CheckStateRole) - + def enable_none(self): for i in xrange(len(self.matches)): index = self.createIndex(i, 0) data = QVariant(False) self.setData(index, data, Qt.CheckStateRole) - + def enable_invert(self): for i in xrange(len(self.matches)): self.toggle_plugin(self.createIndex(i, 0)) @@ -243,6 +243,7 @@ class SearchFilter(SearchQueryParser): 'name': lambda x : x.name.lower(), } q['formats'] = q['format'] + upf = prefs['use_primary_find_in_search'] for sr in self.srs: for locvalue in locations: accessor = q[locvalue] @@ -276,7 +277,7 @@ class SearchFilter(SearchQueryParser): vals = accessor(sr).split(',') else: vals = [accessor(sr)] - if _match(query, vals, m): + if _match(query, vals, m, use_primary_find_in_search=upf): matches.add(sr) break except ValueError: # Unicode errors diff --git a/src/calibre/gui2/store/stores/mobileread/models.py b/src/calibre/gui2/store/stores/mobileread/models.py index 297707e248..60f038c4e2 100644 --- a/src/calibre/gui2/store/stores/mobileread/models.py +++ b/src/calibre/gui2/store/stores/mobileread/models.py @@ -11,13 +11,13 @@ from operator import attrgetter from PyQt4.Qt import (Qt, QAbstractItemModel, QModelIndex, QVariant, pyqtSignal) from calibre.gui2 import NONE -from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ - REGEXP_MATCH +from calibre.db.search import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH +from calibre.utils.config_base import prefs from calibre.utils.icu import sort_key from calibre.utils.search_query_parser import SearchQueryParser class BooksModel(QAbstractItemModel): - + total_changed = pyqtSignal(int) HEADERS = [_('Title'), _('Author(s)'), _('Format')] @@ -37,8 +37,8 @@ class BooksModel(QAbstractItemModel): return self.books[row] else: return None - - def search(self, filter): + + def search(self, filter): self.filter = filter.strip() if not self.filter: self.books = self.all_books @@ -50,7 +50,7 @@ class BooksModel(QAbstractItemModel): self.layoutChanged.emit() self.sort(self.sort_col, self.sort_order) self.total_changed.emit(self.rowCount()) - + def index(self, row, column, parent=QModelIndex()): return self.createIndex(row, column) @@ -64,7 +64,7 @@ class BooksModel(QAbstractItemModel): def columnCount(self, *args): return len(self.HEADERS) - + def headerData(self, section, orientation, role): if role != Qt.DisplayRole: return NONE @@ -112,7 +112,7 @@ class BooksModel(QAbstractItemModel): class SearchFilter(SearchQueryParser): - + USABLE_LOCATIONS = [ 'all', 'author', @@ -161,6 +161,7 @@ class SearchFilter(SearchQueryParser): } for x in ('author', 'format'): q[x+'s'] = q[x] + upf = prefs['use_primary_find_in_search'] for sr in self.srs: for locvalue in locations: accessor = q[locvalue] @@ -182,7 +183,7 @@ class SearchFilter(SearchQueryParser): m = matchkind vals = [accessor(sr)] - if _match(query, vals, m): + if _match(query, vals, m, use_primary_find_in_search=upf): matches.add(sr) break except ValueError: # Unicode errors diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 507305528d..b453c654df 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re, itertools, time, traceback, locale +import itertools, time, traceback, locale from itertools import repeat, izip, imap from datetime import timedelta from threading import Thread @@ -16,10 +16,10 @@ from calibre.utils.date import parse_date, now, UNDEFINED_DATE, clean_date_for_s from calibre.utils.search_query_parser import SearchQueryParser from calibre.utils.pyparsing import ParseException from calibre.utils.localization import (canonicalize_lang, lang_map, get_udc) +from calibre.db.search import CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH, _match from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre import prints -from calibre.utils.icu import primary_find class MetadataBackup(Thread): # {{{ ''' @@ -118,7 +118,6 @@ class MetadataBackup(Thread): # {{{ # }}} - ### Global utility function for get_match here and in gui2/library.py # This is a global for performance pref_use_primary_find_in_search = False @@ -127,47 +126,6 @@ def set_use_primary_find_in_search(toWhat): global pref_use_primary_find_in_search pref_use_primary_find_in_search = toWhat -CONTAINS_MATCH = 0 -EQUALS_MATCH = 1 -REGEXP_MATCH = 2 -def _match(query, value, matchkind): - if query.startswith('..'): - query = query[1:] - sq = query[1:] - internal_match_ok = True - else: - internal_match_ok = False - for t in value: - try: ### ignore regexp exceptions, required because search-ahead tries before typing is finished - t = icu_lower(t) - if (matchkind == EQUALS_MATCH): - if internal_match_ok: - if query == t: - return True - comps = [c.strip() for c in t.split('.') if c.strip()] - for comp in comps: - if sq == comp: - return True - elif query[0] == '.': - if t.startswith(query[1:]): - ql = len(query) - 1 - if (len(t) == ql) or (t[ql:ql+1] == '.'): - return True - elif query == t: - return True - elif matchkind == REGEXP_MATCH: - if re.search(query, t, re.I|re.UNICODE): - return True - elif matchkind == CONTAINS_MATCH: - if pref_use_primary_find_in_search: - if primary_find(query, t)[0] != -1: - return True - elif query in t: - return True - except re.error: - pass - return False - def force_to_bool(val): if isinstance(val, (str, unicode)): try: @@ -576,7 +534,8 @@ class ResultCache(SearchQueryParser): # {{{ continue k = parts[:1] v = parts[1:] - if keyq and not _match(keyq, k, keyq_mkind): + if keyq and not _match(keyq, k, keyq_mkind, + use_primary_find_in_search=pref_use_primary_find_in_search): continue if valq: if valq == 'true': @@ -586,7 +545,8 @@ class ResultCache(SearchQueryParser): # {{{ if v: add_if_nothing_matches = False continue - elif not _match(valq, v, valq_mkind): + elif not _match(valq, v, valq_mkind, + use_primary_find_in_search=pref_use_primary_find_in_search): continue matches.add(id_) @@ -851,7 +811,8 @@ class ResultCache(SearchQueryParser): # {{{ vals = [v.strip() for v in item[loc].split(is_multiple_cols[loc])] else: vals = [item[loc]] ### make into list to make _match happy - if _match(q, vals, matchkind): + if _match(q, vals, matchkind, + use_primary_find_in_search=pref_use_primary_find_in_search): matches.add(item[0]) continue current_candidates -= matches From 93f8de31bda2118d58d1a1c3638d3eb83aca7c8a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 09:12:19 +0530 Subject: [PATCH 20/43] ... --- src/calibre/db/fields.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 34a12c9491..618a8cb1f7 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -114,17 +114,8 @@ class OneToOneField(Field): def iter_searchable_values(self, get_metadata, candidates, default_value=None): cbm = self.table.book_col_map - if (self.name in {'id', 'uuid', 'title'} or - self.metadata['datatype'] == 'datetime'): - # Values are likely to be unique - for book_id in candidates: - yield cbm.get(book_id, default_value), {book_id} - else: - val_map = defaultdict(set) - for book_id in candidates: - val_map[cbm.get(book_id, default_value)].add(book_id) - for val, book_ids in val_map.iteritems(): - yield val, book_ids + for book_id in candidates: + yield cbm.get(book_id, default_value), {book_id} class CompositeField(OneToOneField): From ffbdb63f6ff563a77ac9b850e50f457c028cb0dd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 09:31:03 +0530 Subject: [PATCH 21/43] Update Metro UK --- recipes/metro_uk.recipe | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/recipes/metro_uk.recipe b/recipes/metro_uk.recipe index 934fbab793..78db75daf8 100644 --- a/recipes/metro_uk.recipe +++ b/recipes/metro_uk.recipe @@ -8,13 +8,16 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe): title = u'Metro UK' description = 'News as provided by The Metro -UK' #timefmt = '' - __author__ = 'Dave Asbury' - #last update 9/6/12 - cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/276636_117118184990145_2132092232_n.jpg' - oldest_article = 1 + __author__ = 'fleclerc & Dave Asbury' + #last update 20/1/13 + #cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/276636_117118184990145_2132092232_n.jpg' + + cover_url = 'https://twimg0-a.akamaihd.net/profile_images/1638332595/METRO_LETTERS-01.jpg' remove_empty_feeds = True remove_javascript = True auto_cleanup = True + max_articles_per_feed = 12 + ignore_duplicate_articles = {'title', 'url'} encoding = 'UTF-8' language = 'en_GB' From 45fbe9fa27ace86f0570cba244c80999bb41a1b9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 11:20:22 +0530 Subject: [PATCH 22/43] Keypair searching --- src/calibre/db/fields.py | 6 +++ src/calibre/db/search.py | 83 +++++++++++++++++++++++++++++++-- src/calibre/db/tests/reading.py | 6 +++ 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 618a8cb1f7..e0950fff3b 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -311,6 +311,12 @@ class IdentifiersField(ManyToManyField): (self._default_sort_key,)) for id_, cids in ans.iteritems()} + def iter_searchable_values(self, get_metadata, candidates, default_value=()): + bcm = self.table.book_col_map + for book_id in candidates: + val = bcm.get(book_id, default_value) + if val: + yield val, {book_id} class AuthorsField(ManyToManyField): diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index de95cf69dc..743922e6e4 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -39,6 +39,23 @@ def force_to_bool(val): val = None return val +def _matchkind(query): + matchkind = CONTAINS_MATCH + if (len(query) > 1): + if query.startswith('\\'): + query = query[1:] + elif query.startswith('='): + matchkind = EQUALS_MATCH + query = query[1:] + elif query.startswith('~'): + matchkind = REGEXP_MATCH + query = query[1:] + + if matchkind != REGEXP_MATCH: + # leave case in regexps because it can be significant e.g. \S \W \D + query = icu_lower(query) + return matchkind, query + def _match(query, value, matchkind, use_primary_find_in_search=True): if query.startswith('..'): query = query[1:] @@ -286,7 +303,7 @@ class NumericSearch(object): # {{{ # }}} -class BoolenSearch(object): # {{{ +class BooleanSearch(object): # {{{ def __init__(self): self.local_no = icu_lower(_('no')) @@ -327,16 +344,60 @@ class BoolenSearch(object): # {{{ # }}} +class KeyPairSearch(object): # {{{ + + def __call__(self, query, field_iter, candidates, use_primary_find): + matches = set() + if ':' in query: + q = [q.strip() for q in query.split(':')] + if len(q) != 2: + raise ParseException(query, len(query), + 'Invalid query format for colon-separated search') + keyq, valq = q + keyq_mkind, keyq = _matchkind(keyq) + valq_mkind, valq = _matchkind(valq) + else: + keyq = keyq_mkind = '' + valq_mkind, valq = _matchkind(query) + keyq_mkind + + if valq in {'true', 'false'}: + found = set() + if keyq: + for val, book_ids in field_iter(): + if val and val.get(keyq, False): + found |= book_ids + else: + for val, book_ids in field_iter(): + if val: + found |= book_ids + return found if valq == 'true' else candidates - found + + for m, book_ids in field_iter(): + for key, val in m.iteritems(): + if (keyq and not _match(keyq, (key,), keyq_mkind, + use_primary_find_in_search=use_primary_find)): + continue + if (valq and not _match(valq, (val,), valq_mkind, + use_primary_find_in_search=use_primary_find)): + continue + matches |= book_ids + break + + return matches + +# }}} + class Parser(SearchQueryParser): def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, - bool_search, limit_search_columns, limit_search_columns_to, + bool_search, keypair_search, limit_search_columns, limit_search_columns_to, locations): self.dbcache, self.all_book_ids = dbcache, all_book_ids self.all_search_locations = frozenset(locations) self.grouped_search_terms = gst self.date_search, self.num_search = date_search, num_search - self.bool_search = bool_search + self.bool_search, self.keypair_search = bool_search, keypair_search self.limit_search_columns, self.limit_search_columns_to = ( limit_search_columns, limit_search_columns_to) super(Parser, self).__init__(locations, optimize=True) @@ -372,7 +433,7 @@ class Parser(SearchQueryParser): # get metadata key associated with the search term. Eliminates # dealing with plurals and other aliases - # original_location = location + original_location = location location = self.field_metadata.search_term_to_field_key( icu_lower(location.strip())) # grouped search terms @@ -454,6 +515,16 @@ class Parser(SearchQueryParser): partial(self.field_iter, location, candidates), self.dbcache.pref('bools_are_tristate')) + # special case: colon-separated fields such as identifiers. isbn + # is a special case within the case + if fm.get('is_csp', False): + field_iter = partial(self.field_iter, location, candidates) + upf = prefs['use_primary_find_in_search'] + if location == 'identifiers' and original_location == 'isbn': + return self.keypair_search('=isbn:'+query, field_iter, + candidates, upf) + return self.keypair_search(query, field_iter, candidates, upf) + return matches @@ -463,7 +534,8 @@ class Search(object): self.all_search_locations = all_search_locations self.date_search = DateSearch() self.num_search = NumericSearch() - self.bool_search = BoolenSearch() + self.bool_search = BooleanSearch() + self.keypair_search = KeyPairSearch() def change_locations(self, newlocs): self.all_search_locations = newlocs @@ -492,6 +564,7 @@ class Search(object): sqp = Parser( dbcache, all_book_ids, dbcache.pref('grouped_search_terms'), self.date_search, self.num_search, self.bool_search, + self.keypair_search, prefs[ 'limit_search_columns' ], prefs[ 'limit_search_columns_to' ], self.all_search_locations) try: diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 6069de8026..2fa49033c0 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -212,6 +212,12 @@ class ReadingTest(BaseTest): '#yesno:true', '#yesno:false', '#yesno:yes', '#yesno:no', '#yesno:empty', + # Keypair tests + 'identifiers:true', 'identifiers:false', 'identifiers:test', + 'identifiers:test:false', 'identifiers:test:one', + 'identifiers:t:n', 'identifiers:=test:=two', 'identifiers:x:y', + 'identifiers:z', + # TODO: Tests for searching the size column and # cover:true|false )} From f4fb43875fe83d1556522600767d0b365fde37ed Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 11:25:09 +0530 Subject: [PATCH 23/43] ... --- src/calibre/db/search.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 743922e6e4..5a02a5e5da 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -557,6 +557,9 @@ class Search(object): if not q: return all_book_ids + if not isinstance(q, type(u'')): + q = q.decode('utf-8') + # We construct a new parser instance per search as pyparsing is not # thread safe. On my desktop, constructing a SearchQueryParser instance # takes 0.000975 seconds and restoring it from a pickle takes @@ -567,8 +570,9 @@ class Search(object): self.keypair_search, prefs[ 'limit_search_columns' ], prefs[ 'limit_search_columns_to' ], self.all_search_locations) + try: - ret = sqp.parse(query) + ret = sqp.parse(q) finally: sqp.dbcache = None return ret From 5000f5c13335a97bd4c47803d1c45d1e83473292 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 11:43:57 +0530 Subject: [PATCH 24/43] User category searching --- src/calibre/db/search.py | 27 +++++++++++++++++++++++++++ src/calibre/db/tests/metadata.db | Bin 235520 -> 236544 bytes src/calibre/db/tests/reading.py | 1 + 3 files changed, 28 insertions(+) diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 5a02a5e5da..334bc046d8 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -525,8 +525,35 @@ class Parser(SearchQueryParser): candidates, upf) return self.keypair_search(query, field_iter, candidates, upf) + # check for user categories + if len(location) >= 2 and location.startswith('@'): + return self.get_user_category_matches(location[1:], icu_lower(query), candidates) + return matches + def get_user_category_matches(self, location, query, candidates): + matches = set() + if len(query) < 2: + return matches + + user_cats = self.dbcache.pref('user_categories') + c = set(candidates) + + if query.startswith('.'): + check_subcats = True + query = query[1:] + else: + check_subcats = False + + for key in user_cats: + if key == location or (check_subcats and key.startswith(location + '.')): + for (item, category, ign) in user_cats[key]: + s = self.get_matches(category, '=' + item, candidates=c) + c -= s + matches |= s + if query == 'false': + return candidates - matches + return matches class Search(object): diff --git a/src/calibre/db/tests/metadata.db b/src/calibre/db/tests/metadata.db index 4bd6dfe4f97d037ed7c343a87e6cc929d153b5d9..94748877b6203bb38b302551f2c80ff863b6059e 100644 GIT binary patch delta 1976 zcmeHIUu;ul6u-Bp9WuJDRkx0@;cb^dn2K$jm`vR|-K=4bF0zqoZAxixy4rE?ynig) z7;Ym<2$HbWaN+|JeZU|-fFxdw#t}5BF~bz_?@+eHz%qI9$robcxnG;vE+g^P#H8n@ z_x#TNo!|Yw@BHq>+3JZ4)yFp2bdKW|r-cq)KezkLOw}UX;ZAU$aMvv=_gmI!f4s4Q zw{pJPJ+eHcMq`Q?)s&Q~#WYbNro;!MnyBnm$HrYT zEBA;~5;JO4lcRgYs5&CW#Y8yLELpjLGkYZ06c0xZ)~Yt{!8dDkRaByJqE5<6N>s-x z1;G=L<%FOz96w^w&fmKD25+O?{Z;$;s|46|Hw5_X?TbykmOYen^HTP7u7g((F69m_ z<#=N$XKB#*8|dtT&35*VMOiqUf{p0i0rw+DpgNnsWXB5wP`}Y%TJ_Ird;Cdr&#*lR zb+{!2^_Y1E4x_ObHW@8X!YdXlY0Bel09x_nc35ZB_+b#7WturAIf|RH5zqf)UuHi4rLjNE+iuY= z&g0A!Y&Je?hi)F{CZHPcq~Kw8PLa;3`-8Xl0UO4J?DVM1I5P?ZykXrB$5`?vI(s2y z{B;0;$D_O8J{tT1^!C6#_~$Ng;5Q+f=MOb#<{Jv5TQKRebt;+R`n%0A_Vq%GgIP%l z1NvVB7v}fFhEo&ZI8JSt=d|CY)w0zW&8Oh#TbU6t3oiIBx!wTOwS&A-Pz1#{6P{U;()LuEXfA_GmMLVJ{ZCn!zWvxJd?6*P!zv(LeayxjhOmZw`%FTvGC^CdQ27O*n*w+#2>hXn#hGl0e zPpfEu_@|$Ep$%Bkz~rIFX03m+fc4EnhnV`~4Bs$;Nl*IwKLY3fj}4quV!5*qkvJ7OS@ z8|6A>__{wxu|0}c(18SCDYQ_$0Y=(kNA5-if8&n^`aNZ(B-v5Pqm38`l_s!>wm49r zy1u142TI5p#7=&p86nW`CS=pB0~Qs2N0J*gJnB+~@fzq@1s!n0O2y64tAzJ-trgWg z+=la@O8?rbA9z(eo`Mz=%BlJ`6i;4;MrD7b`eqdJ!(LnmolN18E+9FwxpqeZ$v`h{{ zloKcNxbYZTRv)~;2aY2y(!3kJJa7_VlM*)PW{MNSIwdvuj(JS~3y~6`=l?T8ri@Hw vU&f3AGd?ir4J;#e#vD-7c`l!~CUHa5Mk77J5Is5sGfg?6qhkNsx4-@c!Lsl4 diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 2fa49033c0..627a692860 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -220,6 +220,7 @@ class ReadingTest(BaseTest): # TODO: Tests for searching the size column and # cover:true|false + # TODO: Tests for user categories searching )} old = None From d2866a2208149f432a3cb432cba9936efaf58e89 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 21 Jan 2013 09:35:39 +0100 Subject: [PATCH 25/43] When doing device book matching: on UUID matches, generate covers only if the last_modified date of the book in the db is later than the date on the book on the device, or if existing thumbnail is not the size that the device asked for. --- src/calibre/gui2/device.py | 57 ++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 85e992e264..6f75f4de98 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1661,9 +1661,11 @@ class DeviceMixin(object): # {{{ update_metadata = device_prefs['manage_device_metadata'] == 'on_connect' get_covers = False + desired_thumbnail_height = 0 if update_metadata and self.device_manager.is_device_connected: if self.device_manager.device.WANTS_UPDATED_THUMBNAILS: get_covers = True + desired_thumbnail_height = self.device_manager.device.THUMBNAIL_HEIGHT # Force a reset if the caches are not initialized if reset or not hasattr(self, 'db_book_title_cache'): @@ -1698,17 +1700,28 @@ class DeviceMixin(object): # {{{ # will be used by books_on_device to indicate matches. While we are # going by, update the metadata for a book if automatic management is on + def update_book(id_, book) : + if not update_metadata: + return + mi = db.get_metadata(id_, index_is_id=True, get_cover=get_covers) + book.smart_update(mi, replace_metadata=True) + if get_covers: + if book.cover and os.access(book.cover, os.R_OK): + book.thumbnail = self.cover_to_thumbnail(open(book.cover, 'rb').read()) + else: + book.thumbnail = self.default_thumbnail + for booklist in booklists: for book in booklist: book.in_library = None if getattr(book, 'uuid', None) in self.db_book_uuid_cache: id_ = db_book_uuid_cache[book.uuid] - if (update_metadata and - db.metadata_last_modified(id_, index_is_id=True) != - getattr(book, 'last_modified', None)): - mi = db.get_metadata(id_, index_is_id=True, - get_cover=get_covers) - book.smart_update(mi, replace_metadata=True) + if (db.metadata_last_modified(id_, index_is_id=True) != + getattr(book, 'last_modified', None) + or (not book.thumbnail + or max(book.thumbnail[0], book.thumbnail[1]) != + desired_thumbnail_height)): + update_book(id_, book) book.in_library = 'UUID' # ensure that the correct application_id is set book.application_id = id_ @@ -1721,23 +1734,15 @@ class DeviceMixin(object): # {{{ # will match if any of the db_id, author, or author_sort # also match. if getattr(book, 'application_id', None) in d['db_ids']: - if update_metadata: - id_ = getattr(book, 'application_id', None) - book.smart_update(db.get_metadata(id_, - index_is_id=True, - get_cover=get_covers), - replace_metadata=True) + id_ = getattr(book, 'application_id', None) + update_book(id_, book) book.in_library = 'APP_ID' # app_id already matches a db_id. No need to set it. continue # Sonys know their db_id independent of the application_id # in the metadata cache. Check that as well. if getattr(book, 'db_id', None) in d['db_ids']: - if update_metadata: - book.smart_update(db.get_metadata(book.db_id, - index_is_id=True, - get_cover=get_covers), - replace_metadata=True) + update_book(book.db_id, book) book.in_library = 'DB_ID' book.application_id = book.db_id continue @@ -1752,20 +1757,12 @@ class DeviceMixin(object): # {{{ book_authors = clean_string(authors_to_string(book.authors)) if book_authors in d['authors']: id_ = d['authors'][book_authors] - if update_metadata: - book.smart_update(db.get_metadata(id_, - index_is_id=True, - get_cover=get_covers), - replace_metadata=True) + update_book(id_, book) book.in_library = 'AUTHOR' book.application_id = id_ elif book_authors in d['author_sort']: id_ = d['author_sort'][book_authors] - if update_metadata: - book.smart_update(db.get_metadata(id_, - index_is_id=True, - get_cover=get_covers), - replace_metadata=True) + update_book(id_, book) book.in_library = 'AUTH_SORT' book.application_id = id_ else: @@ -1779,12 +1776,6 @@ class DeviceMixin(object): # {{{ if update_metadata: if self.device_manager.is_device_connected: - if self.device_manager.device.WANTS_UPDATED_THUMBNAILS: - for blist in booklists: - for book in blist: - if book.cover and os.access(book.cover, os.R_OK): - book.thumbnail = \ - self.cover_to_thumbnail(open(book.cover, 'rb').read()) plugboards = self.library_view.model().db.prefs.get('plugboards', {}) self.device_manager.sync_booklists( FunctionDispatcher(self.metadata_synced), booklists, From a4dbc37a90d6f50c6554efcafe27f0a9503d9443 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 14:18:20 +0530 Subject: [PATCH 26/43] Text field searching --- src/calibre/db/cache.py | 5 +- src/calibre/db/search.py | 106 ++++++++++++++++++++++++++++++-- src/calibre/db/tests/reading.py | 8 +++ 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 88a2196a61..e2ecb369ca 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -412,8 +412,9 @@ class Cache(object): return sorted(all_book_ids, key=partial(SortKey, fields, sort_keys)) @read_api - def search(self, query, restriction): - return self._search_api(self, query, restriction) + def search(self, query, restriction, virtual_fields=None): + return self._search_api(self, query, restriction, + virtual_fields=virtual_fields) # }}} diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 334bc046d8..398b153cac 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -14,6 +14,7 @@ from datetime import timedelta from calibre.utils.config_base import prefs from calibre.utils.date import parse_date, UNDEFINED_DATE, now from calibre.utils.icu import primary_find +from calibre.utils.localization import lang_map, canonicalize_lang from calibre.utils.search_query_parser import SearchQueryParser, ParseException # TODO: Thread safety of saved searches @@ -392,7 +393,7 @@ class Parser(SearchQueryParser): def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, bool_search, keypair_search, limit_search_columns, limit_search_columns_to, - locations): + locations, virtual_fields): self.dbcache, self.all_book_ids = dbcache, all_book_ids self.all_search_locations = frozenset(locations) self.grouped_search_terms = gst @@ -400,6 +401,9 @@ class Parser(SearchQueryParser): self.bool_search, self.keypair_search = bool_search, keypair_search self.limit_search_columns, self.limit_search_columns_to = ( limit_search_columns, limit_search_columns_to) + self.virtual_fields = virtual_fields or {} + if 'marked' not in self.virtual_fields: + self.virtual_fields['marked'] = self super(Parser, self).__init__(locations, optimize=True) @property @@ -411,8 +415,15 @@ class Parser(SearchQueryParser): def field_iter(self, name, candidates): get_metadata = partial(self.dbcache._get_metadata, get_user_categories=False) - return self.dbcache.fields[name].iter_searchable_values(get_metadata, - candidates) + try: + field = self.dbcache.fields[name] + except KeyError: + field = self.virtual_fields[name] + return field.iter_searchable_values(get_metadata, candidates) + + def iter_searchable_values(self, *args, **kwargs): + for x in []: + yield x, set() def get_matches(self, location, query, candidates=None, allow_recursion=True): @@ -480,6 +491,8 @@ class Parser(SearchQueryParser): pass return matches + upf = prefs['use_primary_find_in_search'] + if location in self.field_metadata: fm = self.field_metadata[location] dt = fm['datatype'] @@ -519,7 +532,6 @@ class Parser(SearchQueryParser): # is a special case within the case if fm.get('is_csp', False): field_iter = partial(self.field_iter, location, candidates) - upf = prefs['use_primary_find_in_search'] if location == 'identifiers' and original_location == 'isbn': return self.keypair_search('=isbn:'+query, field_iter, candidates, upf) @@ -529,6 +541,87 @@ class Parser(SearchQueryParser): if len(location) >= 2 and location.startswith('@'): return self.get_user_category_matches(location[1:], icu_lower(query), candidates) + # Everything else (and 'all' matches) + matchkind, query = _matchkind(query) + all_locs = set() + text_fields = set() + field_metadata = {} + + for x, fm in self.field_metadata.iteritems(): + if x.startswith('@'): continue + if fm['search_terms'] and x != 'series_sort': + all_locs.add(x) + field_metadata[x] = fm + if fm['datatype'] in { 'composite', 'text', 'comments', 'series', 'enumeration' }: + text_fields.add(x) + + locations = all_locs if location == 'all' else {location} + + current_candidates = set(candidates) + + try: + rating_query = int(float(query)) * 2 + except: + rating_query = None + + try: + int_query = int(float(query)) + except: + int_query = None + + try: + float_query = float(query) + except: + float_query = None + + for location in locations: + current_candidates -= matches + q = query + if location == 'languages': + q = canonicalize_lang(query) + if q is None: + lm = lang_map() + rm = {v.lower():k for k,v in lm.iteritems()} + q = rm.get(query, query) + + if matchkind == CONTAINS_MATCH and q in {'true', 'false'}: + found = set() + for val, book_ids in self.field_iter(location, current_candidates): + if val and (not hasattr(val, 'strip') or val.strip()): + found |= book_ids + matches |= (found if q == 'true' else (current_candidates-found)) + continue + + dt = field_metadata.get(location, {}).get('datatype', None) + if dt == 'rating': + if rating_query is not None: + for val, book_ids in self.field_iter(location, current_candidates): + if val == rating_query: + matches |= book_ids + continue + + if dt == 'float': + if float_query is not None: + for val, book_ids in self.field_iter(location, current_candidates): + if val == float_query: + matches |= book_ids + continue + + if dt == 'int': + if int_query is not None: + for val, book_ids in self.field_iter(location, current_candidates): + if val == int_query: + matches |= book_ids + continue + + if location in text_fields: + for val, book_ids in self.field_iter(location, current_candidates): + if val is not None: + if isinstance(val, basestring): + val = (val,) + if _match(q, val, matchkind, use_primary_find_in_search=upf): + matches |= book_ids + return matches def get_user_category_matches(self, location, query, candidates): @@ -567,7 +660,7 @@ class Search(object): def change_locations(self, newlocs): self.all_search_locations = newlocs - def __call__(self, dbcache, query, search_restriction): + def __call__(self, dbcache, query, search_restriction, virtual_fields=None): ''' Return the set of ids of all records that match the specified query and restriction @@ -596,7 +689,8 @@ class Search(object): self.date_search, self.num_search, self.bool_search, self.keypair_search, prefs[ 'limit_search_columns' ], - prefs[ 'limit_search_columns_to' ], self.all_search_locations) + prefs[ 'limit_search_columns_to' ], self.all_search_locations, + virtual_fields) try: ret = sqp.parse(q) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 627a692860..8a0230704b 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -218,6 +218,14 @@ class ReadingTest(BaseTest): 'identifiers:t:n', 'identifiers:=test:=two', 'identifiers:x:y', 'identifiers:z', + # Text tests + 'title:="Title One"', 'title:~title', '#enum:=one', '#enum:tw', + '#enum:false', '#enum:true', 'series:one', 'tags:one', 'tags:true', + 'tags:false', '2', 'one', '20.02', '"publisher one"', + + # User categories + # '@Good Authors:One', + # TODO: Tests for searching the size column and # cover:true|false # TODO: Tests for user categories searching From acf4811fb67c420328cfeac4c35b907ab3963977 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 15:26:31 +0530 Subject: [PATCH 27/43] Searching done --- src/calibre/db/cache.py | 37 ++++++++++++++++++++++++++++++--- src/calibre/db/search.py | 2 +- src/calibre/db/tests/reading.py | 4 ++-- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index e2ecb369ca..6803ab93ed 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os +import os, traceback from collections import defaultdict from functools import wraps, partial @@ -18,6 +18,7 @@ from calibre.db.tables import VirtualTable from calibre.db.lazy import FormatMetadata, FormatsList from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.date import now +from calibre.utils.icu import sort_key def api(f): f.is_cache_api = True @@ -67,6 +68,36 @@ class Cache(object): lock = self.read_lock if ira else self.write_lock setattr(self, name, wrap_simple(lock, func)) + self.initialize_dynamic() + + def initialize_dynamic(self): + # Reconstruct the user categories, putting them into field_metadata + # Assumption is that someone else will fix them if they change. + self.field_metadata.remove_dynamic_categories() + for user_cat in sorted(self.pref('user_categories', {}).iterkeys(), key=sort_key): + cat_name = '@' + user_cat # add the '@' to avoid name collision + self.field_metadata.add_user_category(label=cat_name, name=user_cat) + + # add grouped search term user categories + muc = frozenset(self.pref('grouped_search_make_user_categories', [])) + for cat in sorted(self.pref('grouped_search_terms', {}).iterkeys(), key=sort_key): + if cat in muc: + # There is a chance that these can be duplicates of an existing + # user category. Print the exception and continue. + try: + self.field_metadata.add_user_category(label=u'@' + cat, name=cat) + except: + traceback.print_exc() + + # TODO: Saved searches + # if len(saved_searches().names()): + # self.field_metadata.add_search_category(label='search', name=_('Searches')) + + self.field_metadata.add_grouped_search_terms( + self.pref('grouped_search_terms', {})) + + self._search_api.change_locations(self.field_metadata.get_search_terms()) + @property def field_metadata(self): return self.backend.field_metadata @@ -319,8 +350,8 @@ class Cache(object): return ans @read_api - def pref(self, name): - return self.backend.prefs[name] + def pref(self, name, default=None): + return self.backend.prefs.get(name, default) @api def get_metadata(self, book_id, diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 398b153cac..7eb747da67 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -650,7 +650,7 @@ class Parser(SearchQueryParser): class Search(object): - def __init__(self, all_search_locations): + def __init__(self, all_search_locations=()): self.all_search_locations = all_search_locations self.date_search = DateSearch() self.num_search = NumericSearch() diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 8a0230704b..b2b63f9e32 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -222,13 +222,13 @@ class ReadingTest(BaseTest): 'title:="Title One"', 'title:~title', '#enum:=one', '#enum:tw', '#enum:false', '#enum:true', 'series:one', 'tags:one', 'tags:true', 'tags:false', '2', 'one', '20.02', '"publisher one"', + '"my comments one"', # User categories - # '@Good Authors:One', + '@Good Authors:One', '@Good Series.good tags:two', # TODO: Tests for searching the size column and # cover:true|false - # TODO: Tests for user categories searching )} old = None From d52a659da28bfacd4cbf025856d26ced4fe8a3d5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 15:27:35 +0530 Subject: [PATCH 28/43] ... --- src/calibre/db/tests/reading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index b2b63f9e32..8183611f91 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -227,7 +227,7 @@ class ReadingTest(BaseTest): # User categories '@Good Authors:One', '@Good Series.good tags:two', - # TODO: Tests for searching the size column and + # TODO: Tests for searching the size and #formats columns and # cover:true|false )} old = None From e0d0eb1973a444e10fd318713ee71d7a4a336e97 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 09:45:10 +0530 Subject: [PATCH 29/43] Fix TSN and St. Louis Post DIspatch --- recipes/st_louis_post_dispatch.recipe | 12 ++++++---- recipes/tsn.recipe | 33 ++++++++------------------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/recipes/st_louis_post_dispatch.recipe b/recipes/st_louis_post_dispatch.recipe index 3b7701cedc..6d22a327ab 100644 --- a/recipes/st_louis_post_dispatch.recipe +++ b/recipes/st_louis_post_dispatch.recipe @@ -7,12 +7,16 @@ class AdvancedUserRecipe1282093204(BasicNewsRecipe): oldest_article = 1 max_articles_per_feed = 15 + use_embedded_content = False + + no_stylesheets = True + auto_cleanup = True masthead_url = 'http://farm5.static.flickr.com/4118/4929686950_0e22e2c88a.jpg' feeds = [ (u'News-Bill McClellan', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fcolumns%2Fbill-mclellan&f=rss&t=article'), (u'News-Columns', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcolumns*&l=50&f=rss&t=article'), - (u'News-Crime & Courtshttp://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcrime-and-courts&l=50&f=rss&t=article'), + (u'News-Crime & Courts', 'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcrime-and-courts&l=50&f=rss&t=article'), (u'News-Deb Peterson', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fcolumns%2Fdeb-peterson&f=rss&t=article'), (u'News-Education', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2feducation&f=rss&t=article'), (u'News-Government & Politics', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fgovt-and-politics&f=rss&t=article'), @@ -62,9 +66,9 @@ class AdvancedUserRecipe1282093204(BasicNewsRecipe): (u'Entertainment-House-O-Fun', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fhouse-o-fun&l=100&f=rss&t=article'), (u'Entertainment-Kevin C. Johnson', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fmusic%2Fkevin-johnson&l=100&f=rss&t=article') ] - remove_empty_feeds = True - remove_tags = [dict(name='div', attrs={'id':'blox-logo'}),dict(name='a')] - keep_only_tags = [dict(name='h1'), dict(name='p', attrs={'class':'byline'}), dict(name="div", attrs={'id':'blox-story-text'})] + #remove_empty_feeds = True + #remove_tags = [dict(name='div', attrs={'id':'blox-logo'}),dict(name='a')] + #keep_only_tags = [dict(name='h1'), dict(name='p', attrs={'class':'byline'}), dict(name="div", attrs={'id':'blox-story-text'})] extra_css = 'p {text-align: left;}' diff --git a/recipes/tsn.recipe b/recipes/tsn.recipe index e822ebc633..6c3dbe5159 100644 --- a/recipes/tsn.recipe +++ b/recipes/tsn.recipe @@ -7,28 +7,15 @@ class AdvancedUserRecipe1289990851(BasicNewsRecipe): language = 'en_CA' __author__ = 'Nexus' no_stylesheets = True + auto_cleanup = True + use_embedded_content = False INDEX = 'http://tsn.ca/nhl/story/?id=nhl' - keep_only_tags = [dict(name='div', attrs={'id':['tsnColWrap']}), - dict(name='div', attrs={'id':['tsnStory']})] - remove_tags = [dict(name='div', attrs={'id':'tsnRelated'}), - dict(name='div', attrs={'class':'textSize'})] - - def parse_index(self): - feeds = [] - soup = self.index_to_soup(self.INDEX) - feed_parts = soup.findAll('div', attrs={'class': 'feature'}) - for feed_part in feed_parts: - articles = [] - if not feed_part.h2: - continue - feed_title = feed_part.h2.string - article_parts = feed_part.findAll('a') - for article_part in article_parts: - article_title = article_part.string - article_date = '' - article_url = 'http://tsn.ca/' + article_part['href'] - articles.append({'title': article_title, 'url': article_url, 'description':'', 'date':article_date}) - if articles: - feeds.append((feed_title, articles)) - return feeds + #keep_only_tags = [dict(name='div', attrs={'id':['tsnColWrap']}), + #dict(name='div', attrs={'id':['tsnStory']})] + #remove_tags = [dict(name='div', attrs={'id':'tsnRelated'}), + #dict(name='div', attrs={'class':'textSize'})] + feeds = [ +('News', + 'http://www.tsn.ca/datafiles/rss/Stories.xml'), +] From a15d236830cb5fb9b83f1d66482d695c2c56e852 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 10:11:28 +0530 Subject: [PATCH 30/43] Fix #1102408 (Untranslated string "(# books)") --- src/calibre/library/database2.py | 2 +- src/calibre/translations/calibre.pot | 293 ++++++++++++++------------- 2 files changed, 157 insertions(+), 138 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index ab85421697..ad15f1a022 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1220,7 +1220,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): loc.append(_('Card A')) if b is not None: loc.append(_('Card B')) - return ', '.join(loc) + ((' (%s books)'%count) if count > 1 else '') + return ', '.join(loc) + ((_(' (%s books)')%count) if count > 1 else '') def set_book_on_device_func(self, func): self.book_on_device_func = func diff --git a/src/calibre/translations/calibre.pot b/src/calibre/translations/calibre.pot index d3f3538c27..9f2bb510fe 100644 --- a/src/calibre/translations/calibre.pot +++ b/src/calibre/translations/calibre.pot @@ -5,8 +5,8 @@ msgid "" msgstr "" "Project-Id-Version: calibre 0.9.15\n" -"POT-Creation-Date: 2013-01-18 09:12+IST\n" -"PO-Revision-Date: 2013-01-18 09:12+IST\n" +"POT-Creation-Date: 2013-01-22 10:10+IST\n" +"PO-Revision-Date: 2013-01-22 10:10+IST\n" "Last-Translator: Automatically generated\n" "Language-Team: LANGUAGE\n" "MIME-Version: 1.0\n" @@ -21,9 +21,9 @@ msgid "Does absolutely nothing" msgstr "" #: /home/kovid/work/calibre/src/calibre/customize/__init__.py:59 -#: /home/kovid/work/calibre/src/calibre/db/cache.py:106 -#: /home/kovid/work/calibre/src/calibre/db/cache.py:109 -#: /home/kovid/work/calibre/src/calibre/db/cache.py:120 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:139 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:142 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:153 #: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:379 #: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:380 #: /home/kovid/work/calibre/src/calibre/devices/hanvon/driver.py:114 @@ -42,8 +42,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/prst1/driver.py:469 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:480 #: /home/kovid/work/calibre/src/calibre/ebooks/chm/metadata.py:57 -#: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:109 -#: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:112 +#: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:183 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/comic_input.py:189 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/fb2_input.py:99 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/fb2_input.py:101 @@ -106,10 +105,10 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/sources/ozon.py:130 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/sources/worker.py:26 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/txt.py:18 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:27 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:95 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:154 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:193 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:28 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:98 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:156 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:195 #: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/mobi6.py:618 #: /home/kovid/work/calibre/src/calibre/ebooks/mobi/utils.py:316 #: /home/kovid/work/calibre/src/calibre/ebooks/mobi/writer2/indexer.py:463 @@ -155,11 +154,11 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/email.py:193 #: /home/kovid/work/calibre/src/calibre/gui2/email.py:208 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:439 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1103 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1319 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1322 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1325 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1413 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1104 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1320 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1323 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1326 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1414 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/basic_widgets.py:85 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/basic_widgets.py:250 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/basic_widgets.py:261 @@ -884,7 +883,7 @@ msgstr "" msgid "Path to library too long. Must be less than %d characters." msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/cache.py:134 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:167 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/book/base.py:666 #: /home/kovid/work/calibre/src/calibre/gui2/custom_column_widgets.py:67 #: /home/kovid/work/calibre/src/calibre/gui2/custom_column_widgets.py:678 @@ -894,23 +893,88 @@ msgstr "" msgid "Yes" msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/fields.py:163 +#: /home/kovid/work/calibre/src/calibre/db/fields.py:186 #: /home/kovid/work/calibre/src/calibre/library/database2.py:1218 msgid "Main" msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/fields.py:165 +#: /home/kovid/work/calibre/src/calibre/db/fields.py:188 #: /home/kovid/work/calibre/src/calibre/gui2/layout.py:77 #: /home/kovid/work/calibre/src/calibre/library/database2.py:1220 msgid "Card A" msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/fields.py:167 +#: /home/kovid/work/calibre/src/calibre/db/fields.py:190 #: /home/kovid/work/calibre/src/calibre/gui2/layout.py:79 #: /home/kovid/work/calibre/src/calibre/library/database2.py:1222 msgid "Card B" msgstr "" +#: /home/kovid/work/calibre/src/calibre/db/search.py:33 +#: /home/kovid/work/calibre/src/calibre/db/search.py:313 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:135 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:577 +msgid "checked" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:33 +#: /home/kovid/work/calibre/src/calibre/db/search.py:311 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:135 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:575 +#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 +msgid "yes" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:35 +#: /home/kovid/work/calibre/src/calibre/db/search.py:310 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:137 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:574 +#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 +msgid "no" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:35 +#: /home/kovid/work/calibre/src/calibre/db/search.py:312 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:137 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:576 +msgid "unchecked" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:110 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:313 +msgid "today" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:111 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:314 +msgid "yesterday" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:112 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:315 +msgid "thismonth" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:113 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:316 +msgid "daysago" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:314 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:578 +msgid "empty" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:315 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:579 +msgid "blank" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:324 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:591 +msgid "Invalid boolean query \"{0}\"" +msgstr "" + #: /home/kovid/work/calibre/src/calibre/debug.py:70 #: /home/kovid/work/calibre/src/calibre/gui2/main.py:47 msgid "Cause a running calibre instance, if any, to be shutdown. Note that if there are running jobs, they will be silently aborted, so use with care." @@ -1123,8 +1187,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/bambook/driver.py:268 #: /home/kovid/work/calibre/src/calibre/devices/bambook/driver.py:324 #: /home/kovid/work/calibre/src/calibre/devices/mtp/driver.py:391 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1134 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1136 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1128 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1130 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:277 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:279 msgid "Transferring books to device..." @@ -1135,8 +1199,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:491 #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:525 #: /home/kovid/work/calibre/src/calibre/devices/mtp/driver.py:430 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1147 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1158 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1141 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1152 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:301 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:332 msgid "Adding books to device metadata listing..." @@ -1158,8 +1222,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/bambook/driver.py:374 #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:479 #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:486 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1190 #: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1196 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1202 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:366 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:371 msgid "Removing books from device metadata listing..." @@ -1668,7 +1732,7 @@ msgid "Communicate with MTP devices" msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/mtp/driver.py:167 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:950 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:952 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:95 msgid "Get device information..." msgstr "" @@ -1967,17 +2031,17 @@ msgstr "" msgid "Too many connection attempts from %s" msgstr "" -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1312 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1306 #, python-format msgid "Invalid port in options: %s" msgstr "" -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1320 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1314 #, python-format msgid "Failed to connect to port %d. Try a different value." msgstr "" -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1332 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1326 msgid "Failed to allocate a random port" msgstr "" @@ -3443,7 +3507,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/quickview.py:85 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/template_dialog.py:222 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:83 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1108 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1109 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:150 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/metadata_sources.py:162 #: /home/kovid/work/calibre/src/calibre/gui2/store/search/models.py:39 @@ -3456,7 +3520,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/book/base.py:770 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:85 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1109 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1110 #: /home/kovid/work/calibre/src/calibre/gui2/store/stores/mobileread/models.py:23 msgid "Author(s)" msgstr "" @@ -3501,7 +3565,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/library/catalogs/epub_mobi_builder.py:982 #: /home/kovid/work/calibre/src/calibre/library/catalogs/epub_mobi_builder.py:1228 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:201 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:802 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:804 msgid "Tags" msgstr "" @@ -3742,7 +3806,7 @@ msgstr "" msgid "Downloads metadata and covers from OZON.ru" msgstr "" -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:58 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:61 msgid "Sample Book" msgstr "" @@ -3778,7 +3842,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1281 #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/transforms/htmltoc.py:15 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:221 -#: /home/kovid/work/calibre/src/calibre/gui2/viewer/toc.py:217 +#: /home/kovid/work/calibre/src/calibre/gui2/viewer/toc.py:219 msgid "Table of Contents" msgstr "" @@ -3863,7 +3927,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/preferences/create_custom_column.py:71 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/metadata_sources.py:160 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:176 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:800 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:802 msgid "Rating" msgstr "" @@ -4985,8 +5049,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/actions/edit_metadata.py:101 #: /home/kovid/work/calibre/src/calibre/gui2/dnd.py:84 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:518 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:830 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:527 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:839 msgid "Download failed" msgstr "" @@ -5018,7 +5082,7 @@ msgid "Download complete" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/actions/edit_metadata.py:123 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:892 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:901 msgid "Download log" msgstr "" @@ -5249,7 +5313,7 @@ msgid "Click the show details button to see which ones." msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/actions/show_book_details.py:16 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:807 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:809 msgid "Show book details" msgstr "" @@ -5799,7 +5863,7 @@ msgid "Click to open" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/book_details.py:180 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:856 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:858 msgid "Ids" msgstr "" @@ -5809,7 +5873,7 @@ msgid "Book %(sidx)s of %(series)s" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/book_details.py:233 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1112 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1113 msgid "Collections" msgstr "" @@ -8315,7 +8379,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/device_drivers/mtp_config.py:421 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:141 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:885 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:894 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/tweaks.py:344 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:227 msgid "Copy to clipboard" @@ -8815,7 +8879,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/delete_matching_from_device.py:77 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:87 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1110 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1111 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/create_custom_column.py:35 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/create_custom_column.py:76 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:365 @@ -8931,7 +8995,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/edit_authors_dialog.py:122 #: /home/kovid/work/calibre/src/calibre/gui2/lrf_renderer/main.py:160 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:527 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:536 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main.py:729 msgid "No matches found" msgstr "" @@ -9110,8 +9174,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:196 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:251 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:950 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1059 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:959 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1074 #: /home/kovid/work/calibre/src/calibre/gui2/proceed.py:48 msgid "View log" msgstr "" @@ -11539,13 +11603,13 @@ msgid "Modified" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:819 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1455 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1456 #: /home/kovid/work/calibre/src/calibre/gui2/tag_browser/model.py:335 msgid "The lookup/search name is \"{0}\"" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:825 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1457 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1458 msgid "This book's UUID is \"{0}\"" msgstr "" @@ -11574,20 +11638,20 @@ msgstr "" msgid "Could not set data, click Show Details to see why." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1107 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1108 msgid "In Library" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1111 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1112 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:355 msgid "Size" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1437 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1438 msgid "Marked for deletion" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1440 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1441 msgid "Double click to edit me

" msgstr "" @@ -11690,7 +11754,7 @@ msgid "Previous Page" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/lrf_renderer/main_ui.py:133 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:947 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:956 #: /home/kovid/work/calibre/src/calibre/gui2/store/web_store_dialog_ui.py:62 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:215 msgid "Back" @@ -12131,7 +12195,7 @@ msgid "Edit Metadata" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/metadata/single.py:63 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:940 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:949 #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:108 #: /home/kovid/work/calibre/src/calibre/web/feeds/templates.py:219 #: /home/kovid/work/calibre/src/calibre/web/feeds/templates.py:410 @@ -12288,62 +12352,62 @@ msgid "" "cover stage, and vice versa." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:292 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:301 msgid "See at" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:446 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:455 msgid "calibre is downloading metadata from: " msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:468 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:477 msgid "Please wait" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:500 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:509 msgid "Query: " msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:519 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:528 msgid "Failed to download metadata. Click Show Details to see details" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:528 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:537 msgid "Failed to find any books that match your search. Try making the search less specific. For example, use only the author's last name and a single distinctive word from the title.

To see the full log, click Show Details." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:636 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:645 msgid "Current cover" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:639 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:648 msgid "Searching..." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:800 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:809 #, python-format msgid "Downloading covers for %s, please wait..." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:831 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:840 msgid "Failed to download any covers, click \"Show details\" for details." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:837 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:846 #, python-format msgid "Could not find any covers for %s" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:839 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:848 #, python-format msgid "Found %(num)d covers of %(title)s. Pick the one you like best." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:928 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:937 msgid "Downloading metadata..." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1043 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1058 msgid "Downloading cover..." msgstr "" @@ -16693,56 +16757,6 @@ msgid "" "

Stanza should see your calibre collection automatically. If not, try adding the URL http://myhostname:8080 as a new catalog in the Stanza reader on your iPhone. Here myhostname should be the fully qualified hostname or the IP address of the computer calibre is running on." msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/caches.py:177 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:617 -msgid "checked" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:177 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:615 -#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 -msgid "yes" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:179 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:614 -#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 -msgid "no" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:179 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:616 -msgid "unchecked" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:355 -msgid "today" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:356 -msgid "yesterday" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:357 -msgid "thismonth" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:358 -msgid "daysago" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:618 -msgid "empty" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:619 -msgid "blank" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:631 -msgid "Invalid boolean query \"{0}\"" -msgstr "" - #: /home/kovid/work/calibre/src/calibre/library/catalogs/bibtex.py:36 #, python-format msgid "" @@ -17766,6 +17780,11 @@ msgstr "" msgid "creating custom column " msgstr "" +#: /home/kovid/work/calibre/src/calibre/library/database2.py:1223 +#, python-format +msgid " (%s books)" +msgstr "" + #: /home/kovid/work/calibre/src/calibre/library/database2.py:3698 #, python-format msgid "

Migrating old database to ebook library in %s

" @@ -17985,19 +18004,19 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/library/server/ajax.py:317 #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:355 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:649 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:651 msgid "All books" msgstr "" #: /home/kovid/work/calibre/src/calibre/library/server/ajax.py:318 #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:354 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:648 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:650 #: /home/kovid/work/calibre/src/calibre/library/server/opds.py:584 msgid "Newest" msgstr "" #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:65 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:518 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:520 msgid "Loading, please wait" msgstr "" @@ -18050,65 +18069,65 @@ msgstr "" msgid "Random book" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:403 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:472 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:405 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:474 msgid "Browse books by" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:408 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:410 msgid "Choose a category to browse by:" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:543 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:545 msgid "Browsing by" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:544 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:546 msgid "Up" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:684 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:686 msgid "in" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:687 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:689 msgid "Books in" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:781 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:783 msgid "Other formats" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:788 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:790 #, python-format msgid "Read %(title)s in the %(fmt)s format" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:793 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:795 msgid "Get" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:806 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:808 msgid "Details" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:808 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:810 msgid "Permalink" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:809 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:811 msgid "A permanent link to this book" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:821 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:823 msgid "This book has been deleted" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:927 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:929 msgid "in search" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:929 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:931 msgid "Matching books" msgstr "" From c64783797e1d90f704e6f5ca1af2c8d3b64130b1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 11:10:35 +0530 Subject: [PATCH 31/43] ... --- src/calibre/ebooks/oeb/base.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index 50df05ed16..e6e499236d 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -98,6 +98,9 @@ _self_closing_pat = re.compile( def close_self_closing_tags(raw): return _self_closing_pat.sub(r'<\g\g>>', raw) +def uuid_id(): + return 'u'+unicode(uuid.uuid4()) + def iterlinks(root, find_links_in_css=True): ''' Iterate over all links in a OEB Document. @@ -1528,7 +1531,7 @@ class TOC(object): if parent is None: parent = etree.Element(NCX('navMap')) for node in self.nodes: - id = node.id or unicode(uuid.uuid4()) + id = node.id or uuid_id() po = node.play_order if po == 0: po = 1 @@ -1634,10 +1637,10 @@ class PageList(object): return self.pages.remove(page) def to_ncx(self, parent=None): - plist = element(parent, NCX('pageList'), id=str(uuid.uuid4())) + plist = element(parent, NCX('pageList'), id=uuid_id()) values = dict((t, count(1)) for t in ('front', 'normal', 'special')) for page in self.pages: - id = page.id or unicode(uuid.uuid4()) + id = page.id or uuid_id() type = page.type value = str(values[type].next()) attrib = {'id': id, 'value': value, 'type': type, 'playOrder': '0'} From b79fdfe65a07b81ee4f5652a736e3f09b74cee0a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 11:11:41 +0530 Subject: [PATCH 32/43] Conversion: Replace all non-ascii characters in CSS class anmes, as they cause problems with some broken EPUB renderers. Fixes #1102587 (ODT-EPUB conversion generates invalid CSS) --- src/calibre/ebooks/oeb/transforms/flatcss.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 12fbd3b7f1..03410e1a65 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -18,7 +18,7 @@ from calibre import guess_type from calibre.ebooks.oeb.base import (XHTML, XHTML_NS, CSS_MIME, OEB_STYLES, namespace, barename, XPath) from calibre.ebooks.oeb.stylizer import Stylizer -from calibre.utils.filenames import ascii_filename +from calibre.utils.filenames import ascii_filename, ascii_text COLLAPSE = re.compile(r'[ \t\r\n\v]+') STRIPNUM = re.compile(r'[-0-9]+$') @@ -437,7 +437,7 @@ class CSSFlattener(object): items.sort() css = u';\n'.join(u'%s: %s' % (key, val) for key, val in items) classes = node.get('class', '').strip() or 'calibre' - klass = STRIPNUM.sub('', classes.split()[0].replace('_', '')) + klass = ascii_text(STRIPNUM.sub('', classes.split()[0].replace('_', ''))) if css in styles: match = styles[css] else: From d45534ccc8553389eb0a7d5a859a24d21e9b1cde Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 11:44:44 +0530 Subject: [PATCH 33/43] Conversion: Do not error out because of an error in user supplied search replace rules. See #1102647 --- src/calibre/ebooks/conversion/preprocess.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/conversion/preprocess.py b/src/calibre/ebooks/conversion/preprocess.py index 72032cb998..82cc4c0f4a 100644 --- a/src/calibre/ebooks/conversion/preprocess.py +++ b/src/calibre/ebooks/conversion/preprocess.py @@ -515,6 +515,7 @@ class HTMLPreProcessor(object): if not getattr(self.extra_opts, 'keep_ligatures', False): html = _ligpat.sub(lambda m:LIGATURES[m.group()], html) + user_sr_rules = {} # Function for processing search and replace def do_search_replace(search_pattern, replace_txt): try: @@ -522,6 +523,7 @@ class HTMLPreProcessor(object): if not replace_txt: replace_txt = '' rules.insert(0, (search_re, replace_txt)) + user_sr_rules[(search_re, replace_txt)] = search_pattern except Exception as e: self.log.error('Failed to parse %r regexp because %s' % (search, as_unicode(e))) @@ -587,7 +589,16 @@ class HTMLPreProcessor(object): #dump(html, 'pre-preprocess') for rule in rules + end_rules: - html = rule[0].sub(rule[1], html) + try: + html = rule[0].sub(rule[1], html) + except re.error as e: + if rule in user_sr_rules: + self.log.error( + 'User supplied search & replace rule: %s -> %s ' + 'failed with error: %s, ignoring.'%( + user_sr_rules[rule], rule[1], e)) + else: + raise if is_pdftohtml and length > -1: # Dehyphenate From c1729160a34acf453f9439683dcfdf587f0aad4b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 11:52:23 +0530 Subject: [PATCH 34/43] PDF Output: Dont error out for open type fonts without OS/2 tables --- src/calibre/utils/fonts/sfnt/metrics.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/fonts/sfnt/metrics.py b/src/calibre/utils/fonts/sfnt/metrics.py index 4f86948ff2..4843893fc3 100644 --- a/src/calibre/utils/fonts/sfnt/metrics.py +++ b/src/calibre/utils/fonts/sfnt/metrics.py @@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en' from future_builtins import map from calibre.utils.fonts.utils import get_all_font_names +from calibre.utils.fonts.sfnt.container import UnsupportedFont class FontMetrics(object): @@ -31,7 +32,10 @@ class FontMetrics(object): self._advance_widths = hhea.advance_widths self.cmap = self.sfnt[b'cmap'] self.units_per_em = self.head.units_per_em - self.os2 = self.sfnt[b'OS/2'] + try: + self.os2 = self.sfnt[b'OS/2'] + except KeyError: + raise UnsupportedFont('This font has no OS/2 table') self.os2.read_data() self.post = self.sfnt[b'post'] self.post.read_data() From 4cf17349af12df50c3f83dd50b6daf8c2237b681 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 22:26:23 +0530 Subject: [PATCH 35/43] Fix #1102403 (Private bug) --- src/calibre/utils/fonts/sfnt/metrics.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/calibre/utils/fonts/sfnt/metrics.py b/src/calibre/utils/fonts/sfnt/metrics.py index 4843893fc3..6f8deff31b 100644 --- a/src/calibre/utils/fonts/sfnt/metrics.py +++ b/src/calibre/utils/fonts/sfnt/metrics.py @@ -20,6 +20,10 @@ class FontMetrics(object): ''' def __init__(self, sfnt): + for table in (b'head', b'hhea', b'hmtx', b'cmap', b'OS/2', b'post', + b'name'): + if table not in sfnt: + raise UnsupportedFont('This font has no %s table'%table) self.sfnt = sfnt self.head = self.sfnt[b'head'] @@ -32,10 +36,7 @@ class FontMetrics(object): self._advance_widths = hhea.advance_widths self.cmap = self.sfnt[b'cmap'] self.units_per_em = self.head.units_per_em - try: - self.os2 = self.sfnt[b'OS/2'] - except KeyError: - raise UnsupportedFont('This font has no OS/2 table') + self.os2 = self.sfnt[b'OS/2'] self.os2.read_data() self.post = self.sfnt[b'post'] self.post.read_data() From 993fdfa34e1b72af5671ea3fbe6ae5d741f024ac Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 22:32:15 +0530 Subject: [PATCH 36/43] New download: Do not convert all downloaded images to JPG format. This fixes the problem of PNG images with transparent backgrounds being rendered with black backgrounds --- src/calibre/web/fetch/simple.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index bea45f1c8d..87f97a3395 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -10,8 +10,6 @@ UTF-8 encoding with any charset declarations removed. import sys, socket, os, urlparse, re, time, copy, urllib2, threading, traceback, imghdr from urllib import url2pathname, quote from httplib import responses -from PIL import Image -from cStringIO import StringIO from base64 import b64decode from calibre import browser, relpath, unicode_path @@ -21,6 +19,8 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag from calibre.ebooks.chardet import xml_to_unicode from calibre.utils.config import OptionParser from calibre.utils.logging import Log +from calibre.utils.magick import Image +from calibre.utils.magick.draw import identify_data class FetchError(Exception): pass @@ -374,8 +374,8 @@ class RecursiveFetcher(object): fname = ascii_filename('img'+str(c)) if isinstance(fname, unicode): fname = fname.encode('ascii', 'replace') - imgpath = os.path.join(diskpath, fname+'.jpg') - if (imghdr.what(None, data) is None and b' Date: Wed, 23 Jan 2013 08:38:35 +0530 Subject: [PATCH 37/43] ... --- src/calibre/devices/prst1/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 7291d5dbcb..72533860d4 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -195,7 +195,7 @@ class PRST1(USBMS): for i, row in enumerate(cursor): try: comp_date = int(os.path.getmtime(self.normalize_path(prefix + row[0])) * 1000); - except (OSError, IOError): + except (OSError, IOError, TypeError): # In case the db has incorrect path info continue device_date = int(row[1]); From 3875cd176ca447d7f8d411836144f27eaa3e69ba Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 12:34:10 +0530 Subject: [PATCH 38/43] News download: Add support for logging in to sites that require javascript for their logins. Fixes #1101809 (Private bug) --- recipes/barrons.recipe | 22 ++++++------ src/calibre/utils/browser.py | 4 +++ src/calibre/web/feeds/news.py | 53 +++++++++++++++++++++++++--- src/calibre/web/jsbrowser/browser.py | 10 ++++++ src/calibre/web/jsbrowser/test.py | 27 ++++++++++++++ 5 files changed, 99 insertions(+), 17 deletions(-) diff --git a/recipes/barrons.recipe b/recipes/barrons.recipe index 41ed7e26ec..58c62e20e9 100644 --- a/recipes/barrons.recipe +++ b/recipes/barrons.recipe @@ -28,6 +28,8 @@ class Barrons(BasicNewsRecipe): ## Don't grab articles more than 7 days old oldest_article = 7 + use_javascript_to_login = True + requires_version = (0, 9, 16) extra_css = ''' .datestamp{font-family:Verdana,Geneva,Kalimati,sans-serif; font-size:x-small;} @@ -40,7 +42,7 @@ class Barrons(BasicNewsRecipe): .insettipUnit{font-size: x-small;} ''' remove_tags = [ - dict(name ='div', attrs={'class':['tabContainer artTabbedNav','rssToolBox hidden','articleToolbox']}), + dict(name ='div', attrs={'class':['sTools sTools-t', 'tabContainer artTabbedNav','rssToolBox hidden','articleToolbox']}), dict(name = 'a', attrs ={'class':'insetClose'}) ] @@ -60,21 +62,17 @@ class Barrons(BasicNewsRecipe): ] ] - def get_browser(self): - br = BasicNewsRecipe.get_browser() - if self.username is not None and self.password is not None: - br.open('http://commerce.barrons.com/auth/login') - br.select_form(nr=0) - br['username'] = self.username - br['password'] = self.password - br.submit() - return br + def javascript_login(self, br, username, password): + br.visit('http://commerce.barrons.com/auth/login') + f = br.select_form(nr=0) + f['username'] = username + f['password'] = password + br.submit(timeout=120) ## Use the print version of a page when available. - def print_version(self, url): main, sep, rest = url.rpartition('?') - return main + '#printmode' + return main + '#text.print' def postprocess_html(self, soup, first): diff --git a/src/calibre/utils/browser.py b/src/calibre/utils/browser.py index de21158ed7..fc04044ad3 100644 --- a/src/calibre/utils/browser.py +++ b/src/calibre/utils/browser.py @@ -32,6 +32,10 @@ class Browser(B): B.set_cookiejar(self, *args, **kwargs) self._clone_actions['set_cookiejar'] = ('set_cookiejar', args, kwargs) + def copy_cookies_from_jsbrowser(self, jsbrowser): + for cookie in jsbrowser.cookies: + self.cookiejar.set_cookie(cookie) + @property def cookiejar(self): return self._clone_actions['set_cookiejar'][1][0] diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 14834ff88c..22901f3ccc 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -332,6 +332,12 @@ class BasicNewsRecipe(Recipe): #: ignore_duplicate_articles = {'title', 'url'} ignore_duplicate_articles = None + #: If you set this True, then calibre will use javascript to login to the + #: website. This is needed for some websites that require the use of + #: javascript to login. If you set this to True you must implement the + #: :meth:`javascript_login` method, to do the actual logging in. + use_javascript_to_login = False + # See the built-in profiles for examples of these settings. def short_title(self): @@ -404,8 +410,7 @@ class BasicNewsRecipe(Recipe): ''' return url - @classmethod - def get_browser(cls, *args, **kwargs): + def get_browser(self, *args, **kwargs): ''' Return a browser instance used to fetch documents from the web. By default it returns a `mechanize `_ @@ -427,9 +432,47 @@ class BasicNewsRecipe(Recipe): return br ''' - br = browser(*args, **kwargs) - br.addheaders += [('Accept', '*/*')] - return br + if self.use_javascript_to_login: + if getattr(self, 'browser', None) is not None: + return self.clone_browser(self.browser) + from calibre.web.jsbrowser.browser import Browser + br = Browser() + with br: + self.javascript_login(br, self.username, self.password) + kwargs['user_agent'] = br.user_agent + ans = browser(*args, **kwargs) + ans.copy_cookies_from_jsbrowser(br) + return ans + else: + br = browser(*args, **kwargs) + br.addheaders += [('Accept', '*/*')] + return br + + def javascript_login(self, browser, username, password): + ''' + This method is used to login to a website that uses javascript for its + login form. After the login is complete, the cookies returned from the + website are copied to a normal (non-javascript) browser and the + download proceeds using those cookies. + + An example implementation:: + + def javascript_login(self, browser, username, password): + browser.visit('http://some-page-that-has-a-login') + form = browser.select_form(nr=0) # Select the first form on the page + form['username'] = username + form['password'] = password + browser.submit(timeout=120) # Submit the form and wait at most two minutes for loading to complete + + Note that you can also select forms with CSS2 selectors, like this:: + + browser.select_form('form#login_form') + browser.select_from('form[name="someform"]') + + ''' + raise NotImplementedError('You must implement the javascript_login()' + ' method if you set use_javascript_to_login' + ' to True') def clone_browser(self, br): ''' diff --git a/src/calibre/web/jsbrowser/browser.py b/src/calibre/web/jsbrowser/browser.py index dd87b000a7..d8f0e79bc4 100644 --- a/src/calibre/web/jsbrowser/browser.py +++ b/src/calibre/web/jsbrowser/browser.py @@ -303,6 +303,10 @@ class Browser(QObject, FormsMixin): self.nam = NetworkAccessManager(log, use_disk_cache=use_disk_cache, parent=self) self.page.setNetworkAccessManager(self.nam) + @property + def user_agent(self): + return self.page.user_agent + def _wait_for_load(self, timeout, url=None): loop = QEventLoop(self) start_time = time.time() @@ -422,3 +426,9 @@ class Browser(QObject, FormsMixin): pass self.nam = self.page = None + def __enter__(self): + pass + + def __exit__(self, *args): + self.close() + diff --git a/src/calibre/web/jsbrowser/test.py b/src/calibre/web/jsbrowser/test.py index 8527f3ec92..6f18d7b850 100644 --- a/src/calibre/web/jsbrowser/test.py +++ b/src/calibre/web/jsbrowser/test.py @@ -11,6 +11,7 @@ import unittest, pprint, threading, time import cherrypy +from calibre import browser from calibre.web.jsbrowser.browser import Browser from calibre.library.server.utils import (cookie_max_age_to_expires, cookie_time_fmt) @@ -105,6 +106,12 @@ class Server(object): import traceback traceback.print_exc() + @cherrypy.expose + def receive_cookies(self): + self.received_cookies = {n:(c.value, dict(c)) for n, c in + dict(cherrypy.request.cookie).iteritems()} + return pprint.pformat(self.received_cookies) + class Test(unittest.TestCase): @classmethod @@ -202,6 +209,26 @@ class Test(unittest.TestCase): if fexp: self.assertEqual(fexp, cexp) + def test_cookie_copy(self): + 'Test copying of cookies from jsbrowser to mechanize' + self.assertEqual(self.browser.visit('http://127.0.0.1:%d/cookies'%self.port), + True) + sent_cookies = self.server.sent_cookies.copy() + self.browser.visit('http://127.0.0.1:%d/receive_cookies'%self.port) + orig_rc = self.server.received_cookies.copy() + br = browser(user_agent=self.browser.user_agent) + br.copy_cookies_from_jsbrowser(self.browser) + br.open('http://127.0.0.1:%d/receive_cookies'%self.port) + for name, vals in sent_cookies.iteritems(): + val = vals[0] + try: + rval = self.server.received_cookies[name][0] + except: + self.fail('The cookie: %s was not received by the server') + self.assertEqual(val, rval, + 'The received value for the cookie: %s, %s != %s'%( + name, rval, val)) + self.assertEqual(orig_rc, self.server.received_cookies) def tests(): return unittest.TestLoader().loadTestsFromTestCase(Test) From 9b6613c192e200e906bc510101d8a01bb69a20d5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 12:37:45 +0530 Subject: [PATCH 39/43] ... --- src/calibre/ebooks/metadata/sources/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 46c6f7a313..e00c2e78d3 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -200,7 +200,7 @@ class Source(Plugin): #: during the identify phase touched_fields = frozenset() - #: Set this to True if your plugin return HTML formatted comments + #: Set this to True if your plugin returns HTML formatted comments has_html_comments = False #: Setting this to True means that the browser object will add From 0ce501fa0d059e6daaf97dab324183f6fc9abce8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 13:39:42 +0530 Subject: [PATCH 40/43] Store caches outside the config directory for non-portable calibre installs --- src/calibre/constants.py | 36 +++++++++++++++++++ src/calibre/devices/apple/driver.py | 6 ++-- .../library/catalogs/epub_mobi_builder.py | 5 ++- src/calibre/web/jsbrowser/browser.py | 7 ++-- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 40d39b0ad4..dff477bbc4 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -79,6 +79,42 @@ def debug(): global DEBUG DEBUG = True +_cache_dir = None + +def _get_cache_dir(): + confcache = os.path.join(config_dir, u'caches') + if isportable: + return confcache + if os.environ.has_key('CALIBRE_CACHE_DIRECTORY'): + return os.path.abspath(os.environ['CALIBRE_CACHE_DIRECTORY']) + + if iswindows: + w = plugins['winutil'][0] + candidate = os.path.join(w.special_folder_path(w.CSIDL_LOCAL_APPDATA), u'%s-cache'%__appname__) + elif isosx: + candidate = os.path.join(os.path.expanduser(u'~/Library/Caches'), __appname__) + else: + candidate = os.environ.get('XDG_CACHE_HOME', u'~/.cache') + candidate = os.path.join(os.path.expanduser(candidate), + __appname__) + if isinstance(candidate, bytes): + try: + candidate = candidate.decode(filesystem_encoding) + except ValueError: + candidate = confcache + if not os.path.exists(candidate): + try: + os.makedirs(candidate) + except: + candidate = confcache + return candidate + +def cache_dir(): + global _cache_dir + if _cache_dir is None: + _cache_dir = _get_cache_dir() + return _cache_dir + # plugins {{{ class Plugins(collections.Mapping): diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index eacb143790..95dedf546a 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' import cStringIO, ctypes, datetime, os, platform, re, shutil, sys, tempfile, time -from calibre.constants import __appname__, __version__, DEBUG +from calibre.constants import __appname__, __version__, DEBUG, cache_dir from calibre import fit_image, confirm_config_name, strftime as _strftime from calibre.constants import isosx, iswindows from calibre.devices.errors import OpenFeedback, UserFeedback @@ -289,9 +289,7 @@ class ITUNES(DriverBase): # Properties cached_books = {} - cache_dir = os.path.join(config_dir, 'caches', 'itunes') calibre_library_path = prefs['library_path'] - archive_path = os.path.join(cache_dir, "thumbs.zip") description_prefix = "added by calibre" ejected = False iTunes = None @@ -884,6 +882,8 @@ class ITUNES(DriverBase): logger().info(" BCD: %s" % ['0x%x' % x for x in sorted(self.BCD)]) logger().info(" PRODUCT_ID: %s" % ['0x%x' % x for x in sorted(self.PRODUCT_ID)]) + self.cache_dir = os.path.join(cache_dir(), 'itunes') + self.archive_path = os.path.join(self.cache_dir, "thumbs.zip") # Confirm/create thumbs archive if not os.path.exists(self.cache_dir): if DEBUG: diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index 8f27db61be..9f946e2ee0 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -9,7 +9,7 @@ from xml.sax.saxutils import escape from calibre import (prepare_string_for_xml, strftime, force_unicode, isbytestring) -from calibre.constants import isosx +from calibre.constants import isosx, cache_dir from calibre.customize.conversion import DummyReporter from calibre.customize.ui import output_profiles from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString @@ -18,7 +18,6 @@ from calibre.ebooks.metadata import author_to_author_sort from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException, \ InvalidGenresSourceFieldException from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.utils.config import config_dir from calibre.utils.date import format_date, is_date_undefined, now as nowf from calibre.utils.filenames import ascii_text, shorten_components_to from calibre.utils.icu import capitalize, collation_order, sort_key @@ -109,7 +108,7 @@ class CatalogBuilder(object): self.plugin = plugin self.reporter = report_progress self.stylesheet = stylesheet - self.cache_dir = os.path.join(config_dir, 'caches', 'catalog') + self.cache_dir = os.path.join(cache_dir(), 'catalog') self.catalog_path = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') self.content_dir = os.path.join(self.catalog_path, "content") self.excluded_tags = self.get_excluded_tags() diff --git a/src/calibre/web/jsbrowser/browser.py b/src/calibre/web/jsbrowser/browser.py index d8f0e79bc4..c22d912128 100644 --- a/src/calibre/web/jsbrowser/browser.py +++ b/src/calibre/web/jsbrowser/browser.py @@ -16,7 +16,7 @@ from PyQt4.Qt import (QObject, QNetworkAccessManager, QNetworkDiskCache, from PyQt4.QtWebKit import QWebPage, QWebSettings, QWebView, QWebElement from calibre import USER_AGENT, prints, get_proxies, get_proxy_info -from calibre.constants import ispy3, config_dir +from calibre.constants import ispy3, cache_dir from calibre.utils.logging import ThreadSafeLog from calibre.gui2 import must_use_qt from calibre.web.jsbrowser.forms import FormsMixin @@ -44,7 +44,7 @@ class WebPage(QWebPage): # {{{ settings = self.settings() if enable_developer_tools: settings.setAttribute(QWebSettings.DeveloperExtrasEnabled, True) - QWebSettings.enablePersistentStorage(os.path.join(config_dir, 'caches', + QWebSettings.enablePersistentStorage(os.path.join(cache_dir(), 'webkit-persistence')) QWebSettings.setMaximumPagesInCache(0) @@ -135,8 +135,7 @@ class NetworkAccessManager(QNetworkAccessManager): # {{{ self.log = log if use_disk_cache: self.cache = QNetworkDiskCache(self) - self.cache.setCacheDirectory(os.path.join(config_dir, 'caches', - 'jsbrowser')) + self.cache.setCacheDirectory(os.path.join(cache_dir(), 'jsbrowser')) self.setCache(self.cache) self.sslErrors.connect(self.on_ssl_errors) self.pf = ProxyFactory(log) From d774d54c2f880d9a14dd75e328265972b1040d4f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 14:01:35 +0530 Subject: [PATCH 41/43] Move the mobileread get books cache into the cache directory --- .../store/stores/mobileread/mobileread_plugin.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py b/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py index 9e41aa45a1..942f345820 100644 --- a/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py +++ b/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py @@ -6,10 +6,12 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import os from threading import Lock from PyQt4.Qt import (QUrl, QCoreApplication) +from calibre.constants import cache_dir from calibre.gui2 import open_url from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig @@ -19,12 +21,20 @@ from calibre.gui2.store.stores.mobileread.models import SearchFilter from calibre.gui2.store.stores.mobileread.cache_progress_dialog import CacheProgressDialog from calibre.gui2.store.stores.mobileread.cache_update_thread import CacheUpdateThread from calibre.gui2.store.stores.mobileread.store_dialog import MobileReadStoreDialog +from calibre.utils.config import JSONConfig + +class Cache(JSONConfig): + + def __init__(self): + JSONConfig.__init__(self, 'mobileread_store') + self.file_path = os.path.join(cache_dir(), 'mobileread_get_books.json') class MobileReadStore(BasicStoreConfig, StorePlugin): def __init__(self, *args, **kwargs): StorePlugin.__init__(self, *args, **kwargs) self.lock = Lock() + self.cache = Cache() def open(self, parent=None, detail_item=None, external=False): url = 'http://www.mobileread.com/' @@ -61,7 +71,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): suppress_progress=False): if self.lock.acquire(False): try: - update_thread = CacheUpdateThread(self.config, self.seralize_books, timeout) + update_thread = CacheUpdateThread(self.cache, self.seralize_books, timeout) if not suppress_progress: progress = CacheProgressDialog(parent) progress.set_message(_('Updating MobileRead book cache...')) @@ -85,7 +95,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): self.lock.release() def get_book_list(self): - return self.deseralize_books(self.config.get('book_list', [])) + return self.deseralize_books(self.cache.get('book_list', [])) def seralize_books(self, books): sbooks = [] From 7e2ad6914db15eb7ffd829019be66e2a43ef4035 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 14:08:01 +0530 Subject: [PATCH 42/43] Delay load MR cache --- .../stores/mobileread/mobileread_plugin.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py b/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py index 942f345820..bf1b2013dd 100644 --- a/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py +++ b/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py @@ -21,20 +21,22 @@ from calibre.gui2.store.stores.mobileread.models import SearchFilter from calibre.gui2.store.stores.mobileread.cache_progress_dialog import CacheProgressDialog from calibre.gui2.store.stores.mobileread.cache_update_thread import CacheUpdateThread from calibre.gui2.store.stores.mobileread.store_dialog import MobileReadStoreDialog -from calibre.utils.config import JSONConfig - -class Cache(JSONConfig): - - def __init__(self): - JSONConfig.__init__(self, 'mobileread_store') - self.file_path = os.path.join(cache_dir(), 'mobileread_get_books.json') class MobileReadStore(BasicStoreConfig, StorePlugin): def __init__(self, *args, **kwargs): StorePlugin.__init__(self, *args, **kwargs) self.lock = Lock() - self.cache = Cache() + + @property + def cache(self): + if not hasattr(self, '_mr_cache'): + from calibre.utils.config import JSONConfig + self._mr_cache = JSONConfig('mobileread_get_books') + self._mr_cache.file_path = os.path.join(cache_dir(), + 'mobileread_get_books.json') + self._mr_cache.refresh() + return self._mr_cache def open(self, parent=None, detail_item=None, external=False): url = 'http://www.mobileread.com/' From 276939c39b4105c98a3ae488d48605d0dd26d3b7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 22:12:50 +0530 Subject: [PATCH 43/43] Fix #1103504 (Crashed by column color tab) --- src/calibre/gui2/preferences/coloring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/preferences/coloring.py b/src/calibre/gui2/preferences/coloring.py index 4b867f347d..72fe4fb12b 100644 --- a/src/calibre/gui2/preferences/coloring.py +++ b/src/calibre/gui2/preferences/coloring.py @@ -413,6 +413,7 @@ class RulesModel(QAbstractListModel): # {{{ rules = list(prefs['column_color_rules']) self.rules = [] for col, template in rules: + if col not in self.fm: continue try: rule = rule_from_template(self.fm, template) except: