From 23307b3aa50a2e5ebc68ec2a02e5af4258745a6e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 30 Jun 2011 18:09:18 +0100 Subject: [PATCH 01/28] Add format_metadata to get_metadata using a cache. Add formatter functions to deal with the information. --- src/calibre/library/database2.py | 17 +++++++++-- src/calibre/utils/formatter_functions.py | 37 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 3ebd63afde..530e5d8adf 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -8,6 +8,7 @@ The database used to store ebook metadata ''' import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \ json, uuid, tempfile, hashlib +from collections import defaultdict import threading, random from itertools import repeat from math import ceil @@ -487,6 +488,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) self.refresh() self.last_update_check = self.last_modified() + self.format_metadata_cache = defaultdict(dict) def break_cycles(self): self.data.break_cycles() @@ -914,11 +916,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.book_size = row[fm['size']] mi.ondevice_col= row[fm['ondevice']] mi.last_modified = row[fm['last_modified']] + id = idx if index_is_id else self.id(idx) formats = row[fm['formats']] + mi.format_metadata = {} if not formats: formats = None else: formats = formats.split(',') + for f in formats: + mi.format_metadata[f] = self.format_metadata(id, f, allow_cache=True) mi.formats = formats tags = row[fm['tags']] if tags: @@ -927,7 +933,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if mi.series: mi.series_index = row[fm['series_index']] mi.rating = row[fm['rating']] - id = idx if index_is_id else self.id(idx) mi.set_identifiers(self.get_identifiers(id, index_is_id=True)) mi.application_id = id mi.id = id @@ -1126,13 +1131,16 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if m: return m['mtime'] - def format_metadata(self, id_, fmt): + def format_metadata(self, id_, fmt, allow_cache=True): + if allow_cache and fmt in self.format_metadata_cache.get(id_, {}): + return self.format_metadata_cache[id_][fmt] path = self.format_abspath(id_, fmt, index_is_id=True) ans = {} if path is not None: stat = os.stat(path) ans['size'] = stat.st_size ans['mtime'] = utcfromtimestamp(stat.st_mtime) + self.format_metadata_cache[id_][fmt] = ans return ans def format_hash(self, id_, fmt): @@ -1254,6 +1262,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ret.name = f.name else: ret = f.read() + try: + self.format_metadata(index if index_is_id else self.id(index), + format, allow_cache=False) + except: + traceback.print_exc() return ret def add_format_with_hooks(self, index, format, fpath, index_is_id=False, diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 1684b9f85b..f3d8370895 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -519,6 +519,41 @@ class BuiltinSelect(BuiltinFormatterFunction): return v[len(key)+1:] return '' +class BuiltinFormatsModtimes(BuiltinFormatterFunction): + name = 'formats_modtimes' + arg_count = 0 + category = 'Get values from metadata' + __doc__ = doc = _('formats_modtimes() -- return a comma-separated list of ' + 'colon_separated items representing modification times ' + 'for the formats of a book. You can use the select ' + 'function to get the mod time for a specific ' + 'format. Note that format names are always uppercase, ' + 'as in EPUB.' + ) + + def evaluate(self, formatter, kwargs, mi, locals): + fmt_data = mi.get('format_metadata', {}) + print fmt_data + return ','.join(k.upper()+':'+format_date(v['mtime'], 'iso') + for k,v in fmt_data.iteritems()) + +class BuiltinFormatsSizes(BuiltinFormatterFunction): + name = 'formats_sizes' + arg_count = 0 + category = 'Get values from metadata' + __doc__ = doc = _('formats_sizes() -- return a comma-separated list of ' + 'colon_separated items representing sizes ' + 'of the formats of a book. You can use the select ' + 'function to get the size for a specific ' + 'format. Note that format names are always uppercase, ' + 'as in EPUB.' + ) + + def evaluate(self, formatter, kwargs, mi, locals): + fmt_data = mi.get('format_metadata', {}) + print fmt_data + return ','.join(k.upper()+':'+str(v['size']) for k,v in fmt_data.iteritems()) + class BuiltinSublist(BuiltinFormatterFunction): name = 'sublist' arg_count = 4 @@ -814,6 +849,8 @@ builtin_eval = BuiltinEval() builtin_first_non_empty = BuiltinFirstNonEmpty() builtin_field = BuiltinField() builtin_format_date = BuiltinFormatDate() +builtin_formats_modt= BuiltinFormatsModtimes() +builtin_formats_size= BuiltinFormatsSizes() builtin_identifier_in_list = BuiltinIdentifierInList() builtin_ifempty = BuiltinIfempty() builtin_in_list = BuiltinInList() From 137093ebb9121b7a0f5bb8864b4c57b5ad68d9d5 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 30 Jun 2011 19:24:18 +0100 Subject: [PATCH 02/28] More changes for formats_ and numeric sorting --- src/calibre/library/caches.py | 9 +++- src/calibre/utils/formatter_functions.py | 66 +++++++++++++++++++----- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index b9dd2f3ed7..dad9ce0bae 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -1024,7 +1024,14 @@ class SortKeyGenerator(object): dt = 'datetime' elif sb == 'number': try: - val = float(val) + val = val.replace(',', '').strip() + p = 1 + for i, candidate in enumerate( + (' B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')): + if val.endswith(candidate): + p = 1024**(i) + val = val[:-len(candidate)] + val = float(val) * p except: val = 0.0 dt = 'float' diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index f3d8370895..f484c617b9 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en' import inspect, re, traceback +from calibre import human_readable from calibre.utils.titlecase import titlecase from calibre.utils.icu import capitalize, strcmp, sort_key from calibre.utils.date import parse_date, format_date, now, UNDEFINED_DATE @@ -521,20 +522,21 @@ class BuiltinSelect(BuiltinFormatterFunction): class BuiltinFormatsModtimes(BuiltinFormatterFunction): name = 'formats_modtimes' - arg_count = 0 + arg_count = 1 category = 'Get values from metadata' - __doc__ = doc = _('formats_modtimes() -- return a comma-separated list of ' - 'colon_separated items representing modification times ' - 'for the formats of a book. You can use the select ' - 'function to get the mod time for a specific ' - 'format. Note that format names are always uppercase, ' - 'as in EPUB.' + __doc__ = doc = _('formats_modtimes(date_format) -- return a comma-separated ' + 'list of colon_separated items representing modification times ' + 'for the formats of a book. The date_format parameter ' + 'specifies how the date is to be formatted. See the ' + 'date_format function for details. You can use the select ' + 'function to get the mod time for a specific ' + 'format. Note that format names are always uppercase, ' + 'as in EPUB.' ) - def evaluate(self, formatter, kwargs, mi, locals): + def evaluate(self, formatter, kwargs, mi, locals, fmt): fmt_data = mi.get('format_metadata', {}) - print fmt_data - return ','.join(k.upper()+':'+format_date(v['mtime'], 'iso') + return ','.join(k.upper()+':'+format_date(v['mtime'], fmt) for k,v in fmt_data.iteritems()) class BuiltinFormatsSizes(BuiltinFormatterFunction): @@ -551,9 +553,47 @@ class BuiltinFormatsSizes(BuiltinFormatterFunction): def evaluate(self, formatter, kwargs, mi, locals): fmt_data = mi.get('format_metadata', {}) - print fmt_data return ','.join(k.upper()+':'+str(v['size']) for k,v in fmt_data.iteritems()) +class BuiltinHumanReadable(BuiltinFormatterFunction): + name = 'human_readable' + arg_count = 1 + category = 'Formatting values' + __doc__ = doc = _('human_readable(v) -- return a string ' + 'representing the number v in KB, MB, GB, etc.' + ) + + def evaluate(self, formatter, kwargs, mi, locals, val): + try: + return human_readable(long(val)) + except: + return '' + +class BuiltinFormatNumber(BuiltinFormatterFunction): + name = 'format_number' + arg_count = 2 + category = 'Formatting values' + __doc__ = doc = _('format_number(v, template) -- format the number v using ' + 'a python formatting template such as "{0:5.2f}" or ' + '"{0:,d}" or "${0:5,.2f}". The field_name part of the ' + 'template must be a 0 (zero), as shown in the examples. See ' + 'the template language and python documentation for more ' + 'examples. Returns the empty string if formatting fails.' + ) + + def evaluate(self, formatter, kwargs, mi, locals, val, template): + if val == '' or val == 'None': + return '' + try: + return template.format(float(val)) + except: + pass + try: + return template.format(int(val)) + except: + pass + return '' + class BuiltinSublist(BuiltinFormatterFunction): name = 'sublist' arg_count = 4 @@ -626,7 +666,7 @@ class BuiltinSubitems(BuiltinFormatterFunction): class BuiltinFormatDate(BuiltinFormatterFunction): name = 'format_date' arg_count = 2 - category = 'Date functions' + category = 'Formatting values' __doc__ = doc = _('format_date(val, format_string) -- format the value, ' 'which must be a date, using the format_string, returning a string. ' 'The formatting codes are: ' @@ -849,8 +889,10 @@ builtin_eval = BuiltinEval() builtin_first_non_empty = BuiltinFirstNonEmpty() builtin_field = BuiltinField() builtin_format_date = BuiltinFormatDate() +builtin_format_numb = BuiltinFormatNumber() builtin_formats_modt= BuiltinFormatsModtimes() builtin_formats_size= BuiltinFormatsSizes() +builtin_human_rable = BuiltinHumanReadable() builtin_identifier_in_list = BuiltinIdentifierInList() builtin_ifempty = BuiltinIfempty() builtin_in_list = BuiltinInList() From d967da0b30b04b7c352eb9210c805dbcad606e37 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 1 Jul 2011 11:07:46 +0100 Subject: [PATCH 03/28] Clear the format metadata cache for a book when a format is deleted. --- src/calibre/library/database2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 530e5d8adf..23642bcec7 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1333,6 +1333,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def remove_format(self, index, format, index_is_id=False, notify=True, commit=True, db_only=False): id = index if index_is_id else self.id(index) + del self.format_metadata_cache[id] name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False) if name: if not db_only: From 13d17b1f11b195dc7648dd5dc75a98dcf5235a0c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 1 Jul 2011 11:22:52 +0100 Subject: [PATCH 04/28] Documentation of the new formatter functions --- src/calibre/manual/template_lang.rst | 4 ++++ src/calibre/utils/formatter_functions.py | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst index f9824187e5..52bf095d01 100644 --- a/src/calibre/manual/template_lang.rst +++ b/src/calibre/manual/template_lang.rst @@ -124,6 +124,8 @@ The functions available are listed below. Note that the definitive documentation * ``capitalize()`` -- return the value with the first letter upper case and the rest lower case. * ``contains(pattern, text if match, text if not match)`` -- checks if field contains matches for the regular expression `pattern`. Returns `text if match` if matches are found, otherwise it returns `text if no match`. * ``count(separator)`` -- interprets the value as a list of items separated by `separator`, returning the number of items in the list. Most lists use a comma as the separator, but authors uses an ampersand. Examples: `{tags:count(,)}`, `{authors:count(&)}` + * ``format_number(template)`` -- interprets the value as a number and format that number using a python formatting template such as "{0:5.2f}" or "{0:,d}" or "${0:5,.2f}". The field_name part of the template must be a 0 (zero) (the "{0:" in the above examples). See the template language and python documentation for more examples. Returns the empty string if formatting fails. + * ``human_readable()`` -- expects the value to be a number and returns a string representing that number in KB, MB, GB, etc. * ``ifempty(text)`` -- if the field is not empty, return the value of the field. Otherwise return `text`. * ``in_list(separator, pattern, found_val, not_found_val)`` -- interpret the field as a list of items separated by `separator`, comparing the `pattern` against each value in the list. If the pattern matches a value, return `found_val`, otherwise return `not_found_val`. * ``list_item(index, separator)`` -- interpret the field as a list of items separated by `separator`, returning the `index`th item. The first item is number zero. The last item can be returned using `list_item(-1,separator)`. If the item is not in the list, then the empty value is returned. The separator has the same meaning as in the `count` function. @@ -257,6 +259,8 @@ The following functions are available in addition to those described in single-f iso : the date with time and timezone. Must be the only format present. * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables. + * ``formats_modtimes(date_format)`` -- return a comma-separated list of colon_separated items representing modification times for the formats of a book. The date_format parameter specifies how the date is to be formatted. See the date_format function for details. You can use the select function to get the mod time for a specific format. Note that format names are always uppercase, as in EPUB. + * ``formats_sizes()`` -- return a comma-separated list of colon_separated items representing sizes in bytes of the formats of a book. You can use the select function to get the size for a specific format. Note that format names are always uppercase, as in EPUB. * ``not(value)`` -- returns the string "1" if the value is empty, otherwise returns the empty string. This function works well with test or first_non_empty. You can have as many values as you want. * ``merge_lists(list1, list2, separator)`` -- return a list made by merging the items in list1 and list2, removing duplicate items using a case-insensitive compare. If items differ in case, the one in list1 is used. The items in list1 and list2 are separated by separator, as are the items in the returned list. * ``multiply(x, y)`` -- returns x * y. Throws an exception if either x or y are not numbers. diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index f484c617b9..654e171339 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -544,7 +544,7 @@ class BuiltinFormatsSizes(BuiltinFormatterFunction): arg_count = 0 category = 'Get values from metadata' __doc__ = doc = _('formats_sizes() -- return a comma-separated list of ' - 'colon_separated items representing sizes ' + 'colon_separated items representing sizes in bytes' 'of the formats of a book. You can use the select ' 'function to get the size for a specific ' 'format. Note that format names are always uppercase, ' @@ -574,11 +574,11 @@ class BuiltinFormatNumber(BuiltinFormatterFunction): arg_count = 2 category = 'Formatting values' __doc__ = doc = _('format_number(v, template) -- format the number v using ' - 'a python formatting template such as "{0:5.2f}" or ' - '"{0:,d}" or "${0:5,.2f}". The field_name part of the ' - 'template must be a 0 (zero), as shown in the examples. See ' - 'the template language and python documentation for more ' - 'examples. Returns the empty string if formatting fails.' + 'a python formatting template such as "{0:5.2f}" or ' + '"{0:,d}" or "${0:5,.2f}". The field_name part of the ' + 'template must be a 0 (zero) (the "{0:" in the above examples). ' + 'See the template language and python documentation for more ' + 'examples. Returns the empty string if formatting fails.' ) def evaluate(self, formatter, kwargs, mi, locals, val, template): From f189ecb1932182a7c32883906a24a4c8045b3a15 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 2 Jul 2011 09:22:25 -0600 Subject: [PATCH 05/28] Szinti Derigisi by thomass --- recipes/sizinti_derigisi.recipe | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 recipes/sizinti_derigisi.recipe diff --git a/recipes/sizinti_derigisi.recipe b/recipes/sizinti_derigisi.recipe new file mode 100644 index 0000000000..d05648170e --- /dev/null +++ b/recipes/sizinti_derigisi.recipe @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +from calibre.web.feeds.news import BasicNewsRecipe + +class TodaysZaman_en(BasicNewsRecipe): + title = u'Sızıntı Dergisi' + __author__ = u'thomass' + description = 'a Turkey based daily for national and international news in the fields of business, diplomacy, politics, culture, arts, sports and economics, in addition to commentaries, specials and features' + oldest_article = 30 + max_articles_per_feed =80 + no_stylesheets = True + #delay = 1 + #use_embedded_content = False + encoding = 'utf-8' + #publisher = ' ' + category = 'dergi, ilim, kültür, bilim,Türkçe' + language = 'tr' + publication_type = 'magazine' + #extra_css = ' body{ font-family: Verdana,Helvetica,Arial,sans-serif } .introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} ' + #keep_only_tags = [dict(name='h1', attrs={'class':['georgia_30']})] + + #remove_attributes = ['aria-describedby'] + #remove_tags = [dict(name='div', attrs={'id':['renk10']}) ] + cover_img_url = 'http://www.sizinti.com.tr/images/sizintiprint.jpg' + masthead_url = 'http://www.sizinti.com.tr/images/sizintiprint.jpg' + remove_tags_before = dict(id='content-right') + + + #remove_empty_feeds= True + #remove_attributes = ['width','height'] + + feeds = [ + ( u'Sızıntı', u'http://www.sizinti.com.tr/rss'), + ] + + #def preprocess_html(self, soup): + # return self.adeify_images(soup) + #def print_version(self, url): #there is a probem caused by table format + #return url.replace('http://www.todayszaman.com/newsDetail_getNewsById.action?load=detay&', 'http://www.todayszaman.com/newsDetail_openPrintPage.action?') + From fcb7fd887e999b1f91634e7588db0559625e3f66 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 2 Jul 2011 09:58:52 -0600 Subject: [PATCH 06/28] Improved Independet and Telegrah UK --- recipes/independent.recipe | 37 ++++++++++++++++++++++++++----------- recipes/telegraph_uk.recipe | 1 + 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/recipes/independent.recipe b/recipes/independent.recipe index 2ce6b24c4f..0a94384b37 100644 --- a/recipes/independent.recipe +++ b/recipes/independent.recipe @@ -6,7 +6,7 @@ class TheIndependent(BasicNewsRecipe): language = 'en_GB' __author__ = 'Krittika Goyal' oldest_article = 1 #days - max_articles_per_feed = 25 + max_articles_per_feed = 30 encoding = 'latin1' no_stylesheets = True @@ -25,24 +25,39 @@ class TheIndependent(BasicNewsRecipe): 'http://www.independent.co.uk/news/uk/rss'), ('World', 'http://www.independent.co.uk/news/world/rss'), - ('Sport', - 'http://www.independent.co.uk/sport/rss'), - ('Arts and Entertainment', - 'http://www.independent.co.uk/arts-entertainment/rss'), ('Business', 'http://www.independent.co.uk/news/business/rss'), - ('Life and Style', - 'http://www.independent.co.uk/life-style/gadgets-and-tech/news/rss'), - ('Science', - 'http://www.independent.co.uk/news/science/rss'), ('People', 'http://www.independent.co.uk/news/people/rss'), + ('Science', + 'http://www.independent.co.uk/news/science/rss'), ('Media', 'http://www.independent.co.uk/news/media/rss'), - ('Health and Families', - 'http://www.independent.co.uk/life-style/health-and-families/rss'), + ('Education', + 'http://www.independent.co.uk/news/education/rss'), ('Obituaries', 'http://www.independent.co.uk/news/obituaries/rss'), + + ('Opinion', + 'http://www.independent.co.uk/opinion/rss'), + + ('Environment', + 'http://www.independent.co.uk/environment/rss'), + + ('Sport', + 'http://www.independent.co.uk/sport/rss'), + + ('Life and Style', + 'http://www.independent.co.uk/life-style/rss'), + + ('Arts and Entertainment', + 'http://www.independent.co.uk/arts-entertainment/rss'), + + ('Travel', + 'http://www.independent.co.uk/travel/rss'), + + ('Money', + 'http://www.independent.co.uk/money/rss'), ] def preprocess_html(self, soup): diff --git a/recipes/telegraph_uk.recipe b/recipes/telegraph_uk.recipe index 5fe5b168b8..157cfa99e9 100644 --- a/recipes/telegraph_uk.recipe +++ b/recipes/telegraph_uk.recipe @@ -56,6 +56,7 @@ class TelegraphUK(BasicNewsRecipe): ,(u'Sport' , u'http://www.telegraph.co.uk/sport/rss' ) ,(u'Earth News' , u'http://www.telegraph.co.uk/earth/earthnews/rss' ) ,(u'Comment' , u'http://www.telegraph.co.uk/comment/rss' ) + ,(u'Travel' , u'http://www.telegraph.co.uk/travel/rss' ) ,(u'How about that?', u'http://www.telegraph.co.uk/news/newstopics/howaboutthat/rss' ) ] From 92ad849bc6c971a6935921d8946e7c696a651a75 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 2 Jul 2011 17:47:48 +0100 Subject: [PATCH 07/28] Minor cleanup of the function list. Get rid of the useless variable declarations. --- src/calibre/utils/formatter_functions.py | 66 ++++++------------------ 1 file changed, 16 insertions(+), 50 deletions(-) diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index fa15498d1f..6916b0903a 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -886,56 +886,22 @@ class BuiltinDaysBetween(BuiltinFormatterFunction): i = d1 - d2 return str('%d.%d'%(i.days, i.seconds/8640)) - -builtin_add = BuiltinAdd() -builtin_and = BuiltinAnd() -builtin_assign = BuiltinAssign() -builtin_booksize = BuiltinBooksize() -builtin_capitalize = BuiltinCapitalize() -builtin_cmp = BuiltinCmp() -builtin_contains = BuiltinContains() -builtin_count = BuiltinCount() -builtin_days_between= BuiltinDaysBetween() -builtin_divide = BuiltinDivide() -builtin_eval = BuiltinEval() -builtin_first_non_empty = BuiltinFirstNonEmpty() -builtin_field = BuiltinField() -builtin_format_date = BuiltinFormatDate() -builtin_format_numb = BuiltinFormatNumber() -builtin_formats_modt= BuiltinFormatsModtimes() -builtin_formats_size= BuiltinFormatsSizes() -builtin_has_cover = BuiltinHasCover() -builtin_human_rable = BuiltinHumanReadable() -builtin_identifier_in_list = BuiltinIdentifierInList() -builtin_ifempty = BuiltinIfempty() -builtin_in_list = BuiltinInList() -builtin_list_item = BuiltinListitem() -builtin_lookup = BuiltinLookup() -builtin_lowercase = BuiltinLowercase() -builtin_merge_lists = BuiltinMergeLists() -builtin_multiply = BuiltinMultiply() -builtin_not = BuiltinNot() -builtin_ondevice = BuiltinOndevice() -builtin_or = BuiltinOr() -builtin_print = BuiltinPrint() -builtin_raw_field = BuiltinRawField() -builtin_re = BuiltinRe() -builtin_select = BuiltinSelect() -builtin_shorten = BuiltinShorten() -builtin_strcat = BuiltinStrcat() -builtin_strcmp = BuiltinStrcmp() -builtin_str_in_list = BuiltinStrInList() -builtin_subitems = BuiltinSubitems() -builtin_sublist = BuiltinSublist() -builtin_substr = BuiltinSubstr() -builtin_subtract = BuiltinSubtract() -builtin_swaparound = BuiltinSwapAroundComma() -builtin_switch = BuiltinSwitch() -builtin_template = BuiltinTemplate() -builtin_test = BuiltinTest() -builtin_titlecase = BuiltinTitlecase() -builtin_today = BuiltinToday() -builtin_uppercase = BuiltinUppercase() +formatter_builtins = [ + BuiltinAdd(), BuiltinAnd(), BuiltinAssign(), BuiltinBooksize(), + BuiltinCapitalize(), BuiltinCmp(), BuiltinContains(), BuiltinCount(), + BuiltinDaysBetween(), BuiltinDivide(), BuiltinEval(), + BuiltinFirstNonEmpty(), BuiltinField(), BuiltinFormatDate(), + BuiltinFormatNumber(), BuiltinFormatsModtimes(), BuiltinFormatsSizes(), + BuiltinHasCover(), BuiltinHumanReadable(), BuiltinIdentifierInList(), + BuiltinIfempty(), BuiltinInList(), BuiltinListitem(), BuiltinLookup(), + BuiltinLowercase(), BuiltinMergeLists(), BuiltinMultiply(), BuiltinNot(), + BuiltinOndevice(), BuiltinOr(), BuiltinPrint(), BuiltinRawField(), + BuiltinRe(), BuiltinSelect(), BuiltinShorten(), BuiltinStrcat(), + BuiltinStrcmp(), BuiltinStrInList(), BuiltinSubitems(), BuiltinSublist(), + BuiltinSubstr(), BuiltinSubtract(), BuiltinSwapAroundComma(), + BuiltinSwitch(), BuiltinTemplate(), BuiltinTest(), BuiltinTitlecase(), + BuiltinToday(), BuiltinUppercase(), +] class FormatterUserFunction(FormatterFunction): def __init__(self, name, doc, arg_count, program_text): From fb5ef977bf1baeb914f25bdbf1fd4fbd2738f153 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 2 Jul 2011 19:14:44 -0600 Subject: [PATCH 08/28] Updated Endgadget --- recipes/endgadget.recipe | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/recipes/endgadget.recipe b/recipes/endgadget.recipe index 8a2181fdc3..83d994a6da 100644 --- a/recipes/endgadget.recipe +++ b/recipes/endgadget.recipe @@ -1,7 +1,7 @@ #!/usr/bin/env python __license__ = 'GPL v3' -__copyright__ = '2008 - 2009, Darko Miletic ' +__copyright__ = 'Copyright 2011 Starson17' ''' engadget.com ''' @@ -9,14 +9,29 @@ engadget.com from calibre.web.feeds.news import BasicNewsRecipe class Engadget(BasicNewsRecipe): - title = u'Engadget' - __author__ = 'Darko Miletic' + title = u'Engadget_Full' + __author__ = 'Starson17' + __version__ = 'v1.00' + __date__ = '02, July 2011' description = 'Tech news' language = 'en' oldest_article = 7 max_articles_per_feed = 100 no_stylesheets = True - use_embedded_content = True + use_embedded_content = False + remove_javascript = True + remove_empty_feeds = True - feeds = [ (u'Posts', u'http://www.engadget.com/rss.xml')] + keep_only_tags = [dict(name='div', attrs={'class':['post_content permalink ','post_content permalink alt-post-full']})] + remove_tags = [dict(name='div', attrs={'class':['filed_under','post_footer']})] + remove_tags_after = [dict(name='div', attrs={'class':['post_footer']})] + + feeds = [(u'Posts', u'http://www.engadget.com/rss.xml')] + + extra_css = ''' + h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;} + h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} + p{font-family:Arial,Helvetica,sans-serif;font-size:small;} + body{font-family:Helvetica,Arial,sans-serif;font-size:small;} + ''' From 42b74a763407004d995d82e3be739f76be06302a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 2 Jul 2011 20:43:07 -0600 Subject: [PATCH 09/28] New db backend: Read custom column tables --- src/calibre/db/backend.py | 261 ++++++++++++++++++++++++++++++++++---- src/calibre/db/tables.py | 13 +- 2 files changed, 242 insertions(+), 32 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index ba683dde50..1bc8bd6c83 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -22,7 +22,7 @@ from calibre.library.field_metadata import FieldMetadata from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.utils.icu import strcmp from calibre.utils.config import to_json, from_json, prefs, tweaks -from calibre.utils.date import utcfromtimestamp +from calibre.utils.date import utcfromtimestamp, parse_date from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, SizeTable, FormatsTable, AuthorsTable, IdentifiersTable) # }}} @@ -248,7 +248,179 @@ class DB(object, SchemaUpgrade): UPDATE authors SET sort=author_to_author_sort(name) WHERE sort IS NULL; ''') - def initialize_prefs(self, default_prefs): + self.initialize_custom_columns() + self.initialize_tables() + + def initialize_custom_columns(self): # {{{ + with self.conn: + # Delete previously marked custom columns + for record in self.conn.get( + 'SELECT id FROM custom_columns WHERE mark_for_delete=1'): + num = record[0] + table, lt = self.custom_table_names(num) + self.conn.execute('''\ + DROP INDEX IF EXISTS {table}_idx; + DROP INDEX IF EXISTS {lt}_aidx; + DROP INDEX IF EXISTS {lt}_bidx; + DROP TRIGGER IF EXISTS fkc_update_{lt}_a; + DROP TRIGGER IF EXISTS fkc_update_{lt}_b; + DROP TRIGGER IF EXISTS fkc_insert_{lt}; + DROP TRIGGER IF EXISTS fkc_delete_{lt}; + DROP TRIGGER IF EXISTS fkc_insert_{table}; + DROP TRIGGER IF EXISTS fkc_delete_{table}; + DROP VIEW IF EXISTS tag_browser_{table}; + DROP VIEW IF EXISTS tag_browser_filtered_{table}; + DROP TABLE IF EXISTS {table}; + DROP TABLE IF EXISTS {lt}; + '''.format(table=table, lt=lt) + ) + self.conn.execute('DELETE FROM custom_columns WHERE mark_for_delete=1') + + # Load metadata for custom columns + self.custom_column_label_map, self.custom_column_num_map = {}, {} + triggers = [] + remove = [] + custom_tables = self.custom_tables + for record in self.conn.get( + 'SELECT label,name,datatype,editable,display,normalized,id,is_multiple FROM custom_columns'): + data = { + 'label':record[0], + 'name':record[1], + 'datatype':record[2], + 'editable':bool(record[3]), + 'display':json.loads(record[4]), + 'normalized':bool(record[5]), + 'num':record[6], + 'is_multiple':bool(record[7]), + } + if data['display'] is None: + data['display'] = {} + # set up the is_multiple separator dict + if data['is_multiple']: + if data['display'].get('is_names', False): + seps = {'cache_to_list': '|', 'ui_to_list': '&', 'list_to_ui': ' & '} + elif data['datatype'] == 'composite': + seps = {'cache_to_list': ',', 'ui_to_list': ',', 'list_to_ui': ', '} + else: + seps = {'cache_to_list': '|', 'ui_to_list': ',', 'list_to_ui': ', '} + else: + seps = {} + data['multiple_seps'] = seps + + table, lt = self.custom_table_names(data['num']) + if table not in custom_tables or (data['normalized'] and lt not in + custom_tables): + remove.append(data) + continue + + self.custom_column_label_map[data['label']] = data['num'] + self.custom_column_num_map[data['num']] = \ + self.custom_column_label_map[data['label']] = data + + # Create Foreign Key triggers + if data['normalized']: + trigger = 'DELETE FROM %s WHERE book=OLD.id;'%lt + else: + trigger = 'DELETE FROM %s WHERE book=OLD.id;'%table + triggers.append(trigger) + + if remove: + with self.conn: + for data in remove: + prints('WARNING: Custom column %r not found, removing.' % + data['label']) + self.conn.execute('DELETE FROM custom_columns WHERE id=?', + (data['num'],)) + + if triggers: + with self.conn: + self.conn.execute('''\ + CREATE TEMP TRIGGER custom_books_delete_trg + AFTER DELETE ON books + BEGIN + %s + END; + '''%(' \n'.join(triggers))) + + # Setup data adapters + def adapt_text(x, d): + if d['is_multiple']: + if x is None: + return [] + if isinstance(x, (str, unicode, bytes)): + x = x.split(d['multiple_seps']['ui_to_list']) + x = [y.strip() for y in x if y.strip()] + x = [y.decode(preferred_encoding, 'replace') if not isinstance(y, + unicode) else y for y in x] + return [u' '.join(y.split()) for y in x] + else: + return x if x is None or isinstance(x, unicode) else \ + x.decode(preferred_encoding, 'replace') + + def adapt_datetime(x, d): + if isinstance(x, (str, unicode, bytes)): + x = parse_date(x, assume_utc=False, as_utc=False) + return x + + def adapt_bool(x, d): + if isinstance(x, (str, unicode, bytes)): + x = x.lower() + if x == 'true': + x = True + elif x == 'false': + x = False + elif x == 'none': + x = None + else: + x = bool(int(x)) + return x + + def adapt_enum(x, d): + v = adapt_text(x, d) + if not v: + v = None + return v + + def adapt_number(x, d): + if x is None: + return None + if isinstance(x, (str, unicode, bytes)): + if x.lower() == 'none': + return None + if d['datatype'] == 'int': + return int(x) + return float(x) + + self.custom_data_adapters = { + 'float': adapt_number, + 'int': adapt_number, + 'rating':lambda x,d : x if x is None else min(10., max(0., float(x))), + 'bool': adapt_bool, + 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}), + 'datetime' : adapt_datetime, + 'text':adapt_text, + 'series':adapt_text, + 'enumeration': adapt_enum + } + + # Create Tag Browser categories for custom columns + for k in sorted(self.custom_column_label_map.iterkeys()): + v = self.custom_column_label_map[k] + if v['normalized']: + is_category = True + else: + is_category = False + is_m = v['multiple_seps'] + tn = 'custom_column_{0}'.format(v['num']) + self.field_metadata.add_custom_field(label=v['label'], + table=tn, column='value', datatype=v['datatype'], + colnum=v['num'], name=v['name'], display=v['display'], + is_multiple=is_m, is_category=is_category, + is_editable=v['editable'], is_csp=False) + + # }}} + + def initialize_prefs(self, default_prefs): # {{{ self.prefs = DBPrefs(self) if default_prefs is not None and not self._exists: @@ -339,6 +511,53 @@ class DB(object, SchemaUpgrade): cats_changed = True if cats_changed: self.prefs.set('user_categories', user_cats) + # }}} + + def initialize_tables(self): # {{{ + tables = self.tables = {} + for col in ('title', 'sort', 'author_sort', 'series_index', 'comments', + 'timestamp', 'published', 'uuid', 'path', 'cover', + 'last_modified'): + metadata = self.field_metadata[col].copy() + if metadata['table'] is None: + metadata['table'], metadata['column'] == 'books', ('has_cover' + if col == 'cover' else col) + tables[col] = OneToOneTable(col, metadata) + + for col in ('series', 'publisher', 'rating'): + tables[col] = ManyToOneTable(col, self.field_metadata[col].copy()) + + for col in ('authors', 'tags', 'formats', 'identifiers'): + cls = { + 'authors':AuthorsTable, + 'formats':FormatsTable, + 'identifiers':IdentifiersTable, + }.get(col, ManyToManyTable) + tables[col] = cls(col, self.field_metadata[col].copy()) + + tables['size'] = SizeTable('size', self.field_metadata['size'].copy()) + + for label, data in self.custom_column_label_map.iteritems(): + metadata = self.field_metadata[label].copy() + link_table = self.custom_table_names(data['num'])[1] + + if data['normalized']: + if metadata['is_multiple']: + tables[label] = ManyToManyTable(label, metadata, + link_table=link_table) + else: + tables[label] = ManyToOneTable(label, metadata, + link_table=link_table) + if metadata['datatype'] == 'series': + # Create series index table + label += '_index' + metadata = self.field_metadata[label].copy() + metadata['column'] = 'extra' + metadata['table'] = link_table + tables[label] = OneToOneTable(label, metadata) + else: + tables[label] = OneToOneTable(label, metadata) + # }}} @property def conn(self): @@ -372,6 +591,15 @@ class DB(object, SchemaUpgrade): # Database layer API {{{ + def custom_table_names(self, num): + return 'custom_column_%d'%num, 'books_custom_column_%d_link'%num + + @property + def custom_tables(self): + return set([x[0] for x in self.conn.get( + 'SELECT name FROM sqlite_master WHERE type="table" AND ' + '(name GLOB "custom_column_*" OR name GLOB "books_custom_column_*")')]) + @classmethod def exists_at(cls, path): return path and os.path.exists(os.path.join(path, 'metadata.db')) @@ -405,39 +633,18 @@ class DB(object, SchemaUpgrade): return utcfromtimestamp(os.stat(self.dbpath).st_mtime) def read_tables(self): - tables = {} - for col in ('title', 'sort', 'author_sort', 'series_index', 'comments', - 'timestamp', 'published', 'uuid', 'path', 'cover', - 'last_modified'): - metadata = self.field_metadata[col].copy() - if metadata['table'] is None: - metadata['table'], metadata['column'] == 'books', ('has_cover' - if col == 'cover' else col) - tables[col] = OneToOneTable(col, metadata) - - for col in ('series', 'publisher', 'rating'): - tables[col] = ManyToOneTable(col, self.field_metadata[col].copy()) - - for col in ('authors', 'tags', 'formats', 'identifiers'): - cls = { - 'authors':AuthorsTable, - 'formats':FormatsTable, - 'identifiers':IdentifiersTable, - }.get(col, ManyToManyTable) - tables[col] = cls(col, self.field_metadata[col].copy()) - - tables['size'] = SizeTable('size', self.field_metadata['size'].copy()) + ''' + Read all data from the db into the python in-memory tables + ''' with self.conn: # Use a single transaction, to ensure nothing modifies # the db while we are reading - for table in tables.itervalues(): + for table in self.tables.itervalues(): try: table.read() except: prints('Failed to read table:', table.name) raise - return tables - # }}} diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index edca43528a..71c7200512 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -32,7 +32,7 @@ def _c_convert_timestamp(val): class Table(object): - def __init__(self, name, metadata): + def __init__(self, name, metadata, link_table=None): self.name, self.metadata = name, metadata # self.adapt() maps values from the db to python objects @@ -46,6 +46,9 @@ class Table(object): # Legacy self.adapt = lambda x: x.replace('|', ',') if x else None + self.link_table = (link_table if link_table else + 'books_%s_link'%self.metadata['table']) + class OneToOneTable(Table): ''' @@ -95,8 +98,8 @@ class ManyToOneTable(Table): def read_maps(self, db): for row in db.conn.execute( - 'SELECT book, {0} FROM books_{1}_link'.format( - self.metadata['link_column'], self.metadata['table'])): + 'SELECT book, {0} FROM {1}'.format( + self.metadata['link_column'], self.link_table)): if row[1] not in self.col_book_map: self.col_book_map[row[1]] = [] self.col_book_map.append(row[0]) @@ -112,8 +115,8 @@ class ManyToManyTable(ManyToOneTable): def read_maps(self, db): for row in db.conn.execute( - 'SELECT book, {0} FROM books_{1}_link'.format( - self.metadata['link_column'], self.metadata['table'])): + 'SELECT book, {0} FROM {1}'.format( + self.metadata['link_column'], self.link_table)): if row[1] not in self.col_book_map: self.col_book_map[row[1]] = [] self.col_book_map.append(row[0]) From 3f8a857af2a0390f572c528411ce761890ac7602 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 2 Jul 2011 21:03:46 -0600 Subject: [PATCH 10/28] South China Morning Post by llam --- recipes/scmp.recipe | 80 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 recipes/scmp.recipe diff --git a/recipes/scmp.recipe b/recipes/scmp.recipe new file mode 100644 index 0000000000..1da7b9e1bc --- /dev/null +++ b/recipes/scmp.recipe @@ -0,0 +1,80 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Darko Miletic ' +''' +scmp.com +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class SCMP(BasicNewsRecipe): + title = 'South China Morning Post' + __author__ = 'llam' + description = "SCMP.com, Hong Kong's premier online English daily provides exclusive up-to-date news, audio video news, podcasts, RSS Feeds, Blogs, breaking news, top stories, award winning news and analysis on Hong Kong and China." + publisher = 'South China Morning Post Publishers Ltd.' + category = 'SCMP, Online news, Hong Kong News, China news, Business news, English newspaper, daily newspaper, Lifestyle news, Sport news, Audio Video news, Asia news, World news, economy news, investor relations news, RSS Feeds' + oldest_article = 2 + delay = 1 + max_articles_per_feed = 200 + no_stylesheets = True + encoding = 'utf-8' + use_embedded_content = False + language = 'en_CN' + remove_empty_feeds = True + needs_subscription = True + publication_type = 'newspaper' + masthead_url = 'http://www.scmp.com/images/logo_scmp_home.gif' + extra_css = ' body{font-family: Arial,Helvetica,sans-serif } ' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + } + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + #br.set_debug_http(True) + #br.set_debug_responses(True) + #br.set_debug_redirects(True) + if self.username is not None and self.password is not None: + br.open('http://www.scmp.com/portal/site/SCMP/') + br.select_form(name='loginForm') + br['Login' ] = self.username + br['Password'] = self.password + br.submit() + return br + + remove_attributes=['width','height','border'] + + keep_only_tags = [ + dict(attrs={'id':['ART','photoBox']}) + ,dict(attrs={'class':['article_label','article_byline','article_body']}) + ] + + preprocess_regexps = [ + (re.compile(r'

).)*', re.DOTALL|re.IGNORECASE), + lambda match: ''), + ] + + feeds = [ + (u'Business' , u'http://www.scmp.com/rss/business.xml' ) + ,(u'Hong Kong' , u'http://www.scmp.com/rss/hong_kong.xml' ) + ,(u'China' , u'http://www.scmp.com/rss/china.xml' ) + ,(u'Asia & World' , u'http://www.scmp.com/rss/news_asia_world.xml') + ,(u'Opinion' , u'http://www.scmp.com/rss/opinion.xml' ) + ,(u'LifeSTYLE' , u'http://www.scmp.com/rss/lifestyle.xml' ) + ,(u'Sport' , u'http://www.scmp.com/rss/sport.xml' ) + ] + + def print_version(self, url): + rpart, sep, rest = url.rpartition('&') + return rpart #+ sep + urllib.quote_plus(rest) + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + items = soup.findAll(src="/images/label_icon.gif") + [item.extract() for item in items] + return self.adeify_images(soup) From ba16dcb128df617fb6493d070d5a07043e88a6f9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 2 Jul 2011 21:08:16 -0600 Subject: [PATCH 11/28] ... --- src/calibre/db/backend.py | 186 +++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 1bc8bd6c83..f30ce01ade 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -251,6 +251,99 @@ class DB(object, SchemaUpgrade): self.initialize_custom_columns() self.initialize_tables() + def initialize_prefs(self, default_prefs): # {{{ + self.prefs = DBPrefs(self) + + if default_prefs is not None and not self._exists: + # Only apply default prefs to a new database + for key in default_prefs: + # be sure that prefs not to be copied are listed below + if key not in frozenset(['news_to_be_synced']): + self.prefs[key] = default_prefs[key] + if 'field_metadata' in default_prefs: + fmvals = [f for f in default_prefs['field_metadata'].values() + if f['is_custom']] + for f in fmvals: + self.create_custom_column(f['label'], f['name'], + f['datatype'], f['is_multiple'] is not None, + f['is_editable'], f['display']) + + defs = self.prefs.defaults + defs['gui_restriction'] = defs['cs_restriction'] = '' + defs['categories_using_hierarchy'] = [] + defs['column_color_rules'] = [] + + # Migrate the bool tristate tweak + defs['bools_are_tristate'] = \ + tweaks.get('bool_custom_columns_are_tristate', 'yes') == 'yes' + if self.prefs.get('bools_are_tristate') is None: + self.prefs.set('bools_are_tristate', defs['bools_are_tristate']) + + # Migrate column coloring rules + if self.prefs.get('column_color_name_1', None) is not None: + from calibre.library.coloring import migrate_old_rule + old_rules = [] + for i in range(1, 6): + col = self.prefs.get('column_color_name_'+str(i), None) + templ = self.prefs.get('column_color_template_'+str(i), None) + if col and templ: + try: + del self.prefs['column_color_name_'+str(i)] + rules = migrate_old_rule(self.field_metadata, templ) + for templ in rules: + old_rules.append((col, templ)) + except: + pass + if old_rules: + self.prefs['column_color_rules'] += old_rules + + # Migrate saved search and user categories to db preference scheme + def migrate_preference(key, default): + oldval = prefs[key] + if oldval != default: + self.prefs[key] = oldval + prefs[key] = default + if key not in self.prefs: + self.prefs[key] = default + + migrate_preference('user_categories', {}) + migrate_preference('saved_searches', {}) + + # migrate grouped_search_terms + if self.prefs.get('grouped_search_terms', None) is None: + try: + ogst = tweaks.get('grouped_search_terms', {}) + ngst = {} + for t in ogst: + ngst[icu_lower(t)] = ogst[t] + self.prefs.set('grouped_search_terms', ngst) + except: + pass + + # Rename any user categories with names that differ only in case + user_cats = self.prefs.get('user_categories', []) + catmap = {} + for uc in user_cats: + ucl = icu_lower(uc) + if ucl not in catmap: + catmap[ucl] = [] + catmap[ucl].append(uc) + cats_changed = False + for uc in catmap: + if len(catmap[uc]) > 1: + prints('found user category case overlap', catmap[uc]) + cat = catmap[uc][0] + suffix = 1 + while icu_lower((cat + unicode(suffix))) in catmap: + suffix += 1 + prints('Renaming user category %s to %s'%(cat, cat+unicode(suffix))) + user_cats[cat + unicode(suffix)] = user_cats[cat] + del user_cats[cat] + cats_changed = True + if cats_changed: + self.prefs.set('user_categories', user_cats) + # }}} + def initialize_custom_columns(self): # {{{ with self.conn: # Delete previously marked custom columns @@ -420,99 +513,6 @@ class DB(object, SchemaUpgrade): # }}} - def initialize_prefs(self, default_prefs): # {{{ - self.prefs = DBPrefs(self) - - if default_prefs is not None and not self._exists: - # Only apply default prefs to a new database - for key in default_prefs: - # be sure that prefs not to be copied are listed below - if key not in frozenset(['news_to_be_synced']): - self.prefs[key] = default_prefs[key] - if 'field_metadata' in default_prefs: - fmvals = [f for f in default_prefs['field_metadata'].values() - if f['is_custom']] - for f in fmvals: - self.create_custom_column(f['label'], f['name'], - f['datatype'], f['is_multiple'] is not None, - f['is_editable'], f['display']) - - defs = self.prefs.defaults - defs['gui_restriction'] = defs['cs_restriction'] = '' - defs['categories_using_hierarchy'] = [] - defs['column_color_rules'] = [] - - # Migrate the bool tristate tweak - defs['bools_are_tristate'] = \ - tweaks.get('bool_custom_columns_are_tristate', 'yes') == 'yes' - if self.prefs.get('bools_are_tristate') is None: - self.prefs.set('bools_are_tristate', defs['bools_are_tristate']) - - # Migrate column coloring rules - if self.prefs.get('column_color_name_1', None) is not None: - from calibre.library.coloring import migrate_old_rule - old_rules = [] - for i in range(1, 6): - col = self.prefs.get('column_color_name_'+str(i), None) - templ = self.prefs.get('column_color_template_'+str(i), None) - if col and templ: - try: - del self.prefs['column_color_name_'+str(i)] - rules = migrate_old_rule(self.field_metadata, templ) - for templ in rules: - old_rules.append((col, templ)) - except: - pass - if old_rules: - self.prefs['column_color_rules'] += old_rules - - # Migrate saved search and user categories to db preference scheme - def migrate_preference(key, default): - oldval = prefs[key] - if oldval != default: - self.prefs[key] = oldval - prefs[key] = default - if key not in self.prefs: - self.prefs[key] = default - - migrate_preference('user_categories', {}) - migrate_preference('saved_searches', {}) - - # migrate grouped_search_terms - if self.prefs.get('grouped_search_terms', None) is None: - try: - ogst = tweaks.get('grouped_search_terms', {}) - ngst = {} - for t in ogst: - ngst[icu_lower(t)] = ogst[t] - self.prefs.set('grouped_search_terms', ngst) - except: - pass - - # Rename any user categories with names that differ only in case - user_cats = self.prefs.get('user_categories', []) - catmap = {} - for uc in user_cats: - ucl = icu_lower(uc) - if ucl not in catmap: - catmap[ucl] = [] - catmap[ucl].append(uc) - cats_changed = False - for uc in catmap: - if len(catmap[uc]) > 1: - prints('found user category case overlap', catmap[uc]) - cat = catmap[uc][0] - suffix = 1 - while icu_lower((cat + unicode(suffix))) in catmap: - suffix += 1 - prints('Renaming user category %s to %s'%(cat, cat+unicode(suffix))) - user_cats[cat + unicode(suffix)] = user_cats[cat] - del user_cats[cat] - cats_changed = True - if cats_changed: - self.prefs.set('user_categories', user_cats) - # }}} - def initialize_tables(self): # {{{ tables = self.tables = {} for col in ('title', 'sort', 'author_sort', 'series_index', 'comments', From 58ca9bc7d0cb8e35641573af395b35d644a227ab Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 3 Jul 2011 10:05:43 -0400 Subject: [PATCH 12/28] Store: Change class name of opensearch store to allow for easily adding more opensearch results for other feed types. Document opensearch module changes. --- src/calibre/gui2/store/opensearch_store.py | 6 ++- .../gui2/store/stores/archive_org_plugin.py | 8 ++-- .../gui2/store/stores/epubbud_plugin.py | 6 +-- .../gui2/store/stores/feedbooks_plugin.py | 6 +-- .../stores/pragmatic_bookshelf_plugin.py | 6 +-- src/calibre/utils/opensearch/__init__.py | 37 +++++++++++++++++++ 6 files changed, 55 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/store/opensearch_store.py b/src/calibre/gui2/store/opensearch_store.py index 54fedbd002..6e8f5de7ba 100644 --- a/src/calibre/gui2/store/opensearch_store.py +++ b/src/calibre/gui2/store/opensearch_store.py @@ -22,7 +22,7 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog from calibre.utils.opensearch.description import Description from calibre.utils.opensearch.query import Query -class OpenSearchStore(StorePlugin): +class OpenSearchOPDSStore(StorePlugin): open_search_url = '' web_url = '' @@ -99,3 +99,7 @@ class OpenSearchStore(StorePlugin): yield s + +class OpenSearchOPDSDetailStore(OpenSearchOPDSStore): + + pass diff --git a/src/calibre/gui2/store/stores/archive_org_plugin.py b/src/calibre/gui2/store/stores/archive_org_plugin.py index 6972c604ce..7439056baa 100644 --- a/src/calibre/gui2/store/stores/archive_org_plugin.py +++ b/src/calibre/gui2/store/stores/archive_org_plugin.py @@ -6,12 +6,11 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' - from calibre.gui2.store.basic_config import BasicStoreConfig -from calibre.gui2.store.opensearch_store import OpenSearchStore +from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore from calibre.gui2.store.search_result import SearchResult -class ArchiveOrgStore(BasicStoreConfig, OpenSearchStore): +class ArchiveOrgStore(BasicStoreConfig, OpenSearchOPDSStore): open_search_url = 'http://bookserver.archive.org/catalog/opensearch.xml' web_url = 'http://www.archive.org/details/texts' @@ -19,7 +18,7 @@ class ArchiveOrgStore(BasicStoreConfig, OpenSearchStore): # http://bookserver.archive.org/catalog/ def search(self, query, max_results=10, timeout=60): - for s in OpenSearchStore.search(self, query, max_results, timeout): + for s in OpenSearchOPDSStore.search(self, query, max_results, timeout): s.detail_item = 'http://www.archive.org/details/' + s.detail_item.split(':')[-1] s.price = '$0.00' s.drm = SearchResult.DRM_UNLOCKED @@ -33,6 +32,7 @@ class ArchiveOrgStore(BasicStoreConfig, OpenSearchStore): from calibre import browser from contextlib import closing from lxml import html + br = browser() with closing(br.open(search_result.detail_item, timeout=timeout)) as nf: idata = html.fromstring(nf.read()) diff --git a/src/calibre/gui2/store/stores/epubbud_plugin.py b/src/calibre/gui2/store/stores/epubbud_plugin.py index b4d642f62b..029b2b3fc9 100644 --- a/src/calibre/gui2/store/stores/epubbud_plugin.py +++ b/src/calibre/gui2/store/stores/epubbud_plugin.py @@ -7,10 +7,10 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' from calibre.gui2.store.basic_config import BasicStoreConfig -from calibre.gui2.store.opensearch_store import OpenSearchStore +from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore from calibre.gui2.store.search_result import SearchResult -class EpubBudStore(BasicStoreConfig, OpenSearchStore): +class EpubBudStore(BasicStoreConfig, OpenSearchOPDSStore): open_search_url = 'http://www.epubbud.com/feeds/opensearch.xml' web_url = 'http://www.epubbud.com/' @@ -18,7 +18,7 @@ class EpubBudStore(BasicStoreConfig, OpenSearchStore): # http://www.epubbud.com/feeds/catalog.atom def search(self, query, max_results=10, timeout=60): - for s in OpenSearchStore.search(self, query, max_results, timeout): + for s in OpenSearchOPDSStore.search(self, query, max_results, timeout): s.price = '$0.00' s.drm = SearchResult.DRM_UNLOCKED s.formats = 'EPUB' diff --git a/src/calibre/gui2/store/stores/feedbooks_plugin.py b/src/calibre/gui2/store/stores/feedbooks_plugin.py index 96d0a10dc7..cac44fd8df 100644 --- a/src/calibre/gui2/store/stores/feedbooks_plugin.py +++ b/src/calibre/gui2/store/stores/feedbooks_plugin.py @@ -7,10 +7,10 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' from calibre.gui2.store.basic_config import BasicStoreConfig -from calibre.gui2.store.opensearch_store import OpenSearchStore +from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore from calibre.gui2.store.search_result import SearchResult -class FeedbooksStore(BasicStoreConfig, OpenSearchStore): +class FeedbooksStore(BasicStoreConfig, OpenSearchOPDSStore): open_search_url = 'http://assets0.feedbooks.net/opensearch.xml?t=1253087147' web_url = 'http://feedbooks.com/' @@ -18,7 +18,7 @@ class FeedbooksStore(BasicStoreConfig, OpenSearchStore): # http://www.feedbooks.com/catalog def search(self, query, max_results=10, timeout=60): - for s in OpenSearchStore.search(self, query, max_results, timeout): + for s in OpenSearchOPDSStore.search(self, query, max_results, timeout): if s.downloads: s.drm = SearchResult.DRM_UNLOCKED s.price = '$0.00' diff --git a/src/calibre/gui2/store/stores/pragmatic_bookshelf_plugin.py b/src/calibre/gui2/store/stores/pragmatic_bookshelf_plugin.py index 671186ba87..99b94778bf 100644 --- a/src/calibre/gui2/store/stores/pragmatic_bookshelf_plugin.py +++ b/src/calibre/gui2/store/stores/pragmatic_bookshelf_plugin.py @@ -7,10 +7,10 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' from calibre.gui2.store.basic_config import BasicStoreConfig -from calibre.gui2.store.opensearch_store import OpenSearchStore +from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore from calibre.gui2.store.search_result import SearchResult -class PragmaticBookshelfStore(BasicStoreConfig, OpenSearchStore): +class PragmaticBookshelfStore(BasicStoreConfig, OpenSearchOPDSStore): open_search_url = 'http://pragprog.com/catalog/search-description' web_url = 'http://pragprog.com/' @@ -18,7 +18,7 @@ class PragmaticBookshelfStore(BasicStoreConfig, OpenSearchStore): # http://pragprog.com/catalog.opds def search(self, query, max_results=10, timeout=60): - for s in OpenSearchStore.search(self, query, max_results, timeout): + for s in OpenSearchOPDSStore.search(self, query, max_results, timeout): s.drm = SearchResult.DRM_UNLOCKED s.formats = 'EPUB, PDF, MOBI' yield s diff --git a/src/calibre/utils/opensearch/__init__.py b/src/calibre/utils/opensearch/__init__.py index e69de29bb2..3d0c4d8787 100644 --- a/src/calibre/utils/opensearch/__init__.py +++ b/src/calibre/utils/opensearch/__init__.py @@ -0,0 +1,37 @@ +''' +Based on the OpenSearch Python module by Ed Summers from +https://github.com/edsu/opensearch . + +This module is heavily modified and does not implement all the features from +the original. The ability for the the module to perform a search and retrieve +search results has been removed. The original module used a modified version +of the Universal feed parser from http://feedparser.org/ . The use of +FeedPaser made getting search results very slow. There is also a bug in the +modified FeedParser that causes the system to run out of file descriptors. + +Instead of fixing the modified feed parser it was decided to remove it and +manually parse the feeds in a set of type specific classes. This is much +faster and as we know in advance the feed format is simpler than using +FeedParser. Also, replacing the modified FeedParser with the newest version +of FeedParser caused some feeds to be parsed incorrectly and result in a loss +of data. + +The module was also rewritten to use lxml instead of MiniDom. + + +Usage: + +description = Description(open_search_url) +url_template = description.get_best_template() +if not url_template: + return +query = Query(url_template) + +# set up initial values. +query.searchTerms = urllib.quote_plus(search_terms) +# Note the count is ignored by some feeds. +query.count = max_results + +search_url = oquery.url() + +''' From 3e0797872c0eaa08f2a4f93927e16be87aa834ae Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 3 Jul 2011 10:59:54 -0400 Subject: [PATCH 13/28] Store: Manybooks uses opds feed (faster, more accurate, fixes covers not showing in many cases, fix formats list). Opensearch: support creating search urls from Stanza catalogs. Store: opensearch based classes don't need to quote the search terms as the opensearch module does this already. --- src/calibre/gui2/store/opensearch_store.py | 7 +- .../gui2/store/stores/manybooks_plugin.py | 144 ++++++++++-------- src/calibre/utils/opensearch/__init__.py | 2 +- src/calibre/utils/opensearch/description.py | 19 ++- 4 files changed, 96 insertions(+), 76 deletions(-) diff --git a/src/calibre/gui2/store/opensearch_store.py b/src/calibre/gui2/store/opensearch_store.py index 6e8f5de7ba..bcc92b25f1 100644 --- a/src/calibre/gui2/store/opensearch_store.py +++ b/src/calibre/gui2/store/opensearch_store.py @@ -7,7 +7,6 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' import mimetypes -import urllib from contextlib import closing from lxml import etree @@ -50,7 +49,7 @@ class OpenSearchOPDSStore(StorePlugin): oquery = Query(url_template) # set up initial values - oquery.searchTerms = urllib.quote_plus(query) + oquery.searchTerms = query oquery.count = max_results url = oquery.url() @@ -99,7 +98,3 @@ class OpenSearchOPDSStore(StorePlugin): yield s - -class OpenSearchOPDSDetailStore(OpenSearchOPDSStore): - - pass diff --git a/src/calibre/gui2/store/stores/manybooks_plugin.py b/src/calibre/gui2/store/stores/manybooks_plugin.py index 829a97012f..c7dbf0a608 100644 --- a/src/calibre/gui2/store/stores/manybooks_plugin.py +++ b/src/calibre/gui2/store/stores/manybooks_plugin.py @@ -6,89 +6,101 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import re -import urllib +import mimetypes from contextlib import closing -from lxml import html +from lxml import etree -from PyQt4.Qt import QUrl - -from calibre import browser, url_slash_cleaner -from calibre.gui2 import open_url -from calibre.gui2.store import StorePlugin +from calibre import browser from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore from calibre.gui2.store.search_result import SearchResult -from calibre.gui2.store.web_store_dialog import WebStoreDialog +from calibre.utils.opensearch.description import Description +from calibre.utils.opensearch.query import Query -class ManyBooksStore(BasicStoreConfig, StorePlugin): +class ManyBooksStore(BasicStoreConfig, OpenSearchOPDSStore): - def open(self, parent=None, detail_item=None, external=False): - url = 'http://manybooks.net/' - - detail_url = None - if detail_item: - detail_url = url + detail_item - - if external or self.config.get('open_external', False): - open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) - else: - d = WebStoreDialog(self.gui, url, parent, detail_url) - d.setWindowTitle(self.name) - d.set_tags(self.config.get('tags', '')) - d.exec_() + open_search_url = 'http://www.manybooks.net/opds/' + web_url = 'http://manybooks.net' def search(self, query, max_results=10, timeout=60): - # ManyBooks website separates results for title and author. - # It also doesn't do a clear job of references authors and - # secondary titles. Google is also faster. - # Using a google search so we can search on both fields at once. - url = 'http://www.google.com/xhtml?q=site:manybooks.net+' + urllib.quote_plus(query) + ''' + Manybooks uses a very strange opds feed. The opds + main feed is structured like a stanza feed. The + search result entries give very little information + and requires you to go to a detail link. The detail + link has the wrong type specified (text/html instead + of application/atom+xml). + ''' + if not hasattr(self, 'open_search_url'): + return - br = browser() + description = Description(self.open_search_url) + url_template = description.get_best_template() + if not url_template: + return + oquery = Query(url_template) + # set up initial values + oquery.searchTerms = query + oquery.count = max_results + url = oquery.url() + counter = max_results + br = browser() with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) - for data in doc.xpath('//div[@class="edewpi"]//div[@class="r ld"]'): + doc = etree.fromstring(f.read()) + for data in doc.xpath('//*[local-name() = "entry"]'): if counter <= 0: break - - url = '' - url_a = data.xpath('div[@class="jd"]/a') - if url_a: - url_a = url_a[0] - url = url_a.get('href', None) - if url: - url = url.split('u=')[-1][:-2] - if '/titles/' not in url: - continue - id = url.split('/')[-1] - id = id.strip() - - url_a = html.fromstring(html.tostring(url_a)) - heading = ''.join(url_a.xpath('//text()')) - title, _, author = heading.rpartition('by ') - author = author.split('-')[0] - price = '$0.00' - - cover_url = '' - mo = re.match('^\D+', id) - if mo: - cover_name = mo.group() - cover_name = cover_name.replace('etext', '') - cover_id = id.split('.')[0] - cover_url = 'http://www.manybooks.net/images/' + id[0] + '/' + cover_name + '/' + cover_id + '-thumb.jpg' - + counter -= 1 - + s = SearchResult() - s.cover_url = cover_url - s.title = title.strip() - s.author = author.strip() - s.price = price.strip() - s.detail_item = '/titles/' + id + + detail_links = data.xpath('./*[local-name() = "link" and @type = "text/html"]') + if not detail_links: + continue + detail_link = detail_links[0] + detail_href = detail_link.get('href') + if not detail_href: + continue + + s.detail_item = 'http://manybooks.net/titles/' + detail_href.split('tid=')[-1] + '.html' + # These can have HTML inside of them. We are going to get them again later + # just in case. + s.title = ''.join(data.xpath('./*[local-name() = "title"]//text()')).strip() + s.author = ', '.join(data.xpath('./*[local-name() = "author"]//text()')).strip() + + # Follow the detail link to get the rest of the info. + with closing(br.open(detail_href, timeout=timeout/4)) as df: + ddoc = etree.fromstring(df.read()) + ddata = ddoc.xpath('//*[local-name() = "entry"][1]') + if ddata: + ddata = ddata[0] + + # This is the real title and author info we want. We got + # it previously just in case it's not specified here for some reason. + s.title = ''.join(ddata.xpath('./*[local-name() = "title"]//text()')).strip() + s.author = ', '.join(ddata.xpath('./*[local-name() = "author"]//text()')).strip() + if s.author.startswith(','): + s.author = s.author[1:] + if s.author.endswith(','): + s.author = s.author[:-1] + + s.cover_url = ''.join(ddata.xpath('./*[local-name() = "link" and @rel = "http://opds-spec.org/thumbnail"][1]/@href')).strip() + + for link in ddata.xpath('./*[local-name() = "link" and @rel = "http://opds-spec.org/acquisition"]'): + type = link.get('type') + href = link.get('href') + if type: + ext = mimetypes.guess_extension(type) + if ext: + ext = ext[1:].upper().strip() + s.downloads[ext] = href + + s.price = '$0.00' s.drm = SearchResult.DRM_UNLOCKED - s.formts = 'EPUB, PDB (eReader, PalmDoc, zTXT, Plucker, iSilo), FB2, ZIP, AZW, MOBI, PRC, LIT, PKG, PDF, TXT, RB, RTF, LRF, TCR, JAR' + s.formats = 'EPUB, PDB (eReader, PalmDoc, zTXT, Plucker, iSilo), FB2, ZIP, AZW, MOBI, PRC, LIT, PKG, PDF, TXT, RB, RTF, LRF, TCR, JAR' yield s diff --git a/src/calibre/utils/opensearch/__init__.py b/src/calibre/utils/opensearch/__init__.py index 3d0c4d8787..62bd0e0236 100644 --- a/src/calibre/utils/opensearch/__init__.py +++ b/src/calibre/utils/opensearch/__init__.py @@ -28,7 +28,7 @@ if not url_template: query = Query(url_template) # set up initial values. -query.searchTerms = urllib.quote_plus(search_terms) +query.searchTerms = search_terms # Note the count is ignored by some feeds. query.count = max_results diff --git a/src/calibre/utils/opensearch/description.py b/src/calibre/utils/opensearch/description.py index 0b5afd8a7e..d5922d0c2b 100644 --- a/src/calibre/utils/opensearch/description.py +++ b/src/calibre/utils/opensearch/description.py @@ -40,7 +40,7 @@ class Description(object): with closing(br.open(url, timeout=15)) as f: doc = etree.fromstring(f.read()) - # version 1.1 has repeating Url elements + # version 1.1 has repeating Url elements. self.urls = [] for element in doc.xpath('//*[local-name() = "Url"]'): template = element.get('template') @@ -50,9 +50,22 @@ class Description(object): url.template = template url.type = type self.urls.append(url) + # Stanza catalogs. + for element in doc.xpath('//*[local-name() = "link"]'): + if element.get('rel') != 'search': + continue + href = element.get('href') + type = element.get('type') + if href and type: + url = URL() + url.template = href + url.type = type + self.urls.append(url) - # this is version 1.0 specific - self.url = ''.join(doc.xpath('//*[local-name() = "Url"][1]//text()')) + # this is version 1.0 specific. + self.url = '' + if not self.urls: + self.url = ''.join(doc.xpath('//*[local-name() = "Url"][1]//text()')) self.format = ''.join(doc.xpath('//*[local-name() = "Format"][1]//text()')) self.shortname = ''.join(doc.xpath('//*[local-name() = "ShortName"][1]//text()')) From 38e8eb3616b8ebe37748b2d671e7261a2ddd89fe Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 3 Jul 2011 11:08:13 -0400 Subject: [PATCH 14/28] Store: Use title and format to construct filename instead of relying on url. Makes the downloading notice look better and easier to understand what is being downloaded. --- src/calibre/gui2/store/search/search.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index fd20669f09..f6fa423e23 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -349,7 +349,8 @@ class SearchDialog(QDialog, Ui_Dialog): d = ChooseFormatDialog(self, _('Choose format to download to your library.'), result.downloads.keys()) if d.exec_() == d.Accepted: ext = d.format() - self.gui.download_ebook(result.downloads[ext]) + fname = result.title + '.' + ext.lower() + self.gui.download_ebook(result.downloads[ext], filename=fname) def open_store(self, result): self.gui.istores[result.store_name].open(self, result.detail_item, self.open_external.isChecked()) From fb0210dff84dd4c9e0f4b66c9b43222a0a365b45 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2011 09:26:40 -0600 Subject: [PATCH 15/28] ... --- src/calibre/db/tables.py | 12 ++++++------ src/calibre/manual/faq.rst | 4 ++++ src/calibre/manual/gui.rst | 4 +++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 71c7200512..398afac5d2 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -35,8 +35,8 @@ class Table(object): def __init__(self, name, metadata, link_table=None): self.name, self.metadata = name, metadata - # self.adapt() maps values from the db to python objects - self.adapt = \ + # self.unserialize() maps values from the db to python objects + self.unserialize = \ { 'datetime': _c_convert_timestamp, 'bool': bool @@ -44,7 +44,7 @@ class Table(object): metadata['datatype'], lambda x: x) if name == 'authors': # Legacy - self.adapt = lambda x: x.replace('|', ',') if x else None + self.unserialize = lambda x: x.replace('|', ',') if x else None self.link_table = (link_table if link_table else 'books_%s_link'%self.metadata['table']) @@ -62,7 +62,7 @@ class OneToOneTable(Table): idcol = 'id' if self.metadata['table'] == 'books' else 'book' for row in db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol, self.metadata['column'], self.metadata['table'])): - self.book_col_map[row[0]] = self.adapt(row[1]) + self.book_col_map[row[0]] = self.unserialize(row[1]) class SizeTable(OneToOneTable): @@ -71,7 +71,7 @@ class SizeTable(OneToOneTable): for row in db.conn.execute( 'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data ' 'WHERE data.book=books.id) FROM books'): - self.book_col_map[row[0]] = self.adapt(row[1]) + self.book_col_map[row[0]] = self.unserialize(row[1]) class ManyToOneTable(Table): @@ -94,7 +94,7 @@ class ManyToOneTable(Table): for row in db.conn.execute('SELECT id, {0} FROM {1}'.format( self.metadata['name'], self.metadata['table'])): if row[1]: - self.id_map[row[0]] = self.adapt(row[1]) + self.id_map[row[0]] = self.unserialize(row[1]) def read_maps(self, db): for row in db.conn.execute( diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index c67d44b7d5..ee72d0d442 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -340,6 +340,10 @@ When you first run |app|, it will ask you for a folder in which to store your bo Metadata about the books is stored in the file ``metadata.db`` at the top level of the library folder This file is is a sqlite database. When backing up your library make sure you copy the entire folder and all its sub-folders. +The library folder and all it's contents make up what is called a *|app| library*. You can have multiple such libraries. To manage the libraries, click the |app| icon on the toolbar. You can create new libraries, remove/rename existing ones and switch between libraries easily. + +You can copy or move books between different libraries (once you have more than one library setup) by right clicking on a book and selecting the :guilabel:`Copy to library` action. + How does |app| manage author names and sorting? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/calibre/manual/gui.rst b/src/calibre/manual/gui.rst index e5e789d9dd..0e3e6e7eb9 100644 --- a/src/calibre/manual/gui.rst +++ b/src/calibre/manual/gui.rst @@ -164,7 +164,7 @@ Library .. |lii| image:: images/library.png :class: float-right-img -|lii| The :guilabel:`Library` action allows you to create, switch between, rename or delete a Library. |app| allows you to create as many libraries as you wish. You could for instance create a fiction library, a non fiction library, a foreign language library, a project library, basically any structure that suits your needs. Libraries are the highest organizational structure within |app|, each library has its own set of books, tags, categories and base storage location. +|lii| The :guilabel:`Library` action allows you to create, switch between, rename or remove a Library. |app| allows you to create as many libraries as you wish. You could for instance create a fiction library, a non fiction library, a foreign language library, a project library, basically any structure that suits your needs. Libraries are the highest organizational structure within |app|, each library has its own set of books, tags, categories and base storage location. 1. **Switch/Create library**: This action allows you to; a) connect to a pre-existing |app| library at another location from your currently open library, b) Create and empty library at a new location or, c) Move the current Library to a newly specified location. 2. **Quick Switch**: This action allows you to switch between libraries that have been registered or created within |app|. @@ -175,6 +175,8 @@ Library .. note:: Metadata about your ebooks like title/author/tags/etc. is stored in a single file in your |app| library folder called metadata.db. If this file gets corrupted (a very rare event), you can lose the metadata. Fortunately, |app| automatically backs up the metadata for every individual book in the book's folder as an .opf file. By using the Restore Library action under Library Maintenance described above, you can have |app| rebuild the metadata.db file from the individual .opf files for you. +You can copy or move books between different libraries (once you have more than one library setup) by right clicking on the book and selecting the action :guilabel:`Copy to library`. + .. _device: Device From 5b805ffc2fa56082bcce154fc9150c7ef4db8f8e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2011 09:28:09 -0600 Subject: [PATCH 16/28] ... --- src/calibre/manual/gui.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/manual/gui.rst b/src/calibre/manual/gui.rst index 0e3e6e7eb9..1cd48a8dce 100644 --- a/src/calibre/manual/gui.rst +++ b/src/calibre/manual/gui.rst @@ -270,6 +270,7 @@ Preferences .. |cbi| image:: images/preferences.png The Preferences Action allows you to change the way various aspects of |app| work. To access it, click the |cbi|. +You can also re-run the Welcome Wizard by clicking the arrow next to the preferences button. .. _catalogs: From 6df5c994d9955e7e0de057435a7703c2c378d6c1 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 3 Jul 2011 11:40:24 -0400 Subject: [PATCH 17/28] Store: Gutenberg, rewrite plugin to use gutenberg's search and allow for direct downloading. --- .../gui2/store/stores/gutenberg_plugin.py | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/src/calibre/gui2/store/stores/gutenberg_plugin.py b/src/calibre/gui2/store/stores/gutenberg_plugin.py index 85d1f3966a..ad30f2067d 100644 --- a/src/calibre/gui2/store/stores/gutenberg_plugin.py +++ b/src/calibre/gui2/store/stores/gutenberg_plugin.py @@ -6,6 +6,7 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import mimetypes import urllib from contextlib import closing @@ -23,70 +24,67 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog class GutenbergStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): - url = 'http://m.gutenberg.org/' - ext_url = 'http://gutenberg.org/' + url = 'http://gutenberg.org/' + + if detail_item: + detail_item = url_slash_cleaner(url + detail_item) if external or self.config.get('open_external', False): - if detail_item: - ext_url = ext_url + detail_item - open_url(QUrl(url_slash_cleaner(ext_url))) + open_url(QUrl(detail_item if detail_item else url)) else: - detail_url = None - if detail_item: - detail_url = url + detail_item - d = WebStoreDialog(self.gui, url, parent, detail_url) + d = WebStoreDialog(self.gui, url, parent, detail_item) d.setWindowTitle(self.name) d.set_tags(self.config.get('tags', '')) d.exec_() def search(self, query, max_results=10, timeout=60): - # Gutenberg's website does not allow searching both author and title. - # Using a google search so we can search on both fields at once. - url = 'http://www.google.com/xhtml?q=site:gutenberg.org+' + urllib.quote_plus(query) + url = 'http://m.gutenberg.org/ebooks/search.mobile/?default_prefix=all&sort_order=title&query=' + urllib.quote_plus(query) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: doc = html.fromstring(f.read()) - for data in doc.xpath('//div[@class="edewpi"]//div[@class="r ld"]'): + for data in doc.xpath('//ol[@class="results"]//li[contains(@class, "icon_title")]'): if counter <= 0: break + + id = ''.join(data.xpath('./a/@href')) + id = id.split('.mobile')[0] - url = '' - url_a = data.xpath('div[@class="jd"]/a') - if url_a: - url_a = url_a[0] - url = url_a.get('href', None) - if url: - url = url.split('u=')[-1].split('&')[0] - if '/ebooks/' not in url: - continue - id = url.split('/')[-1] - - url_a = html.fromstring(html.tostring(url_a)) - heading = ''.join(url_a.xpath('//text()')) - title, _, author = heading.rpartition('by ') - author = author.split('-')[0] - price = '$0.00' + title = ''.join(data.xpath('.//span[@class="title"]/text()')) + author = ''.join(data.xpath('.//span[@class="subtitle"]/text()')) counter -= 1 s = SearchResult() s.cover_url = '' + + s.detail_item = id.strip() s.title = title.strip() s.author = author.strip() - s.price = price.strip() - s.detail_item = '/ebooks/' + id.strip() + s.price = '$0.00' s.drm = SearchResult.DRM_UNLOCKED yield s def get_details(self, search_result, timeout): - url = 'http://m.gutenberg.org/' + url = url_slash_cleaner('http://m.gutenberg.org/' + search_result.detail_item + '.mobile') br = browser() - with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: - idata = html.fromstring(nf.read()) - search_result.formats = ', '.join(idata.xpath('//a[@type!="application/atom+xml"]//span[@class="title"]/text()')) - return True \ No newline at end of file + with closing(br.open(url, timeout=timeout)) as nf: + doc = html.fromstring(nf.read()) + + for save_item in doc.xpath('//li[contains(@class, "icon_save")]/a'): + type = save_item.get('type') + href = save_item.get('href') + + if type: + ext = mimetypes.guess_extension(type) + if ext: + ext = ext[1:].upper().strip() + search_result.downloads[ext] = href + + search_result.formats = ', '.join(search_result.downloads.keys()) + + return True From 18c3d48a5ed4c998454644c8c31d40f32bbb66b3 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 3 Jul 2011 11:51:51 -0400 Subject: [PATCH 18/28] Store: Remove openlibrary plugin. It uses Archive.org as it's backend for digital texts. Since we filter out results that do not have downloadable files from open library we're left wit the same results given by the archive.org plugin. Thus making open library redundant and not necessary. --- src/calibre/customize/builtins.py | 9 -- .../gui2/store/stores/open_library_plugin.py | 84 ------------------- 2 files changed, 93 deletions(-) delete mode 100644 src/calibre/gui2/store/stores/open_library_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 4858b585ae..dcec4dbc6b 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1387,15 +1387,6 @@ class StoreOpenBooksStore(StoreBase): drm_free_only = True headquarters = 'US' -class StoreOpenLibraryStore(StoreBase): - name = 'Open Library' - description = u'One web page for every book ever published. The goal is to be a true online library. Over 20 million records from a variety of large catalogs as well as single contributions, with more on the way.' - actual_plugin = 'calibre.gui2.store.stores.open_library_plugin:OpenLibraryStore' - - drm_free_only = True - headquarters = 'US' - formats = ['DAISY', 'DJVU', 'EPUB', 'MOBI', 'PDF', 'TXT'] - class StoreOReillyStore(StoreBase): name = 'OReilly' description = u'Programming and tech ebooks from OReilly.' diff --git a/src/calibre/gui2/store/stores/open_library_plugin.py b/src/calibre/gui2/store/stores/open_library_plugin.py deleted file mode 100644 index b95f1bf930..0000000000 --- a/src/calibre/gui2/store/stores/open_library_plugin.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import (unicode_literals, division, absolute_import, print_function) - -__license__ = 'GPL 3' -__copyright__ = '2011, John Schember ' -__docformat__ = 'restructuredtext en' - -import urllib2 -from contextlib import closing - -from lxml import html - -from PyQt4.Qt import QUrl - -from calibre import browser, url_slash_cleaner -from calibre.gui2 import open_url -from calibre.gui2.store import StorePlugin -from calibre.gui2.store.basic_config import BasicStoreConfig -from calibre.gui2.store.search_result import SearchResult -from calibre.gui2.store.web_store_dialog import WebStoreDialog - -class OpenLibraryStore(BasicStoreConfig, StorePlugin): - - def open(self, parent=None, detail_item=None, external=False): - url = 'http://openlibrary.org/' - - if external or self.config.get('open_external', False): - if detail_item: - url = url + detail_item - open_url(QUrl(url_slash_cleaner(url))) - else: - detail_url = None - if detail_item: - detail_url = url + detail_item - d = WebStoreDialog(self.gui, url, parent, detail_url) - d.setWindowTitle(self.name) - d.set_tags(self.config.get('tags', '')) - d.exec_() - - def search(self, query, max_results=10, timeout=60): - url = 'http://openlibrary.org/search?q=' + urllib2.quote(query) + '&has_fulltext=true' - - br = browser() - - counter = max_results - with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) - for data in doc.xpath('//div[@id="searchResults"]/ul[@id="siteSearch"]/li'): - if counter <= 0: - break - - # Don't include books that don't have downloadable files. - if not data.xpath('boolean(./span[@class="actions"]//span[@class="label" and contains(text(), "Read")])'): - continue - id = ''.join(data.xpath('./span[@class="bookcover"]/a/@href')) - if not id: - continue - cover_url = ''.join(data.xpath('./span[@class="bookcover"]/a/img/@src')) - - title = ''.join(data.xpath('.//h3[@class="booktitle"]/a[@class="results"]/text()')) - author = ''.join(data.xpath('.//span[@class="bookauthor"]/a/text()')) - price = '$0.00' - - counter -= 1 - - s = SearchResult() - s.cover_url = cover_url - s.title = title.strip() - s.author = author.strip() - s.price = price - s.detail_item = id.strip() - s.drm = SearchResult.DRM_UNLOCKED - - yield s - - def get_details(self, search_result, timeout): - url = 'http://openlibrary.org/' - - br = browser() - with closing(br.open(url_slash_cleaner(url + search_result.detail_item), timeout=timeout)) as nf: - idata = html.fromstring(nf.read()) - search_result.formats = ', '.join(list(set(idata.xpath('//a[contains(@title, "Download")]/text()')))) - return True From c535dbbabf2aa77480e47b852737d848ee7e96e2 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 3 Jul 2011 12:11:25 -0400 Subject: [PATCH 19/28] ... --- src/calibre/customize/builtins.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index dcec4dbc6b..82d1d2ff01 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1505,7 +1505,6 @@ plugins += [ StoreMobileReadStore, StoreNextoStore, StoreOpenBooksStore, - StoreOpenLibraryStore, StoreOReillyStore, StorePragmaticBookshelfStore, StoreSmashwordsStore, From 6481066d732748a2debdfca33925d9f2d0913302 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2011 10:16:09 -0600 Subject: [PATCH 20/28] New db backend now reads all metadata from the db correctly --- src/calibre/db/backend.py | 45 +++++++++++++++------------ src/calibre/db/tables.py | 10 +++--- src/calibre/library/field_metadata.py | 4 +-- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index f30ce01ade..fa428a718f 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -31,6 +31,7 @@ from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, Differences in semantics from pysqlite: 1. execute/executemany/executescript operate in autocommit mode + 2. There is no fetchone() method on cursor objects, instead use next() ''' @@ -128,32 +129,31 @@ class Connection(apsw.Connection): # {{{ self.setbusytimeout(self.BUSY_TIMEOUT) self.execute('pragma cache_size=5000') - self.conn.execute('pragma temp_store=2') + self.execute('pragma temp_store=2') - encoding = self.execute('pragma encoding').fetchone()[0] - self.conn.create_collation('PYNOCASE', partial(pynocase, + encoding = self.execute('pragma encoding').next()[0] + self.createcollation('PYNOCASE', partial(pynocase, encoding=encoding)) - self.conn.create_function('title_sort', 1, title_sort) - self.conn.create_function('author_to_author_sort', 1, - _author_to_author_sort) - - self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4())) + self.createscalarfunction('title_sort', title_sort, 1) + self.createscalarfunction('author_to_author_sort', + _author_to_author_sort, 1) + self.createscalarfunction('uuid4', lambda : str(uuid.uuid4()), + 0) # Dummy functions for dynamically created filters - self.conn.create_function('books_list_filter', 1, lambda x: 1) - self.conn.create_collation('icucollate', icu_collator) + self.createscalarfunction('books_list_filter', lambda x: 1, 1) + self.createcollation('icucollate', icu_collator) def create_dynamic_filter(self, name): f = DynamicFilter(name) - self.conn.create_function(name, 1, f) + self.createscalarfunction(name, f, 1) def get(self, *args, **kw): ans = self.cursor().execute(*args) if kw.get('all', True): return ans.fetchall() - for row in ans: - return ans[0] + return ans.next()[0] def execute(self, sql, bindings=None): cursor = self.cursor() @@ -169,7 +169,7 @@ class Connection(apsw.Connection): # {{{ return self.cursor().execute(sql) # }}} -class DB(object, SchemaUpgrade): +class DB(SchemaUpgrade): PATH_LIMIT = 40 if iswindows else 100 WINDOWS_LIBRARY_PATH_LIMIT = 75 @@ -516,12 +516,14 @@ class DB(object, SchemaUpgrade): def initialize_tables(self): # {{{ tables = self.tables = {} for col in ('title', 'sort', 'author_sort', 'series_index', 'comments', - 'timestamp', 'published', 'uuid', 'path', 'cover', + 'timestamp', 'pubdate', 'uuid', 'path', 'cover', 'last_modified'): metadata = self.field_metadata[col].copy() - if metadata['table'] is None: - metadata['table'], metadata['column'] == 'books', ('has_cover' + if not metadata['table']: + metadata['table'], metadata['column'] = 'books', ('has_cover' if col == 'cover' else col) + if not metadata['column']: + metadata['column'] = col tables[col] = OneToOneTable(col, metadata) for col in ('series', 'publisher', 'rating'): @@ -538,6 +540,7 @@ class DB(object, SchemaUpgrade): tables['size'] = SizeTable('size', self.field_metadata['size'].copy()) for label, data in self.custom_column_label_map.iteritems(): + label = '#' + label metadata = self.field_metadata[label].copy() link_table = self.custom_table_names(data['num'])[1] @@ -562,11 +565,11 @@ class DB(object, SchemaUpgrade): @property def conn(self): if self._conn is None: - self._conn = apsw.Connection(self.dbpath) + self._conn = Connection(self.dbpath) if self._exists and self.user_version == 0: self._conn.close() os.remove(self.dbpath) - self._conn = apsw.Connection(self.dbpath) + self._conn = Connection(self.dbpath) return self._conn @dynamic_property @@ -641,9 +644,11 @@ class DB(object, SchemaUpgrade): # the db while we are reading for table in self.tables.itervalues(): try: - table.read() + table.read(self) except: prints('Failed to read table:', table.name) + import pprint + pprint.pprint(table.metadata) raise # }}} diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 398afac5d2..cbb3ce0006 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -92,7 +92,7 @@ class ManyToOneTable(Table): def read_id_maps(self, db): for row in db.conn.execute('SELECT id, {0} FROM {1}'.format( - self.metadata['name'], self.metadata['table'])): + self.metadata['column'], self.metadata['table'])): if row[1]: self.id_map[row[0]] = self.unserialize(row[1]) @@ -102,7 +102,7 @@ class ManyToOneTable(Table): self.metadata['link_column'], self.link_table)): if row[1] not in self.col_book_map: self.col_book_map[row[1]] = [] - self.col_book_map.append(row[0]) + self.col_book_map[row[1]].append(row[0]) self.book_col_map[row[0]] = row[1] class ManyToManyTable(ManyToOneTable): @@ -119,7 +119,7 @@ class ManyToManyTable(ManyToOneTable): self.metadata['link_column'], self.link_table)): if row[1] not in self.col_book_map: self.col_book_map[row[1]] = [] - self.col_book_map.append(row[0]) + self.col_book_map[row[1]].append(row[0]) if row[0] not in self.book_col_map: self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append(row[1]) @@ -145,7 +145,7 @@ class FormatsTable(ManyToManyTable): if row[1] is not None: if row[1] not in self.col_book_map: self.col_book_map[row[1]] = [] - self.col_book_map.append(row[0]) + self.col_book_map[row[1]].append(row[0]) if row[0] not in self.book_col_map: self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append((row[1], row[2])) @@ -160,7 +160,7 @@ class IdentifiersTable(ManyToManyTable): if row[1] is not None and row[2] is not None: if row[1] not in self.col_book_map: self.col_book_map[row[1]] = [] - self.col_book_map.append(row[0]) + self.col_book_map[row[1]].append(row[0]) if row[0] not in self.book_col_map: self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append((row[1], row[2])) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 231af23038..f8ed0b035e 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -220,8 +220,8 @@ class FieldMetadata(dict): 'is_custom':False, 'is_category':False, 'is_csp': False}), - ('comments', {'table':None, - 'column':None, + ('comments', {'table':'comments', + 'column':'text', 'datatype':'text', 'is_multiple':{}, 'kind':'field', From e501c5262230a8de3650d556e4759a5e7ff5acca Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2011 10:24:16 -0600 Subject: [PATCH 21/28] ... --- src/calibre/db/backend.py | 2 ++ src/calibre/library/field_metadata.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index fa428a718f..5c0b8aaae7 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -519,6 +519,8 @@ class DB(SchemaUpgrade): 'timestamp', 'pubdate', 'uuid', 'path', 'cover', 'last_modified'): metadata = self.field_metadata[col].copy() + if col == 'comments': + metadata['table'], metadata['column'] = 'comments', 'text' if not metadata['table']: metadata['table'], metadata['column'] = 'books', ('has_cover' if col == 'cover' else col) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index f8ed0b035e..231af23038 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -220,8 +220,8 @@ class FieldMetadata(dict): 'is_custom':False, 'is_category':False, 'is_csp': False}), - ('comments', {'table':'comments', - 'column':'text', + ('comments', {'table':None, + 'column':None, 'datatype':'text', 'is_multiple':{}, 'kind':'field', From 33fa268fe7a88518bd1cab573639f63d179565eb Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 3 Jul 2011 12:45:31 -0400 Subject: [PATCH 22/28] Updates to quick start guide. --- resources/quick_start.epub | Bin 130640 -> 130575 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/resources/quick_start.epub b/resources/quick_start.epub index 3b289537a68d390883048ea0c099950fd46893c6..b1e793604ce19900042e954c3514aa94af5b8fed 100644 GIT binary patch delta 15289 zcmZX*V_;=Vw*{IV+v?c1&5rGkJL%YVvSW5^JL%X~$F^Ho z_1PI=2QQZy46|?4-tK*A-e5-D&P||v0|!gb+ECDWsst)+RHHf^mt=A}Ml~H=mQtMR zPY(2kl(SXWFqr7iq7-7%JtklO63>Y3>EYpEoMF3OF8OWCT_tH>TUwf&P{z&(eq^X3 z&`_$&X!+xU*17;JK~EP2mEiYP3a*|CqoKXY6l&t#0--8fO1SjWTtP zI31>V8oGKDU!oFVPlCEXs~mw5{#iY5ItG}@e2A_D!l^0T$N~|vTe@fn6#^YTx#!ua zl91DrH>6C5#)zP+CVb;x6G#~x%(72IT@yvCh-vo$6lW77WwAL?mpTr|>A$#SuBCx% zpqhRmLC*hqK}Hh}usp${_^P7W%MeKX1(LK#%5r(WN3jq^Fg00adw&SD-U4=^0Yzs=CljyWtrUn64X<*5lnzJ_@%xSr2z<^4X*Zdp;V zWA$#$Itn{R;2_E1P%-WpuCwxS1_Jy^TVDyIKe$p385O*I1Tm5QVA4UFK3J;B*=n&6 zIUtr6dFXy&Z(7=~xvA@GyHMKtZj!sfzT?WigJeZ^WI=|#H1w-1cKWyYfNtbZC=Q49 z`JwyY;uN}(vXC4s=N3jh)v=lyq;-CgV{JI8NS1w<9#fC32|)dfxWuP*O+jryIU zE<5nc#{t&P_qhMt93Y_oX%BUh;GE$9w1@zI=q7(?0CL_x7fp$!fE57ZKP{x5cgOV= z00I&QmRx-epld2A2Oxm{#k5ocI>G*)dDH>~ArSsSFZop~8-as>OeR-d1L%`)n*f-w zF~h%v{djT{id>S>ngN_Z>7?WQzyfNkk};pedIQ%RH4y=56cuQe!mtO&=ntzcR-b#6!~Nsq<1NtL_P9Kt4yg=*>Xd^wMg0no zlwf0Na~`K~vk;@U+6W&m{E{lIqgY~mn_67aa{83zF)JZsVgfOJyTXuEhqMhL1^Npl zAGkYOtyxU}x9OejWsfBMLy60YiEP`fF5U?frngoUPpK{y`Zc}O_zHyX1%;#0JEb?v zp4nrnrGt-K!L2MHeiyJyoXiL>ChyMbmv@q_^|DHC5R`tukPvFO+9*_XGE9j>B2~bG z#Zr`Gb9eDdpJzPefg=#iku!s@2q2(KInI)@LReOF1PI^~IjW^IkDkB+> zq7{u5yDz)}O=TFk!fp!QwpjO&V4dqf>KBY9yA>?6N`li)2UYstvy`E|1E$pH&q>r27* z+-JJwnVj@iW{rm7uiA`}LmDn|(kVzQMbwKN>HQ_*=+T@X1hB zg!Ctk;U&;|Iwh|bGwN3P;%iwiU-Bo-9GwYaa)K$HQ7K_k<2h*WunwKd=v&jIB4BVwbi8sGhkw<0K-7j(W5B*=Ju-#Lm`;z6dy3 z=m(iQG1NR4_C~`fhB%`L&6x7Yx2kKoKv)$@YdasBP_fb_q_@|RbdEZYI35P#Vrwj00;`j<YGQk@F zIvKGK01bViZ$FP^8$_B+&@_NyIHs}X@;&na2m`4D8^yCI{%}Wfe1AX$=V95SEn_aM+sFCjW`~D6;;>Xwho?i9 zF;*Ge)BfxHsRon6p{9#_qAbWC?s(BmYOqdkR8T(USUxE}i-I3JbSeM>)>H{EiUEaENurH7ip; zfTW=uoh6g2JFT9bG0hUBp%zW^%=V3cVO#E!B|5X{BOYptxeoqmeI2#iuutaM>~^*N zwqxb|ux-H69l5z+NV*L~T7qp-Ixw;ptzI>~QhN#gJh_tVLbvjM`cUA@;(vRx4@;W* zfkj=E<=moxX{miiwAe$c=ME**)2$XAvN#{sIS=y-v={xuB#Tbt32rWlN^T#%g1j~z{02}W&*!~FwIETJOcfWo>}lrie| z3k`I&)_R-OLWYG#7%;6TUyMF_aFn8{Qbn<;7i`dlYT0d|NMfPW4ET}Nc5G46RSveg zxDRp{%h1Hk62C@V7Du59>^J+&d$f=>?vq0N}@q=TK)Em%18k7b&{5 zbI3!Y!nTJ3D`IO!IiX)~0va|EN+&90Np(@t*DSOxPbSGAj~Od&G73jn+9EQ}fxB9w z7qL~9YLg?NZ!T?TXv5%)My}~$@%kM(?#3YXG>ga#O-1*@%-Sd>&#+{@|MU<_H9_F{ zuP1nPoHO2J$G&Ai9!#Qhjnb|6tp3u1Sp@j%A$Gd{9df*YNEEq5kIcS4Hj1c`dX);;QN*$!LYpo&n?79+^ggDWygeSZh*X(CyZkG8N6Z*o>>o%a&U3 zzDxu_Cov^{c*Lz&s0uNMYgblF|adY)UZi^wuCw|(76f+GsP>a0|t zpLXM;!!^G$lf(z~IXzqF4v!Q7@G%qPqy!iE*E0?FQ&W~ z-KigsfWiDwIvJ678l33}$ADCm6S3`u#{L|ny&r6O$emLoTz0nu>oeZ{21c16_dUXp zngEOjC^hjpc7HpCz!VK3`aRjRz)-V24Sm!u7Fr-9SN%_uam9BbO2L(y_GF6?HHWokyR4Kx5YL z6+TZz(k`gN-Z)^yn9?e*js!sl`eNzDpf;)*BkW$g_u83f&~+$fSkJDt^Go(OF^lbY z`BQ8nsV%p$U9A`iMCI>|x6nT@hzaW(dyS6OeFq?kzl7o22ZWAP^{Wqj0TJ8QlI1=By&MR~_&MW4#YG60W}^hj+sZ zOV@t&+w%7{n?A6X8j3r=5|wiumxWo^?a*Dvj^f}DA*^A(mlhJzeSF=Jcd!Y#gyhp?S2bf(3d` zDiGj(ZojQB_it_jZPn?|fuFk(i&Yb28{1Y7v+;GyXKy=eZ*RT*>5B(u9&G8}pDvev z%${Y971c!+OWPbV|wI8APioS72> zjp;r8ntOz_hb!ioyIJNs*H*Sqc9Zq#X-`wF3g|Z2W7{kANuP)$(4#;kPvFZUOy*Zk z*uuNJ)b{u8YrZ|)cnl8IE*68|1ZYhjYru9My2^0=T}P7Hfzd%vO02tHJ|Bw-kwOn4 zN&x{t&EWln{kP7W>8a1Kp^1rzNOO&e1meb`2%BjPG|>?fLZjSrxJPqC%!x*PDNzMx zgPNT5q67bw=*?i(6Ghh`Yv51Pi9i{{&T;cXb5R>kr^4OD;=*@!)q|?e5jf86Xn|mh z9|BhvorKvP)p8i)gw~os(~V&6utWMxZgxTSh^cyAV{Hv9iXzyG8nIlXuci*J`>a^c z4gypwGpRDI^oFr919ND&(pahZL0#xOL()I1_QZ7WQZ+-0hQ>YiLxC@HN3zGsWeapi zTxe}p#-S-QjeS#QJ|2!d%(feX+msnuS#slLH0zvSB!@*Bj)@i%xT;)SXGr`bdcLhrt35|LHiJ7GLK`yCMd|vf0{;(?u@2OfNA;FT* z0XoqkVLA8$zSH?&eJqV!Qer)IgNngwuXDz)p(C~;5!S0Rj=(!AR7c(T0Ws!Cj7(uR zV=J4A)HauMokR{qk=C%q#2WVt?a-_!vo!}JIwd!mri8bF@HO1))h34U?pWlZ6%S{_ z*tzv~BK&btbN+0Y&BD_#*Snl>N!{2QUe?W&22wdtEE*r!=RV296DZv_Si|{=v(M$} zS-a`Z>$^E4RG>9&^{|68zCXTYaqC zAh^^)bCq>u=aho8ugHb<3js5+sOk6vv&&g4?2S!}RY}``$v0|gKFZr$giM-EB=O_b zqj0gyf-v}Iah*i?VEA~aT&-5#hP!ot*(~Y_E3wjs%5b7Awucia@9Kl_f^)6L!Hw_( zBON-F2B3j<`8mY{9%DQOb-?mOW1g6;jmm96yld#~_zMV>oMVn(qT0ez?DKAH>PUdO zpFheboj)0yG}wBG+!;4n046 zFq*)dmDhvM^X{I`c0|-OrIVk`nogH>m6!Wb2DrJ>qtM=PFjEvju zoPVON@vwMmBnWPZtjPN+jpzh?-6n;23JijyvEq(Z{CXoepa9-z4TEyLDCrMN(nezJkv|+axWegoX=qL0=HnA4L6@L)6akgjN+F zQGftf+YsBw^h`}u8&7W=q33&DZ~389Kpdu+QI@9yD}Dqv4bBD2C?ge-2V_JnUXt0s zJjMBzDFn(Bus6GtL1x%z(izts)RngfQ=4+-NF`%8&O+5oE|e!c-VJ%GXB^Chh^ z!_E#0WPDgShOHoNTBuCB8YqZGmFbFiol7^|4kM%}EtbS2{4E4aVI#*CgbWI)T^4nc z7Zdyzv+2=aHpf8`Q9rdis8=PuIgH+>Jv-h9l=9suqc%QPrSi+A)8N)#4`(cM5$JLP zacsx~Y;DZCj)JjGz&9?{AMS*(3ZK`rUw7|ckxbFveAt+=Eyjsx%OR2>?t8`O+Pk$bh_ z3%#~QMq^$INep@!(VvC1hdVBC4_iiPTO^d7oaE`Kba-OzMXvJR^3`I2S_cjN!*v?4 z63Iqx_LBSq7uRg4M?la{QtuovDngoea00#T5ptrG3%c;}bEjygB~j=bAzvk)6P_Cq zW`e7@rnI9qAC7`nYt5gXU)c1hCcbG^q4D_gQN4~wS8Dkn2sVs!b7vihO{mji<(rbK{YqNUIKb50=;Lkl43Z2%-6jy^ z1(5c7_H#FG28XBq82?(@yxC(8dyh`LwmEx~LfaFi;?9$aCkm&={2N&$x18<#K1PpV zEM%%@*$8m|q`0p)#qSWhHIBV#3>*%+&WGuY)YJuvrpjtbljLjZ4L?M&!=zJ#=9BCt zzuuQ#dEL@;W&ousz2e*Q;-6Kx)+>W*$faY3nG@g((H;&f`>AC~0)5__CsmZ)DwxiI zPKi$%!L+YKHmmmR1qb2K?w6h!Jg5p`1#4=dFuLOlaRnz+(Eigoz&_3Pin6L!4{%sT9~mZa0Ud4Usi9+^{LqmGkxKe#FqlOOIWPJmMU48DW+{_HI_q6Y7z5en5@B^pay0ztNb`;QBX{L=AjWL9vXGY9im;| z`#J298?dHT!3RcVs7D_N-T;J>FhDPcACb*|>L9 z+AmUz&LLg`V|QSs#8ahJp%RuoV8LFlYKDeJV5{BMB20QO-|v1Z&@e34C9!y)Z`tON z9YEk3*4`sF-&f;0*+#KnYpIc=BdhS1cr{tTXw?ftL{CPJ{^hy!qVH?K5|%sjRufP) z(r)-_cz=(sA}%TcRm~#cIy|WOl08Lnq+BsYSHwd7!5)02>zgDyuv2QDTE~;p_67Mw zf*-aZ<_E8e#(<_`-SPb?hv!#+tLc?doIjkdw3IqUPGwrkWgxBmVZVlzsMj6-!=v5+n?+^08AuB9C$`~1XZoQjuk6ydYnzl zCChXg$GVlZ(N`a8LR&_J5?H9*-2<6)m|1mL=DI;HhDxX*pXTjEM`EgR9E4}4WD@{& zhV*;C7J9R8_kEsy#`AQu6dg$Fvs;t6cNd{vv4crcOF%^2PAV#@ltvpMM=ExaOI_mC zMNUi1#TXT1Kjc>8QAkG|tmjGSu$ofO0#fvKwm%9E&fA+r8=`d^B++vI%P`}~N-i88 zqslLgMJA{5myCd!r}kp2w(F4D{I+ff&q4M|7YjJGjoM`q?sspc_}@S!D(oJyUk^&| zP=R;;ps7UyqyFsQw;6v93TA^*;5)ByZCpkOz8*{7FrT>6Flcc6ayoIXUCC7ndvn{< zArB13T_HatD)`J;H!21XR!dnFG(50pzSw{&274=(>C-@W(7tE|M47pWNxg&9$ z@8CP<(B_g_F9#}zcv4@bGfz>3C>vTYw7;Y&2F{d4TS|;Gto&6%ye6tAL(BNWx;L&<(_;!mPtquIQXo3?=GX~P^G-&8>_J{&hw?Er zEGu0rX{%EHifyft444+rGa%Wf?|XmvLhnj&IqR@JQP}+WdlxJPEccNXan;3#yyBn~ z|Gdm-sN7k*aHxCP_7S8fwm4@!MD|> z=GO{d1^no5lCI`W4(IM=}y2 zLp+gHNs}S1)<%nCEwlw9GAPR_jH}dg{Vvg1!N8&=_^?Hz`c@4nmW@fGr#%IuflIBs zbj7K?SgK(rK}m(QFYf`QbtHnxg(5Pabkw6DO*bXRLntC1AsMBHugu&-bBS`H-?3Rv z45g`+;0lm|7sf>c)_Y6i2*r`5?vh5JN<|;|^vS=S{FzX|sbj}kvRS`Wt-MpEfYr9c z3drG+kF>(>T9iNOFzTK;S)MJ*XZ2DP0 z|0Ri64MoCYYjz-R9p=qSPAAMi2Zq%`-(ppeRj-Vpup~|SUEh0LH>Tv7R|*j`&txRE z;hRktl{-h{bNU*NH^n;v<*5WY!p~w~Q0jJr+8PV~u3&bEXroV#srAK+-;INtXX3aV zRI0FBz%_&C745@(NP(+bjaPCOJ@^4%@0HgV*`qZZ8QI|NeC!36~S!Ah29lfW8z-DHxpSYde8Tm>%o(!sNA`W{7Y2$M0 z!Hd7T6dr%2ZE6ncklC{OqmV`}r7lnL5NW^RsV{TUkJ}!{COHhw#~d}Hk`wLr4ZAYo zi)3C5^e`m1U~4_0qc-P_c(A8U6Bo?H$(Wc`<8NOci+qA(y%ZOxc)67pQc7`OrkZ{? z1d0^#SS@L_empq}5h89lB@HIy5Fl2sPySxXik3D`F( zpT-K1=u+KJRj`X5R||~{o||%64Raj6HNh~px=@AflW`-3$-SU>rWj)1Pu>+xZ$b}= z8jv5G>!CUP9`;EIQP`DLo8vDa)8Z*G2d25l<9)-lK#%`!FDe1GPoo6$lU(&mJ3T!~ zHuc(9_I*om@d%nnd93(%xuc0WGKMbj#zwMAs=ZpqzSND`wv+fRs)nD6wTL~xSmdh ztXrVq5*>G*!HsMq;Xmptxf8@S)q%Z2=KeS^8ifLvTrcyMEYES;lv=Yr)B2>%>RWodRg(GJ3dscqq~-lsj|kl~1yOXIKqq1S;|rx*d50H# z@8z&M72Js3lR@2#PtQ*%X-Q!nP1b2hED`?kX@k0j1pFldrnPrCG z8K8;GXC%K!fm_}3?29NO3R7CRKtUfq{3aH-HUhtp`TJZ84Y|dSnMH<2^-{FE)wA}o zGbYr}V5?rW#7k*M@du*$=~+6tfi7-dentKs=a~V2!CYOLm8iH#+uBN_PvhmH80iL{ zSB6+!q6@GPCm;H0A!g!77Q6!N1eS)k*8@G~CeYLgbF!f(6_61pBN;clNqJ!T#^W zdfp4Ah&}`eNHpC4Eax6=12BPb5*Bd?W;}zssIJzHA6mP)wBdOI|#3v zt`bV&yC(gx=u&-cyKSzQS!~`ub?Vi^rv=l+!v-&8fVmdx0=-O%NJgYAimmpkAVu+A z^E|m5FOD>Ul&-dKULK#IGz!m`pIkt1cfhc4k~d%HQ0=oJBCu1d-5rP!a%_;);g0H%YGBA?m~wL9Bg{9| z;^FgN@V7<^0zTlz6CJowu6pQ_V0vWj;4D-&y4_Yw1@`7s-6d%-84hE7SyPx5nj+ZkV+mL z0u3mOeAF0-ZuE;6u2lLXC_KS{7V`!ce@Z0V8fVoG;K8J+z6!ZdndZ1jh*In; zeAC+*ALYMIdUype%GhF5K<~5%{;O5S4lUn#=v@UP9OAB zY78wHjivz_3yBjQc0YcLkz@T@Rp?@KsZKMWpO~6lA4ZcGf$ByJaQ^a zOelt?W}C-5!ZP>yyNp?DmQI~hOJYOuxf4EEPrx)c;y9YzPx=kG3bH<#z?r3nD_N5F z0ZypjUmdBU$oL%b3G(H(6U}4OGOmdIrt7bDf}=&Refvk$_CsJ>G#7!9hkK)O*~s?Q zEM8CuCSkpi8J%G4sw@U)u1vmnKP22I96v^!@(H_&_xE6?tp?ESela~V2}m4_FWKj; z=OQ)reV1uU89-|u>Q1rfmYFjP^F}Kv*vU{qqSmz{Iu!`^-_V}@<)7|(d-28+=#rKR zj>k9z(JRW}WqgMz5NHI%lHXKra7vs1Y*E4r&!l0yfqQ9(RgE-QjuAGq@rxcpJL<}X zfg9yd!MQ&V4EDBj4-(7Aev*(vbq+3Uz&D!sg1@Mi@y+0yE_nkQ;_!(sazNZ-^XN@l zfS70&dHYzeG*T0)H)y*K6tj(@%)$?{`eYcfQ|6Kcbzjo{?b$ni)y* z6ze!x*V(K607I3hhFqT^Z64Ue0QM9YB4)biUzX_IzRAD$|NLd-=t;`BQ&&b}`1|=y zB61etY)Ei3ZtQZh#VZ5#*!Q4qehqkd^)aN?=>$W>GiVIdV0qF~OKlU`2NhnfA%%EU z*A|sfr?+3TkFIIleOC+0Ir24dm?*EAg;vW+EYNkz`a1831ukml8Xdx4$ZdbvC&_qL z=Ch{Lc4&o1nsfk!lpQp$PJ>ZYG;_4gp!N#4f)*PwF;*{%yv!s@BhZUUy=%J+N8fly zdV~ix*+q09Y?t0^sS(pjH4EVJ;S*zo(Yx1O&zK$gPK~o~>!oV6TC@0IH=VLT~WFCzk4F9vsX7J5>E5ZP$^j=i{_S%?pV7`|H9w@cPbb*5#HbvC~MKOC-m$8k&f zE;iq8?^jP{q$xBXs_I7R+O%}0ySg;r-s$CL&67Hw8Muk1X)af9>9ngBk6-B6T@vXx zj32GO^X$|1UYc)582LD`z%`;|yI@7@WOaHyv*>kiYm$pGha{(JbSB-(D3ui^-bKsr zH{ET(ogSacV>l%BgL%~h)nAfmXh$Rd?5lb7QISb*xzZ;uRB${*t*z~pbU{M9Nyp@J zbHA*!BZ6z}3EOqOjXzsHiy&mI(xcaVf1=~;U#~rcU_O&6{mKLFzKYp-hFS$GkRP`$eZ1avC!+MI=%b0C=mm;>i5c8qdeAb94IJ6KFhLtvN2+M^xT$e5m!o&^=bYBprG;?Z&@!%J;s zm){A`ZCv4k@I==K3YhPf6c^xOWbToMY!}r33MAyVs2EF(zYkuR53bDT#2%N^9ZiwD zWGNID#mGq3)r~(z2KA?nB=;+alXC5$Ip#DPfg*@4bl5>o7BrKO zHA$%RA>6pTZ7gkjH9DYyr)w@JdV|wLzn3{u2zK1Wvh_p{1gY3h*}&}}R;BYEr0^4C zK=IPf_0e^^=&w!MD6A`w#8Wd3`Kj}+B9;2 zM^^TPp_JeOzm#MAMzN+1B9HXn5yBpywp8y9O_IbNOh8oVU{=hmR+NaG=y40|MUvxD z71Il9iu`>ZV_A_AjUKGXO=7@#rX1*MWlKxm1i=+uO&puD7DHDNgb3h;DR=3E|MsBx zHq&iMIH|nX_So^sd{OrzTHOsIn(gS%<}q1hpNq``#Pc=U6ANFQBm6JxvVg?Iofej|! zR87LdKxTR;kRncKsS%-bMtGH}#rR5HIBi+2e5aV7P0~jgX~i>9ZE-m|%k4=D(MGMN z`HGbqiJs*+VBfrqsU(;*tNI7-IUXFfrP;o=n8vPCXU|x6BM<<&RkOtj#S+aK43lgO z?uK?Lv!8(4)6`9QvyL8)$Ytj4F9tZjy-U<}ff>Qs=b~_us7ygFIc+9iz{BB3lpR8f zVyGpC?U79O8$*0q=O??k-X6BFSp%H-Ery%N`*ZkKW|`h;!+~K zfTp~z)_m@CN7hiKOF!Ff$q$IIWPJ0FnX;Ho)M=y28cdAEc0FcS?S!(Ey;YerR`&dO zvAzmdvgqn=Q8ypJNEVu@R{Y6)ZW$nn3FJh2QV{E|bOjBTiX71$Y7E?pg!42l^l^OuXoE$-d6a@oj`i`ld@18&)x6>ZuT* zQP_UrtE)LdpV?P$Kq-_TYS30RDjcbiyqv76oXPFc7uccd+|afYlKh1~c(7zmaz zgWTt4x`Vce4jLh`bHg(BYnyEaP2-1E{wkGcd5iUYK-Y#%0PJP4Dv}DAH+;ZeH3K#= z!I&shwl5(jF%=$WHvPT<_QMG8kp`GVdk+m%<8EdDE~GxMNSS^JL#MXolq;^ESk!W_ z7I+;J29hOF(0gJ z(Dgxkp%P6@RQM6Qdf}Oi36cd7Wv9c_mKVA#_)Hj@4xhbU{xC7Hji9Kb2JL}@a`)zL zNFKR}F&(*q)z8oGn0_xJ({j6sRA1VAO6N)6iGoY@^8_pj(HyLT#LMf!5q18qU&@#2sc2)-@KKG*DT56UMXkm3Z{p* zx39#}&`*`9g@O9jsLmzlRF0uG&ai{m!jEcC56nT89t59q`(961F{o(HXNfaM?b-a& zEE-nCoIUaS?zSMOeI>nJ2J9@By+T-IDo^{&GKWsTuc^lp!KM7yP#3fsYbkbm+05ii zvC2c}I`Dwy2+YThhQ}SYew|q7BX#&|0FHDY2Q~!nGKd zy08VD;zoJN)GS)qO}=YNmNNBKcRMBR-^ut}78Tgf0vxQ(x)f$AoR`{Wc(vydr9in? zEL54=I-=0@f*@cbrs-k%lh|JVYYgpd>if;?k6@w<8U%#lUjb5b>mQ*7@X~heEAqP! zuzyaUL@lgILQE`tl9-IX!i=~u1h$l>zH#vIyuwxYl4rv2Y16&!p-@VdgzMO8*v+T& z(Ig{Nry-4@#dRc2tyY$hysxM}ktJP?<>}=pJuoA=C~gHu?~mGo@|ncb?&Eg(iBZr|8h^ORU|{4^`1Z%8aMEIQG&e7RkXW1_kdgT zA*K@qK57iT{F$lVDSgvV645f9ZC}O+&KMmXU5w0SiZDtekvK+M#Vr`Gg~30YE}W?C zacX>d93rfi5jITRY+>9ucc)Vo$A)41E?}e{A%Y)-R6UDZbsjAaq|{ zZ~Tn3yT0lT!An)NKS9rrFn!EKs;m5=-;$-%B#*p9)e{EfIJ*GiIyrXrQ|oGI$w9wZ!N(P7FCku(~1 z3!S;}$8HPkd~XhFRB4nkQ?-Q>jgu~IuY?bife}1O$!->QDJXG}(@0_J(M92mP?>8A zxSv6}>_i(guxbO(wU5|SYeoJ0)JL>>ny^Le-op3n6#rO?9~4GeGpUHQ*!NeUFI+jl z-0QHynlMR4L8u7#bACGwQWQEHA*@T{Xj@C3-*)!Vx*66-}HN zC~G5!TWf!H>7Zw8x*(Vs4L)pqqrzA>T)oi442m?P3e)8`&dwH_BMBg$URR36Kp%-;;hlWk67W)Ct<&j2OU z_o}K8n5|T&p4O5oEY%EGi#?H5E|)VJm*K4e6GDdzOG=In1~397@lgW<6I|;cP^P`h zqA$~Wz*3r-m5UwBk(n}?g|=qH=uNWF-82M&=l4AXz$L4P%6n0H_@p7j&z)zj%B zypp9lgmhXRb--nE>F17EgwCKIpB31g@b6#S`Iak_ z#1M7L)x4VnQD8qXr`E0qX!e0St4QS_M%(tr+_e3c?{)ou#Tx?d^kYqy4}em@KLU?Z*0-xK*K0Jk(c*gs;*{~4wJw|w%?4}XsWKaxL^vHzc#Qc(r~`v0k(eEbB!8NA?H2(5UzfzMf9i^~|6AAe^RwmNmhCT@c(uApPAYJ232eWlDoG7jL92s z{|5ihOv8U;za9P?ob?@m_Sf%!vZepVE?xY^QYJ^e1JM7zr~mHR{_{}Y$=&b&z9+y3 Y02jcVjQ;_khW+yA(``Ne`Gz3>56j9ZrvLx| delta 15359 zcmZX*1yo(T);7$>-KDq{cX!v~?(Po7eV5|y?yzxphvM!Om*Q^4{p&gB{_h><{uyKK zm6bf1nLBG`&WvO}v9B;QuP{hTvJjA%V1K3WYH9IETo8ZVr1pWLikX7HZ!FfQz`?+7 zkifvuzzDz;B}A1}nM@rWT+JO^86BN0)Yag?AP@vv<^Q_eJQ2VEkVg<;V1NHKrOLZx zGNXoEK4IbM>!KPB8wO^l7qeyGE`D{nhpiims3VbPpX#|JBM?(9F&1rGu0891snuyj z-6TMvc>+QOr=;tsY~4`Naieqkn3tE`^^7s zn}w@0#b98kGAC8SCfT%?Bp9OH`k+IwkjhZ*9vLW;3~(7RF$R4xb{kj4#S->UQaKx4 zz3Pi|r1&0kcIJYN2IZ891sRMjkz&H(X-&N{hDi`3p%qC;Kt?T?c)6l5t9~bI5T$Vi znvpKnqo7#)3N^-?H9buvKN}X6*{+V*o*)_V;?r0;CdwcqZU||DDF-3wUQ90JJXCl+ zkq-2^Sx&c;YOoTFe#I5%)F$5mjgM0&dxuggrw(~@F0R(;kdv{RD;a%}2&p&{+=HWt z&=g)2hU+*wZ`&;&x0!B|+iUw!KqK|B4rnKCP0*b=_)LjJFj5VsXPpSA8BX35Tn7Ua zM&U8xVo)WYQ=;M&*mrg`Kq(rlxJ-qlDGNNg_$IMb=|+`oaoSi#PiJ`_C+4hdYoDXK zFn8|=W2Bew#_+hhsKMdHKB2>8_1=hcinYS8G1)$=-r&H#30m>YIVAEZZs!@_7`&d; zQBU2XU+5t5*F*&x#pTgSGc%8r(v4Ud#eLO`yZ%A4q%Tqp1D);)m5zwWdJfM@_z7sX zh-JpfJkm)w;;V()b2@c`(%JB=@1gMlJgdC_Nj1!~r0|?tXA4enTt!24qDM=@-F;fO zAu0?up{Dq-G`eiwqtJf=QRg4KoF{cwDs|Q>b>9_<0p+ld3xhy3c4ZU(>5ax7&+4Lr zu&VI((R!3al+zjwuSc(;cL(SD2<70fOu*-ZpMx0*^YkxI*SzCXT!**gEt!`Fvt;^a zE0LFpxXLQQGyJ3V)u(e8Uu`x7*E**hxPO@RGS2(ygs^H)T6jev`3^A&?R0YU@!{(k zWofD;0qM*}1L1Y57eyXLP}Jr3l_! zt9-$o>w5{xS{UEEHx|uz{2>;{MH&gM9fid}lv8D~^40&=!SCJo=5^yMXW;fy^s?-8 zw$1kaSeAo5Y_q?pVR7t^Ax4gZR4?a4?g!xE4EXr=f<;EU`wg?mQtZ6J*LFMyo`L_7 z78*POwpxeZvr}3+d0ZT~Q6n(KufCmoecmeQMHNQs@J>iNZ?`KGv zE#M2uX_7(9LHD)l39Yp>)GAHk3|EKKz8&bID0TcQr3k8;xrJjB%ZYZqh#qy)XA?t;*8 zDG@TC(yrQ27VFxY2Zc}a?Yzd4p4g`|K7e+kP*ln}NyNS(uOj&|i(AL1qr)ywW6kZ5 zqycbfA$!rkc`z}}ng=kg~oYZh0N;LhS9`QXc z;ZnES0W!d9j?2xqUd}8RrPA`RbAg-vdFfwtk?9akFZN4o!_}pVIyEF9$Hx}HK96^p z(Ku-F1XNe;DvKKB`Jnouijqgsnw$ddXHyI6vHfpUF2pPP`E4e*z_+{HqaUD&u0_8I z4uu$F7&kJ{go+H7V8_KI_B4&L*z>sss&qY*2s8qLsmLi}X9vE^$~TH^){{iKHG~=r zi@5N)V&vTno`)AYo1*jYbf#DR3cMrNVKi8k1O8g@69!arMi=X?;+IT<=sx)*lVqZY zE}L_Nxo6^N9B1o!p9EcP41=wm8S5U4`W@kvLhaFn=PY@V+ch*Rmt+>2Tyqswzb_FA zYRpq`8u&u(SpKlNNBNf>X;JJ1kVE}#9o%{WeBggu2m9Xi z94BZnuma?Nti$FD00uQ!)82876ZJjE@as<;M+vf2@BsiNu#~GqADg6>KH=f_y36!z z$Mv9c)tl0tw0wrNRa2M{`4Z z;jnJT)iG>-ZEfZsBVr`-an@uTi_yH#KI&*%n@ax=~u+ z)=^FhiGa6hLR~2wY%uL=dbI%tluiN5xZ$>gD0ONC;NINW=jI18A%A=(5$} zADXI-xk2`8ybYnPv}Xjr#~l7;n_i(Z3LZfTyx^jx&SyE8S)l1;X#+(!=)x6HJLNL# zgNZo>CvQ%Ge+nNr-<`n~O*owMkqPrPrI#jP*QI5cm(cjIL}N1fRG6Cfd<`3~nsCm$ zhmF5AMgorpnKb1Fw7`4}(rjI?)J#~**iHm0hw2EMAUo}0hm`S{uw|GuKiv23n7!}tjV_VzA z{!kyRz=~dF#9ll2Kod{?Jt)FtOnPvge$;)dKZ{WT4OPCh@>GYDc;Vt}r-4S1D)q_nH+(n$GmLX`23S_(ste4)V zSff(Ds%=_8(~_?g_V87-mlJ+x?&6Szjzk)!_X~Sbi;gDai7>*u{^^}~2PQ9*h1Uhdk&v1FJup*b~ACQiZ4f;l*)G}@>< z;mOcR6NzNNzHT7`KXAK#c7oawm<{~y8TUzIpP4)25@K(x8O_gmN)p9ly3|HbS>!`$ z%)NkW#Euw$LvBzr-HNUyV{FpKg&}yn!eLdpk+8hzD&R!ExOQDEx!o8>1oa781rCIY zyTeGyGQeePZntT*&3uy8E})?_?p&(x5mx=_lYTUKq>BX3!lr8`slVGYkQ^Oz55bn} z$YX<=@czIFUMP-aF1aXhSMov{G19>xvEwwnV-qb=KPFqQE4ol=ZiJhX2(|$MF>b+t zXbOSgLqJ!x^~`2KfOQ=&gkWRQr76sG#9C3m?=9SPl#BPV)F$PwxQ2@uHv5r4?J~_% zzE`PWV9w1^@4T1p(&kne=&=)UJAt@f-<2A98ULNM5e*a%SBxBFBmsoa}CGs{2-v;o=*Wm?L0_fR7%3&QXm6Cw>A1>*T^|knS3G;HGNq zXrm~%xs^GW@6FK0+|NN{E9GzA+a!^d-y)s^WynwPlv}C9<>*NwfJZ{`9Q$D%J}j3G zYCHx=i=N@XU7x{}Rs~ZY*b?u3d6Q?Is;eA#Heg^{QbK1Zr+9~^tJ6=}e7)ByS!h7S zer*X1$a;wG&FJ{PCw)S&*)^o|x+lW^(6cmVt$mxV^&B5U*)4m-W79J16l>UtYhsx?9IEz3-HfKSy8V9OW*mO;=t##r9O?rKIJtLQp)0h zvvX39il2w!52*PwG}oUVENewZ`43%9EO4p%}?j*RHQ6+~C$w63sweWUt4@K=v zru_)(wEKCHd&9JHlRYoM2F7GUYaStNsE~W=!N~7>*G9h(ZQbeibG+*9dY-|&d}6;j zqi5Zw-|n)5VL!0iS-*9xbG>|h5m-@36$<2jaq_+!f3j_YZTxsGN%U}3{JFSk_WSzj z*v#48t=Rv4`{a$UNoX@y&fufK_GX@W`Si4g6_7Bd-K> zuy;VFst1!r(ABSyFtuQ68QL8+*TNc7wZQO@I z>5E;n{CaUTx>wOJ^tyX{_IUD0Y$%^I)Avz&w_f_8QRHyYyqejLs^^v0jVrYO@pD_^ znYPvY?FKk|KdSlhdWe=S4&;I_y1fnWd>h?f*r!XzTE*yLHU5E((c-!2*SSpR5W!&X zLZPgiS<QoH7t!7hzon(> zPvb9X@L2cX_%v;wN996|wnjYi*lX_bBs_*K7}0t=?#g%x1D%jO&(1qpc?@FD<&)UD zAam2E@x0k0oZsJCKCYD)%|@0LZAl#8$;PsJ^4>@&4+TWjL#knsS*@hKimNHK6EkWn z8Ah*y1Yzm%W@@&fHEk~Rki9#}_q7={UIlxc7)Gz#oyR>Tr%gqNqgj4@|>3gM3ABsyM-V#Hq&~Mu8hCA#6>i> zL*TWE(`V@VW`%q$YBxRxX?Yh-n{mrNkHNbE0dFM<5aY;lbVl{UTpVKn0+vv)Z$XL6 zE{wevcb!dP^M~D$b*q)Gwzo}v#4kM-ii9sz8wqVeK-uuMlq8sUQLYrXOm2f+NzF`n zyrRMj&qDnA%ykw13@J7}odR0_UdL&MdBu#Dd5x7mC0G)XQ6XNAq_xxxjTJE35g-kf z8MXzs`+R*cjZNIQeY-X-7LF?oq^F3fEq0SNdMOQi0~o%%>@{&8k&hvf+YPO3oLd_R z7|5l3AjGNJC5^s%I=w5w+++JF)-_e72^+;!DbghDZ!UDudhF5h9(0Pl=gV zbH3{S%O;WiRdJfzX8OoC-2wGcjSdGcIzUZGo?7)jxc+r4XIrTM@hP7^E854>dS!> zkb73j=J0IijWM3+%;3PreW}+hax7k?s(a60YdZg##R^61^h?9lO_h`VLH%NXj{Cib zqm+F@6osvP);84T#=;_F2j^&5nBkI6*N4+@4gSNah7H}`oa}XEL4lsu&!^Max4k4} z^Cdo8f`09M?WaRaka{m2PW?9<$hlFmK87hS|BuAAzH5hhp z*Y3u!#JgqKYY>DGBQGdeYol$h^k&w;PyiO;eJ&w_#Pa|d*`}YSCigS*K{D?Y(9y{E zjMO{W7YdbqUi(v>y7R;m%Ws(^BD(IX#6O;CpF+|(>}3rlwRWPRC%Z0;4z7nvqQfJ9 zG`StzCdmrziE+ht=6Y7$TXTJ6+YQ2;T0lDCk-9sRJ~n+Zt^Jpy7J9Na^3*{x<^_bG zh6*hy7*q(PYsWcPr?z;3saxd%kc48pV2VmtSwK_VxMG#tcY4N6=8?f`;xdr zWOgp0OPV`G=eVMUl-Bax&PJmb&DT*9cJl9gi}N=crgzVRo(A9xDX_FU1L+MV`d`H3 z&dZIsjb(q!z+qUs!&@twX_IUXtf5h*Cx`1e#iTU3O_9kEP#&)@X9?gr0wZQ`jg{Qw zdL01qh^{hUM(AKIx0Z!b{85m)S=1F^smk%hT4!3xoUuNI!Iq+}uwS#F?Pu4bo{8v9 zB8g-WxC=j^Cfz0_!{3%wUdQX78DsO>Y%Hi*Sgaf?AQk3+iiPb+AIp^P1@k6J0u$Ia zr_tSJBfr}D76CD`W%3oG0f?a_3=^LOjx76K()GXx*I*fcDr>@dz_9?cm)!-Yjgbf5 z99h;^38OqJ0w}6Sb7zUoC6^HPx^2g#ZO$&G(hrY!xcVgeZ7(D8;&)g}k|ujnUwpi3 zmhANffzcw9(>4B5<9=8_U#+DA(RXpO;3(*JLXto|CD%>oiO`|XFF;CIktfWRe%Js) z2mZFeDTp{`JCcgv^_!Hz-ig7xY;KI|GxY(Da*v&>pJO$?Kayiz=BKxdb5Z8q>fK~G z-U5+vE3;OH*e0U1IQOOY3hf89N9-I*SRV4}3Gn*8a1diZ-oIjs-9HT;!}eUpF8qAj zI(}&2qCc(3tubRF;{*;nrqD^Q`)o|l?!g3oooFj~LMMwidC=3i&GN!n=dRC(X^txIq zS6-C(eCu^58Pva?J}zAwi8O1)x1~=wS_g89w>xM?b>n&_qAX~kRCbwtnF{HPbjE9a zE*oVZkSn1_%M>S+`Q^gISf^Mw3KfU9gTWt846)gdtVcmh^piV#wILcJm9f^JA|6X> zkZk4$bd@4X$s=%Y`{TGB_nQ-9pNsx{3|IeiNcJ5KH6IyowgX10B|^g&LQ9QndBUTp z8gJ$SxNKvk^y4dJcHg*1uW?ieaEjQnc;Ok91Zn>n{TXb0F8)JWavU|T?x&0d(Fk|i zWYZ_T?_JuT@cM++;!M_r;DUDF%<#MIEf!EQhok z*P|y;PfvhHnCWw%C0jd*FC7d-VMcIoI1&@`^r4z@u0vO1{O}Q9P~X7gbC;=6>W{5pzYFv(2R5(2CSU?NyB_e>7pe&nlUK`WSOyv*mYNf( zG%n*ZqkM8g1&qN5oaRbl67haE^EkA}0bw3^;C>_5n~wsqyWaH8-|wy!Bp32qfpGA9 zgwcDr9a_LZzmNN;Zy+(AC26YwrWJn}cWLe8Xcj26;*$P?22svWvPU?@pI=CtW^t)y zqrdBLd2b$hiwvRuq_mbh%ZU$-HPH4o=pm<7gp4hd%a11|g_j&RLF~*?YqfIldhiuj%Dh7Rs$Iz zsc(;;1Lj3)T>v|-;YR0vGB7fd2IH?@yp1 zH5f{FjWMSDfV5zvIa1Ut6AZQ`-jm1qa-Js-m_>?acxF=w%1|${dcdV%HH#iHSk%&% zq3tIMW)u-|Nh%i#S<9cJ1S`ZQfB*py%_mofFqB_uL|^!3kwpX`q{j2We}dA0pf3V} zo6{@4N#Hk=Eu7}{m{Qw61P9o(q8;00K?a|#6BR%$wvBMx=>b*$ zY|{6X6<LT8?(dMlQl1L7Stn^ZQ}A!0lB>!bd=?(Y>iM|u0e)5e zcyauIVG5#-zF3wjg`Lsez>1S8JkEs>mZLt+xY>%?9AJ+nqO&DN3o26S>Vw8U46He< z;M%0PLp#Nh)bVqrC$rQYMBtez*o5X@YvP>Fg3Mh|Nw%ub84 zNBD$|@cIT_#k?Y7Su-MWMys{Hq?ql!2zbE(ysrbI#6n~X#m3N0jv75eI|f0?$jpNk zi!EpO9Vg<=POkloHcQHupZiN~0}pC?6!JB)e7D$6>yc%4Zer^1nu!%24v_oLeKqJk zkaZ`1r~JelJHEJKfg%tG1_-LO+Tb^NkVw-}$625Jrq_bN_BY|)++BRB{{bqbY>sZ} zn45OzP0*8b<-^zLH_Z|ESA&ejeOKk93UF@^#7E%)hN6 zkINR|Z26>ygD$yJ%1Qzz&5@aDECED%+Oc^ARNHs4zv3>r%n*M9b}h%`tb%(IHbAuL z(@}!b6CojBw3rC{C1R4n+as(|d6iUJk|+7s^E5{`msR`O(b*+q26Ei^L6L!M82!ru z(w0aky*64u8Dj|I;AAHQF%U2rcJ)VdO$|scR}qC(3cCrjXJ{-NKHu|OaQ1hYw}zSg zw%B5Efn++InuH?=(TSFm`$Pyf%#BE}Mi`Ay_97S?X#A-OR_*1fSNVYc3A@gz@3@>^rRU)ji-8`ZOb z)s*<=VbBRfk;q6xaoSI#UDlsE?eE_hJC9L{h@&Z76#b59fI6-!^rYBkelkcG=?6o0 z#p}{4yH^Ba8{*`L-v|zd{mKW`g(DjD9Xr0_if?ZQMWfxhQl_mc8tVXFQiZ0T? z&ksWF$*j(-XE^TI?HVTHV`x~z)v9JDOTvoB2>9yAn^m9rOx3kdtQ_iS@gV{T2q1EQ z3Bi*vCXejd0@YTtZ6nlkA)=x2c}L^W`~YH99VxgSx*}HMb&)U$pI18Kx|tEqV8l-3 z-zjDHV=l~92ms!DsJXT4gXZB{AtjziP>9T#J}W z4~v4&%aSaEaqTBA-xe+jFTsnpi0^0#0um3W9gY(8uy_ro5kVfp+*ANpS-*Va-lEyeFkdHXMfYrwJDsZu& z?%6cq&D2nv=P^ak&X!k<&i!0uW(z0OQ~rcK0rXWwe(gDV-oA$YDTw$CdufIJ5yHE zo`>o?+qXo{Tes4z&z7Wh%3PDkVy=NX>YdiiF-$1I(V&u*G;oLkw?Z~wyu*-!GsdE- zxp=wx{))X$R?R|R2%l43YHHMQ86(8J@6$RPO=c5OFv2GmXKmCL4yc+m-0#yTY98_0rlt_|s^G7_b zpeB-u<+zj291+Ad5?yy!0mHJOj*plnGdP^bW)64 zAaur-BR~+S;PF+*{J-!^DnYE|*en)K_?b{!fHH0uhBU{S4u3N(*~CI_N)7ZElh5Pj zxm$9~>RFK%OYIdz*l{##%#RV2*a2zNw>h z;kM!-$QUAP5XXa|b*u=Rx0;WVOV&CQ%+k- z5a-%B0U%z0cMQ2rq#y#5mgwx1?jrnDBaCIfN=x(N7qr_4GyLLmJKb{CdYCdPB1>!Z zx3}YKhXryyDgp9Gj8%@3^U{Q6N?X}`!wh?PZPl=b16JN)^7HkfLt(G!a{ZyfvyO{) zDmP>$!FUG=xt)0`B@ugE`#W})*!-T-u^NJo*CAV38n@<4$} zYB=*HQl-$N*4K9;wYsTd#cjYH_Cq8+W?s)j0j+2e&j%L1lIw_d3fFRp!MvnwiaU)J z@5>D%7M|KL3DG`LiiA0sq7!p0GL{vD_p1orh=1nd2yyAx4_(8G<^y))ZA1^xa`rrk z#mdF@kl?o7%rpTzDw$A>*I@FGK~w6;LaL8{pWh04=b!f)K!Jh9ApCPy`+FOJ1&kE8 zNkEe39o9$puy4K{#zWCp#)M+XrYB>kiA%zAT>j|hi%=opqSMHeCfg+3>hgI2jsDrx z`_7?56)h-_JZA})HziMc_nY0?w+)!}?M4(w6gRpgoO3kt*y^{{{e{W<@tbTZ|9;QM z?(4~K@Ps$|7f;uC^M)tHcV7XZ3lIwCSR|#(H!}(>VeIAEE2YUkvstRFZrtvjv|}F} z1}1CnBL}-%y0~!m^8%&sYPMjUuamR$T#?chNK{$wI3+yQVEa~-x~&$rhl{l*z?P{p zR5P01DZvduP!8%27&Gz809m`AbHVE$^u@x@mbAIRX{nWrZuCRG7E==O12>=PAd76&ui?-t{6W`nGD(-D&RL`#c>rrniT0wI4NKa-Yc^PhO@! z@^8$-qPz;0 zRM9dddd?Yc75v6zKs#zsNJH}DxYeWliKsDr>gw(sd@Pjo`J}tGA!%5!+MMGzg@~|E z?7hKr8CbAd2z2pNa7k>@le0xmL#5Q2GL0JE7gvv=V$^2Z1rp~k@Y0_A%|VTlT>j{( zNP7SxpsIvOn7-=LkP11?1kFZ)5=lhUOb&-|7;W7Ft6wNx{*#fIQG1GBgHu~_Q^~$7 zA)qgCmIt{DLl&{Br#TtYGe8Ni13iZawDoxKEmaG7Fy7VDq0h9DrjwIFCDzAR+K8Qj zKXrRoiQlNsS^Ftp)Iq~-m*=T|k7@v)9%)Mi=%W40iM+&71SwfA?O6bkq+)@r?zU0xpkxT1Wc7f=LDqmcb#jK21_(1Jl`I0{=TO|A()KiZUV zfDzHO?7tD7JK@!$jc4P8tQ-PjN6?RY^5GE11=8^DHiAOjoV9-7Iv2+`C}0tp*Md+@j-U10 zGcQ-NI>>-?57wb{$lJG%GNVB+1S*kHbEpo>hmPjUHp$oEqRR=CQ19Bt;xd}dO5k;b z`30SKZKbFZ`wwOwOO0jgAlez3IYwT^f|E9c@1%`u!~HpPS*7tRL~x5;WM|?| znZ33;3B7cyKwf`72_`s$JN=EUdEl544c`9cHx*oP;Vi#Xf&Ot)qmCZ$P5AS%N9R?y z748r4CDvh>hggB=0$KyOu7~q(r2Q8+{3x$+M~?}ftaV)1q}Sq#%_E0}-3A|fZcqcv zuU%{;_B*x=&Mw~P2s+If&A~gGppypI~2_uV#h6|eJ9|@{bkM8rk=8MhWQ6~&shEG z6UW=l+LPhd+~;hwmCVaT@&b@^b=OG!Pn+cGDb;y~hNq6R03C z&MRx5r-A*^@l>XMEP@;j6-+kIbl!a%o(}XT9oK~S#fsOMO}<&4=VHy^EQ@V0;;Q8C z!t%s@^8{e=icd>*(PZUEO~PDJTN9jHeYadQ3WoD3Hb3{%FIFJQP4jDbzehvi*~)EX zZgsWcq~Y5wJzrmMPooLVF$70}F0EfX^Hlo5_xCKphYy{HpI2k)?jKK5-Hy8XsvQ%G zRJQ?dS=g*x*-p}1P^2VYC*@PO<0s_Emec<4UeLgvA2>VU%GyRJR_)A#8NYS3YBOQl z(TQc;n!WV07a!b2%Y~KtHoK%PTwbM^7>^|RR>Idss2WwO$-=cJ2#?UA79hDS*@pP+ zQGBx~8t~;OGw_NKoRjzH7OiZGtVQzg+DzE&c(s~7)2BCcXzWDfH|yv^`Cu9lwQMiI z(Z~WNn4U%ivyu*yoycYFsa(v>yNcDDiZg#9NxG;mv5I5p7qeTHe!r=&v8@*%`4jWX ziyqb<=gP!Ov5_J7G(ukBIu!0L$KpKpQk}Tk>(v;Pr{sOR^Hex4H0I9VDR+$IuiX(b zo1ASey(!!p+^N~tDE*v$%kK{(YRgmnAXx*1!;;-<_~H~V`^}MVq2_KnTWeu%#Wkae zRO;uG{}2@)jUv@3q7dUzvquhbK1`$Z>gw%=pr_gUYhrp2t4`nnRW*yk2RoPinwYLB z3P-6iiBd9%J93p9ox@O1b;%Y<>sJRNG&mD|@CfT%hnAxqES~-g2Du6knHEhP`Kksm zB8DA4B7ixwaH;Y;8Ure449$5iuW26dtOy|oFT%|>iV&x@GM3l=7AJU%Uu9E0BWOm< zT)aTHL3Vqqt}8m}EaVseOo*_klUQvd`YJ>-x}H4-n-gx?V^io-m?i1&^OqxF)Jvs5 z$|Anei3@U^}0u#FiSrkuM2P`)6K;s&}xSx^xXq_CkpOjuTDr zDPJxr%0>q^yL3dA)GysnIeQ-UB42_EfLUn_IV?YZ?ZstDkEPrTY};4-g6n$E3jp_! zbke1*Vg`p_r{hBmgt!?&BNP8M<;u+tVw9|pE?}mUFF2jBF?0z0E~<3|mSw;M{7mC# z12oU*?2dVHr)b}4Bl62DPfb=E{rs$i_y9}ST6u)`daR9`76pG_Rp3~?5x zJWbYGvohss4i*fk{bRDJ6lA{Ye^^VwY%QOddsKN-Ya$ z{CGaYa)3BDSsDPTgd)2;DwS3&^(5ifk(@u2M=RnOXb02?l+)RoJHqZW^|X2}?1zt5 zFo`d|Wf*RO1G{s?yVL!Ip&>k>P{1F&=AICdI~J;8l2do&rzIcD68W_bs8|2i%r>sT zj$P<7%{%wY%oV+IiSmJ`R~UG)<^+8q%}9`a_7|;G*(Y7AlXQZ`1XU1L|ggyBA8hdz3HM@39W=- z;aO2`%sr5&N5|*bKQJo?IIO^P23}O=^@t)JhZHI}WB`FZy@*l2PZ&hSi=8jJ+No;) zt9yyGnxT!#NJ%6(es@!__ktP1{ZamyVKF#PXj`|*NPh&TSHNEm`1*dSvYz=z;56~8 z{~H?0hXk?sCGGr5?4O$4q!6y|3I@+hh*APnPP1nSf;ljgP7dI8xGnsTzWCZzL<~Fb z=!3x~yo|Fyf&LkpIs%W8?)VG9EaUiM=;u#E6Wto=q(o~FX^Jx_k*Ju+k<-}E%(R8S z@7q`6$PJv5C7Sywhx%c+L`5sMLWp|wy=Ri~12|$ELW|%mi0Q6AMLs6=&_Hv)&y%G$ zW%pT*;N>EDa=ic(4VLfe&ZZv5ZaMEaFwkWU?1JugX~KGIx_q*2_#4aC+NJQfVTAK% zKWTe^Jqw-=N556=;Aw71inC{4OE1(EOov+za!4f@kM3f4ZppZLYOv#)Xroxhf>am~?2&fUI0gp>D&0 zUBC1NLLG=(?D2Y<>V7>AxszeRJ(^qa3U?y(H;7N zw2G#IwXS84YMKv$#1A}6MJ3J}h^}gP$282)n16rBPOJy$baS`+JX~zt@bIivB37F3 zHZ`bnhNFCv)@@Z;vl-LTi!x-pqPC-nYH%K_G_|x&I-3-luFqY%6e0a-2=w#tVu9d1 zczv0Ty7A%+B_?jDkWCxFnA!rGpn<6Z!|rGf^_K=Wl+Q9EEV+C!U?~%oeqRx%YSptx zB;;h?Y74z7UhnPgnNgEE6@aK!PZIJZ$>-CVc^yCdTD#Q+>0%=Fg=(a-ZbOg=M>oqV zMeE6F6Wr*o;~lqMqEJu_1wSC-Sr?xf#sM5`e*^ZRy-8WSOthfsHkBB8Gn| zo)>H)yGB|G_COb1ztCfIwSFMc=dM(Sxhr)hry{yw5-vB?%QaDeHmdZIIa%tvdd*5i zQ#IrqIXNfG5hjm)l&k_xYL*a>-`xMOBl=LnTcVf6UD))R?!vOE5B#!gTa&b@uy%;Al50Yq0g@_{L~Oi{nzhntcG z&vl`XJKzh>v?4VR=FXq3Yf^=!lH0Y0yK$UGxl~d9HyUkJis*+8a?+u%a>FoX=UgUl zk*$beF$f;8<>uUfi_=uUMeU-cb!q<+*8SyjSn-hflGtm_ z&-|da+kg^88#dN1`;{x%J8!}HuVdSN(g@+@>D)Ut^DZ4H(DqJ zdllZr47R|QRPgnm~qgF700s2*rT~|-y+)akNd!8~LaeZnud)&fG ziXX&U0{z)9YYYCuMrSFN2UHs=DB(#&9BUvQiW2Y_>NY*PLuFMS$~c(exP~=_&mC20 zW0$G^e5o&n?3thj?79jvvL$mSLR50eS`o*M>jo(t*(D6M=2r%o=scx^QW9yjd;`KAgTOviHIM~8X+ieMxtfZ!Yac8ELlO6m~%9#76mURFNm*Rg+5j~)KtL4 z8~%x1&Mm&ZZ#gN=Sk$#2g~=9iI-lD((nyP)r00z(&2r|#kxg*~t`Kv$^Nw5^xI$X6 zDGHCJi5b9(?1m}d#xSdWI%60hD87)(B9#TEK@YaDa@vQqrC>`H5%n#8SK0VXf7A?n z;jL@sK)B(qfs%OXZ^S2auk@e7{=S=`K%GiD7v;8=Ie7cY4mKP)gs`ZqyYy4-1`rOW=!P1)2JZ$n+7) z=b{eTB%)#iU7n_94iy$ZMBfDMUB?T`&@c$ULyEfTDrm>@to&- zXhAQ?;y83>!bqfJuwFFGbsg%Oke!RX)| zn#uVIV8r*f4U_x;3R#?SH-f&p+wXH)ZYQh`tt5IqNQE}VbtB@Q(}NEfL>9D$um9$M zwFuq=s{dwsrCvS&*#07Lq*6Qrcu0vL{vuSug8ly~l%M|MS%LvlfsX(@pt>5=Kjg{( zAxClN zzf?+fdII2sn^&ghKLcn|qbmV0sa;P1#J@*i;;*X8>VK-xp8*&ofAyhp*))3pM_XEW zFfhh{^S;Q|MF@6S88AXKlnckn|~DeZ|w8`Lj6-wUjYJt%lq%1@Be~8+yBT* zdHv6S|G%9J)c=Jl9R35%KK!HV)9Zg!{qHL4fAM!0sR3_)MqJ_TKdS!MOa3o3E%n>m cf5z&^8vq}mle+$=B|OWYcMSLV=VgQaKQQ(AApigX From 075da685e606bab4e1b7a8064119a24261d66657 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2011 12:00:18 -0600 Subject: [PATCH 23/28] Ensure link column cannot be added to table without updating user_version --- src/calibre/library/schema_upgrades.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index 2907e43098..a9cd36a588 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -606,6 +606,7 @@ class SchemaUpgrade(object): ''' script = ''' + BEGIN TRANSACTION; ALTER TABLE authors ADD COLUMN link TEXT NOT NULL DEFAULT ""; ''' self.conn.executescript(script) From 83de27a06ba62a78911171cea6d653a69ea19b8c Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 3 Jul 2011 19:29:52 -0400 Subject: [PATCH 24/28] More updates to the quick start guide. --- resources/quick_start.epub | Bin 130575 -> 130580 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/resources/quick_start.epub b/resources/quick_start.epub index b1e793604ce19900042e954c3514aa94af5b8fed..882ad767651e010ef58e94514c07a4ec16060f7f 100644 GIT binary patch delta 1540 zcmZ8hdo-tRfj_dVzR<9%`j(zgWC81Dm;NL2)U z>I!l(HWH$`Z(3A;GXX0SFyfImND-!y&~y?SC>4u(dVD1q1#m;G2ClG*UQi2KCE?~o zJ#dneuPb;*Fi6;v9fm|8M$ynu)xa1UqJtgC3|2xTgnA5c%1<6d%rfhL&fD4VCgXl& z*ym0-Wb+v40BPvhW3Vm7mNxC49~%h8-crnq`vq1+A*XHUg;K>YvNdwwRAglyqj==`7~Z{~ zbLf2|?@kUB&~+UX;OsCqg6v*$jbI_mh9OW)kunvuN&QyJ<)n{tFoE={=*4Y1B7mR z6Q`p0Y1+fnI9&Rh4^wHK-83S@{TLU=`y7yg|P#p zgSk$vQG90G?uGyvI|jdo>nh)jmSu-+*w9%D)S6G^enC+kGO|- zH2*v{TzSAPPN9}yV(@o>R?J#CUwB&^*qgJA*{$!`1-~s&tiKc$7KvZ%~`vqwSTCi zlfoZqYDVf&Wt6T=X3gi$>h0-5wn&YdkVjFyB$C^?30jfIR5U42S1%`hTE|p2YjOw! z8V&|C)KgH5i%7G4>}>h4a@G+R!RtCQwm8)G0?`dCyezO{aGe_emN0K^w00>t5HSse zhkX>1y}csM>IG{vCzBYPP&Jso=nujdmWu}Z+?T@7YY38RNcFEhn%|cLr(Qd~-jXYJX za5JMP$jQ$0+aSJ^`Ds8d{OgA_o}w_2;HOYmxNyq-h2x*7d<$;vnvo}}EVtU^xQSg2#8EcQ!ZvXCi`;SS++05~Yq;2^7_g-vqo}mQwYv1MU zRdVI9aGHde%F1>NTWDC&$L>Uyw>P74v*y}{fmJ)O)li9X+%8`q8zJAq?EDft*fQ$! z<`72<+B9?SbDCB~w|TO#V@WSrr8-oI_%|2OqFi`KhSzJVtvPr zI1;#U6aoRsi+6l{6-d+P+d&Kp){(n_mpa@q*q~zlC20mEOT*0M94J{2j`S~}9?loK z!y_8+<>*2LUj4Z=m4{2$1_<072O+ zzzdd4tN<+uHzss{1sFjFt3U-N?N&u(RGXNrX%hud`YOQS;2-`Q4}maQ4@q$;rB=0z z^`e1QbYbb4qL5fAbp(QF_r3HKq$m(i;3W__)w9Jf^95r2DwqtM5tD^8B744I!+wGm z+w3oaK=35L|8r~(qOOT%@Lb!lj}5)#M;3)`75aN=Ds*X0Y~Q*Dw15@FS_8(YsU;v9 FL6Qm_lf)_~&43Yv+EtYUb;sY=MVB-4-IKs0Sw;zLM zS(pUY0)pJ8^6+6iFTKW_ghU`X3ebs100(hI00n(w07InPU~&^A&jk8v{(TLU;9u(8 z@&zUI(%mtYooSFI6X*gt=ok|i(oEPefx6TlMi%+ z#SHCF`&)JNn?N8Kk{Nz>qd3sOUDvlKKGmrHqm`>&;{#@~#m`K_o&Z0#%~oZeujyIG zyqu?gH_H2O_mBnq_4efDZpS_IeCPVy?X;eex=7vznS`#5yw-NS_Y1#2nU=3J>O9rm zT^Lu;Y~6ioub8Ep;@a$k8w|dcQ?>CW*K@4n`w=nuRiD6^H?lfsH&me0Olx*BByGD^ z{5_^3H9s!4-{;;XR7e|vp!U+Q1CtQUU*tyj)Od0+87bPCo6j|!>P(tzS^0zX zywWgJxPxH*!7{*Qsw|?3NgIk9@G)~rek;rr?usY><>$|{GL4O>*BD!%pb$w$T=mmv z7ia5<3PzQ|fJ+9QAV|?8-F-j79kfVu3);D{l3*RK&Kkj#I+CwMKfsnT8p}MQ4!@z+ z=hY{URbgkazJ?V!+GB<<(l$R1)GxY_NU112y+vuZvgq38+<-bC+OlT6_3MaaDtn3W zZ!(c^qG9T-=mBaIxk+^}FfmuGx6Mo((SJS~ddO31mc)6P<~&13W5vG7D)beC z`2fW2lwB<;)fE>N@JOVkWlr;AJVx1HtCauYpWEcg!)`-&pBxLVsJ+7;HL-Lu=#h`{ zQWF+=UpAk%5Pp)|eusswZ2VI}V{3aaeIo4h%`MZ>)eanJo&_`&$iY|N^Qa+bAoZuf znHCiAwuIh7NVHoF^(7iyH3^-f#rEDTuGP$%^ACQO)N_dlX zQsZRf-rfw?w)>hrSvk!OZ1Xtlgp2;qMf$d{X^;xnK4$d%aql@#pQtWfaH?L(< zXm9uecTOPm54P`0WiemO;sjM|y(JLcCmGC~o%7b5*!8+c-1@`4X4$Nuap} z@Dq%?7XcB**hOFq-aF+>i5$lmaOkt*nncuk~6ED<>lb*-n$E0UdfLI4%cNJM;A02nuqG2*e>71Va0VyO=}#6@Zf;T99-R{{jPxm#Y8( From a100068b59c43c5b3ca6ed2a983ea873e8729109 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2011 19:03:52 -0600 Subject: [PATCH 25/28] New db: Port schema upgrades and fix various bugs with initializing a new database --- src/calibre/db/__init__.py | 1 - src/calibre/db/backend.py | 119 +++++- src/calibre/db/schema_upgrades.py | 618 ++++++++++++++++++++++++++++++ src/calibre/utils/date.py | 3 +- src/calibre/utils/filenames.py | 21 + 5 files changed, 740 insertions(+), 22 deletions(-) create mode 100644 src/calibre/db/schema_upgrades.py diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 59c27eea8e..3c7c86b932 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -63,5 +63,4 @@ Various things that require other things before they can be migrated: columns/categories/searches info into self.field_metadata. Finally, implement metadata dirtied functionality. - 2. Test Schema upgrades ''' diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 5c0b8aaae7..0716cf691c 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -17,12 +17,13 @@ from calibre import isbytestring, force_unicode, prints from calibre.constants import (iswindows, filesystem_encoding, preferred_encoding) from calibre.ptempfile import PersistentTemporaryFile -from calibre.library.schema_upgrades import SchemaUpgrade +from calibre.db.schema_upgrades import SchemaUpgrade from calibre.library.field_metadata import FieldMetadata from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.utils.icu import strcmp from calibre.utils.config import to_json, from_json, prefs, tweaks from calibre.utils.date import utcfromtimestamp, parse_date +from calibre.utils.filenames import is_case_sensitive from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, SizeTable, FormatsTable, AuthorsTable, IdentifiersTable) # }}} @@ -30,8 +31,9 @@ from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, ''' Differences in semantics from pysqlite: - 1. execute/executemany/executescript operate in autocommit mode + 1. execute/executemany operate in autocommit mode 2. There is no fetchone() method on cursor objects, instead use next() + 3. There is no executescript ''' @@ -120,6 +122,66 @@ def icu_collator(s1, s2): return strcmp(force_unicode(s1, 'utf-8'), force_unicode(s2, 'utf-8')) # }}} +# Unused aggregators {{{ +def Concatenate(sep=','): + '''String concatenation aggregator for sqlite''' + + def step(ctxt, value): + if value is not None: + ctxt.append(value) + + def finalize(ctxt): + if not ctxt: + return None + return sep.join(ctxt) + + return ([], step, finalize) + +def SortedConcatenate(sep=','): + '''String concatenation aggregator for sqlite, sorted by supplied index''' + + def step(ctxt, ndx, value): + if value is not None: + ctxt[ndx] = value + + def finalize(ctxt): + if len(ctxt) == 0: + return None + return sep.join(map(ctxt.get, sorted(ctxt.iterkeys()))) + + return ({}, step, finalize) + +def IdentifiersConcat(): + '''String concatenation aggregator for the identifiers map''' + + def step(ctxt, key, val): + ctxt.append(u'%s:%s'%(key, val)) + + def finalize(ctxt): + return ','.join(ctxt) + + return ([], step, finalize) + +def AumSortedConcatenate(): + '''String concatenation aggregator for the author sort map''' + + def step(ctxt, ndx, author, sort, link): + if author is not None: + ctxt[ndx] = ':::'.join((author, sort, link)) + + def finalize(ctxt): + keys = list(ctxt.iterkeys()) + l = len(keys) + if l == 0: + return None + if l == 1: + return ctxt[keys[0]] + return ':#:'.join([ctxt[v] for v in sorted(keys)]) + + return ({}, step, finalize) + +# }}} + class Connection(apsw.Connection): # {{{ BUSY_TIMEOUT = 2000 # milliseconds @@ -145,6 +207,18 @@ class Connection(apsw.Connection): # {{{ self.createscalarfunction('books_list_filter', lambda x: 1, 1) self.createcollation('icucollate', icu_collator) + # Legacy aggregators (never used) but present for backwards compat + self.createaggregatefunction('sortconcat', SortedConcatenate, 2) + self.createaggregatefunction('sortconcat_bar', + partial(SortedConcatenate, sep='|'), 2) + self.createaggregatefunction('sortconcat_amper', + partial(SortedConcatenate, sep='&'), 2) + self.createaggregatefunction('identifiers_concat', + IdentifiersConcat, 2) + self.createaggregatefunction('concat', Concatenate, 1) + self.createaggregatefunction('aum_sortconcat', + AumSortedConcatenate, 4) + def create_dynamic_filter(self, name): f = DynamicFilter(name) self.createscalarfunction(name, f, 1) @@ -153,7 +227,10 @@ class Connection(apsw.Connection): # {{{ ans = self.cursor().execute(*args) if kw.get('all', True): return ans.fetchall() - return ans.next()[0] + try: + return ans.next()[0] + except (StopIteration, IndexError): + return None def execute(self, sql, bindings=None): cursor = self.cursor() @@ -162,14 +239,9 @@ class Connection(apsw.Connection): # {{{ def executemany(self, sql, sequence_of_bindings): return self.cursor().executemany(sql, sequence_of_bindings) - def executescript(self, sql): - with self: - # Use an explicit savepoint so that even if this is called - # while a transaction is active, it is atomic - return self.cursor().execute(sql) # }}} -class DB(SchemaUpgrade): +class DB(object): PATH_LIMIT = 40 if iswindows else 100 WINDOWS_LIBRARY_PATH_LIMIT = 75 @@ -213,25 +285,24 @@ class DB(SchemaUpgrade): shutil.copyfile(self.dbpath, pt.name) self.dbpath = pt.name - self.is_case_sensitive = (not iswindows and - not os.path.exists(self.dbpath.replace('metadata.db', - 'MeTAdAtA.dB'))) + if not os.path.exists(os.path.dirname(self.dbpath)): + os.makedirs(os.path.dirname(self.dbpath)) self._conn = None - if self.user_version == 0: self.initialize_database() - with self.conn: - SchemaUpgrade.__init__(self) + if not os.path.exists(self.library_path): + os.makedirs(self.library_path) + self.is_case_sensitive = is_case_sensitive(self.library_path) + + SchemaUpgrade(self.conn, self.library_path, self.field_metadata) # Guarantee that the library_id is set self.library_id - self.initialize_prefs(default_prefs) - # Fix legacy triggers and columns - self.conn.executescript(''' + self.conn.execute(''' DROP TRIGGER IF EXISTS author_insert_trg; CREATE TEMP TRIGGER author_insert_trg AFTER INSERT ON authors @@ -248,6 +319,7 @@ class DB(SchemaUpgrade): UPDATE authors SET sort=author_to_author_sort(name) WHERE sort IS NULL; ''') + self.initialize_prefs(default_prefs) self.initialize_custom_columns() self.initialize_tables() @@ -589,7 +661,14 @@ class DB(SchemaUpgrade): def initialize_database(self): metadata_sqlite = P('metadata_sqlite.sql', data=True, allow_user_override=False).decode('utf-8') - self.conn.executescript(metadata_sqlite) + cur = self.conn.cursor() + cur.execute('BEGIN EXCLUSIVE TRANSACTION') + try: + cur.execute(metadata_sqlite) + except: + cur.execute('ROLLBACK') + else: + cur.execute('COMMIT') if self.user_version == 0: self.user_version = 1 # }}} @@ -629,7 +708,7 @@ class DB(SchemaUpgrade): self.conn.execute(''' DELETE FROM library_id; INSERT INTO library_id (uuid) VALUES (?); - ''', self._library_id_) + ''', (self._library_id_,)) return property(doc=doc, fget=fget, fset=fset) diff --git a/src/calibre/db/schema_upgrades.py b/src/calibre/db/schema_upgrades.py new file mode 100644 index 0000000000..f3ca6f9852 --- /dev/null +++ b/src/calibre/db/schema_upgrades.py @@ -0,0 +1,618 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os + +from calibre import prints +from calibre.utils.date import isoformat, DEFAULT_DATE + +class SchemaUpgrade(object): + + def __init__(self, conn, library_path, field_metadata): + conn.execute('BEGIN EXCLUSIVE TRANSACTION') + self.conn = conn + self.library_path = library_path + self.field_metadata = field_metadata + # Upgrade database + try: + while True: + uv = self.conn.execute('pragma user_version').next()[0] + meth = getattr(self, 'upgrade_version_%d'%uv, None) + if meth is None: + break + else: + prints('Upgrading database to version %d...'%(uv+1)) + meth() + self.conn.execute('pragma user_version=%d'%(uv+1)) + except: + self.conn.execute('ROLLBACK') + raise + else: + self.conn.execute('COMMIT') + finally: + self.conn = self.field_metadata = None + + def upgrade_version_1(self): + ''' + Normalize indices. + ''' + self.conn.execute('''\ + DROP INDEX IF EXISTS authors_idx; + CREATE INDEX authors_idx ON books (author_sort COLLATE NOCASE, sort COLLATE NOCASE); + DROP INDEX IF EXISTS series_idx; + CREATE INDEX series_idx ON series (name COLLATE NOCASE); + DROP INDEX IF EXISTS series_sort_idx; + CREATE INDEX series_sort_idx ON books (series_index, id); + ''') + + def upgrade_version_2(self): + ''' Fix Foreign key constraints for deleting from link tables. ''' + script = '''\ + DROP TRIGGER IF EXISTS fkc_delete_books_%(ltable)s_link; + CREATE TRIGGER fkc_delete_on_%(table)s + BEFORE DELETE ON %(table)s + BEGIN + SELECT CASE + WHEN (SELECT COUNT(id) FROM books_%(ltable)s_link WHERE %(ltable_col)s=OLD.id) > 0 + THEN RAISE(ABORT, 'Foreign key violation: %(table)s is still referenced') + END; + END; + DELETE FROM %(table)s WHERE (SELECT COUNT(id) FROM books_%(ltable)s_link WHERE %(ltable_col)s=%(table)s.id) < 1; + ''' + self.conn.execute(script%dict(ltable='authors', table='authors', ltable_col='author')) + self.conn.execute(script%dict(ltable='publishers', table='publishers', ltable_col='publisher')) + self.conn.execute(script%dict(ltable='tags', table='tags', ltable_col='tag')) + self.conn.execute(script%dict(ltable='series', table='series', ltable_col='series')) + + def upgrade_version_3(self): + ' Add path to result cache ' + self.conn.execute(''' + DROP VIEW IF EXISTS meta; + CREATE VIEW meta AS + SELECT id, title, + (SELECT concat(name) FROM authors WHERE authors.id IN (SELECT author from books_authors_link WHERE book=books.id)) authors, + (SELECT name FROM publishers WHERE publishers.id IN (SELECT publisher from books_publishers_link WHERE book=books.id)) publisher, + (SELECT rating FROM ratings WHERE ratings.id IN (SELECT rating from books_ratings_link WHERE book=books.id)) rating, + timestamp, + (SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size, + (SELECT concat(name) FROM tags WHERE tags.id IN (SELECT tag from books_tags_link WHERE book=books.id)) tags, + (SELECT text FROM comments WHERE book=books.id) comments, + (SELECT name FROM series WHERE series.id IN (SELECT series FROM books_series_link WHERE book=books.id)) series, + series_index, + sort, + author_sort, + (SELECT concat(format) FROM data WHERE data.book=books.id) formats, + isbn, + path + FROM books; + ''') + + def upgrade_version_4(self): + 'Rationalize books table' + self.conn.execute(''' + CREATE TEMPORARY TABLE + books_backup(id,title,sort,timestamp,series_index,author_sort,isbn,path); + INSERT INTO books_backup SELECT id,title,sort,timestamp,series_index,author_sort,isbn,path FROM books; + DROP TABLE books; + CREATE TABLE books ( id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL DEFAULT 'Unknown' COLLATE NOCASE, + sort TEXT COLLATE NOCASE, + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + pubdate TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + series_index REAL NOT NULL DEFAULT 1.0, + author_sort TEXT COLLATE NOCASE, + isbn TEXT DEFAULT "" COLLATE NOCASE, + lccn TEXT DEFAULT "" COLLATE NOCASE, + path TEXT NOT NULL DEFAULT "", + flags INTEGER NOT NULL DEFAULT 1 + ); + INSERT INTO + books (id,title,sort,timestamp,pubdate,series_index,author_sort,isbn,path) + SELECT id,title,sort,timestamp,timestamp,series_index,author_sort,isbn,path FROM books_backup; + DROP TABLE books_backup; + + DROP VIEW IF EXISTS meta; + CREATE VIEW meta AS + SELECT id, title, + (SELECT concat(name) FROM authors WHERE authors.id IN (SELECT author from books_authors_link WHERE book=books.id)) authors, + (SELECT name FROM publishers WHERE publishers.id IN (SELECT publisher from books_publishers_link WHERE book=books.id)) publisher, + (SELECT rating FROM ratings WHERE ratings.id IN (SELECT rating from books_ratings_link WHERE book=books.id)) rating, + timestamp, + (SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size, + (SELECT concat(name) FROM tags WHERE tags.id IN (SELECT tag from books_tags_link WHERE book=books.id)) tags, + (SELECT text FROM comments WHERE book=books.id) comments, + (SELECT name FROM series WHERE series.id IN (SELECT series FROM books_series_link WHERE book=books.id)) series, + series_index, + sort, + author_sort, + (SELECT concat(format) FROM data WHERE data.book=books.id) formats, + isbn, + path, + lccn, + pubdate, + flags + FROM books; + ''') + + def upgrade_version_5(self): + 'Update indexes/triggers for new books table' + self.conn.execute(''' + CREATE INDEX authors_idx ON books (author_sort COLLATE NOCASE); + CREATE INDEX books_idx ON books (sort COLLATE NOCASE); + CREATE TRIGGER books_delete_trg + AFTER DELETE ON books + BEGIN + DELETE FROM books_authors_link WHERE book=OLD.id; + DELETE FROM books_publishers_link WHERE book=OLD.id; + DELETE FROM books_ratings_link WHERE book=OLD.id; + DELETE FROM books_series_link WHERE book=OLD.id; + DELETE FROM books_tags_link WHERE book=OLD.id; + DELETE FROM data WHERE book=OLD.id; + DELETE FROM comments WHERE book=OLD.id; + DELETE FROM conversion_options WHERE book=OLD.id; + END; + CREATE TRIGGER books_insert_trg + AFTER INSERT ON books + BEGIN + UPDATE books SET sort=title_sort(NEW.title) WHERE id=NEW.id; + END; + CREATE TRIGGER books_update_trg + AFTER UPDATE ON books + BEGIN + UPDATE books SET sort=title_sort(NEW.title) WHERE id=NEW.id; + END; + + UPDATE books SET sort=title_sort(title) WHERE sort IS NULL; + ''' + ) + + + def upgrade_version_6(self): + 'Show authors in order' + self.conn.execute(''' + DROP VIEW IF EXISTS meta; + CREATE VIEW meta AS + SELECT id, title, + (SELECT sortconcat(bal.id, name) FROM books_authors_link AS bal JOIN authors ON(author = authors.id) WHERE book = books.id) authors, + (SELECT name FROM publishers WHERE publishers.id IN (SELECT publisher from books_publishers_link WHERE book=books.id)) publisher, + (SELECT rating FROM ratings WHERE ratings.id IN (SELECT rating from books_ratings_link WHERE book=books.id)) rating, + timestamp, + (SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size, + (SELECT concat(name) FROM tags WHERE tags.id IN (SELECT tag from books_tags_link WHERE book=books.id)) tags, + (SELECT text FROM comments WHERE book=books.id) comments, + (SELECT name FROM series WHERE series.id IN (SELECT series FROM books_series_link WHERE book=books.id)) series, + series_index, + sort, + author_sort, + (SELECT concat(format) FROM data WHERE data.book=books.id) formats, + isbn, + path, + lccn, + pubdate, + flags + FROM books; + ''') + + def upgrade_version_7(self): + 'Add uuid column' + self.conn.execute(''' + ALTER TABLE books ADD COLUMN uuid TEXT; + DROP TRIGGER IF EXISTS books_insert_trg; + DROP TRIGGER IF EXISTS books_update_trg; + UPDATE books SET uuid=uuid4(); + + CREATE TRIGGER books_insert_trg AFTER INSERT ON books + BEGIN + UPDATE books SET sort=title_sort(NEW.title),uuid=uuid4() WHERE id=NEW.id; + END; + + CREATE TRIGGER books_update_trg AFTER UPDATE ON books + BEGIN + UPDATE books SET sort=title_sort(NEW.title) WHERE id=NEW.id; + END; + + DROP VIEW IF EXISTS meta; + CREATE VIEW meta AS + SELECT id, title, + (SELECT sortconcat(bal.id, name) FROM books_authors_link AS bal JOIN authors ON(author = authors.id) WHERE book = books.id) authors, + (SELECT name FROM publishers WHERE publishers.id IN (SELECT publisher from books_publishers_link WHERE book=books.id)) publisher, + (SELECT rating FROM ratings WHERE ratings.id IN (SELECT rating from books_ratings_link WHERE book=books.id)) rating, + timestamp, + (SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size, + (SELECT concat(name) FROM tags WHERE tags.id IN (SELECT tag from books_tags_link WHERE book=books.id)) tags, + (SELECT text FROM comments WHERE book=books.id) comments, + (SELECT name FROM series WHERE series.id IN (SELECT series FROM books_series_link WHERE book=books.id)) series, + series_index, + sort, + author_sort, + (SELECT concat(format) FROM data WHERE data.book=books.id) formats, + isbn, + path, + lccn, + pubdate, + flags, + uuid + FROM books; + ''') + + def upgrade_version_8(self): + 'Add Tag Browser views' + def create_tag_browser_view(table_name, column_name): + self.conn.execute(''' + DROP VIEW IF EXISTS tag_browser_{tn}; + CREATE VIEW tag_browser_{tn} AS SELECT + id, + name, + (SELECT COUNT(id) FROM books_{tn}_link WHERE {cn}={tn}.id) count + FROM {tn}; + '''.format(tn=table_name, cn=column_name)) + + for tn in ('authors', 'tags', 'publishers', 'series'): + cn = tn[:-1] + if tn == 'series': + cn = tn + create_tag_browser_view(tn, cn) + + def upgrade_version_9(self): + 'Add custom columns' + self.conn.execute(''' + CREATE TABLE custom_columns ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT NOT NULL, + name TEXT NOT NULL, + datatype TEXT NOT NULL, + mark_for_delete BOOL DEFAULT 0 NOT NULL, + editable BOOL DEFAULT 1 NOT NULL, + display TEXT DEFAULT "{}" NOT NULL, + is_multiple BOOL DEFAULT 0 NOT NULL, + normalized BOOL NOT NULL, + UNIQUE(label) + ); + CREATE INDEX IF NOT EXISTS custom_columns_idx ON custom_columns (label); + CREATE INDEX IF NOT EXISTS formats_idx ON data (format); + ''') + + def upgrade_version_10(self): + 'Add restricted Tag Browser views' + def create_tag_browser_view(table_name, column_name, view_column_name): + script = (''' + DROP VIEW IF EXISTS tag_browser_{tn}; + CREATE VIEW tag_browser_{tn} AS SELECT + id, + {vcn}, + (SELECT COUNT(id) FROM books_{tn}_link WHERE {cn}={tn}.id) count + FROM {tn}; + DROP VIEW IF EXISTS tag_browser_filtered_{tn}; + CREATE VIEW tag_browser_filtered_{tn} AS SELECT + id, + {vcn}, + (SELECT COUNT(books_{tn}_link.id) FROM books_{tn}_link WHERE + {cn}={tn}.id AND books_list_filter(book)) count + FROM {tn}; + '''.format(tn=table_name, cn=column_name, vcn=view_column_name)) + self.conn.execute(script) + + for field in self.field_metadata.itervalues(): + if field['is_category'] and not field['is_custom'] and 'link_column' in field: + table = self.conn.get( + 'SELECT name FROM sqlite_master WHERE type="table" AND name=?', + ('books_%s_link'%field['table'],), all=False) + if table is not None: + create_tag_browser_view(field['table'], field['link_column'], field['column']) + + def upgrade_version_11(self): + 'Add average rating to tag browser views' + def create_std_tag_browser_view(table_name, column_name, + view_column_name, sort_column_name): + script = (''' + DROP VIEW IF EXISTS tag_browser_{tn}; + CREATE VIEW tag_browser_{tn} AS SELECT + id, + {vcn}, + (SELECT COUNT(id) FROM books_{tn}_link WHERE {cn}={tn}.id) count, + (SELECT AVG(ratings.rating) + FROM books_{tn}_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.{cn}={tn}.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0) avg_rating, + {scn} AS sort + FROM {tn}; + DROP VIEW IF EXISTS tag_browser_filtered_{tn}; + CREATE VIEW tag_browser_filtered_{tn} AS SELECT + id, + {vcn}, + (SELECT COUNT(books_{tn}_link.id) FROM books_{tn}_link WHERE + {cn}={tn}.id AND books_list_filter(book)) count, + (SELECT AVG(ratings.rating) + FROM books_{tn}_link AS tl, books_ratings_link AS bl, ratings + WHERE tl.{cn}={tn}.id AND bl.book=tl.book AND + ratings.id = bl.rating AND ratings.rating <> 0 AND + books_list_filter(bl.book)) avg_rating, + {scn} AS sort + FROM {tn}; + + '''.format(tn=table_name, cn=column_name, + vcn=view_column_name, scn= sort_column_name)) + self.conn.execute(script) + + def create_cust_tag_browser_view(table_name, link_table_name): + script = ''' + DROP VIEW IF EXISTS tag_browser_{table}; + CREATE VIEW tag_browser_{table} AS SELECT + id, + value, + (SELECT COUNT(id) FROM {lt} WHERE value={table}.id) count, + (SELECT AVG(r.rating) + FROM {lt}, + books_ratings_link AS bl, + ratings AS r + WHERE {lt}.value={table}.id AND bl.book={lt}.book AND + r.id = bl.rating AND r.rating <> 0) avg_rating, + value AS sort + FROM {table}; + + DROP VIEW IF EXISTS tag_browser_filtered_{table}; + CREATE VIEW tag_browser_filtered_{table} AS SELECT + id, + value, + (SELECT COUNT({lt}.id) FROM {lt} WHERE value={table}.id AND + books_list_filter(book)) count, + (SELECT AVG(r.rating) + FROM {lt}, + books_ratings_link AS bl, + ratings AS r + WHERE {lt}.value={table}.id AND bl.book={lt}.book AND + r.id = bl.rating AND r.rating <> 0 AND + books_list_filter(bl.book)) avg_rating, + value AS sort + FROM {table}; + '''.format(lt=link_table_name, table=table_name) + self.conn.execute(script) + + for field in self.field_metadata.itervalues(): + if field['is_category'] and not field['is_custom'] and 'link_column' in field: + table = self.conn.get( + 'SELECT name FROM sqlite_master WHERE type="table" AND name=?', + ('books_%s_link'%field['table'],), all=False) + if table is not None: + create_std_tag_browser_view(field['table'], field['link_column'], + field['column'], field['category_sort']) + + db_tables = self.conn.get('''SELECT name FROM sqlite_master + WHERE type='table' + ORDER BY name''') + tables = [] + for (table,) in db_tables: + tables.append(table) + for table in tables: + link_table = 'books_%s_link'%table + if table.startswith('custom_column_') and link_table in tables: + create_cust_tag_browser_view(table, link_table) + + self.conn.execute('UPDATE authors SET sort=author_to_author_sort(name)') + + def upgrade_version_12(self): + 'DB based preference store' + script = ''' + DROP TABLE IF EXISTS preferences; + CREATE TABLE preferences(id INTEGER PRIMARY KEY, + key TEXT NON NULL, + val TEXT NON NULL, + UNIQUE(key)); + ''' + self.conn.execute(script) + + def upgrade_version_13(self): + 'Dirtied table for OPF metadata backups' + script = ''' + DROP TABLE IF EXISTS metadata_dirtied; + CREATE TABLE metadata_dirtied(id INTEGER PRIMARY KEY, + book INTEGER NOT NULL, + UNIQUE(book)); + INSERT INTO metadata_dirtied (book) SELECT id FROM books; + ''' + self.conn.execute(script) + + def upgrade_version_14(self): + 'Cache has_cover' + self.conn.execute('ALTER TABLE books ADD COLUMN has_cover BOOL DEFAULT 0') + data = self.conn.get('SELECT id,path FROM books', all=True) + def has_cover(path): + if path: + path = os.path.join(self.library_path, path.replace('/', os.sep), + 'cover.jpg') + return os.path.exists(path) + return False + + ids = [(x[0],) for x in data if has_cover(x[1])] + self.conn.executemany('UPDATE books SET has_cover=1 WHERE id=?', ids) + + def upgrade_version_15(self): + 'Remove commas from tags' + self.conn.execute("UPDATE OR IGNORE tags SET name=REPLACE(name, ',', ';')") + self.conn.execute("UPDATE OR IGNORE tags SET name=REPLACE(name, ',', ';;')") + self.conn.execute("UPDATE OR IGNORE tags SET name=REPLACE(name, ',', '')") + + def upgrade_version_16(self): + self.conn.execute(''' + DROP TRIGGER IF EXISTS books_update_trg; + CREATE TRIGGER books_update_trg + AFTER UPDATE ON books + BEGIN + UPDATE books SET sort=title_sort(NEW.title) + WHERE id=NEW.id AND OLD.title <> NEW.title; + END; + ''') + + def upgrade_version_17(self): + 'custom book data table (for plugins)' + script = ''' + DROP TABLE IF EXISTS books_plugin_data; + CREATE TABLE books_plugin_data(id INTEGER PRIMARY KEY, + book INTEGER NON NULL, + name TEXT NON NULL, + val TEXT NON NULL, + UNIQUE(book,name)); + DROP TRIGGER IF EXISTS books_delete_trg; + CREATE TRIGGER books_delete_trg + AFTER DELETE ON books + BEGIN + DELETE FROM books_authors_link WHERE book=OLD.id; + DELETE FROM books_publishers_link WHERE book=OLD.id; + DELETE FROM books_ratings_link WHERE book=OLD.id; + DELETE FROM books_series_link WHERE book=OLD.id; + DELETE FROM books_tags_link WHERE book=OLD.id; + DELETE FROM data WHERE book=OLD.id; + DELETE FROM comments WHERE book=OLD.id; + DELETE FROM conversion_options WHERE book=OLD.id; + DELETE FROM books_plugin_data WHERE book=OLD.id; + END; + ''' + self.conn.execute(script) + + def upgrade_version_18(self): + ''' + Add a library UUID. + Add an identifiers table. + Add a languages table. + Add a last_modified column. + NOTE: You cannot downgrade after this update, if you do + any changes you make to book isbns will be lost. + ''' + script = ''' + DROP TABLE IF EXISTS library_id; + CREATE TABLE library_id ( id INTEGER PRIMARY KEY, + uuid TEXT NOT NULL, + UNIQUE(uuid) + ); + + DROP TABLE IF EXISTS identifiers; + CREATE TABLE identifiers ( id INTEGER PRIMARY KEY, + book INTEGER NON NULL, + type TEXT NON NULL DEFAULT "isbn" COLLATE NOCASE, + val TEXT NON NULL COLLATE NOCASE, + UNIQUE(book, type) + ); + + DROP TABLE IF EXISTS languages; + CREATE TABLE languages ( id INTEGER PRIMARY KEY, + lang_code TEXT NON NULL COLLATE NOCASE, + UNIQUE(lang_code) + ); + + DROP TABLE IF EXISTS books_languages_link; + CREATE TABLE books_languages_link ( id INTEGER PRIMARY KEY, + book INTEGER NOT NULL, + lang_code INTEGER NOT NULL, + item_order INTEGER NOT NULL DEFAULT 0, + UNIQUE(book, lang_code) + ); + + DROP TRIGGER IF EXISTS fkc_delete_on_languages; + CREATE TRIGGER fkc_delete_on_languages + BEFORE DELETE ON languages + BEGIN + SELECT CASE + WHEN (SELECT COUNT(id) FROM books_languages_link WHERE lang_code=OLD.id) > 0 + THEN RAISE(ABORT, 'Foreign key violation: language is still referenced') + END; + END; + + DROP TRIGGER IF EXISTS fkc_delete_on_languages_link; + CREATE TRIGGER fkc_delete_on_languages_link + BEFORE INSERT ON books_languages_link + BEGIN + SELECT CASE + WHEN (SELECT id from books WHERE id=NEW.book) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: book not in books') + WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages') + END; + END; + + DROP TRIGGER IF EXISTS fkc_update_books_languages_link_a; + CREATE TRIGGER fkc_update_books_languages_link_a + BEFORE UPDATE OF book ON books_languages_link + BEGIN + SELECT CASE + WHEN (SELECT id from books WHERE id=NEW.book) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: book not in books') + END; + END; + DROP TRIGGER IF EXISTS fkc_update_books_languages_link_b; + CREATE TRIGGER fkc_update_books_languages_link_b + BEFORE UPDATE OF lang_code ON books_languages_link + BEGIN + SELECT CASE + WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL + THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages') + END; + END; + + DROP INDEX IF EXISTS books_languages_link_aidx; + CREATE INDEX books_languages_link_aidx ON books_languages_link (lang_code); + DROP INDEX IF EXISTS books_languages_link_bidx; + CREATE INDEX books_languages_link_bidx ON books_languages_link (book); + DROP INDEX IF EXISTS languages_idx; + CREATE INDEX languages_idx ON languages (lang_code COLLATE NOCASE); + + DROP TRIGGER IF EXISTS books_delete_trg; + CREATE TRIGGER books_delete_trg + AFTER DELETE ON books + BEGIN + DELETE FROM books_authors_link WHERE book=OLD.id; + DELETE FROM books_publishers_link WHERE book=OLD.id; + DELETE FROM books_ratings_link WHERE book=OLD.id; + DELETE FROM books_series_link WHERE book=OLD.id; + DELETE FROM books_tags_link WHERE book=OLD.id; + DELETE FROM books_languages_link WHERE book=OLD.id; + DELETE FROM data WHERE book=OLD.id; + DELETE FROM comments WHERE book=OLD.id; + DELETE FROM conversion_options WHERE book=OLD.id; + DELETE FROM books_plugin_data WHERE book=OLD.id; + DELETE FROM identifiers WHERE book=OLD.id; + END; + + INSERT INTO identifiers (book, val) SELECT id,isbn FROM books WHERE isbn; + + ALTER TABLE books ADD COLUMN last_modified TIMESTAMP NOT NULL DEFAULT "%s"; + + '''%isoformat(DEFAULT_DATE, sep=' ') + # Sqlite does not support non constant default values in alter + # statements + self.conn.execute(script) + + def upgrade_version_19(self): + recipes = self.conn.get('SELECT id,title,script FROM feeds') + if recipes: + from calibre.web.feeds.recipes import (custom_recipes, + custom_recipe_filename) + bdir = os.path.dirname(custom_recipes.file_path) + for id_, title, script in recipes: + existing = frozenset(map(int, custom_recipes.iterkeys())) + if id_ in existing: + id_ = max(existing) + 1000 + id_ = str(id_) + fname = custom_recipe_filename(id_, title) + custom_recipes[id_] = (title, fname) + if isinstance(script, unicode): + script = script.encode('utf-8') + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) + + def upgrade_version_20(self): + ''' + Add a link column to the authors table. + ''' + + script = ''' + ALTER TABLE authors ADD COLUMN link TEXT NOT NULL DEFAULT ""; + ''' + self.conn.execute(script) + + diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index c93e69874c..2c973da224 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -121,7 +121,8 @@ def isoformat(date_time, assume_utc=False, as_utc=True, sep='T'): date_time = date_time.replace(tzinfo=_utc_tz if assume_utc else _local_tz) date_time = date_time.astimezone(_utc_tz if as_utc else _local_tz) - return unicode(date_time.isoformat(sep)) + # str(sep) because isoformat barfs with unicode sep on python 2.x + return unicode(date_time.isoformat(str(sep))) def as_local_time(date_time, assume_utc=True): if not hasattr(date_time, 'tzinfo'): diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index cca75915b5..8c6daa5adf 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -93,3 +93,24 @@ def find_executable_in_path(name, path=None): q = os.path.abspath(os.path.join(x, name)) if os.access(q, os.X_OK): return q + +def is_case_sensitive(path): + ''' + Return True if the filesystem is case sensitive. + + path must be the path to an existing directory. You must have permission + to create and delete files in this directory. The results of this test + apply to the filesystem containing the directory in path. + ''' + is_case_sensitive = False + if not iswindows: + name1, name2 = ('calibre_test_case_sensitivity.txt', + 'calibre_TesT_CaSe_sensitiVitY.Txt') + f1, f2 = os.path.join(path, name1), os.path.join(path, name2) + if os.path.exists(f1): + os.remove(f1) + open(f1, 'w').close() + is_case_sensitive = not os.path.exists(f2) + os.remove(f1) + return is_case_sensitive + From 980909069f3ebe67ef62a4e398e4faffa8dabe93 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2011 19:17:50 -0600 Subject: [PATCH 26/28] Fix #802083 ("ExpirationStatus" ?) --- src/calibre/devices/kobo/driver.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index e3019b8ced..6580d92367 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -137,7 +137,7 @@ class KOBO(USBMS): bl_cache[lpath] = None if ImageID is not None: imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - NickelBookCover.parsed') - if not os.path.exists(imagename): + if not os.path.exists(imagename): # Try the Touch version if the image does not exist imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - N3_LIBRARY_FULL.parsed') @@ -210,7 +210,14 @@ class KOBO(USBMS): query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus, ___ExpirationStatus, "-1" as FavouritesIndex from content where BookID is Null' - cursor.execute (query) + try: + cursor.execute (query) + except Exception as e: + if '___ExpirationStatus' not in str(e): + raise + query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ + 'ImageID, ReadStatus, "-1" as ___ExpirationStatus, "-1" as FavouritesIndex from content where BookID is Null' + cursor.execute(query) changed = False for i, row in enumerate(cursor): @@ -577,7 +584,7 @@ class KOBO(USBMS): for book in books: # debug_print('Title:', book.title, 'lpath:', book.path) if 'Im_Reading' not in book.device_collections: - book.device_collections.append('Im_Reading') + book.device_collections.append('Im_Reading') extension = os.path.splitext(book.path)[1] ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path) @@ -621,7 +628,7 @@ class KOBO(USBMS): for book in books: # debug_print('Title:', book.title, 'lpath:', book.path) if 'Read' not in book.device_collections: - book.device_collections.append('Read') + book.device_collections.append('Read') extension = os.path.splitext(book.path)[1] ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path) @@ -658,7 +665,7 @@ class KOBO(USBMS): for book in books: # debug_print('Title:', book.title, 'lpath:', book.path) if 'Closed' not in book.device_collections: - book.device_collections.append('Closed') + book.device_collections.append('Closed') extension = os.path.splitext(book.path)[1] ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path) @@ -695,8 +702,8 @@ class KOBO(USBMS): for book in books: # debug_print('Title:', book.title, 'lpath:', book.path) if 'Shortlist' not in book.device_collections: - book.device_collections.append('Shortlist') - # debug_print ("Shortlist found for: ", book.title) + book.device_collections.append('Shortlist') + # debug_print ("Shortlist found for: ", book.title) extension = os.path.splitext(book.path)[1] ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path) From 500935e3fad9a7736a8240d5d97873ca946e7d0f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2011 19:40:50 -0600 Subject: [PATCH 27/28] Fix #804790 (some of News categories(languages) are not translated) --- src/calibre/utils/localization.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py index aab0f29995..eaf8ac675a 100644 --- a/src/calibre/utils/localization.py +++ b/src/calibre/utils/localization.py @@ -150,7 +150,9 @@ def get_language(lang): global _iso639 lang = _lcase_map.get(lang, lang) if lang in _extra_lang_codes: - return _extra_lang_codes[lang] + # The translator was not active when _extra_lang_codes was defined, so + # re-translate + return _(_extra_lang_codes[lang]) ip = P('localization/iso639.pickle') if not os.path.exists(ip): return lang From 37cddda5ca2c2b5eb39a58b15e6a6ad9e8474920 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Sun, 3 Jul 2011 22:48:26 -0300 Subject: [PATCH 28/28] Fix ExpirationStatus message by database version --- src/calibre/devices/kobo/driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 6580d92367..62e15452f1 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -206,9 +206,12 @@ class KOBO(USBMS): if self.dbversion >= 14: query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex from content where BookID is Null' - else: + elif self.dbversion < 14 and self.dbversion >= 8: query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus, ___ExpirationStatus, "-1" as FavouritesIndex from content where BookID is Null' + else: + query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ + 'ImageID, ReadStatus, "-1" as ___ExpirationStatus, "-1" as FavouritesIndex from content where BookID is Null' try: cursor.execute (query)