diff --git a/recipes/adventuregamers.recipe b/recipes/adventuregamers.recipe index d08eca1723..b82bb7d02d 100644 --- a/recipes/adventuregamers.recipe +++ b/recipes/adventuregamers.recipe @@ -1,5 +1,5 @@ __license__ = 'GPL v3' -__copyright__ = '2009-2010, Darko Miletic ' +__copyright__ = '2009-2012, Darko Miletic ' ''' www.adventuregamers.com ''' @@ -14,24 +14,24 @@ class AdventureGamers(BasicNewsRecipe): publisher = 'Adventure Gamers' category = 'news, games, adventure, technology' oldest_article = 10 - delay = 10 + #delay = 10 max_articles_per_feed = 100 no_stylesheets = True - encoding = 'cp1252' + encoding = 'utf8' remove_javascript = True use_embedded_content = False INDEX = u'http://www.adventuregamers.com' extra_css = """ .pageheader_type{font-size: x-large; font-weight: bold; color: #828D74} - .pageheader_title{font-size: xx-large; color: #394128} + .pageheader_title,.page_title{font-size: xx-large; color: #394128} .pageheader_byline{font-size: small; font-weight: bold; color: #394128} .score_bg {display: inline; width: 100%; margin-bottom: 2em} .score_column_1{ padding-left: 10px; font-size: small; width: 50%} .score_column_2{ padding-left: 10px; font-size: small; width: 50%} .score_column_3{ padding-left: 10px; font-size: small; width: 50%} - .score_header{font-size: large; color: #50544A} - .bodytext{display: block} - body{font-family: Helvetica,Arial,sans-serif} + .score_header{font-size: large; color: #50544A} + img{margin-bottom: 1em;} + body{font-family: 'Open Sans',Helvetica,Arial,sans-serif} """ conversion_options = { @@ -41,35 +41,38 @@ class AdventureGamers(BasicNewsRecipe): , 'language' : language } - keep_only_tags = [ - dict(name='div', attrs={'class':'content_middle'}) - ] - + keep_only_tags = [dict(name='div', attrs={'class':'cleft_inn'})] remove_tags = [ - dict(name=['object','link','embed','form']) - ,dict(name='div', attrs={'class':['related-stories','article_leadout','prev','next','both']}) + dict(name=['object','link','embed','form','iframe','meta']) + ,dict(name='a', attrs={'href':'http://www.adventuregamers.com/about/scoring'}) + ,dict(name='a', attrs={'href':'http://www.adventuregamers.com/about/policies'}) ] - - remove_tags_after = [dict(name='div', attrs={'class':'toolbar_fat'})] + remove_tags_after = [dict(name='div', attrs={'class':'bodytext'})] remove_attributes = ['width','height'] - feeds = [(u'Articles', u'http://feeds2.feedburner.com/AdventureGamers')] + feeds = [(u'Articles', u'http://www.adventuregamers.com/rss/')] def get_article_url(self, article): - return article.get('guid', None) + url = BasicNewsRecipe.get_article_url(self, article) + if '/videos/' in url or '/hypeometer/' in url: + return None + return url def append_page(self, soup, appendtag, position): - pager = soup.find('div',attrs={'class':'toolbar_fat_next'}) + pager = soup.find('div', attrs={'class':'pagination_big'}) if pager: - nexturl = self.INDEX + pager.a['href'] - soup2 = self.index_to_soup(nexturl) - texttag = soup2.find('div', attrs={'class':'bodytext'}) - for it in texttag.findAll(style=True): - del it['style'] - newpos = len(texttag.contents) - self.append_page(soup2,texttag,newpos) - texttag.extract() - appendtag.insert(position,texttag) + nextpage = soup.find('a', attrs={'class':'next-page'}) + if nextpage: + nexturl = nextpage['href'] + soup2 = self.index_to_soup(nexturl) + texttag = soup2.find('div', attrs={'class':'bodytext'}) + for it in texttag.findAll(style=True): + del it['style'] + newpos = len(texttag.contents) + self.append_page(soup2,texttag,newpos) + texttag.extract() + pager.extract() + appendtag.insert(position,texttag) def preprocess_html(self, soup): @@ -78,7 +81,7 @@ class AdventureGamers(BasicNewsRecipe): for item in soup.findAll('div', attrs={'class':'floatright'}): item.extract() self.append_page(soup, soup.body, 3) - pager = soup.find('div',attrs={'class':'toolbar_fat'}) + pager = soup.find('div',attrs={'class':'pagination_big'}) if pager: pager.extract() return self.adeify_images(soup) diff --git a/recipes/cosmopolitan_uk.recipe b/recipes/cosmopolitan_uk.recipe index 7a34d56865..ae23be224d 100644 --- a/recipes/cosmopolitan_uk.recipe +++ b/recipes/cosmopolitan_uk.recipe @@ -1,13 +1,13 @@ -import re from calibre.web.feeds.news import BasicNewsRecipe -#from calibre import __appname__ -from calibre.utils.magick import Image +import re +from calibre import browser + class AdvancedUserRecipe1306097511(BasicNewsRecipe): title = u'Cosmopolitan UK' - description = 'Fashion, beauty and Gossip for women from COSMOPOLITAN -UK' + description = 'Author : D.Asbury : Womens Fashion, beauty and Gossip for women from COSMOPOLITAN -UK' __author__ = 'Dave Asbury' - #last update 21/12/11 + #last update 7/7/12 hopefully get current cover from itunes # greyscale code by Starson cover_url = 'http://www.cosmopolitan.magazine.co.uk/files/4613/2085/8988/Cosmo_Cover3.jpg' no_stylesheets = True @@ -39,14 +39,19 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe): feeds = [ (u'Love & Sex', u'http://www.cosmopolitan.co.uk/love-sex/rss/'), (u'Men', u'http://cosmopolitan.co.uk/men/rss/'), (u'Fashion', u'http://cosmopolitan.co.uk/fashion/rss/'), (u'Hair & Beauty', u'http://cosmopolitan.co.uk/beauty-hair/rss/'), (u'LifeStyle', u'http://cosmopolitan.co.uk/lifestyle/rss/'), (u'Cosmo On Campus', u'http://cosmopolitan.co.uk/campus/rss/'), (u'Celebrity Gossip', u'http://cosmopolitan.co.uk/celebrity-gossip/rss/')] - def postprocess_html(self, soup, first): - #process all the images - for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')): - iurl = tag['src'] - img = Image() - img.open(iurl) - if img < 0: - raise RuntimeError('Out of memory') - img.type = "GrayscaleType" - img.save(iurl) - return soup + def get_cover_url(self): + soup = self.index_to_soup('http://itunes.apple.com/gb/app/cosmopolitan-uk/id461363572?mt=8') + # look for the block containing the sun button and url + cov = soup.find(attrs={'alt' : 'iPhone Screenshot 1'}) + cov2 = str(cov['src']) + br = browser() + br.set_handle_redirect(False) + try: + br.open_novisit(cov2) + cover_url = cov2 + except: + cover_url = 'http://www.cosmopolitan.magazine.co.uk/files/4613/2085/8988/Cosmo_Cover3.jpg' + + return cover_url + + diff --git a/recipes/empire_magazine.recipe b/recipes/empire_magazine.recipe new file mode 100644 index 0000000000..138b7bffd1 --- /dev/null +++ b/recipes/empire_magazine.recipe @@ -0,0 +1,51 @@ +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1341650280(BasicNewsRecipe): + + title = u'Empire Magazine' + description = 'Author D.Asbury. Film articles from Empire Mag. ' + __author__ = 'Dave Asbury' + # last updated 7/7/12 + remove_empty_feeds = True + remove_javascript = True + no_stylesheets = True + #oldest_article = 7 + max_articles_per_feed = 20 + cover_url = 'http://www.empireonline.com/images/magazine/cover.jpg' + conversion_options = { + 'linearize_tables' : True, + } + #auto_cleanup = True + preprocess_regexps = [ + (re.compile(r'By Title - pTag = Tag(soup, "p") - pTag['class'] = 'title' + # Kindle don't need this because it shows section titles in Periodical format aTag = Tag(soup, "a") - aTag['name'] = "bytitle" - pTag.insert(0,aTag) - pTag.insert(1,NavigableString('Titles')) - body.insert(btc,pTag) - btc += 1 + aTag['id'] = "bytitle" + pTag.insert(ptc,aTag) + ptc += 1 + pTag.insert(ptc,NavigableString('Titles')) + + body.insert(btc,pTag) + btc += 1 divTag = Tag(soup, "div") dtc = 0 @@ -955,7 +954,7 @@ Author '{0}': # Incoming title : if not self.useSeriesPrefixInTitlesSection: nspt = deepcopy(self.booksByTitle) - nspt = sorted(nspt, key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) + nspt = sorted(nspt, key=lambda x: sort_key(x['title_sort'].upper())) self.booksByTitle_noSeriesPrefix = nspt # Loop through the books by title @@ -977,11 +976,14 @@ Author '{0}': if dtc > 0: divRunningTag['class'] = "initial_letter" drtc = 0 - current_letter = self.letter_or_symbol(book['title_sort'][0]) pIndexTag = Tag(soup, "p") pIndexTag['class'] = "author_title_letter_index" aTag = Tag(soup, "a") - aTag['name'] = "%s" % self.letter_or_symbol(book['title_sort'][0]) + current_letter = self.letter_or_symbol(book['title_sort'][0]) + if current_letter == self.SYMBOLS: + aTag['id'] = self.SYMBOLS + else: + aTag['id'] = "%s" % self.generateUnicodeName(current_letter) pIndexTag.insert(0,aTag) pIndexTag.insert(1,NavigableString(self.letter_or_symbol(book['title_sort'][0]))) divRunningTag.insert(dtc,pIndexTag) @@ -1074,19 +1076,6 @@ Author '{0}': btc = 0 - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - body.insert(btc, aTag) - btc += 1 - divTag = Tag(soup, "div") dtc = 0 divOpeningTag = None @@ -1117,7 +1106,6 @@ Author '{0}': drtc = 0 divRunningTag = None - current_letter = self.letter_or_symbol(book['author_sort'][0].upper()) author_count = 0 divOpeningTag = Tag(soup, 'div') if dtc > 0: @@ -1126,7 +1114,11 @@ Author '{0}': pIndexTag = Tag(soup, "p") pIndexTag['class'] = "author_title_letter_index" aTag = Tag(soup, "a") - aTag['name'] = "%sauthors" % self.letter_or_symbol(current_letter) + current_letter = self.letter_or_symbol(book['author_sort'][0].upper()) + if current_letter == self.SYMBOLS: + aTag['id'] = self.SYMBOLS + else: + aTag['id'] = "%s_authors" % self.generateUnicodeName(current_letter) pIndexTag.insert(0,aTag) pIndexTag.insert(1,NavigableString(self.letter_or_symbol(book['author_sort'][0].upper()))) divOpeningTag.insert(dotc,pIndexTag) @@ -1158,7 +1150,7 @@ Author '{0}': pAuthorTag = Tag(soup, "p") pAuthorTag['class'] = "author_index" aTag = Tag(soup, "a") - aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) + aTag['id'] = "%s" % self.generateAuthorAnchor(current_author) aTag.insert(0,NavigableString(current_author)) pAuthorTag.insert(0,aTag) if author_count == 1: @@ -1247,19 +1239,25 @@ Author '{0}': # Loop ends here + pTag = Tag(soup, "p") + pTag['class'] = 'title' + ptc = 0 + aTag = Tag(soup,'a') + aTag['id'] = 'section_start' + pTag.insert(ptc, aTag) + ptc += 1 + if not self.__generateForKindle: - # Insert the <h2> tag with book_count at the head - #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2> - pTag = Tag(soup, "p") - pTag['class'] = 'title' + # Kindle don't need this because it shows section titles in Periodical format aTag = Tag(soup, "a") anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - pTag.insert(0,aTag) - #h2Tag.insert(1,NavigableString('%s (%d)' % (friendly_name, book_count))) - pTag.insert(1,NavigableString('%s' % (friendly_name))) - body.insert(btc,pTag) - btc += 1 + aTag['id'] = anchor_name.replace(" ","") + pTag.insert(ptc,aTag) + ptc += 1 + pTag.insert(ptc,NavigableString('%s' % (friendly_name))) + + body.insert(btc,pTag) + btc += 1 if author_count == 1: divTag.insert(dtc, divOpeningTag) @@ -1294,7 +1292,7 @@ Author '{0}': pIndexTag = Tag(soup, "p") pIndexTag['class'] = "date_index" aTag = Tag(soup, "a") - aTag['name'] = "bda_%s-%s" % (current_date.year, current_date.month) + aTag['id'] = "bda_%s-%s" % (current_date.year, current_date.month) pIndexTag.insert(0,aTag) pIndexTag.insert(1,NavigableString(date_string)) divTag.insert(dtc,pIndexTag) @@ -1312,7 +1310,7 @@ Author '{0}': pAuthorTag['class'] = "author_index" aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) + aTag['id'] = "%s" % self.generateAuthorAnchor(current_author) aTag.insert(0,NavigableString(current_author)) pAuthorTag.insert(0,aTag) divTag.insert(dtc,pAuthorTag) @@ -1386,7 +1384,7 @@ Author '{0}': pIndexTag = Tag(soup, "p") pIndexTag['class'] = "date_index" aTag = Tag(soup, "a") - aTag['name'] = "bda_%s" % date_range.replace(' ','') + aTag['id'] = "bda_%s" % date_range.replace(' ','') pIndexTag.insert(0,aTag) pIndexTag.insert(1,NavigableString(date_range)) divTag.insert(dtc,pIndexTag) @@ -1457,30 +1455,27 @@ Author '{0}': btc = 0 - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 + pTag = Tag(soup, "p") + pTag['class'] = 'title' + ptc = 0 - # Insert the anchor - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - body.insert(btc, aTag) - btc += 1 + aTag = Tag(soup,'a') + aTag['id'] = 'section_start' + pTag.insert(ptc, aTag) + ptc += 1 if not self.__generateForKindle: - #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2> - pTag = Tag(soup, "p") - pTag['class'] = 'title' + # Kindle don't need this because it shows section titles in Periodical format aTag = Tag(soup, "a") anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - pTag.insert(0,aTag) - pTag.insert(1,NavigableString('%s' % friendly_name)) - body.insert(btc,pTag) - btc += 1 + aTag['id'] = anchor_name.replace(" ","") + + pTag.insert(ptc,aTag) + ptc += 1 + pTag.insert(ptc, NavigableString('%s' % friendly_name)) + + body.insert(btc,pTag) + btc += 1 divTag = Tag(soup, "div") dtc = 0 @@ -1895,11 +1890,10 @@ Author '{0}': self.updateProgressFullStep("'Genres'") self.genre_tags_dict = self.filterDbTags(self.db.all_tags()) - # Extract books matching filtered_tags genre_list = [] - for friendly_tag in sorted(self.genre_tags_dict): - #print "\ngenerateHTMLByTags(): looking for books with friendly_tag '%s'" % friendly_tag + for friendly_tag in sorted(self.genre_tags_dict, key=sort_key): + #print("\ngenerateHTMLByTags(): looking for books with friendly_tag '%s'" % friendly_tag) # tag_list => { normalized_genre_tag : [{book},{},{}], # normalized_genre_tag : [{book},{},{}] } @@ -2268,7 +2262,7 @@ Author '{0}': navPointTag.insert(1, contentTag) cmiTag = Tag(soup, '%s' % 'calibre:meta-img') - cmiTag['name'] = "mastheadImage" + cmiTag['id'] = "mastheadImage" cmiTag['src'] = "images/mastheadImage.gif" navPointTag.insert(2,cmiTag) navMapTag.insert(0,navPointTag) @@ -2552,7 +2546,10 @@ Author '{0}': navLabelTag.insert(0, textTag) navPointByLetterTag.insert(0,navLabelTag) contentTag = Tag(soup, 'content') - contentTag['src'] = "content/%s.html#%s" % (output, title_letters[i]) + if title_letters[i] == self.SYMBOLS: + contentTag['src'] = "content/%s.html#%s" % (output, title_letters[i]) + else: + contentTag['src'] = "content/%s.html#%s" % (output, self.generateUnicodeName(title_letters[i])) navPointByLetterTag.insert(1,contentTag) if self.generateForKindle: @@ -2640,7 +2637,7 @@ Author '{0}': navLabelTag.insert(0, textTag) navPointByLetterTag.insert(0,navLabelTag) contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#%sauthors" % (HTML_file, authors_by_letter[1]) + contentTag['src'] = "%s#%s_authors" % (HTML_file, self.generateUnicodeName(authors_by_letter[1])) navPointByLetterTag.insert(1,contentTag) @@ -3213,7 +3210,7 @@ Author '{0}': ans = '%s%d %s:\n' % (' ' * indent, len(tags), header) ans += ' ' * (indent + 1) out_str = '' - sorted_tags = sorted(tags) + sorted_tags = sorted(tags, key=sort_key) for tag in next_tag(sorted_tags): out_str += tag if len(out_str) >= line_break: @@ -3234,7 +3231,7 @@ Author '{0}': if tag == ' ': continue - normalized_tags.append(re.sub('\W','',tag).lower()) + normalized_tags.append(re.sub('\W','',ascii_text(tag)).lower()) friendly_tags.append(tag) genre_tags_dict = dict(zip(friendly_tags,normalized_tags)) @@ -3293,18 +3290,24 @@ Author '{0}': body = soup.find('body') btc = 0 + divTag = Tag(soup, 'div') + dtc = 0 + # Insert section tag if this is the section start - first article only if section_head: aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 + aTag['id'] = 'section_start' + divTag.insert(dtc, aTag) + dtc += 1 + #body.insert(btc, aTag) + #btc += 1 # Create an anchor from the tag aTag = Tag(soup, 'a') - aTag['name'] = "Genre_%s" % genre - body.insert(btc,aTag) + aTag['id'] = "Genre_%s" % genre + divTag.insert(dtc, aTag) + body.insert(btc,divTag) btc += 1 titleTag = body.find(attrs={'class':'title'}) @@ -3477,7 +3480,7 @@ Author '{0}': for (i, tag) in enumerate(sorted(book.get('tags', []))): aTag = Tag(_soup,'a') if self.opts.generate_genres: - aTag['href'] = "Genre_%s.html" % re.sub("\W","",tag.lower()) + aTag['href'] = "Genre_%s.html" % re.sub("\W","",ascii_text(tag).lower()) aTag.insert(0,escape(NavigableString(tag))) genresTag.insert(gtc, aTag) gtc += 1 @@ -3544,8 +3547,10 @@ Author '{0}': btc = 0 # Insert the title anchor for inbound links aTag = Tag(soup, "a") - aTag['name'] = "book%d" % int(book['id']) - body.insert(btc, aTag) + aTag['id'] = "book%d" % int(book['id']) + divTag = Tag(soup, 'div') + divTag.insert(0, aTag) + body.insert(btc, divTag) btc += 1 # Insert the link to the series or remove <a class="series"> @@ -3770,7 +3775,7 @@ Author '{0}': else: word = '%10.0f' % (float(word)) translated.append(word) - return ascii_text(' '.join(translated)) + return ' '.join(translated) def generateThumbnail(self, title, image_dir, thumb_file): ''' @@ -3824,6 +3829,14 @@ Author '{0}': with zf: zf.writestr(title['uuid']+cover_crc, thumb_data) + def generateUnicodeName(self, c): + ''' + Generate an anchor name string + ''' + fullname = unicodedata.name(unicode(c)) + terms = fullname.split() + return "_".join(terms) + def getFriendlyGenreTag(self, genre): # Find the first instance of friendly_tag matching genre for friendly_tag in self.genre_tags_dict: @@ -3837,8 +3850,8 @@ Author '{0}': return markerTags def letter_or_symbol(self,char): - if not re.search('[a-zA-Z]',char): - return 'Symbols' + if not re.search('[a-zA-Z]', ascii_text(char)): + return self.SYMBOLS else: return char diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index e60350b307..905eb84fa4 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -3737,4 +3737,42 @@ books_series_link feeds 'SELECT {0}, count(*) FROM books_{1}_link GROUP BY {0}'.format( fm['link_column'], fm['table'])) + def all_author_names(self): + ai = self.FIELD_MAP['authors'] + ans = set() + for rec in self.data.iterall(): + auts = rec[ai] + if auts: + for x in auts.split(','): + ans.add(x.replace('|', ',')) + return ans + + def all_tag_names(self): + ai = self.FIELD_MAP['tags'] + ans = set() + for rec in self.data.iterall(): + auts = rec[ai] + if auts: + for x in auts.split(','): + ans.add(x) + return ans + + def all_publisher_names(self): + ai = self.FIELD_MAP['publisher'] + ans = set() + for rec in self.data.iterall(): + auts = rec[ai] + if auts: + ans.add(auts) + return ans + + def all_series_names(self): + ai = self.FIELD_MAP['series'] + ans = set() + for rec in self.data.iterall(): + auts = rec[ai] + if auts: + ans.add(auts) + return ans + diff --git a/src/calibre/utils/config_base.py b/src/calibre/utils/config_base.py index a50d0fd153..198c09bfcd 100644 --- a/src/calibre/utils/config_base.py +++ b/src/calibre/utils/config_base.py @@ -427,7 +427,9 @@ def _prefs(): 'accented versions, based on the language you have chosen ' 'for the calibre interface. For example, in ' u' English, searching for n will match ñ and n, but if ' - 'your language is Spanish it will only match n.')) + 'your language is Spanish it will only match n. Note that ' + 'this is much slower than a simple search on very large ' + 'libraries.')) c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.') return c diff --git a/src/calibre/utils/icu.c b/src/calibre/utils/icu.c index 8e8a8e9ec8..c451e9cdac 100644 --- a/src/calibre/utils/icu.c +++ b/src/calibre/utils/icu.c @@ -272,6 +272,44 @@ icu_Collator_contractions(icu_Collator *self, PyObject *args, PyObject *kwargs) return Py_BuildValue("O", ans); } // }}} +// Collator.startswith {{{ +static PyObject * +icu_Collator_startswith(icu_Collator *self, PyObject *args, PyObject *kwargs) { + PyObject *a_, *b_; + size_t asz, bsz; + int32_t actual_a, actual_b; + UChar *a, *b; + wchar_t *aw, *bw; + UErrorCode status = U_ZERO_ERROR; + int ans = 0; + + if (!PyArg_ParseTuple(args, "UU", &a_, &b_)) return NULL; + asz = PyUnicode_GetSize(a_); bsz = PyUnicode_GetSize(b_); + if (asz < bsz) Py_RETURN_FALSE; + if (bsz == 0) Py_RETURN_TRUE; + + a = (UChar*)calloc(asz*4 + 2, sizeof(UChar)); + b = (UChar*)calloc(bsz*4 + 2, sizeof(UChar)); + aw = (wchar_t*)calloc(asz*4 + 2, sizeof(wchar_t)); + bw = (wchar_t*)calloc(bsz*4 + 2, sizeof(wchar_t)); + + if (a == NULL || b == NULL || aw == NULL || bw == NULL) return PyErr_NoMemory(); + + actual_a = (int32_t)PyUnicode_AsWideChar((PyUnicodeObject*)a_, aw, asz*4+1); + actual_b = (int32_t)PyUnicode_AsWideChar((PyUnicodeObject*)b_, bw, bsz*4+1); + if (actual_a > -1 && actual_b > -1) { + u_strFromWCS(a, asz*4 + 1, &actual_a, aw, -1, &status); + u_strFromWCS(b, bsz*4 + 1, &actual_b, bw, -1, &status); + + if (U_SUCCESS(status) && ucol_equal(self->collator, a, actual_b, b, actual_b)) + ans = 1; + } + + free(a); free(b); free(aw); free(bw); + if (ans) Py_RETURN_TRUE; + Py_RETURN_FALSE; +} // }}} + static PyObject* icu_Collator_clone(icu_Collator *self, PyObject *args, PyObject *kwargs); @@ -296,6 +334,10 @@ static PyMethodDef icu_Collator_methods[] = { "clone() -> returns a clone of this collator." }, + {"startswith", (PyCFunction)icu_Collator_startswith, METH_VARARGS, + "startswith(a, b) -> returns True iff a startswith b, following the current collation rules." + }, + {NULL} /* Sentinel */ }; diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index 76a374d085..0dab76cd30 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -12,7 +12,7 @@ from functools import partial from calibre.constants import plugins from calibre.utils.config_base import tweaks -_icu = _collator = _primary_collator = None +_icu = _collator = _primary_collator = _secondary_collator = None _locale = None _none = u'' @@ -55,6 +55,13 @@ def primary_collator(): _primary_collator.strength = _icu.UCOL_PRIMARY return _primary_collator +def secondary_collator(): + global _secondary_collator + if _secondary_collator is None: + _secondary_collator = _collator.clone() + _secondary_collator.strength = _icu.UCOL_SECONDARY + return _secondary_collator + def py_sort_key(obj): if not obj: return _none @@ -63,7 +70,10 @@ def py_sort_key(obj): def icu_sort_key(collator, obj): if not obj: return _none2 - return collator.sort_key(lower(obj)) + try: + return _secondary_collator.sort_key(obj) + except AttributeError: + return secondary_collator().sort_key(obj) def py_find(pattern, source): pos = source.find(pattern) @@ -77,6 +87,12 @@ def icu_find(collator, pattern, source): except TypeError: return collator.find(unicode(pattern), unicode(source)) +def icu_startswith(collator, a, b): + try: + return collator.startswith(a, b) + except TypeError: + return collator.startswith(unicode(a), unicode(b)) + def py_case_sensitive_sort_key(obj): if not obj: return _none @@ -129,7 +145,7 @@ sort_key = py_sort_key if _icu_not_ok else partial(icu_sort_key, _collator) strcmp = py_strcmp if _icu_not_ok else partial(icu_strcmp, _collator) case_sensitive_sort_key = py_case_sensitive_sort_key if _icu_not_ok else \ - icu_case_sensitive_sort_key + partial(icu_case_sensitive_sort_key, _collator) case_sensitive_strcmp = cmp if _icu_not_ok else icu_case_sensitive_strcmp @@ -155,14 +171,39 @@ def primary_strcmp(a, b): if _icu_not_ok: from calibre.utils.filenames import ascii_text return py_strcmp(ascii_text(a), ascii_text(b)) - return primary_collator().strcmp(a, b) + try: + return _primary_collator.strcmp(a, b) + except AttributeError: + return primary_collator().strcmp(a, b) def primary_find(pat, src): 'find that ignores case and accents on letters' if _icu_not_ok: from calibre.utils.filenames import ascii_text return py_find(ascii_text(pat), ascii_text(src)) - return icu_find(primary_collator(), pat, src) + try: + return icu_find(_primary_collator, pat, src) + except AttributeError: + return icu_find(primary_collator(), pat, src) + +def primary_sort_key(val): + 'A sort key that ignores case and diacritics' + if _icu_not_ok: + from calibre.utils.filenames import ascii_text + return ascii_text(val).lower() + try: + return _primary_collator.sort_key(val) + except AttributeError: + return primary_collator().sort_key(val) + +def primary_startswith(a, b): + if _icu_not_ok: + from calibre.utils.filenames import ascii_text + return ascii_text(a).lower().startswith(ascii_text(b).lower()) + try: + return icu_startswith(_primary_collator, a, b) + except AttributeError: + return icu_startswith(primary_collator(), a, b) ################################################################################ @@ -283,8 +324,8 @@ pêché''' print print '\nTesting primary collation' - for k, v in {u'pèché': u'peche', u'flüße':u'flusse', - u'Štepánek':u'Štepanek'}.iteritems(): + for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', + u'Štepánek':u'ŠtepaneK'}.iteritems(): if primary_strcmp(k, v) != 0: prints('primary_strcmp() failed with %s != %s'%(k, v)) return @@ -293,10 +334,12 @@ pêché''' return global _primary_collator + orig = _primary_collator _primary_collator = _icu.Collator('es') if primary_strcmp(u'peña', u'pena') == 0: print 'Primary collation in Spanish locale failed' return + _primary_collator = orig print '\nTesting contractions' c = _icu.Collator('cs') @@ -306,6 +349,13 @@ pêché''' print 'Contractions for the Czech language failed' return + print '\nTesting startswith' + p = primary_startswith + if (not p('asd', 'asd') or not p('asd', 'A') or + not p('x', '')): + print 'startswith() failed' + return + # }}} if __name__ == '__main__':