diff --git a/Changelog.yaml b/Changelog.yaml index 04cea3af3d..b6831e340b 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -5,7 +5,7 @@ # Also, each release can have new and improved recipes. # - version: ?.?.? -# date: 2011-??-?? +# date: 2012-??-?? # # new features: # - title: @@ -19,8 +19,201 @@ # new recipes: # - title: +- version: 0.8.38 + date: 2012-02-03 + + new features: + - title: "Implement the ability to automatically add books to calibre from a specified folder." + type: major + description: "calibre can now watch a folder on your computer and instantly add any files you put there to the calibre library as new books. You can tell calibre which folder to watch via Preferences->Adding Books->Automatic Adding." + tickets: [920249] + + - title: "Conversion: When automatically inserting page breaks, do not put a page break before a

or

tag if it is immediately preceded by another

or

tag." + + - title: "Driver for EZReader T730 and Point-of-View PlayTab Pro" + tickets: [923283, 922969] + + bug fixes: + - title: "Fix device entry not visible in menubar even when it has been added via Preferences->Toolbars." + tickets: [923175] + + - title: "Fix metadata plugboards not applied when auto sending news by email" + + - title: "Fix regression in 0.8.34 that broke recipes that used skip_ad_pages() but not get_browser(). " + tickets: [923724] + + - title: "Restore device support on FreeBSD, by using HAL" + tickets: [924503] + + - title: "Get books: Show no more than 10 results from the Gandalf store" + + - title: "Content server: Fix metadata not being updated when sending for some MOBI files." + tickets: [923130] + + - title: "Heuristic processing: Fix the italicize common patterns algorithm breaking on some HTML markup." + tickets: [922317] + + - title: "When trying to find an ebook inside a zip file, do not fail if the zip file itself contains other zip files." + tickets: [925670] + + - title: "EPUB Input: Handle EPUBs with duplicate entries in the manifest." + tickets: [925831] + + - title: "MOBI Input: Handle files that have extra tags sprinkled through out their markup." + tickets: [925833] + + improved recipes: + - Metro Nieuws NL + - FHM UK + + new recipes: + - title: Strange Horizons + author: Jim DeVona + + - title: Telegraph India and Live Mint + author: Krittika Goyal + + - title: High Country News + author: Armin Geller + + - title: Countryfile + author: Dave Asbury + + - title: Liberation (subscription version) + author: Remi Vanicat + + - title: Various Italian news sources + author: faber1971 + + +- version: 0.8.37 + date: 2012-01-27 + + new features: + - title: "Allow calibre to be run simultaneously in two different user accounts on windows." + tickets: [919856] + + - title: "Driver for Motorola Photon and Point of View PlayTab" + tickets: [920582, 919080] + + - title: "Add a checkbox to preferences->plugins to show only user installed plugins" + + - title: "Add a restart calibre button to the warning dialog that pops up after changing some preference that requires a restart" + + bug fixes: + - title: "Fix regression in 0.8.36 that caused the remove format from book function to only delete the entry from the database and not delete the actual file from the disk" + tickets: [921721] + + - title: "Fix regression in 0.8.36 that caused the calibredb command to not properly refresh the format information in the GUI" + tickets: [919494] + + - title: "E-book viewer: Preserve the current position more accurately when changing font size/other preferences." + tickets: [912406] + + - title: "Conversion pipeline: Fix items in the that refer to files with URL unsafe filenames being ignored." + tickets: [920804] + + - title: "Fix calibre not running on linux systems that set LANG to an empty string" + + - title: "On first run of calibre, ensure the columns are sized appropriately" + + - title: "MOBI Output: Do not collapse whitespace when setting the comments metadata in newly created MOBI files" + + - title: "HTML Input: Fix handling of files with ä characters in their filenames." + tickets: [919931] + + - title: "Fix the sort on startup tweak ignoring more than three levels" + tickets: [919584] + + - title: "Edit metadata dialog: Fix a bug that broke adding of a file to the book that calibre did not previously know about in the books directory while simultaneously changing the author or title of the book." + tickets: [922003] + + improved recipes: + - People's Daily + - Plus Info + - grantland.com + - Eret es irodalom + - Sueddeutsche.de + + new recipes: + - title: Mumbai Mirror + author: Krittika Goyal + + - title: Real Clear + author: TMcN + + - title: Gazeta Wyborcza + author: ravcio + + - title: The Daily News Egypt and al masry al youm + author: Omm Mishmishah + + - title: Klip.me + author: Ken Sun + + +- version: 0.8.36 + date: 2012-01-20 + + new features: + - title: "Decrease startup time for large libraries with at least one composite custom column by reading format info on demand" + + - title: "When automatically deleting news older than x days, from the calibre library, only delete the book if it both has the tag News and the author calibre. This prevents accidental deletion of books tagged with News by the user." + + - title: "Driver for Infibeam Pi 2" + + - title: "Add a Tag Editor for tags like custom columns to the edit metadata dialog" + + bug fixes: + - title: "E-book viewer: Fix regression in 0.8.35 that caused viewer to raise an error on books that did not define a language" + + - title: "Content server: Fix grouping for categories based on custom columns." + tickets: [919011] + + - title: "Edit metadata dialog: When setting the series from a format or via metadata download, ensure that the series index is not automatically changed, when closing the dialog." + tickets: [918751] + + - title: "When reading metadata from Topaz (azw1) files, handle non ascii metadata correctly." + tickets: [917419] + + - title: "CHM Input: Do not choke on CHM files with non ascii internal filenames on windows." + tickets: [917696] + + - title: "Fix reading metadata from CHM files with non-ascii titles" + + - title: "Fix HTML 5 parser choking on comments" + + - title: "If calibre is started from a directory that does not exist, automatically use the home directory as the working directory, instead of crashing" + + - title: "Fix iriver story HD Wi-Fi device and external SD card swapped" + tickets: [916364] + + - title: "Content server: Fix ugly URLs for specific format download in the book details and permalink panels" + + - title: "When adding FB2 files do not set the date field from the metadata in the file" + + improved recipes: + - OReilly Premuim + - Variety + - Blic + - New Journal of Physics + - Der Tagesspiegel + + new recipes: + - title: Tweakers.net + author: Roedi06 + + - title: Village Voice + author: Barty + + - title: Edge.org Conversations + author: levien + + - title: Novi list - printed edition + author: Darko Miletic + - version: 0.8.35 - date: 2011-01-13 + date: 2012-01-13 new features: - title: "Metadata plugboards: Allow creation of plugboards for email delivery." diff --git a/recipes/al_masry_al_youm.recipe b/recipes/al_masry_al_youm.recipe new file mode 100644 index 0000000000..6ee085e412 --- /dev/null +++ b/recipes/al_masry_al_youm.recipe @@ -0,0 +1,50 @@ +__license__ = 'GPL v3' +__copyright__ = '2011, Pat Stapleton ' +''' +abc.net.au/news +''' +import re +from calibre.web.feeds.recipes import BasicNewsRecipe + +class TheDailyNewsEG(BasicNewsRecipe): + title = u'al-masry al-youm' + __author__ = 'Omm Mishmishah' + description = 'Independent News from Egypt' + masthead_url = 'http://www.almasryalyoum.com/sites/default/files/img/english_logo.png' + cover_url = 'http://www.almasryalyoum.com/sites/default/files/img/english_logo.png' + + auto_cleanup = True + oldest_article = 7 + max_articles_per_feed = 100 + no_stylesheets = False + #delay = 1 + use_embedded_content = False + encoding = 'utf8' + publisher = 'Independent News Egypt' + category = 'News, Egypt, World' + language = 'en_EG' + publication_type = 'newsportal' +# preprocess_regexps = [(re.compile(r'', re.DOTALL), lambda m: '')] +#Remove annoying map links (inline-caption class is also used for some image captions! hence regex to match maps.google) + preprocess_regexps = [(re.compile(r'' + summary - - articles.append({'title': title, 'date': None, 'url': url, 'description': description}) - else: - continue - else: - continue - - answer.append(('Magazine', articles)) - - ul = content.find('ul') - if ul: + soup = self.index_to_soup(self.FRONTPAGE) + sec_start = soup.findAll('div', attrs={'class':'panel-separator'}) + for sec in sec_start: + content = sec.nextSibling + if content: + section = self.tag_to_string(content.find('h2')) articles = [] - for li in ul.findAll('li'): - tag = li.find('div', attrs = {'class': 'views-field-title'}) - if tag: - a = tag.find('a') - if a: - title = self.tag_to_string(a) - url = self.INDEX + a['href'] - description = '' - tag = li.find('div', attrs = {'class': 'views-field-field-article-display-authors-value'}) - if tag: - description = self.tag_to_string(tag) - articles.append({'title': title, 'date': None, 'url': url, 'description': description}) - else: - continue + tags = [] + for div in content.findAll('div', attrs = {'class': re.compile(r'view-row\s+views-row-[0-9]+\s+views-row-[odd|even].*')}): + tags.append(div) + ul = content.find('ul') + for li in content.findAll('li'): + tags.append(li) + + for div in tags: + title = url = description = author = None + + if self.INCLUDE_PREMIUM: + found_premium = False else: - continue - - answer.append(('Letters to the Editor', articles)) + found_premium = div.findAll('span', attrs={'class': + 'premium-icon'}) + if not found_premium: + tag = div.find('div', attrs={'class': 'views-field-title'}) + if tag: + a = tag.find('a') + if a: + title = self.tag_to_string(a) + url = self.INDEX + a['href'] + author = self.tag_to_string(div.find('div', attrs = {'class': 'views-field-field-article-display-authors-value'})) + tag_summary = div.find('span', attrs = {'class': 'views-field-field-article-summary-value'}) + description = self.tag_to_string(tag_summary) + articles.append({'title':title, 'date':None, 'url':url, + 'description':description, 'author':author}) + if articles: + answer.append((section, articles)) return answer def preprocess_html(self, soup): diff --git a/recipes/grantland.recipe b/recipes/grantland.recipe index e169f87f25..2cee9b2077 100644 --- a/recipes/grantland.recipe +++ b/recipes/grantland.recipe @@ -2,105 +2,75 @@ import re from calibre.web.feeds.news import BasicNewsRecipe class GrantLand(BasicNewsRecipe): - title = u"Grantland" - description = 'Writings on Sports & Pop Culture' - language = 'en' - __author__ = 'barty on mobileread.com forum' - max_articles_per_feed = 100 - no_stylesheets = False - # auto_cleanup is too aggressive sometimes and we end up with blank articles - auto_cleanup = False - timefmt = ' [%a, %d %b %Y]' - oldest_article = 365 + title = u"Grantland" + description = 'Writings on Sports & Pop Culture' + language = 'en' + __author__ = 'barty on mobileread.com forum' + max_articles_per_feed = 100 + no_stylesheets = True + # auto_cleanup is too aggressive sometimes and we end up with blank articles + auto_cleanup = False + timefmt = ' [%a, %d %b %Y]' + oldest_article = 90 - cover_url = 'http://cdn0.sbnation.com/imported_assets/740965/blog_grantland_grid_3.jpg' - masthead_url = 'http://a1.espncdn.com/prod/assets/grantland/grantland-logo.jpg' + cover_url = 'http://cdn0.sbnation.com/imported_assets/740965/blog_grantland_grid_3.jpg' + masthead_url = 'http://a1.espncdn.com/prod/assets/grantland/grantland-logo.jpg' - INDEX = 'http://www.grantland.com' - CATEGORIES = [ - # comment out categories you don't want - # (user friendly name, url suffix, max number of articles to load) - ('Today in Grantland','',20), - ('In Case You Missed It','incaseyoumissedit',35), - ] + INDEX = 'http://www.grantland.com' + CATEGORIES = [ + # comment out second line if you don't want older articles + # (user friendly name, url suffix, max number of articles to load) + ('Today in Grantland','',20), + ('In Case You Missed It','incaseyoumissedit',35), + ] - remove_tags = [ - {'name':['head','style','script']}, - {'id':['header']}, - {'class':re.compile(r'\bside|\bad\b|floatright|tags')} - ] - remove_tags_before = {'class':'wrapper'} - remove_tags_after = [{'id':'content'}] + remove_tags = [ + {'name':['style','aside','nav','footer','script']}, + {'name':'h1','text':'Grantland'}, + {'id':['header','col-right']}, + {'class':['connect_widget']}, + {'name':'section','class':re.compile(r'\b(ad|module)\b')}, + ] - preprocess_regexps = [ - #
tags with an img inside are just blog banners, don't need them - # note: there are other useful
tags so we don't want to just strip all of them - (re.compile(r'
.+?
', re.DOTALL|re.IGNORECASE),lambda m: ''), - # delete everything between the *last*
and - (re.compile(r'
', re.DOTALL|re.IGNORECASE),lambda m: '
'), - ] - extra_css = """cite, time { font-size: 0.8em !important; margin-right: 1em !important; } - img + cite { display:block; text-align:right}""" + preprocess_regexps = [ + # remove blog banners + (re.compile(r'. skip + if not title: + continue + self.log('\tFound article:', title) + self.log('\t', url) + articles.append({'title':title,'url':url}) + seen_urls.add(url) - if len(articles) >= max_articles: - break + if len(articles) >= max_articles: + break - if articles: - feeds.append((cat_name, articles)) + if articles: + feeds.append((cat_name, articles)) - return feeds - - def print_version(self, url): - return url+'?view=print' + return feeds diff --git a/recipes/high_country_news.recipe b/recipes/high_country_news.recipe new file mode 100644 index 0000000000..15db60a957 --- /dev/null +++ b/recipes/high_country_news.recipe @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal , Armin Geller' + +''' +Fetch High Country News +''' +from calibre.web.feeds.news import BasicNewsRecipe +class HighCountryNews(BasicNewsRecipe): + + title = u'High Country News' + description = u'News from the American West' + __author__ = 'Armin Geller' # 2012-01-31 + publisher = 'High Country News' + timefmt = ' [%a, %d %b %Y]' + language = 'en-Us' + encoding = 'UTF-8' + publication_type = 'newspaper' + oldest_article = 7 + max_articles_per_feed = 100 + no_stylesheets = True + auto_cleanup = True + remove_javascript = True + use_embedded_content = False + masthead_url = 'http://www.hcn.org/logo.jpg' # 2012-01-31 AGe add + cover_source = 'http://www.hcn.org' # 2012-01-31 AGe add + + def get_cover_url(self): # 2012-01-31 AGe add + cover_source_soup = self.index_to_soup(self.cover_source) + preview_image_div = cover_source_soup.find(attrs={'class':' portaltype-Plone Site content--hcn template-homepage_view'}) + return preview_image_div.div.img['src'] + + feeds = [ + (u'Most recent', u'http://feeds.feedburner.com/hcn/most-recent'), + (u'Current Issue', u'http://feeds.feedburner.com/hcn/current-issue'), + + (u'Writers on the Range', u'http://feeds.feedburner.com/hcn/wotr'), + (u'High Country Views', u'http://feeds.feedburner.com/hcn/HighCountryViews'), + ] + + def print_version(self, url): + return url + '/print_view' + diff --git a/recipes/icons/metro_news_nl.png (PNG Image, 16x16 pixels).png b/recipes/icons/metro_news_nl.png similarity index 100% rename from recipes/icons/metro_news_nl.png (PNG Image, 16x16 pixels).png rename to recipes/icons/metro_news_nl.png diff --git a/recipes/ilmanifesto.recipe b/recipes/ilmanifesto.recipe new file mode 100644 index 0000000000..d7428cebb2 --- /dev/null +++ b/recipes/ilmanifesto.recipe @@ -0,0 +1,110 @@ +from calibre import strftime +from calibre.web.feeds.recipes import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup + +MANIFESTO_BASEURL = 'http://www.ilmanifesto.it/' + +class IlManifesto(BasicNewsRecipe): + title = 'Il Manifesto' + __author__ = 'Giacomo Lacava' + description = 'quotidiano comunista - ultima edizione html disponibile' + publication_type = 'newspaper' + publisher = 'il manifesto coop. editrice a r.l.' + language = 'it' + + oldest_article = 2 + max_articles_per_feed = 100 + delay = 1 + no_stylesheets = True + simultaneous_downloads = 5 + timeout = 30 + auto_cleanup = True + remove_tags = [dict(name='div', attrs={'class':'column_1 float_left'})] + remove_tags_before = dict(name='div',attrs={'class':'column_2 float_right'}) + remove_tags_after = dict(id='myPrintArea') + + manifesto_index = None + manifesto_datestr = None + + def _set_manifesto_index(self): + if self.manifesto_index == None: + startUrl = MANIFESTO_BASEURL + 'area-abbonati/in-edicola/' + startSoup = self.index_to_soup(startUrl) + lastEdition = startSoup.findAll('div',id='accordion_inedicola')[1].find('a')['href'] + del(startSoup) + self.manifesto_index = MANIFESTO_BASEURL + lastEdition + urlsplit = lastEdition.split('/') + self.manifesto_datestr = urlsplit[-1] + if urlsplit[-1] == '': + self.manifesto_datestr = urlsplit[-2] + + + + def get_cover_url(self): + self._set_manifesto_index() + url = MANIFESTO_BASEURL + 'fileadmin/archivi/in_edicola/%sprimapagina.gif' % self.manifesto_datestr + return url + + def parse_index(self): + self._set_manifesto_index() + soup = self.index_to_soup(self.manifesto_index) + feedLinks = soup.find('div',id='accordion_inedicola').findAll('a') + result = [] + for feed in feedLinks: + articles = [] + feedName = feed.find('h2').string + feedUrl = MANIFESTO_BASEURL + feed['href'] + feedSoup = self.index_to_soup(feedUrl) + indexRoot = feedSoup.find('div',attrs={'class':'column1'}) + for div in indexRoot.findAll('div',attrs={'class':'strumenti1_inedicola'}): + artLink = div.find('a') + if artLink is None: continue # empty div + title = artLink.string + url = MANIFESTO_BASEURL + artLink['href'] + + description = '' + descNode = div.find('div',attrs={'class':'text_12'}) + if descNode is not None: + description = descNode.string + + author = '' + authNode = div.find('div',attrs={'class':'firma'}) + if authNode is not None: + author = authNode.string + + articleText = '' + article = { + 'title':title, + 'url':url, + 'date': strftime('%d %B %Y'), + 'description': description, + 'content': articleText, + 'author': author + } + articles.append(article) + result.append((feedName,articles)) + return result + + + def extract_readable_article(self, html, url): + + bs = BeautifulSoup(html) + col1 = bs.find('div',attrs={'class':'column1'}) + + content = col1.find('div',attrs={'class':'bodytext'}) + title = bs.find(id='titolo_articolo').string + author = col1.find('span',attrs={'class':'firma'}) + subtitle = '' + subNode = col1.findPrevious('div',attrs={'class':'occhiello_rosso'}) + if subNode is not None: + subtitle = subNode + summary = '' + sommNode = bs.find('div',attrs={'class':'sommario'}) + if sommNode is not None: + summary = sommNode + + template = "%(title)s

%(title)s

%(subtitle)s

%(author)s

%(summary)s
%(content)s
" + del(bs) + return template % dict(title=title,subtitle=subtitle,author=author,summary=summary,content=content) + + diff --git a/recipes/klip_me.recipe b/recipes/klip_me.recipe new file mode 100644 index 0000000000..71918dc78b --- /dev/null +++ b/recipes/klip_me.recipe @@ -0,0 +1,72 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1299694372(BasicNewsRecipe): + title = u'Klipme' + __author__ = 'Ken Sun' + publisher = 'Klip.me' + category = 'info, custom, Klip.me' + oldest_article = 365 + max_articles_per_feed = 100 + no_stylesheets = True + remove_javascript = True + remove_tags = [ + dict(name='div', attrs={'id':'text_controls_toggle'}) + ,dict(name='script') + ,dict(name='div', attrs={'id':'text_controls'}) + ,dict(name='div', attrs={'id':'editing_controls'}) + ,dict(name='div', attrs={'class':'bar bottom'}) + ] + use_embedded_content = False + needs_subscription = True + INDEX = u'http://www.klip.me' + LOGIN = INDEX + u'/fav/signin?callback=/fav' + + + feeds = [ + (u'Klip.me unread', u'http://www.klip.me/fav'), + (u'Klip.me started', u'http://www.klip.me/fav?s=starred') + ] + + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + if self.username is not None: + br.open(self.LOGIN) + br.select_form(nr=0) + br['Email'] = self.username + if self.password is not None: + br['Passwd'] = self.password + br.submit() + return br + + def parse_index(self): + totalfeeds = [] + lfeeds = self.get_feeds() + for feedobj in lfeeds: + feedtitle, feedurl = feedobj + self.report_progress(0, 'Fetching feed'+' %s...'%(feedtitle if feedtitle else feedurl)) + articles = [] + soup = self.index_to_soup(feedurl) + for item in soup.findAll('table',attrs={'class':['item','item new']}): + atag = item.a + if atag and atag.has_key('href'): + url = atag['href'] + articles.append({ + 'url' :url + }) + totalfeeds.append((feedtitle, articles)) + return totalfeeds + + def print_version(self, url): + return 'http://www.klip.me' + url + + def populate_article_metadata(self, article, soup, first): + article.title = soup.find('title').contents[0].strip() + + def postprocess_html(self, soup, first_fetch): + for link_tag in soup.findAll(attrs={"id" : "story"}): + link_tag.insert(0,'

'+soup.find('title').contents[0].strip()+'

') + print link_tag + + return soup + diff --git a/recipes/la_voce.recipe b/recipes/la_voce.recipe new file mode 100644 index 0000000000..140adbb84c --- /dev/null +++ b/recipes/la_voce.recipe @@ -0,0 +1,15 @@ +__license__ = 'GPL v3' +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1324114228(BasicNewsRecipe): + title = u'La Voce' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + masthead_url = 'http://www.lavoce.info/binary/la_voce/testata/lavoce.1184661635.gif' + feeds = [(u'La Voce', u'http://www.lavoce.info/feed_rss.php?id_feed=1')] + __author__ = 'faber1971' + description = 'Italian website on Economy - v1.01 (17, December 2011)' + language = 'it' + + diff --git a/recipes/liberation_sub.recipe b/recipes/liberation_sub.recipe new file mode 100644 index 0000000000..3ea933f364 --- /dev/null +++ b/recipes/liberation_sub.recipe @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2012, Rémi Vanicat ' +''' +liberation.fr +''' +# The cleanning is from the Liberation recipe, by Darko Miletic + +from calibre.web.feeds.news import BasicNewsRecipe + +class Liberation(BasicNewsRecipe): + + title = u'Libération: Édition abonnés' + __author__ = 'Rémi Vanicat' + description = u'Actualités' + category = 'Actualités, France, Monde' + language = 'fr' + needs_subscription = True + + use_embedded_content = False + no_stylesheets = True + remove_empty_feeds = True + + extra_css = ''' + h1, h2, h3 {font-size:xx-large; font-family:Arial,Helvetica,sans-serif;} + p.subtitle {font-size:xx-small; font-family:Arial,Helvetica,sans-serif;} + h4, h5, h2.rubrique, {font-size:xx-small; color:#4D4D4D; font-family:Arial,Helvetica,sans-serif;} + .ref, .date, .author, .legende {font-size:xx-small; color:#4D4D4D; font-family:Arial,Helvetica,sans-serif;} + .mna-body, entry-body {font-size:medium; font-family:Arial,Helvetica,sans-serif;} + ''' + + keep_only_tags = [ + dict(name='div', attrs={'class':'article'}) + ,dict(name='div', attrs={'class':'text-article m-bot-s1'}) + ,dict(name='div', attrs={'class':'entry'}) + ,dict(name='div', attrs={'class':'col_contenu'}) + ] + + remove_tags_after = [ + dict(name='div',attrs={'class':['object-content text text-item', 'object-content', 'entry-content', 'col01', 'bloc_article_01']}) + ,dict(name='p',attrs={'class':['chapo']}) + ,dict(id='_twitter_facebook') + ] + + remove_tags = [ + dict(name='iframe') + ,dict(name='a', attrs={'class':'lnk-comments'}) + ,dict(name='div', attrs={'class':'toolbox'}) + ,dict(name='ul', attrs={'class':'share-box'}) + ,dict(name='ul', attrs={'class':'tool-box'}) + ,dict(name='ul', attrs={'class':'rub'}) + ,dict(name='p',attrs={'class':['chapo']}) + ,dict(name='p',attrs={'class':['tag']}) + ,dict(name='div',attrs={'class':['blokLies']}) + ,dict(name='div',attrs={'class':['alire']}) + ,dict(id='_twitter_facebook') + ] + + index = 'http://www.liberation.fr/abonnes/' + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + if self.username is not None and self.password is not None: + br.open('http://www.liberation.fr/jogger/login/') + br.select_form(nr=0) + br['email'] = self.username + br['password'] = self.password + br.submit() + return br + + def parse_index(self): + soup=self.index_to_soup(self.index) + + content = soup.find('div', { 'class':'block-content' }) + + articles = [] + cat_articles = [] + + for tag in content.findAll(recursive=False): + if(tag['class']=='headrest headrest-basic-rounded'): + cat_articles = [] + articles.append((tag.find('h5').contents[0],cat_articles)) + else: + title = tag.find('h3').contents[0] + url = tag.find('a')['href'] + print(url) + descripion = tag.find('p',{ 'class':'subtitle' }).contents[0] + article = { + 'title': title, + 'url': url, + 'descripion': descripion, + 'content': '' + } + cat_articles.append(article) + return articles + + + +# Local Variables: +# mode: python +# End: diff --git a/recipes/livemint.recipe b/recipes/livemint.recipe index 4c45fa95d8..12f7b5c470 100644 --- a/recipes/livemint.recipe +++ b/recipes/livemint.recipe @@ -1,41 +1,26 @@ -#!/usr/bin/env python - -__license__ = 'GPL v3' -__copyright__ = '2009, Darko Miletic ' -''' -www.livemint.com -''' - from calibre.web.feeds.news import BasicNewsRecipe class LiveMint(BasicNewsRecipe): - title = u'Livemint' - __author__ = 'Darko Miletic' - description = 'The Wall Street Journal' - publisher = 'The Wall Street Journal' - category = 'news, games, adventure, technology' - language = 'en' + title = u'Live Mint' + language = 'en_IN' + __author__ = 'Krittika Goyal' + #encoding = 'cp1252' + oldest_article = 1 #days + max_articles_per_feed = 25 + use_embedded_content = True - oldest_article = 15 - max_articles_per_feed = 100 - no_stylesheets = True - encoding = 'utf-8' - use_embedded_content = False - extra_css = ' #dvArtheadline{font-size: x-large} #dvArtAbstract{font-size: large} ' + no_stylesheets = True + auto_cleanup = True - keep_only_tags = [dict(name='div', attrs={'class':'innercontent'})] - remove_tags = [dict(name=['object','link','embed','form','iframe'])] + feeds = [ +('Latest News', + 'http://www.livemint.com/StoryRss.aspx?LN=Latestnews'), + ('Gallery', + 'http://www.livemint.com/GalleryRssfeed.aspx'), + ('Top Stories', + 'http://www.livemint.com/StoryRss.aspx?ts=Topstories'), + ('Banking', + 'http://www.livemint.com/StoryRss.aspx?Id=104'), +] - feeds = [(u'Articles', u'http://www.livemint.com/SectionRssfeed.aspx?Mid=1')] - - def print_version(self, url): - link = url - msoup = self.index_to_soup(link) - mlink = msoup.find(attrs={'id':'ctl00_bodyplaceholdercontent_cntlArtTool_printUrl'}) - if mlink: - link = 'http://www.livemint.com/Articles/' + mlink['href'].rpartition('/Articles/')[2] - return link - - def preprocess_html(self, soup): - return self.adeify_images(soup) diff --git a/recipes/marketing_magazine.recipe b/recipes/marketing_magazine.recipe new file mode 100644 index 0000000000..55b6ea2584 --- /dev/null +++ b/recipes/marketing_magazine.recipe @@ -0,0 +1,16 @@ +__license__ = 'GPL v3' +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1327062445(BasicNewsRecipe): + title = u'Marketing Magazine' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + remove_javascript = True + masthead_url = 'http://www.simrendeogun.com/wp-content/uploads/2011/06/New-Marketing-Magazine-Logo.jpg' + feeds = [(u'My Marketing', u'http://feed43.com/0537744466058428.xml'), (u'My Marketing_', u'http://feed43.com/8126723074604845.xml'), (u'Venturini', u'http://robertoventurini.blogspot.com/feeds/posts/default?alt=rss'), (u'Ninja Marketing', u'http://feeds.feedburner.com/NinjaMarketing'), (u'Comunitàzione', u'http://www.comunitazione.it/feed/novita.asp'), (u'Brandforum news', u'http://www.brandforum.it/rss/news'), (u'Brandforum papers', u'http://www.brandforum.it/rss/papers'), (u'Disambiguando', u'http://giovannacosenza.wordpress.com/feed/')] + __author__ = 'faber1971' + description = 'Collection of Italian marketing websites - v1.00 (28, January 2012)' + language = 'it' + + diff --git a/recipes/metro_news_nl.recipe b/recipes/metro_news_nl.recipe index 79e450491c..ac3e23869b 100644 --- a/recipes/metro_news_nl.recipe +++ b/recipes/metro_news_nl.recipe @@ -38,18 +38,23 @@ except: removed keep_only tags Version 1.8 26-11-2022 added remove tag: article-slideshow + Version 1.9 31-1-2012 + removed some left debug settings + extended timeout from 2 to 10 + changed oldest article from 10 to 1.2 + changed max articles from 15 to 25 ''' class AdvancedUserRecipe1306097511(BasicNewsRecipe): title = u'Metro Nieuws NL' - oldest_article = 10 - max_articles_per_feed = 15 + oldest_article = 1.2 + max_articles_per_feed = 25 __author__ = u'DrMerry' description = u'Metro Nederland' language = u'nl' - simultaneous_downloads = 5 + simultaneous_downloads = 3 masthead_url = 'http://blog.metronieuws.nl/wp-content/themes/metro/images/header.gif' - timeout = 2 + timeout = 10 center_navbar = True timefmt = ' [%A, %d %b %Y]' no_stylesheets = True diff --git a/recipes/montreal_gazette.recipe b/recipes/montreal_gazette.recipe index 3061cc37e4..52399e45bd 100644 --- a/recipes/montreal_gazette.recipe +++ b/recipes/montreal_gazette.recipe @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- __license__ = 'GPL v3' @@ -6,15 +7,77 @@ __license__ = 'GPL v3' www.canada.com ''' +import string, re +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe + +import string, re +from calibre import strftime from calibre.web.feeds.recipes import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class CanWestPaper(BasicNewsRecipe): - # un-comment the following three lines for the Montreal Gazette + # un-comment the following four lines for the Victoria Times Colonist +## title = u'Victoria Times Colonist' +## url_prefix = 'http://www.timescolonist.com' +## description = u'News from Victoria, BC' +## fp_tag = 'CAN_TC' + + # un-comment the following four lines for the Vancouver Province +## title = u'Vancouver Province' +## url_prefix = 'http://www.theprovince.com' +## description = u'News from Vancouver, BC' +## fp_tag = 'CAN_VP' + + # un-comment the following four lines for the Vancouver Sun +## title = u'Vancouver Sun' +## url_prefix = 'http://www.vancouversun.com' +## description = u'News from Vancouver, BC' +## fp_tag = 'CAN_VS' + + # un-comment the following four lines for the Edmonton Journal +## title = u'Edmonton Journal' +## url_prefix = 'http://www.edmontonjournal.com' +## description = u'News from Edmonton, AB' +## fp_tag = 'CAN_EJ' + + # un-comment the following four lines for the Calgary Herald +## title = u'Calgary Herald' +## url_prefix = 'http://www.calgaryherald.com' +## description = u'News from Calgary, AB' +## fp_tag = 'CAN_CH' + + # un-comment the following four lines for the Regina Leader-Post +## title = u'Regina Leader-Post' +## url_prefix = 'http://www.leaderpost.com' +## description = u'News from Regina, SK' +## fp_tag = '' + + # un-comment the following four lines for the Saskatoon Star-Phoenix +## title = u'Saskatoon Star-Phoenix' +## url_prefix = 'http://www.thestarphoenix.com' +## description = u'News from Saskatoon, SK' +## fp_tag = '' + + # un-comment the following four lines for the Windsor Star +## title = u'Windsor Star' +## url_prefix = 'http://www.windsorstar.com' +## description = u'News from Windsor, ON' +## fp_tag = 'CAN_' + + # un-comment the following four lines for the Ottawa Citizen +## title = u'Ottawa Citizen' +## url_prefix = 'http://www.ottawacitizen.com' +## description = u'News from Ottawa, ON' +## fp_tag = 'CAN_OC' + + # un-comment the following four lines for the Montreal Gazette title = u'Montreal Gazette' url_prefix = 'http://www.montrealgazette.com' description = u'News from Montreal, QC' + fp_tag = 'CAN_MG' language = 'en_CA' @@ -46,6 +109,80 @@ class CanWestPaper(BasicNewsRecipe): del(div['id']) return soup + def get_cover_url(self): + from datetime import timedelta, datetime, date + if self.fp_tag=='': + return None + cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'+str(date.today().day)+'/lg/'+self.fp_tag+'.jpg' + br = BasicNewsRecipe.get_browser() + daysback=1 + try: + br.open(cover) + except: + while daysback<7: + cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'+str((date.today() - timedelta(days=daysback)).day)+'/lg/'+self.fp_tag+'.jpg' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + daysback = daysback+1 + continue + break + if daysback==7: + self.log("\nCover unavailable") + cover = None + return cover + + def fixChars(self,string): + # Replace lsquo (\x91) + fixed = re.sub("\x91","‘",string) + # Replace rsquo (\x92) + fixed = re.sub("\x92","’",fixed) + # Replace ldquo (\x93) + fixed = re.sub("\x93","“",fixed) + # Replace rdquo (\x94) + fixed = re.sub("\x94","”",fixed) + # Replace ndash (\x96) + fixed = re.sub("\x96","–",fixed) + # Replace mdash (\x97) + fixed = re.sub("\x97","—",fixed) + fixed = re.sub("’","’",fixed) + return fixed + + def massageNCXText(self, description): + # Kindle TOC descriptions won't render certain characters + if description: + massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + # Replace '&' with '&' + massaged = re.sub("&","&", massaged) + return self.fixChars(massaged) + else: + return description + + def populate_article_metadata(self, article, soup, first): + if first: + picdiv = soup.find('body').find('img') + if picdiv is not None: + self.add_toc_thumbnail(article,re.sub(r'links\\link\d+\\','',picdiv['src'])) + xtitle = article.text_summary.strip() + if len(xtitle) == 0: + desc = soup.find('meta',attrs={'property':'og:description'}) + if desc is not None: + article.summary = article.text_summary = desc['content'] + + def strip_anchors(self,soup): + paras = soup.findAll(True) + for para in paras: + aTags = para.findAll('a') + for a in aTags: + if a.img is None: + a.replaceWith(a.renderContents().decode('cp1252','replace')) + return soup + + def preprocess_html(self, soup): + return self.strip_anchors(soup) + + def parse_index(self): soup = self.index_to_soup(self.url_prefix+'/news/todays-paper/index.html') diff --git a/recipes/mumbai_mirror.recipe b/recipes/mumbai_mirror.recipe new file mode 100644 index 0000000000..4af37ab01b --- /dev/null +++ b/recipes/mumbai_mirror.recipe @@ -0,0 +1,59 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class MumbaiMirror(BasicNewsRecipe): + title = u'Mumbai Mirror' + oldest_article = 2 + max_articles_per_feed = 100 + __author__ = 'Krittika Goyal' + + description = 'People Daily Newspaper' + language = 'en_IN' + category = 'News, Mumbai, India' + remove_javascript = True + use_embedded_content = False + auto_cleanup = True + no_stylesheets = True + #encoding = 'GB2312' + conversion_options = {'linearize_tables':True} + + + feeds = [ +('Cover Story', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=latest'), +('City Diary', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=citydiary'), +('Columnists', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=mmcolumnists'), +('Mumbai, The City', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=city'), +('Nation', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=nation'), +('Top Stories', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=topstories'), +('Business', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=business'), +('World', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=world'), +(' Chai Time', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=chaitime'), +('Technology', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=technology'), +('Entertainment', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=entertainment'), +('Style', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=style'), +('Ask the Sexpert', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=askthesexpert'), +('Television', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=television'), +('Lifestyle', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=lifestyle'), +('Sports', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=sports'), +('Travel: Travelers Diary', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=travellersdiaries'), +('Travel: Domestic', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=traveldomestic'), +('Travel: International', +'http://www.mumbaimirror.com/rssfeeds.aspx?feed=travelinternational') +] diff --git a/recipes/mwjournal.recipe b/recipes/mwjournal.recipe index 0eacee6703..65fb948eaa 100644 --- a/recipes/mwjournal.recipe +++ b/recipes/mwjournal.recipe @@ -1,58 +1,53 @@ +#!/usr/bin/env python ## -## Title: Microwave Journal RSS recipe +## Title: Microwave Journal ## Contact: Kiavash (use Mobile Read) ## ## License: GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html ## Copyright: Kiavash ## ## Written: Jan 2012 -## Last Edited: Jan 2012 +## Last Edited: Feb 2012 ## +# Feb 2012: New Recipe compatible with the MWJournal 2.0 website + __license__ = 'GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html' __copyright__ = 'Kiavash' __author__ = 'Kaivash' ''' -Microwave Journal Monthly Magazine -You need to sign up (free) and get username/password. +microwavejournal.com ''' -import re # Import the regular expressions module. -from calibre.ptempfile import TemporaryFile # we need this for saving to a temp file +import re from calibre.web.feeds.news import BasicNewsRecipe +from calibre.utils.magick import Image class MWJournal(BasicNewsRecipe): - # Title to use for the ebook. - title = u'Microwave Journal' - __author__ = 'Kiavash' - language = 'en' - #A brief description for the ebook. - description = u'Microwave Journal web site ebook created using rss feeds.' - - # Set publisher and publication type. - publisher = 'Horizon House' + title = u'Microwave Journal' + description = u'Microwave Journal Monthly Magazine' + publisher = 'Horizon House' publication_type = 'magazine' + INDEX = 'http://www.microwavejournal.com/publications/' - oldest_article = 31 # monthly published magazine. Some months are 31 days! - max_articles_per_feed = 100 - remove_empty_feeds = True - auto_cleanup = True - - # Disable stylesheets and javascript from site. - no_stylesheets = True - remove_javascript = True - - asciiize = True # Converts all none ascii characters to their ascii equivalents - - needs_subscription = True # oh yeah... we need to login btw. - - # Timeout for fetching files from the server in seconds. The default of 120 seconds, seems somewhat excessive. + language = 'en' timeout = 30 - # Specify extra CSS - overrides ALL other CSS (IE. Added last). + Convert_Grayscale = False # Convert images to gray scale or not + keep_only_tags = [dict(name='div', attrs={'class':'record'})] + no_stylesheets = True + remove_javascript = True + remove_tags = [ + dict(name='font', attrs={'class':'footer'}), # remove fonts + ] + + remove_attributes = [ 'border', 'cellspacing', 'align', 'cellpadding', 'colspan', + 'valign', 'vspace', 'hspace', 'alt', 'width', 'height' ] + + # Specify extra CSS - overrides ALL other CSS (IE. Added last). extra_css = 'body { font-family: verdana, helvetica, sans-serif; } \ .introduction, .first { font-weight: bold; } \ .cross-head { font-weight: bold; font-size: 125%; } \ @@ -72,72 +67,75 @@ class MWJournal(BasicNewsRecipe): h3 { font-size: 125%; font-weight: bold; } \ h4, h5, h6 { font-size: 100%; font-weight: bold; }' - remove_tags = [ - dict(name='div', attrs={'class':'boxadzonearea350'}), # Removes banner ads - dict(name='font', attrs={'class':'footer'}), # remove fonts if you do like your fonts more! Comment out to use website's fonts - dict(name='div', attrs={'class':'newsarticlead'}) - ] - - # Remove various tag attributes to improve the look of the ebook pages. - remove_attributes = [ 'border', 'cellspacing', 'align', 'cellpadding', 'colspan', - 'valign', 'vspace', 'hspace', 'alt', 'width', 'height' ] - - # Remove the line breaks as well as href links. Books don't have links generally speaking + # Remove the line breaks, href links and float left/right and picture width/height. preprocess_regexps = [(re.compile(r'', re.IGNORECASE), lambda m: ''), (re.compile(r'', re.IGNORECASE), lambda m: ''), (re.compile(r''), lambda h1: ''), - (re.compile(r''), lambda h2: '') + (re.compile(r''), lambda h2: ''), + (re.compile(r'float:.*?'), lambda h3: ''), + (re.compile(r'width:.*?px'), lambda h4: ''), + (re.compile(r'height:.*?px'), lambda h5: '') ] - # Select the feeds that you are interested. - feeds = [ - (u'Current Issue', u'http://www.mwjournal.com/rss/Rss.asp?type=99'), - (u'Industry News', u'http://www.mwjournal.com/rss/Rss.asp?type=1'), - (u'Resources', u'http://www.mwjournal.com/rss/Rss.asp?type=3'), - (u'Buyer\'s Guide', u'http://www.mwjournal.com/rss/Rss.asp?type=5'), - (u'Events', u'http://www.mwjournal.com/rss/Rss.asp?type=2'), - (u'All Updates', u'http://www.mwjournal.com/rss/Rss.asp?type=0'), - ] - - # No magazine is complete without cover. Let's get it then! - # The function is adapted from the Economist recipe - def get_cover_url(self): - cover_url = None - cover_page_location = 'http://www.mwjournal.com/Journal/' # Cover image is located on this page - soup = self.index_to_soup(cover_page_location) - cover_item = soup.find('img',attrs={'src':lambda x: x and '/IssueImg/3_MWJ_CurrIss_CoverImg' in x}) # There are three files named cover, we want the highest resolution which is the 3rd image. So we look for the pattern. Remember that the name of the cover image changes every month so we cannot search for the complete name. Instead we are searching for the pattern - if cover_item: - cover_url = 'http://www.mwjournal.com' + cover_item['src'].strip() # yeah! we found it. Let's fetch the image file and pass it as cover to calibre - return cover_url def print_version(self, url): - if url.find('/Journal/article.asp?HH_ID=') >= 0: - return self.browser.open_novisit(url).geturl().replace('/Journal/article.asp?HH_ID=', '/Journal/Print.asp?Id=') - elif url.find('/News/article.asp?HH_ID=') >= 0: - return self.browser.open_novisit(url).geturl().replace('/News/article.asp?HH_ID=', '/Journal/Print.asp?Id=') - elif url.find('/Resources/TechLib.asp?HH_ID=') >= 0: - return self.browser.open_novisit(url).geturl().replace('/Resources/TechLib.asp?HH_ID=', '/Resources/PrintRessource.asp?Id=') + return url.replace('/articles/', '/articles/print/') - def get_browser(self): - ''' - Microwave Journal website, directs the login page to omeda.com once login info is submitted, omeda.com redirects to mwjournal.com with again the browser logs in into that site (hidden from the user). To overcome this obsticle, first login page is fetch and its output is stored to an HTML file. Then the HTML file is opened again and second login form is submitted (Many thanks to Barty which helped with second page login). - ''' - br = BasicNewsRecipe.get_browser() - if self.username is not None and self.password is not None: - url = ('http://www.omeda.com/cgi-win/mwjreg.cgi?m=login') # main login page. - br.open(url) # fetch the 1st login page - br.select_form('login') # finds the login form - br['EMAIL_ADDRESS'] = self.username # fills the username - br['PASSWORD'] = self.password # fills the password - raw = br.submit().read() # submit the form and read the 2nd login form - # save it to an htm temp file (from ESPN recipe written by Kovid Goyal kovid@kovidgoyal.net - with TemporaryFile(suffix='.htm') as fname: - with open(fname, 'wb') as f: - f.write(raw) - br.open_local_file(fname) - br.select_form(nr=0) # finds submit on the 2nd form - didwelogin = br.submit().read() # submit it and read the return html - if 'Welcome ' not in didwelogin: # did it login successfully? Is Username/password correct? - raise Exception('Failed to login, are you sure your username and password are correct?') - #login is done - return br + def parse_index(self): + articles = [] + + soup = self.index_to_soup(self.INDEX) + ts = soup.find('div', attrs={'class':'box1 article publications-show'}) + ds = self.tag_to_string(ts.find('h2')) + self.log('Found Current Issue:', ds) + self.timefmt = ' [%s]'%ds + + cover = ts.find('img', src=True) + if cover is not None: + self.cover_url = 'http://www.microwavejournal.com' + cover['src'] + self.log('Found Cover image:', self.cover_url) + + feeds = [] + seen_titles = set([]) # This is used to remove duplicant articles + sections = soup.find('div', attrs={'class':'box2 publication'}) + for section in sections.findAll('div', attrs={'class':'records'}): + section_title = self.tag_to_string(section.find('h3')) + self.log('Found section:', section_title) + articles = [] + for post in section.findAll('div', attrs={'class':'record'}): + h = post.find('h2') + title = self.tag_to_string(h) + if title.find('The MWJ Puzzler') >=0: #Let's get rid of the useless Puzzler! + continue + if title in seen_titles: + continue + seen_titles.add(title) + a = post.find('a', href=True) + url = a['href'] + if url.startswith('/'): + url = 'http://www.microwavejournal.com'+url + abstract = post.find('div', attrs={'class':'abstract'}) + p = abstract.find('p') + desc = None + self.log('\tFound article:', title, 'at', url) + if p is not None: + desc = self.tag_to_string(p) + self.log('\t\t', desc) + articles.append({'title':title, 'url':url, 'description':desc, + 'date':self.timefmt}) + if articles: + feeds.append((section_title, articles)) + return feeds + + def postprocess_html(self, soup, first): + if self.Convert_Grayscale: + #process all the images + for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')): + iurl = tag['src'] + img = Image() + img.open(iurl) + if img < 0: + raise RuntimeError('Out of memory') + img.type = "GrayscaleType" + img.save(iurl) + return soup diff --git a/recipes/new_scientist.recipe b/recipes/new_scientist.recipe index 434c41f525..1bfe27685f 100644 --- a/recipes/new_scientist.recipe +++ b/recipes/new_scientist.recipe @@ -1,16 +1,35 @@ -__license__ = 'GPL v3' -__copyright__ = '2008-2010, AprilHare, Darko Miletic ' +## +## Title: Microwave Journal RSS recipe +## Contact: AprilHare, Darko Miletic +## +## License: GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html +## Copyright: 2008-2010, AprilHare, Darko Miletic +## +## Written: 2008 +## Last Edited: Jan 2012 +## + +''' +01-19-2012: Added GrayScale Image conversion and Duplicant article removals +''' + +__license__ = 'GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html' +__copyright__ = '2008-2012, AprilHare, Darko Miletic ' +__version__ = 'v0.5.0' +__date__ = '2012-01-19' +__author__ = 'Darko Miletic' + ''' newscientist.com ''' import re import urllib +from calibre.utils.magick import Image from calibre.web.feeds.news import BasicNewsRecipe class NewScientist(BasicNewsRecipe): title = 'New Scientist - Online News w. subscription' - __author__ = 'Darko Miletic' description = 'Science news and science articles from New Scientist.' language = 'en' publisher = 'Reed Business Information Ltd.' @@ -39,10 +58,19 @@ class NewScientist(BasicNewsRecipe): keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})] + # Whether to omit duplicates of articles (typically arsing when articles are indexed in + # more than one section). If True, only the first occurance will be downloaded. + filterDuplicates = True + + # Whether to convert images to grayscale for eInk readers. + Convert_Grayscale = False + + url_list = [] # This list is used to check if an article had already been included. + def get_browser(self): br = BasicNewsRecipe.get_browser() br.open('http://www.newscientist.com/') - if self.username is not None and self.password is not None: + if self.username is not None and self.password is not None: br.open('https://www.newscientist.com/user/login') data = urllib.urlencode({ 'source':'form' ,'redirectURL':'' @@ -80,6 +108,10 @@ class NewScientist(BasicNewsRecipe): return article.get('guid', None) def print_version(self, url): + if self.filterDuplicates: + if url in self.url_list: + return + self.url_list.append(url) return url + '?full=true&print=true' def preprocess_html(self, soup): @@ -91,7 +123,7 @@ class NewScientist(BasicNewsRecipe): item.name='p' for item in soup.findAll(['xref','figref']): tstr = item.string - item.replaceWith(tstr) + item.replaceWith(tstr) for tg in soup.findAll('a'): if tg.string == 'Home': tg.parent.extract() @@ -101,3 +133,16 @@ class NewScientist(BasicNewsRecipe): tg.replaceWith(tstr) return soup + # Converts images to Gray Scale + def postprocess_html(self, soup, first): + if self.Convert_Grayscale: + #process all the images + for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')): + iurl = tag['src'] + img = Image() + img.open(iurl) + if img < 0: + raise RuntimeError('Out of memory') + img.type = "GrayscaleType" + img.save(iurl) + return soup diff --git a/recipes/onda_rock.recipe b/recipes/onda_rock.recipe new file mode 100644 index 0000000000..72e93088f2 --- /dev/null +++ b/recipes/onda_rock.recipe @@ -0,0 +1,21 @@ +__license__ = 'GPL v3' +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1328535130(BasicNewsRecipe): + title = u'Onda Rock' + __author__ = 'faber1971' + description = 'Italian rock webzine' + language = 'it' + + + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = False + remove_tags = [ + dict(name='div', attrs={'id':['boxHeader','boxlinks_med','footer','boxinterviste','box_special_med','boxdiscografia_head','path']}), + dict(name='div', attrs={'align':'left'}), + dict(name='div', attrs={'style':'text-align: center'}), + ] + no_stylesheets = True + feeds = [(u'Onda Rock', u'http://www.ondarock.it/feed.php')] + masthead_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/71135_45820579767_4993043_n.jpg' diff --git a/recipes/oreilly_premium.recipe b/recipes/oreilly_premium.recipe index 94d24c1e8e..9dc11059c4 100644 --- a/recipes/oreilly_premium.recipe +++ b/recipes/oreilly_premium.recipe @@ -14,6 +14,7 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup class OReillyPremium(BasicNewsRecipe): title = u'OReilly Premium' __author__ = 'TMcN' + language = 'en' description = 'Retrieves Premium and News Letter content from BillOReilly.com. Requires a Bill OReilly Premium Membership.' cover_url = 'http://images.billoreilly.com/images/headers/billgray_header.png' auto_cleanup = True diff --git a/recipes/ottawa_citizen.recipe b/recipes/ottawa_citizen.recipe index 5465212d4c..a79b8f7567 100644 --- a/recipes/ottawa_citizen.recipe +++ b/recipes/ottawa_citizen.recipe @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- __license__ = 'GPL v3' @@ -6,20 +7,77 @@ __license__ = 'GPL v3' www.canada.com ''' +import string, re +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe + +import string, re +from calibre import strftime from calibre.web.feeds.recipes import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class CanWestPaper(BasicNewsRecipe): - # un-comment the following three lines for the Ottawa Citizen + # un-comment the following four lines for the Victoria Times Colonist +## title = u'Victoria Times Colonist' +## url_prefix = 'http://www.timescolonist.com' +## description = u'News from Victoria, BC' +## fp_tag = 'CAN_TC' + + # un-comment the following four lines for the Vancouver Province +## title = u'Vancouver Province' +## url_prefix = 'http://www.theprovince.com' +## description = u'News from Vancouver, BC' +## fp_tag = 'CAN_VP' + + # un-comment the following four lines for the Vancouver Sun +## title = u'Vancouver Sun' +## url_prefix = 'http://www.vancouversun.com' +## description = u'News from Vancouver, BC' +## fp_tag = 'CAN_VS' + + # un-comment the following four lines for the Edmonton Journal +## title = u'Edmonton Journal' +## url_prefix = 'http://www.edmontonjournal.com' +## description = u'News from Edmonton, AB' +## fp_tag = 'CAN_EJ' + + # un-comment the following four lines for the Calgary Herald +## title = u'Calgary Herald' +## url_prefix = 'http://www.calgaryherald.com' +## description = u'News from Calgary, AB' +## fp_tag = 'CAN_CH' + + # un-comment the following four lines for the Regina Leader-Post +## title = u'Regina Leader-Post' +## url_prefix = 'http://www.leaderpost.com' +## description = u'News from Regina, SK' +## fp_tag = '' + + # un-comment the following four lines for the Saskatoon Star-Phoenix +## title = u'Saskatoon Star-Phoenix' +## url_prefix = 'http://www.thestarphoenix.com' +## description = u'News from Saskatoon, SK' +## fp_tag = '' + + # un-comment the following four lines for the Windsor Star +## title = u'Windsor Star' +## url_prefix = 'http://www.windsorstar.com' +## description = u'News from Windsor, ON' +## fp_tag = 'CAN_' + + # un-comment the following four lines for the Ottawa Citizen title = u'Ottawa Citizen' url_prefix = 'http://www.ottawacitizen.com' description = u'News from Ottawa, ON' + fp_tag = 'CAN_OC' - # un-comment the following three lines for the Montreal Gazette - #title = u'Montreal Gazette' - #url_prefix = 'http://www.montrealgazette.com' - #description = u'News from Montreal, QC' + # un-comment the following four lines for the Montreal Gazette +## title = u'Montreal Gazette' +## url_prefix = 'http://www.montrealgazette.com' +## description = u'News from Montreal, QC' +## fp_tag = 'CAN_MG' language = 'en_CA' @@ -51,6 +109,80 @@ class CanWestPaper(BasicNewsRecipe): del(div['id']) return soup + def get_cover_url(self): + from datetime import timedelta, datetime, date + if self.fp_tag=='': + return None + cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'+str(date.today().day)+'/lg/'+self.fp_tag+'.jpg' + br = BasicNewsRecipe.get_browser() + daysback=1 + try: + br.open(cover) + except: + while daysback<7: + cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'+str((date.today() - timedelta(days=daysback)).day)+'/lg/'+self.fp_tag+'.jpg' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + daysback = daysback+1 + continue + break + if daysback==7: + self.log("\nCover unavailable") + cover = None + return cover + + def fixChars(self,string): + # Replace lsquo (\x91) + fixed = re.sub("\x91","‘",string) + # Replace rsquo (\x92) + fixed = re.sub("\x92","’",fixed) + # Replace ldquo (\x93) + fixed = re.sub("\x93","“",fixed) + # Replace rdquo (\x94) + fixed = re.sub("\x94","”",fixed) + # Replace ndash (\x96) + fixed = re.sub("\x96","–",fixed) + # Replace mdash (\x97) + fixed = re.sub("\x97","—",fixed) + fixed = re.sub("’","’",fixed) + return fixed + + def massageNCXText(self, description): + # Kindle TOC descriptions won't render certain characters + if description: + massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + # Replace '&' with '&' + massaged = re.sub("&","&", massaged) + return self.fixChars(massaged) + else: + return description + + def populate_article_metadata(self, article, soup, first): + if first: + picdiv = soup.find('body').find('img') + if picdiv is not None: + self.add_toc_thumbnail(article,re.sub(r'links\\link\d+\\','',picdiv['src'])) + xtitle = article.text_summary.strip() + if len(xtitle) == 0: + desc = soup.find('meta',attrs={'property':'og:description'}) + if desc is not None: + article.summary = article.text_summary = desc['content'] + + def strip_anchors(self,soup): + paras = soup.findAll(True) + for para in paras: + aTags = para.findAll('a') + for a in aTags: + if a.img is None: + a.replaceWith(a.renderContents().decode('cp1252','replace')) + return soup + + def preprocess_html(self, soup): + return self.strip_anchors(soup) + + def parse_index(self): soup = self.index_to_soup(self.url_prefix+'/news/todays-paper/index.html') diff --git a/recipes/people_daily.recipe b/recipes/people_daily.recipe index 4dec2452e2..76ee599e39 100644 --- a/recipes/people_daily.recipe +++ b/recipes/people_daily.recipe @@ -1,10 +1,11 @@ from calibre.web.feeds.news import BasicNewsRecipe +import os, time class AdvancedUserRecipe1277129332(BasicNewsRecipe): - title = u'People Daily - China' + title = u'人民日报' oldest_article = 2 max_articles_per_feed = 100 - __author__ = 'rty' + __author__ = 'zzh' pubisher = 'people.com.cn' description = 'People Daily Newspaper' @@ -14,21 +15,65 @@ class AdvancedUserRecipe1277129332(BasicNewsRecipe): use_embedded_content = False no_stylesheets = True encoding = 'GB2312' + language = 'zh' conversion_options = {'linearize_tables':True} + masthead_url = 'http://www.people.com.cn/img/2010wb/images/logo.gif' - feeds = [(u'\u56fd\u5185\u65b0\u95fb', u'http://www.people.com.cn/rss/politics.xml'), - (u'\u56fd\u9645\u65b0\u95fb', u'http://www.people.com.cn/rss/world.xml'), - (u'\u7ecf\u6d4e\u65b0\u95fb', u'http://www.people.com.cn/rss/finance.xml'), - (u'\u4f53\u80b2\u65b0\u95fb', u'http://www.people.com.cn/rss/sports.xml'), - (u'\u53f0\u6e7e\u65b0\u95fb', u'http://www.people.com.cn/rss/haixia.xml')] + feeds = [ + (u'时政', u'http://www.people.com.cn/rss/politics.xml'), + (u'国际', u'http://www.people.com.cn/rss/world.xml'), + (u'经济', u'http://www.people.com.cn/rss/finance.xml'), + (u'体育', u'http://www.people.com.cn/rss/sports.xml'), + (u'教育', u'http://www.people.com.cn/rss/edu.xml'), + (u'文化', u'http://www.people.com.cn/rss/culture.xml'), + (u'社会', u'http://www.people.com.cn/rss/society.xml'), + (u'传媒', u'http://www.people.com.cn/rss/media.xml'), + (u'娱乐', u'http://www.people.com.cn/rss/ent.xml'), + # (u'汽车', u'http://www.people.com.cn/rss/auto.xml'), + (u'海峡两岸', u'http://www.people.com.cn/rss/haixia.xml'), + # (u'IT频道', u'http://www.people.com.cn/rss/it.xml'), + # (u'环保', u'http://www.people.com.cn/rss/env.xml'), + # (u'科技', u'http://www.people.com.cn/rss/scitech.xml'), + # (u'新农村', u'http://www.people.com.cn/rss/nc.xml'), + # (u'天气频道', u'http://www.people.com.cn/rss/weather.xml'), + (u'生活提示', u'http://www.people.com.cn/rss/life.xml'), + (u'卫生', u'http://www.people.com.cn/rss/medicine.xml'), + # (u'人口', u'http://www.people.com.cn/rss/npmpc.xml'), + # (u'读书', u'http://www.people.com.cn/rss/booker.xml'), + # (u'食品', u'http://www.people.com.cn/rss/shipin.xml'), + # (u'女性新闻', u'http://www.people.com.cn/rss/women.xml'), + # (u'游戏', u'http://www.people.com.cn/rss/game.xml'), + # (u'家电频道', u'http://www.people.com.cn/rss/homea.xml'), + # (u'房产', u'http://www.people.com.cn/rss/house.xml'), + # (u'健康', u'http://www.people.com.cn/rss/health.xml'), + # (u'科学发展观', u'http://www.people.com.cn/rss/kxfz.xml'), + # (u'知识产权', u'http://www.people.com.cn/rss/ip.xml'), + # (u'高层动态', u'http://www.people.com.cn/rss/64094.xml'), + # (u'党的各项工作', u'http://www.people.com.cn/rss/64107.xml'), + # (u'党建聚焦', u'http://www.people.com.cn/rss/64101.xml'), + # (u'机关党建', u'http://www.people.com.cn/rss/117094.xml'), + # (u'事业党建', u'http://www.people.com.cn/rss/117095.xml'), + # (u'国企党建', u'http://www.people.com.cn/rss/117096.xml'), + # (u'非公党建', u'http://www.people.com.cn/rss/117097.xml'), + # (u'社区党建', u'http://www.people.com.cn/rss/117098.xml'), + # (u'高校党建', u'http://www.people.com.cn/rss/117099.xml'), + # (u'农村党建', u'http://www.people.com.cn/rss/117100.xml'), + # (u'军队党建', u'http://www.people.com.cn/rss/117101.xml'), + # (u'时代先锋', u'http://www.people.com.cn/rss/78693.xml'), + # (u'网友声音', u'http://www.people.com.cn/rss/64103.xml'), + # (u'反腐倡廉', u'http://www.people.com.cn/rss/64371.xml'), + # (u'综合报道', u'http://www.people.com.cn/rss/64387.xml'), + # (u'中国人大新闻', u'http://www.people.com.cn/rss/14576.xml'), + # (u'中国政协新闻', u'http://www.people.com.cn/rss/34948.xml'), + ] keep_only_tags = [ - dict(name='div', attrs={'class':'left_content'}), + dict(name='div', attrs={'class':'text_c'}), ] remove_tags = [ - dict(name='table', attrs={'class':'title'}), + dict(name='div', attrs={'class':'tools'}), ] remove_tags_after = [ - dict(name='table', attrs={'class':'bianji'}), + dict(name='div', attrs={'id':'p_content'}), ] def append_page(self, soup, appendtag, position): @@ -36,7 +81,7 @@ class AdvancedUserRecipe1277129332(BasicNewsRecipe): if pager: nexturl = self.INDEX + pager.a['href'] soup2 = self.index_to_soup(nexturl) - texttag = soup2.find('div', attrs={'class':'left_content'}) + texttag = soup2.find('div', attrs={'class':'text_c'}) #for it in texttag.findAll(style=True): # del it['style'] newpos = len(texttag.contents) @@ -44,9 +89,15 @@ class AdvancedUserRecipe1277129332(BasicNewsRecipe): texttag.extract() appendtag.insert(position,texttag) + def skip_ad_pages(self, soup): + if ('advertisement' in soup.find('title').string.lower()): + href = soup.find('a').get('href') + return self.browser.open(href).read().decode('GB2312', 'ignore') + else: + return None def preprocess_html(self, soup): - mtag = '\n' + mtag = '\n' soup.head.insert(0,mtag) for item in soup.findAll(style=True): del item['form'] @@ -55,3 +106,19 @@ class AdvancedUserRecipe1277129332(BasicNewsRecipe): #if pager: # pager.extract() return soup + + def get_cover_url(self): + cover = None + os.environ['TZ'] = 'Asia/Shanghai' + time.tzset() + year = time.strftime('%Y') + month = time.strftime('%m') + day = time.strftime('%d') + cover = 'http://paper.people.com.cn/rmrb/page/'+year+'-'+month+'/'+day+'/01/RMRB'+year+month+day+'B001_b.jpg' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + self.log("\nCover unavailable: " + cover) + cover = None + return cover diff --git a/recipes/plus_info.recipe b/recipes/plus_info.recipe index e95a3e7359..ef365e38ca 100644 --- a/recipes/plus_info.recipe +++ b/recipes/plus_info.recipe @@ -1,47 +1,50 @@ -#!/usr/bin/env python - -__author__ = 'Darko Spasovski' -__license__ = 'GPL v3' -__copyright__ = '2011, Darko Spasovski ' - -''' -www.plusinfo.mk -''' - -from calibre.web.feeds.news import BasicNewsRecipe - -class PlusInfo(BasicNewsRecipe): - - INDEX = 'www.plusinfo.mk' - title = u'+info' - __author__ = 'Darko Spasovski' - description = 'Macedonian news portal' - publication_type = 'newsportal' - category = 'news, Macedonia' - language = 'mk' - masthead_url = 'http://www.plusinfo.mk/style/images/logo.jpg' - remove_javascript = True - no_stylesheets = True - use_embedded_content = False - remove_empty_feeds = True - oldest_article = 1 - max_articles_per_feed = 100 - - keep_only_tags = [dict(name='div', attrs={'class': 'vest'})] - remove_tags = [dict(name='div', attrs={'class':['komentari_holder', 'objava']})] - - feeds = [(u'Македонија', u'http://www.plusinfo.mk/rss/makedonija'), - (u'Бизнис', u'http://www.plusinfo.mk/rss/biznis'), - (u'Скопје', u'http://www.plusinfo.mk/rss/skopje'), - (u'Култура', u'http://www.plusinfo.mk/rss/kultura'), - (u'Свет', u'http://www.plusinfo.mk/rss/svet'), - (u'Сцена', u'http://www.plusinfo.mk/rss/scena'), - (u'Здравје', u'http://www.plusinfo.mk/rss/zdravje'), - (u'Магазин', u'http://www.plusinfo.mk/rss/magazin'), - (u'Спорт', u'http://www.plusinfo.mk/rss/sport')] - - # uncomment the following block if you want the print version (note: it lacks photos) -# def print_version(self,url): -# segments = url.split('/') -# printURL = '/'.join(segments[0:3]) + '/print/' + '/'.join(segments[5:]) -# return printURL +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__author__ = 'Darko Spasovski' +__license__ = 'GPL v3' +__copyright__ = '2011, Darko Spasovski ' + +''' +www.plusinfo.mk +''' +from calibre.web.feeds.news import BasicNewsRecipe + +class PlusInfo(BasicNewsRecipe): + + INDEX = 'www.plusinfo.mk' + title = u'+info' + __author__ = 'Darko Spasovski' + description = 'Macedonian news portal' + publication_type = 'newsportal' + category = 'news, Macedonia' + language = 'mk' + masthead_url = 'http://www.plusinfo.mk/style/images/logo.jpg' + remove_javascript = True + no_stylesheets = True + use_embedded_content = False + remove_empty_feeds = True + oldest_article = 1 + max_articles_per_feed = 100 + + remove_tags = [] + remove_tags.append(dict(name='div', attrs={'class':['komentari_holder', 'objava', 'koment']})) + remove_tags.append(dict(name='ul', attrs={'class':['vest_meni']})) + remove_tags.append(dict(name='a', attrs={'name': ['fb_share']})) + keep_only_tags = [dict(name='div', attrs={'class': 'vest1'})] + + feeds = [(u'Македонија', u'http://www.plusinfo.mk/rss/makedonija'), + (u'Бизнис', u'http://www.plusinfo.mk/rss/biznis'), + (u'Скопје', u'http://www.plusinfo.mk/rss/skopje'), + (u'Култура', u'http://www.plusinfo.mk/rss/kultura'), + (u'Свет', u'http://www.plusinfo.mk/rss/svet'), + (u'Сцена', u'http://www.plusinfo.mk/rss/scena'), + (u'Здравје', u'http://www.plusinfo.mk/rss/zdravje'), + (u'Магазин', u'http://www.plusinfo.mk/rss/magazin'), + (u'Спорт', u'http://www.plusinfo.mk/rss/sport')] + + # uncomment the following block if you want the print version (note: it lacks photos) +# def print_version(self,url): +# segments = url.split('/') +# printURL = '/'.join(segments[0:3]) + '/print/' + '/'.join(segments[5:]) +# return printURL diff --git a/recipes/readitlater.recipe b/recipes/readitlater.recipe index ea9c92868b..38f7ec1a9a 100644 --- a/recipes/readitlater.recipe +++ b/recipes/readitlater.recipe @@ -1,30 +1,36 @@ +""" +readitlaterlist.com +""" __license__ = 'GPL v3' __copyright__ = ''' 2010, Darko Miletic 2011, Przemyslaw Kryger -''' -''' -readitlaterlist.com +2012, tBunnyMan ''' from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe + class Readitlater(BasicNewsRecipe): - title = 'Read It Later' - __author__ = 'Darko Miletic, Przemyslaw Kryger' - description = '''Personalized news feeds. Go to readitlaterlist.com to - setup up your news. Fill in your account - username, and optionally you can add password.''' - publisher = 'readitlater.com' + title = 'ReadItLater' + __author__ = 'Darko Miletic, Przemyslaw Kryger, Keith Callenberg, tBunnyMan' + description = '''Personalized news feeds. Go to readitlaterlist.com to setup \ + up your news. This version displays pages of articles from \ + oldest to newest, with max & minimum counts, and marks articles \ + read after downloading.''' + publisher = 'readitlaterlist.com' category = 'news, custom' oldest_article = 7 - max_articles_per_feed = 100 + max_articles_per_feed = 50 + minimum_articles = 1 no_stylesheets = True use_embedded_content = False needs_subscription = True INDEX = u'http://readitlaterlist.com' LOGIN = INDEX + u'/l' + readList = [] + def get_browser(self): br = BasicNewsRecipe.get_browser() @@ -33,41 +39,46 @@ class Readitlater(BasicNewsRecipe): br.select_form(nr=0) br['feed_id'] = self.username if self.password is not None: - br['password'] = self.password + br['password'] = self.password br.submit() return br def get_feeds(self): - self.report_progress(0, ('Fetching list of feeds...')) + self.report_progress(0, ('Fetching list of pages...')) lfeeds = [] i = 1 feedurl = self.INDEX + u'/unread/1' while True: title = u'Unread articles, page ' + str(i) - lfeeds.append((title, feedurl)) - self.report_progress(0, ('Got ') + str(i) + (' feeds')) + lfeeds.insert(0, (title, feedurl)) + self.report_progress(0, ('Got ') + str(i) + (' pages')) i += 1 soup = self.index_to_soup(feedurl) - ritem = soup.find('a',attrs={'id':'next', 'class':'active'}) + ritem = soup.find('a', attrs={'id':'next', 'class':'active'}) if ritem is None: break feedurl = self.INDEX + ritem['href'] - if self.test: - return lfeeds[:2] return lfeeds def parse_index(self): totalfeeds = [] + articlesToGrab = self.max_articles_per_feed lfeeds = self.get_feeds() for feedobj in lfeeds: + if articlesToGrab < 1: + break feedtitle, feedurl = feedobj self.report_progress(0, ('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl)) articles = [] soup = self.index_to_soup(feedurl) - ritem = soup.find('ul',attrs={'id':'list'}) - for item in ritem.findAll('li'): + ritem = soup.find('ul', attrs={'id':'list'}) + for item in reversed(ritem.findAll('li')): + if articlesToGrab < 1: + break + else: + articlesToGrab -= 1 description = '' - atag = item.find('a',attrs={'class':'text'}) + atag = item.find('a', attrs={'class':'text'}) if atag and atag.has_key('href'): url = self.INDEX + atag['href'] title = self.tag_to_string(item.div) @@ -78,6 +89,20 @@ class Readitlater(BasicNewsRecipe): ,'url' :url ,'description':description }) + readLink = item.find('a', attrs={'class':'check'})['href'] + self.readList.append(readLink) totalfeeds.append((feedtitle, articles)) + if len(self.readList) < self.minimum_articles: + raise Exception("Not enough articles in RIL! Change minimum_articles or add more.") return totalfeeds + def mark_as_read(self, markList): + br = self.get_browser() + for link in markList: + url = self.INDEX + link + response = br.open(url) + response + + def cleanup(self): + self.mark_as_read(self.readList) + diff --git a/recipes/real_clear.recipe b/recipes/real_clear.recipe new file mode 100644 index 0000000000..19add74fcd --- /dev/null +++ b/recipes/real_clear.recipe @@ -0,0 +1,170 @@ +# Test with "\Program Files\Calibre2\ebook-convert.exe" RealClear.recipe .epub --test -vv --debug-pipeline debug +import time +from calibre.web.feeds.recipes import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import NavigableString + +class RealClear(BasicNewsRecipe): + title = u'Real Clear' + __author__ = 'TMcN' + description = 'Real Clear Politics/Science/etc... aggregation of news\n' + cover_url = 'http://www.realclearpolitics.com/dev/mt-static/images/logo.gif' + custom_title = 'Real Clear - '+ time.strftime('%d %b %Y') + auto_cleanup = True + encoding = 'utf8' + language = 'en' + needs_subscription = False + no_stylesheets = True + oldest_article = 7 + remove_javascript = True + remove_tags = [dict(name='img', attrs={})] + # Don't go down + recursions = 0 + max_articles_per_feed = 400 + debugMessages = False + + # Numeric parameter is type, controls whether we look for + feedsets = [ + ["Politics", "http://www.realclearpolitics.com/index.xml", 0], + ["Science", "http://www.realclearscience.com/index.xml", 0], + ["Tech", "http://www.realcleartechnology.com/index.xml", 0], + # The feedburner is essentially the same as the top feed, politics. + # ["Politics Burner", "http://feeds.feedburner.com/realclearpolitics/qlMj", 1], + # ["Commentary", "http://feeds.feedburner.com/Realclearpolitics-Articles", 1], + ["Markets Home", "http://www.realclearmarkets.com/index.xml", 0], + ["Markets", "http://www.realclearmarkets.com/articles/index.xml", 0], + ["World", "http://www.realclearworld.com/index.xml", 0], + ["World Blog", "http://www.realclearworld.com/blog/index.xml", 2] + ] + # Hints to extractPrintURL. + # First column is the URL snippet. Then the string to search for as text, and the attributes to look for above it. Start with attributes and drill down. + printhints = [ + ["billoreilly.com", "Print this entry", 'a', ''], + ["billoreilly.com", "Print This Article", 'a', ''], + ["politico.com", "Print", 'a', 'share-print'], + ["nationalreview.com", ">Print<", 'a', ''], + ["reason.com", "", 'a', 'printer'] + # The following are not supported due to JavaScripting, and would require obfuscated_article to handle + # forbes, + # usatoday - just prints with all current crap anyhow + + ] + + # Returns the best-guess print url. + # The second parameter (pageURL) is returned if nothing is found. + def extractPrintURL(self, pageURL): + tagURL = pageURL + hintsCount =len(self.printhints) + for x in range(0,hintsCount): + if pageURL.find(self.printhints[x][0])== -1 : + continue + print("Trying "+self.printhints[x][0]) + # Only retrieve the soup if we have a match to check for the printed article with. + soup = self.index_to_soup(pageURL) + if soup is None: + return pageURL + if len(self.printhints[x][3])>0 and len(self.printhints[x][1]) == 0: + if self.debugMessages == True : + print("search1") + printFind = soup.find(self.printhints[x][2], attrs=self.printhints[x][3]) + elif len(self.printhints[x][3])>0 : + if self.debugMessages == True : + print("search2") + printFind = soup.find(self.printhints[x][2], attrs=self.printhints[x][3], text=self.printhints[x][1]) + else : + printFind = soup.find(self.printhints[x][2], text=self.printhints[x][1]) + if printFind is None: + if self.debugMessages == True : + print("Not Found") + continue + print(printFind) + if isinstance(printFind, NavigableString)==False: + if printFind['href'] is not None: + return printFind['href'] + tag = printFind.parent + print(tag) + if tag['href'] is None: + if self.debugMessages == True : + print("Not in parent, trying skip-up") + if tag.parent['href'] is None: + if self.debugMessages == True : + print("Not in skip either, aborting") + continue; + return tag.parent['href'] + return tag['href'] + return tagURL + + def get_browser(self): + if self.debugMessages == True : + print("In get_browser") + br = BasicNewsRecipe.get_browser() + return br + + def parseRSS(self, index) : + if self.debugMessages == True : + print("\n\nStarting "+self.feedsets[index][0]) + articleList = [] + soup = self.index_to_soup(self.feedsets[index][1]) + for div in soup.findAll("item"): + title = div.find("title").contents[0] + urlEl = div.find("originalLink") + if urlEl is None or len(urlEl.contents)==0 : + urlEl = div.find("originallink") + if urlEl is None or len(urlEl.contents)==0 : + urlEl = div.find("link") + if urlEl is None or len(urlEl.contents)==0 : + urlEl = div.find("guid") + if urlEl is None or title is None or len(urlEl.contents)==0 : + print("Error in feed "+ self.feedsets[index][0]) + print(div) + continue + print(title) + print(urlEl) + url = urlEl.contents[0].encode("utf-8") + description = div.find("description") + if description is not None and description.contents is not None and len(description.contents)>0: + description = description.contents[0] + else : + description="None" + pubDateEl = div.find("pubDate") + if pubDateEl is None : + pubDateEl = div.find("pubdate") + if pubDateEl is None : + pubDate = time.strftime('%a, %d %b') + else : + pubDate = pubDateEl.contents[0] + if self.debugMessages == True : + print("Article"); + print(title) + print(description) + print(pubDate) + print(url) + url = self.extractPrintURL(url) + print(url) + #url +=re.sub(r'\?.*', '', div['href']) + pubdate = time.strftime('%a, %d %b') + articleList.append(dict(title=title, url=url, date=pubdate, description=description, content='')) + return articleList + + # calibre.web.feeds.news.BasicNewsRecipe.parse_index() fetches the list of articles. + # returns a list of tuple ('feed title', list of articles) + # { + # 'title' : article title, + # 'url' : URL of print version, + # 'date' : The publication date of the article as a string, + # 'description' : A summary of the article + # 'content' : The full article (can be an empty string). This is used by FullContentProfile + # } + # this is used instead of BasicNewsRecipe.parse_feeds(). + def parse_index(self): + # Parse the page into Python Soup + + ans = [] + feedsCount = len(self.feedsets) + for x in range(0,feedsCount): # should be ,4 + feedarticles = self.parseRSS(x) + if feedarticles is not None: + ans.append((self.feedsets[x][0], feedarticles)) + if self.debugMessages == True : + print(ans) + return ans + diff --git a/recipes/regina_leader_post.recipe b/recipes/regina_leader_post.recipe index 9efec51848..fc12c80079 100644 --- a/recipes/regina_leader_post.recipe +++ b/recipes/regina_leader_post.recipe @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- __license__ = 'GPL v3' @@ -6,35 +7,77 @@ __license__ = 'GPL v3' www.canada.com ''' +import string, re +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe + +import string, re +from calibre import strftime from calibre.web.feeds.recipes import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class CanWestPaper(BasicNewsRecipe): - # un-comment the following three lines for the Regina Leader-Post + # un-comment the following four lines for the Victoria Times Colonist +## title = u'Victoria Times Colonist' +## url_prefix = 'http://www.timescolonist.com' +## description = u'News from Victoria, BC' +## fp_tag = 'CAN_TC' + + # un-comment the following four lines for the Vancouver Province +## title = u'Vancouver Province' +## url_prefix = 'http://www.theprovince.com' +## description = u'News from Vancouver, BC' +## fp_tag = 'CAN_VP' + + # un-comment the following four lines for the Vancouver Sun +## title = u'Vancouver Sun' +## url_prefix = 'http://www.vancouversun.com' +## description = u'News from Vancouver, BC' +## fp_tag = 'CAN_VS' + + # un-comment the following four lines for the Edmonton Journal +## title = u'Edmonton Journal' +## url_prefix = 'http://www.edmontonjournal.com' +## description = u'News from Edmonton, AB' +## fp_tag = 'CAN_EJ' + + # un-comment the following four lines for the Calgary Herald +## title = u'Calgary Herald' +## url_prefix = 'http://www.calgaryherald.com' +## description = u'News from Calgary, AB' +## fp_tag = 'CAN_CH' + + # un-comment the following four lines for the Regina Leader-Post title = u'Regina Leader-Post' url_prefix = 'http://www.leaderpost.com' description = u'News from Regina, SK' + fp_tag = '' - # un-comment the following three lines for the Saskatoon Star-Phoenix - #title = u'Saskatoon Star-Phoenix' - #url_prefix = 'http://www.thestarphoenix.com' - #description = u'News from Saskatoon, SK' + # un-comment the following four lines for the Saskatoon Star-Phoenix +## title = u'Saskatoon Star-Phoenix' +## url_prefix = 'http://www.thestarphoenix.com' +## description = u'News from Saskatoon, SK' +## fp_tag = '' - # un-comment the following three lines for the Windsor Star - #title = u'Windsor Star' - #url_prefix = 'http://www.windsorstar.com' - #description = u'News from Windsor, ON' + # un-comment the following four lines for the Windsor Star +## title = u'Windsor Star' +## url_prefix = 'http://www.windsorstar.com' +## description = u'News from Windsor, ON' +## fp_tag = 'CAN_' - # un-comment the following three lines for the Ottawa Citizen - #title = u'Ottawa Citizen' - #url_prefix = 'http://www.ottawacitizen.com' - #description = u'News from Ottawa, ON' + # un-comment the following four lines for the Ottawa Citizen +## title = u'Ottawa Citizen' +## url_prefix = 'http://www.ottawacitizen.com' +## description = u'News from Ottawa, ON' +## fp_tag = 'CAN_OC' - # un-comment the following three lines for the Montreal Gazette - #title = u'Montreal Gazette' - #url_prefix = 'http://www.montrealgazette.com' - #description = u'News from Montreal, QC' + # un-comment the following four lines for the Montreal Gazette +## title = u'Montreal Gazette' +## url_prefix = 'http://www.montrealgazette.com' +## description = u'News from Montreal, QC' +## fp_tag = 'CAN_MG' language = 'en_CA' @@ -66,6 +109,80 @@ class CanWestPaper(BasicNewsRecipe): del(div['id']) return soup + def get_cover_url(self): + from datetime import timedelta, datetime, date + if self.fp_tag=='': + return None + cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'+str(date.today().day)+'/lg/'+self.fp_tag+'.jpg' + br = BasicNewsRecipe.get_browser() + daysback=1 + try: + br.open(cover) + except: + while daysback<7: + cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'+str((date.today() - timedelta(days=daysback)).day)+'/lg/'+self.fp_tag+'.jpg' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + daysback = daysback+1 + continue + break + if daysback==7: + self.log("\nCover unavailable") + cover = None + return cover + + def fixChars(self,string): + # Replace lsquo (\x91) + fixed = re.sub("\x91","‘",string) + # Replace rsquo (\x92) + fixed = re.sub("\x92","’",fixed) + # Replace ldquo (\x93) + fixed = re.sub("\x93","“",fixed) + # Replace rdquo (\x94) + fixed = re.sub("\x94","”",fixed) + # Replace ndash (\x96) + fixed = re.sub("\x96","–",fixed) + # Replace mdash (\x97) + fixed = re.sub("\x97","—",fixed) + fixed = re.sub("’","’",fixed) + return fixed + + def massageNCXText(self, description): + # Kindle TOC descriptions won't render certain characters + if description: + massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + # Replace '&' with '&' + massaged = re.sub("&","&", massaged) + return self.fixChars(massaged) + else: + return description + + def populate_article_metadata(self, article, soup, first): + if first: + picdiv = soup.find('body').find('img') + if picdiv is not None: + self.add_toc_thumbnail(article,re.sub(r'links\\link\d+\\','',picdiv['src'])) + xtitle = article.text_summary.strip() + if len(xtitle) == 0: + desc = soup.find('meta',attrs={'property':'og:description'}) + if desc is not None: + article.summary = article.text_summary = desc['content'] + + def strip_anchors(self,soup): + paras = soup.findAll(True) + for para in paras: + aTags = para.findAll('a') + for a in aTags: + if a.img is None: + a.replaceWith(a.renderContents().decode('cp1252','replace')) + return soup + + def preprocess_html(self, soup): + return self.strip_anchors(soup) + + def parse_index(self): soup = self.index_to_soup(self.url_prefix+'/news/todays-paper/index.html') diff --git a/recipes/saskatoon_star_phoenix.recipe b/recipes/saskatoon_star_phoenix.recipe index 25330478d4..346590b357 100644 --- a/recipes/saskatoon_star_phoenix.recipe +++ b/recipes/saskatoon_star_phoenix.recipe @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- __license__ = 'GPL v3' @@ -6,30 +7,77 @@ __license__ = 'GPL v3' www.canada.com ''' +import string, re +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe + +import string, re +from calibre import strftime from calibre.web.feeds.recipes import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class CanWestPaper(BasicNewsRecipe): - # un-comment the following three lines for the Saskatoon Star-Phoenix + # un-comment the following four lines for the Victoria Times Colonist +## title = u'Victoria Times Colonist' +## url_prefix = 'http://www.timescolonist.com' +## description = u'News from Victoria, BC' +## fp_tag = 'CAN_TC' + + # un-comment the following four lines for the Vancouver Province +## title = u'Vancouver Province' +## url_prefix = 'http://www.theprovince.com' +## description = u'News from Vancouver, BC' +## fp_tag = 'CAN_VP' + + # un-comment the following four lines for the Vancouver Sun +## title = u'Vancouver Sun' +## url_prefix = 'http://www.vancouversun.com' +## description = u'News from Vancouver, BC' +## fp_tag = 'CAN_VS' + + # un-comment the following four lines for the Edmonton Journal +## title = u'Edmonton Journal' +## url_prefix = 'http://www.edmontonjournal.com' +## description = u'News from Edmonton, AB' +## fp_tag = 'CAN_EJ' + + # un-comment the following four lines for the Calgary Herald +## title = u'Calgary Herald' +## url_prefix = 'http://www.calgaryherald.com' +## description = u'News from Calgary, AB' +## fp_tag = 'CAN_CH' + + # un-comment the following four lines for the Regina Leader-Post +## title = u'Regina Leader-Post' +## url_prefix = 'http://www.leaderpost.com' +## description = u'News from Regina, SK' +## fp_tag = '' + + # un-comment the following four lines for the Saskatoon Star-Phoenix title = u'Saskatoon Star-Phoenix' url_prefix = 'http://www.thestarphoenix.com' description = u'News from Saskatoon, SK' + fp_tag = '' - # un-comment the following three lines for the Windsor Star - #title = u'Windsor Star' - #url_prefix = 'http://www.windsorstar.com' - #description = u'News from Windsor, ON' + # un-comment the following four lines for the Windsor Star +## title = u'Windsor Star' +## url_prefix = 'http://www.windsorstar.com' +## description = u'News from Windsor, ON' +## fp_tag = 'CAN_' - # un-comment the following three lines for the Ottawa Citizen - #title = u'Ottawa Citizen' - #url_prefix = 'http://www.ottawacitizen.com' - #description = u'News from Ottawa, ON' + # un-comment the following four lines for the Ottawa Citizen +## title = u'Ottawa Citizen' +## url_prefix = 'http://www.ottawacitizen.com' +## description = u'News from Ottawa, ON' +## fp_tag = 'CAN_OC' - # un-comment the following three lines for the Montreal Gazette - #title = u'Montreal Gazette' - #url_prefix = 'http://www.montrealgazette.com' - #description = u'News from Montreal, QC' + # un-comment the following four lines for the Montreal Gazette +## title = u'Montreal Gazette' +## url_prefix = 'http://www.montrealgazette.com' +## description = u'News from Montreal, QC' +## fp_tag = 'CAN_MG' language = 'en_CA' @@ -61,6 +109,80 @@ class CanWestPaper(BasicNewsRecipe): del(div['id']) return soup + def get_cover_url(self): + from datetime import timedelta, datetime, date + if self.fp_tag=='': + return None + cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'+str(date.today().day)+'/lg/'+self.fp_tag+'.jpg' + br = BasicNewsRecipe.get_browser() + daysback=1 + try: + br.open(cover) + except: + while daysback<7: + cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'+str((date.today() - timedelta(days=daysback)).day)+'/lg/'+self.fp_tag+'.jpg' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + daysback = daysback+1 + continue + break + if daysback==7: + self.log("\nCover unavailable") + cover = None + return cover + + def fixChars(self,string): + # Replace lsquo (\x91) + fixed = re.sub("\x91","‘",string) + # Replace rsquo (\x92) + fixed = re.sub("\x92","’",fixed) + # Replace ldquo (\x93) + fixed = re.sub("\x93","“",fixed) + # Replace rdquo (\x94) + fixed = re.sub("\x94","”",fixed) + # Replace ndash (\x96) + fixed = re.sub("\x96","–",fixed) + # Replace mdash (\x97) + fixed = re.sub("\x97","—",fixed) + fixed = re.sub("’","’",fixed) + return fixed + + def massageNCXText(self, description): + # Kindle TOC descriptions won't render certain characters + if description: + massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + # Replace '&' with '&' + massaged = re.sub("&","&", massaged) + return self.fixChars(massaged) + else: + return description + + def populate_article_metadata(self, article, soup, first): + if first: + picdiv = soup.find('body').find('img') + if picdiv is not None: + self.add_toc_thumbnail(article,re.sub(r'links\\link\d+\\','',picdiv['src'])) + xtitle = article.text_summary.strip() + if len(xtitle) == 0: + desc = soup.find('meta',attrs={'property':'og:description'}) + if desc is not None: + article.summary = article.text_summary = desc['content'] + + def strip_anchors(self,soup): + paras = soup.findAll(True) + for para in paras: + aTags = para.findAll('a') + for a in aTags: + if a.img is None: + a.replaceWith(a.renderContents().decode('cp1252','replace')) + return soup + + def preprocess_html(self, soup): + return self.strip_anchors(soup) + + def parse_index(self): soup = self.index_to_soup(self.url_prefix+'/news/todays-paper/index.html') diff --git a/recipes/satira.recipe b/recipes/satira.recipe new file mode 100644 index 0000000000..bb91b87eb2 --- /dev/null +++ b/recipes/satira.recipe @@ -0,0 +1,14 @@ +__license__ = 'GPL v3' +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1327351409(BasicNewsRecipe): + title = u'Satira' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + feeds = [(u'spinoza', u'http://feeds.feedburner.com/Spinoza'), (u'umore maligno', u'http://www.umoremaligno.it/feed/rss/'), (u'fed-ex', u'http://exfed.tumblr.com/rss'), (u'metilparaben', u'http://feeds.feedburner.com/metil'), (u'freddy nietzsche', u'http://feeds.feedburner.com/FreddyNietzsche')] + __author__ = 'faber1971' + description = 'Collection of Italian satiric blogs - v1.00 (28, January 2012)' + language = 'it' + + diff --git a/recipes/strange_horizons.recipe b/recipes/strange_horizons.recipe new file mode 100644 index 0000000000..e946d97543 --- /dev/null +++ b/recipes/strange_horizons.recipe @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +import urlparse +from collections import OrderedDict + +from calibre.web.feeds.news import BasicNewsRecipe + +class StrangeHorizons(BasicNewsRecipe): + + # Recipe metadata + # Any issue archive page is an acceptable index as well. + # However, reviews will not be included in older issues. + # (Using the reviews archive instead of the recent reviews page would fix.) + INDEX = 'http://www.strangehorizons.com/' + title = 'Strange Horizons' + description = 'A magazine of speculative fiction and related nonfiction. Best downloaded on weekends' + masthead_url = 'http://strangehorizons.com/images/sh_head.gif' + publication_type = 'magazine' + language = 'en' + __author__ = 'Jim DeVona' + __version__ = '1.0' + + # Cruft filters + keep_only_tags = [dict(name='div', id='content')] + remove_tags = [dict(name='p', attrs={'class': 'forum-links'}), dict(name='p', attrs={'class': 'top-link'})] + remove_tags_after = [dict(name='p', attrs={'class': 'author-bio'})] + + # Styles + no_stylesheets = True + extra_css = '''div.image-left { margin: 0.5em auto 1em auto; } div.image-right { margin: 0.5em auto 1em auto; } div.illustration { margin: 0.5em auto 1em auto; text-align: center; } p.image-caption { margin-top: 0.25em; margin-bottom: 1em; font-size: 75%; text-align: center; } h1 { font-size: 160%; } h2 { font-size: 110%; } h3 { font-size: 85%; } h4 { font-size: 80%; } p { font-size: 90%; margin: 1em 1em 1em 15px; } p.author-bio { font-size: 75%; font-style: italic; margin: 1em 1em 1em 15px; } p.author-bio i, p.author-bio cite, p.author-bio .foreign { font-style: normal; } p.author-copyright { font-size: 75%; text-align: center; margin: 3em 1em 1em 15px; } p.content-date { font-weight: bold; } p.dedication { font-style: italic; } div.stanza { margin-bottom: 1em; } div.stanza p { margin: 0px 1em 0px 15px; font-size: 90%; } p.verse-line { margin-bottom: 0px; margin-top: 0px; } p.verse-line-indent-1 { margin-bottom: 0px; margin-top: 0px; text-indent: 2em; } p.verse-line-indent-2 { margin-bottom: 0px; margin-top: 0px; text-indent: 4em; } p.verse-stanza-break { margin-bottom: 0px; margin-top: 0px; } .foreign { font-style: italic; } .thought { font-style: italic; } .thought cite { font-style: normal; } .thought em { font-style: normal; } blockquote { font-size: 90%; font-style: italic; } blockquote cite { font-style: normal; } blockquote em { font-style: normal; } blockquote .foreign { font-style: normal; } blockquote .thought { font-style: normal; } .speaker { font-weight: bold; } pre { margin-left: 15px; } div.screenplay { font-family: monospace; } blockquote.screenplay-dialogue { font-style: normal; font-size: 100%; } .screenplay p.dialogue-first { margin-top: 0; } .screenplay p.speaker { margin-bottom: 0; text-align: center; font-weight: normal; } blockquote.typed-letter { font-style: normal; font-size: 100%; font-family: monospace; } .no-italics { font-style: normal; }''' + + def parse_index(self): + + sections = OrderedDict() + strange_soup = self.index_to_soup(self.INDEX) + + # Find the heading that marks the start of this issue. + issue_heading = strange_soup.find('h2') + issue_date = self.tag_to_string(issue_heading) + self.title = self.title + " - " + issue_date + + # Examine subsequent headings for information about this issue. + heading_tag = issue_heading.findNextSibling(['h2','h3']) + while heading_tag != None: + + # An h2 indicates the start of the next issue. + if heading_tag.name == 'h2': + break + + # The heading begins with a word indicating the article category. + section = self.tag_to_string(heading_tag).split(':', 1)[0].title() + + # Reviews aren't linked from the index, so we need to look them up + # separately. Currently using Recent Reviews page. The reviews + # archive page lists all reviews, but is >500k. + if section == 'Review': + + # Get the list of recent reviews. + review_soup = self.index_to_soup('http://www.strangehorizons.com/reviews/') + review_titles = review_soup.findAll('p', attrs={'class': 'contents-title'}) + + # Get the list of reviews included in this issue. (Kludgey.) + reviews_summary = heading_tag.findNextSibling('p', attrs={'class': 'contents-pullquote'}) + for br in reviews_summary.findAll('br'): + br.replaceWith('----') + review_summary_text = self.tag_to_string(reviews_summary) + review_lines = review_summary_text.split(' ----') + + # Look for each of the needed reviews (there are 3, right?)... + for review_info in review_lines[0:3]: + + # Get the review's release day (unused), title, and author. + day, tna = review_info.split(': ', 1) + article_title, article_author = tna.split(', reviewed by ') + + # ... in the list of recent reviews. + for review_title_tag in review_titles: + review_title = self.tag_to_string(review_title_tag) + if review_title != article_title: + continue + + # Extract review information from heading and surrounding text. + article_summary = self.tag_to_string(review_title_tag.findNextSibling('p', attrs={'class': 'contents-pullquote'})) + review_date = self.tag_to_string(review_title_tag.findNextSibling('p', attrs={'class': 'contents-date'})) + article_url = review_title_tag.find('a')['href'] + + # Add this review to the Review section. + if section not in sections: + sections[section] = [] + sections[section].append({ + 'title': article_title, + 'author': article_author, + 'url': article_url, + 'description': article_summary, + 'date': review_date}) + + break + + else: + # Try http://www.strangehorizons.com/reviews/archives.shtml + self.log("Review not found in Recent Reviews:", article_title) + + else: + + # Extract article information from the heading and surrounding text. + link = heading_tag.find('a') + article_title = self.tag_to_string(link) + article_url = urlparse.urljoin(self.INDEX, link['href']) + article_author = link.nextSibling.replace(', by ', '') + article_summary = self.tag_to_string(heading_tag.findNextSibling('p', attrs={'class':'contents-pullquote'})) + + # Add article to the appropriate collection of sections. + if section not in sections: + sections[section] = [] + sections[section].append({ + 'title': article_title, + 'author': article_author, + 'url': article_url, + 'description': article_summary, + 'date': issue_date}) + + heading_tag = heading_tag.findNextSibling(['h2','h3']) + + # Manually insert standard info about the magazine. + sections['About'] = [{ + 'title': 'Strange Horizons', + 'author': 'Niall Harrison, Editor-in-Chief', + 'url': 'http://www.strangehorizons.com/AboutUs.shtml', + 'description': 'Strange Horizons is a magazine of and about speculative fiction and related nonfiction. Speculative fiction includes science fiction, fantasy, horror, slipstream, and all other flavors of fantastika. Work published in Strange Horizons has been shortlisted for or won Hugo, Nebula, Rhysling, Theodore Sturgeon, James Tiptree Jr., and World Fantasy Awards.', + 'date': ''}] + + return sections.items() + diff --git a/recipes/sueddeutsche.recipe b/recipes/sueddeutsche.recipe index 4b5fb4e7bf..4e683ef0a9 100644 --- a/recipes/sueddeutsche.recipe +++ b/recipes/sueddeutsche.recipe @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- __license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal ' +__copyright__ = '2012, Kovid Goyal ' # 2012-01-26 AGe change to actual Year ''' Fetch sueddeutsche.de @@ -8,19 +8,30 @@ Fetch sueddeutsche.de from calibre.web.feeds.news import BasicNewsRecipe class Sueddeutsche(BasicNewsRecipe): - title = u'sueddeutsche.de' - description = 'News from Germany' - __author__ = 'Oliver Niesner and Armin Geller' #Update AGe 2011-12-16 - use_embedded_content = False - timefmt = ' [%d %b %Y]' - oldest_article = 7 - max_articles_per_feed = 50 - no_stylesheets = True - language = 'de' - encoding = 'utf-8' - remove_javascript = True - auto_cleanup = True - cover_url = 'http://polpix.sueddeutsche.com/polopoly_fs/1.1237395.1324054345!/image/image.jpg_gen/derivatives/860x860/image.jpg' # 2011-12-16 AGe + title = u'Süddeutsche.de' # 2012-01-26 AGe Correct Title + description = 'News from Germany, Access to online content' # 2012-01-26 AGe + __author__ = 'Oliver Niesner and Armin Geller' #Update AGe 2012-01-26 + publisher = 'Süddeutsche Zeitung' # 2012-01-26 AGe add + category = 'news, politics, Germany' # 2012-01-26 AGe add + timefmt = ' [%a, %d %b %Y]' # 2012-01-26 AGe add %a + oldest_article = 7 + max_articles_per_feed = 100 + language = 'de' + encoding = 'utf-8' + publication_type = 'newspaper' # 2012-01-26 add + cover_source = 'http://www.sueddeutsche.de/verlag' # 2012-01-26 AGe add from Darko Miletic paid content source + masthead_url = 'http://www.sueddeutsche.de/static_assets/build/img/sdesiteheader/logo_homepage.441d531c.png' # 2012-01-26 AGe add + + use_embedded_content = False + no_stylesheets = True + remove_javascript = True + auto_cleanup = True + + def get_cover_url(self): # 2012-01-26 AGe add from Darko Miletic paid content source + cover_source_soup = self.index_to_soup(self.cover_source) + preview_image_div = cover_source_soup.find(attrs={'class':'preview-image'}) + return preview_image_div.div.img['src'] + feeds = [ (u'Politik', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5EPolitik%24?output=rss'), (u'Wirtschaft', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5EWirtschaft%24?output=rss'), @@ -29,6 +40,9 @@ class Sueddeutsche(BasicNewsRecipe): (u'Sport', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5ESport%24?output=rss'), (u'Leben', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5ELeben%24?output=rss'), (u'Karriere', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5EKarriere%24?output=rss'), + (u'Bildung', u'http://rss.sueddeutsche.de/rss/bildung'), #2012-01-26 AGe New + (u'Gesundheit', u'http://rss.sueddeutsche.de/rss/gesundheit'), #2012-01-26 AGe New + (u'Stil', u'http://rss.sueddeutsche.de/rss/stil'), #2012-01-26 AGe New (u'München & Region', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5EMünchen&Region%24?output=rss'), (u'Bayern', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5EBayern%24?output=rss'), (u'Medien', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5EMedien%24?output=rss'), @@ -42,6 +56,7 @@ class Sueddeutsche(BasicNewsRecipe): (u'Job', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5EJob%24?output=rss'), # sometimes only (u'Service', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5EService%24?output=rss'), # sometimes only (u'Verlag', u'http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5EVerlag%24?output=rss'), # sometimes only + ] # AGe 2011-12-16 Problem of Handling redirections solved by a solution of Recipes-Re-usable code from kiklop74. # Feed is: http://suche.sueddeutsche.de/query/%23/sort/-docdatetime/drilldown/%C2%A7ressort%3A%5ESport%24?output=rss diff --git a/recipes/tech_economy.recipe b/recipes/tech_economy.recipe new file mode 100644 index 0000000000..6d21800e4e --- /dev/null +++ b/recipes/tech_economy.recipe @@ -0,0 +1,15 @@ +__license__ = 'GPL v3' +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1327051385(BasicNewsRecipe): + title = u'Tech Economy' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + masthead_url = 'http://www.techeconomy.it/wp-content/uploads/2012/01/Logo-TE9.png' + feeds = [(u'Tech Economy', u'http://www.techeconomy.it/feed/')] + remove_tags_after = [dict(name='div', attrs={'class':'cab-author-name'})] + __author__ = 'faber1971' + description = 'Italian website on technology - v1.00 (28, January 2012)' + language = 'it' + diff --git a/recipes/telegraph_in.recipe b/recipes/telegraph_in.recipe new file mode 100644 index 0000000000..5771e71131 --- /dev/null +++ b/recipes/telegraph_in.recipe @@ -0,0 +1,37 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class Telegraph(BasicNewsRecipe): + title = u'The Telegraph India' + language = 'en_IN' + __author__ = 'Krittika Goyal' + oldest_article = 1 #days + max_articles_per_feed = 25 + use_embedded_content = False + + no_stylesheets = True + auto_cleanup = True + + + feeds = [ +('Front Page', + 'http://www.telegraphindia.com/feeds/rss.jsp?id=3'), + ('Nation', + 'http://www.telegraphindia.com/feeds/rss.jsp?id=4'), + ('Calcutta', + 'http://www.telegraphindia.com/feeds/rss.jsp?id=5'), + ('Bengal', + 'http://www.telegraphindia.com/feeds/rss.jsp?id=8'), + ('Bihar', + 'http://www.telegraphindia.com/feeds/rss.jsp?id=22'), + ('Sports', + 'http://www.telegraphindia.com/feeds/rss.jsp?id=7'), + ('International', + 'http://www.telegraphindia.com/feeds/rss.jsp?id=13'), + ('Business', + 'http://www.telegraphindia.com/feeds/rss.jsp?id=9'), + ('Entertainment', + 'http://www.telegraphindia.com/feeds/rss.jsp?id=20'), + ('Opinion', + 'http://www.telegraphindia.com/feeds/rss.jsp?id=6'), +] + diff --git a/recipes/the_daily_news_egypt.recipe b/recipes/the_daily_news_egypt.recipe new file mode 100644 index 0000000000..540cd2f806 --- /dev/null +++ b/recipes/the_daily_news_egypt.recipe @@ -0,0 +1,46 @@ +__license__ = 'GPL v3' +__copyright__ = '2011, Pat Stapleton ' +''' +abc.net.au/news +''' +import re +from calibre.web.feeds.recipes import BasicNewsRecipe + +class TheDailyNewsEG(BasicNewsRecipe): + title = u'The Daily News Egypt' + __author__ = 'Omm Mishmishah' + description = 'News from Egypt' + masthead_url = 'http://www.thedailynewsegypt.com/images/DailyNews-03_05.gif' + cover_url = 'http://www.thedailynewsegypt.com/images/DailyNews-03_05.gif' + + auto_cleanup = True + oldest_article = 7 + max_articles_per_feed = 100 + no_stylesheets = False + #delay = 1 + use_embedded_content = False + encoding = 'utf8' + publisher = 'The Daily News Egypt' + category = 'News, Egypt, World' + language = 'en_EG' + publication_type = 'newsportal' +# preprocess_regexps = [(re.compile(r'', re.DOTALL), lambda m: '')] +#Remove annoying map links (inline-caption class is also used for some image captions! hence regex to match maps.google) + preprocess_regexps = [(re.compile(r' - + ..:: calibre {library} ::.. {title} @@ -58,7 +58,7 @@ method="post" title="Donate to support the development of calibre">
- + diff --git a/resources/quick_start.epub b/resources/quick_start.epub index 882ad76765..cf323ba5de 100644 Binary files a/resources/quick_start.epub and b/resources/quick_start.epub differ diff --git a/setup/hosting.py b/setup/hosting.py index e5fc3c878d..cdfd333f2a 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -26,7 +26,11 @@ def login_to_google(username, password): br.form['Email'] = username br.form['Passwd'] = password raw = br.submit().read() - if b'Account overview - Account Settings' not in raw: + if re.search(br'.*?Account Settings', raw) is None: + x = re.search(br'(?is).*?', raw) + if x is not None: + print ('Title of post login page: %s'%x.group()) + #open('/tmp/goog.html', 'wb').write(raw) raise ValueError(('Failed to login to google with credentials: %s %s' '\nGoogle sometimes requires verification when logging in from a ' 'new IP address. Use lynx to login and supply the verification, ' diff --git a/setup/iso_639/de.po b/setup/iso_639/de.po index dbb5613d60..77bd0f4695 100644 --- a/setup/iso_639/de.po +++ b/setup/iso_639/de.po @@ -18,14 +18,14 @@ msgstr "" "Report-Msgid-Bugs-To: Debian iso-codes team \n" "POT-Creation-Date: 2011-11-25 14:01+0000\n" -"PO-Revision-Date: 2012-01-08 20:03+0000\n" -"Last-Translator: Simeon \n" +"PO-Revision-Date: 2012-01-14 02:30+0000\n" +"Last-Translator: Wolfgang Rohdewald \n" "Language-Team: German \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Launchpad-Export-Date: 2012-01-09 04:49+0000\n" -"X-Generator: Launchpad (build 14640)\n" +"X-Launchpad-Export-Date: 2012-01-15 05:18+0000\n" +"X-Generator: Launchpad (build 14664)\n" "Language: de\n" #. name for aaa diff --git a/setup/iso_639/en_GB.po b/setup/iso_639/en_GB.po index 11d0bbf361..9b7ac9484b 100644 --- a/setup/iso_639/en_GB.po +++ b/setup/iso_639/en_GB.po @@ -8,14 +8,14 @@ msgstr "" "Project-Id-Version: calibre\n" "Report-Msgid-Bugs-To: FULL NAME \n" "POT-Creation-Date: 2011-11-25 14:01+0000\n" -"PO-Revision-Date: 2012-01-07 09:51+0000\n" +"PO-Revision-Date: 2012-01-28 05:12+0000\n" "Last-Translator: Vibhav Pant \n" "Language-Team: English (United Kingdom) \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Launchpad-Export-Date: 2012-01-08 05:02+0000\n" -"X-Generator: Launchpad (build 14640)\n" +"X-Launchpad-Export-Date: 2012-01-29 05:21+0000\n" +"X-Generator: Launchpad (build 14727)\n" #. name for aaa msgid "Ghotuo" @@ -1623,7 +1623,7 @@ msgstr "Attié" #. name for atj msgid "Atikamekw" -msgstr "" +msgstr "Atikamekw" #. name for atk msgid "Ati" @@ -1739,7 +1739,7 @@ msgstr "Asu (Nigeria)" #. name for aun msgid "One; Molmo" -msgstr "" +msgstr "One; Molmo" #. name for auo msgid "Auyokawa" @@ -1747,7 +1747,7 @@ msgstr "Auyokawa" #. name for aup msgid "Makayam" -msgstr "" +msgstr "Makayam" #. name for auq msgid "Anus" @@ -1803,5283 +1803,5283 @@ msgstr "Avikam" #. name for avk msgid "Kotava" -msgstr "" +msgstr "Kotava" #. name for avl msgid "Arabic; Eastern Egyptian Bedawi" -msgstr "" +msgstr "Arabic; Eastern Egyptian Bedawi" #. name for avn msgid "Avatime" -msgstr "" +msgstr "Avatime" #. name for avo msgid "Agavotaguerra" -msgstr "" +msgstr "Agavotaguerra" #. name for avs msgid "Aushiri" -msgstr "" +msgstr "Aushiri" #. name for avt msgid "Au" -msgstr "" +msgstr "Au" #. name for avu msgid "Avokaya" -msgstr "" +msgstr "Avokaya" #. name for avv msgid "Avá-Canoeiro" -msgstr "" +msgstr "Avá-Canoeiro" #. name for awa msgid "Awadhi" -msgstr "" +msgstr "Awadhi" #. name for awb msgid "Awa (Papua New Guinea)" -msgstr "" +msgstr "Awa (Papua New Guinea)" #. name for awc msgid "Cicipu" -msgstr "" +msgstr "Cicipu" #. name for awe msgid "Awetí" -msgstr "" +msgstr "Awetí" #. name for awh msgid "Awbono" -msgstr "" +msgstr "Awbono" #. name for awi msgid "Aekyom" -msgstr "" +msgstr "Aekyom" #. name for awk msgid "Awabakal" -msgstr "" +msgstr "Awabakal" #. name for awm msgid "Arawum" -msgstr "" +msgstr "Arawum" #. name for awn msgid "Awngi" -msgstr "" +msgstr "Awngi" #. name for awo msgid "Awak" -msgstr "" +msgstr "Awak" #. name for awr msgid "Awera" -msgstr "" +msgstr "Awera" #. name for aws msgid "Awyu; South" -msgstr "" +msgstr "Awyu; South" #. name for awt msgid "Araweté" -msgstr "" +msgstr "Araweté" #. name for awu msgid "Awyu; Central" -msgstr "" +msgstr "Awyu; Central" #. name for awv msgid "Awyu; Jair" -msgstr "" +msgstr "Awyu; Jair" #. name for aww msgid "Awun" -msgstr "" +msgstr "Awun" #. name for awx msgid "Awara" -msgstr "" +msgstr "Awara" #. name for awy msgid "Awyu; Edera" -msgstr "" +msgstr "Awyu; Edera" #. name for axb msgid "Abipon" -msgstr "" +msgstr "Abipon" #. name for axg msgid "Arára; Mato Grosso" -msgstr "" +msgstr "Arára; Mato Grosso" #. name for axk msgid "Yaka (Central African Republic)" -msgstr "" +msgstr "Yaka (Central African Republic)" #. name for axm msgid "Armenian; Middle" -msgstr "" +msgstr "Armenian; Middle" #. name for axx msgid "Xaragure" -msgstr "" +msgstr "Xaragure" #. name for aya msgid "Awar" -msgstr "" +msgstr "Awar" #. name for ayb msgid "Gbe; Ayizo" -msgstr "" +msgstr "Gbe; Ayizo" #. name for ayc msgid "Aymara; Southern" -msgstr "" +msgstr "Aymara; Southern" #. name for ayd msgid "Ayabadhu" -msgstr "" +msgstr "Ayabadhu" #. name for aye msgid "Ayere" -msgstr "" +msgstr "Ayere" #. name for ayg msgid "Ginyanga" -msgstr "" +msgstr "Ginyanga" #. name for ayh msgid "Arabic; Hadrami" -msgstr "" +msgstr "Arabic; Hadrami" #. name for ayi msgid "Leyigha" -msgstr "" +msgstr "Leyigha" #. name for ayk msgid "Akuku" -msgstr "" +msgstr "Akuku" #. name for ayl msgid "Arabic; Libyan" -msgstr "" +msgstr "Arabic; Libyan" #. name for aym msgid "Aymara" -msgstr "" +msgstr "Aymara" #. name for ayn msgid "Arabic; Sanaani" -msgstr "" +msgstr "Arabic; Sanaani" #. name for ayo msgid "Ayoreo" -msgstr "" +msgstr "Ayoreo" #. name for ayp msgid "Arabic; North Mesopotamian" -msgstr "" +msgstr "Arabic; North Mesopotamian" #. name for ayq msgid "Ayi (Papua New Guinea)" -msgstr "" +msgstr "Ayi (Papua New Guinea)" #. name for ayr msgid "Aymara; Central" -msgstr "" +msgstr "Aymara; Central" #. name for ays msgid "Ayta; Sorsogon" -msgstr "" +msgstr "Ayta; Sorsogon" #. name for ayt msgid "Ayta; Magbukun" -msgstr "" +msgstr "Ayta; Magbukun" #. name for ayu msgid "Ayu" -msgstr "" +msgstr "Ayu" #. name for ayy msgid "Ayta; Tayabas" -msgstr "" +msgstr "Ayta; Tayabas" #. name for ayz msgid "Mai Brat" -msgstr "" +msgstr "Mai Brat" #. name for aza msgid "Azha" -msgstr "" +msgstr "Azha" #. name for azb msgid "Azerbaijani; South" -msgstr "" +msgstr "Azerbaijani; South" #. name for aze msgid "Azerbaijani" -msgstr "" +msgstr "Azerbaijani" #. name for azg msgid "Amuzgo; San Pedro Amuzgos" -msgstr "" +msgstr "Amuzgo; San Pedro Amuzgos" #. name for azj msgid "Azerbaijani; North" -msgstr "" +msgstr "Azerbaijani; North" #. name for azm msgid "Amuzgo; Ipalapa" -msgstr "" +msgstr "Amuzgo; Ipalapa" #. name for azo msgid "Awing" -msgstr "" +msgstr "Awing" #. name for azt msgid "Atta; Faire" -msgstr "" +msgstr "Atta; Faire" #. name for azz msgid "Nahuatl; Highland Puebla" -msgstr "" +msgstr "Nahuatl; Highland Puebla" #. name for baa msgid "Babatana" -msgstr "" +msgstr "Babatana" #. name for bab msgid "Bainouk-Gunyuño" -msgstr "" +msgstr "Bainouk-Gunyuño" #. name for bac msgid "Badui" -msgstr "" +msgstr "Badui" #. name for bae msgid "Baré" -msgstr "" +msgstr "Baré" #. name for baf msgid "Nubaca" -msgstr "" +msgstr "Nubaca" #. name for bag msgid "Tuki" -msgstr "" +msgstr "Tuki" #. name for bah msgid "Creole English; Bahamas" -msgstr "" +msgstr "Creole English; Bahamas" #. name for baj msgid "Barakai" -msgstr "" +msgstr "Barakai" #. name for bak msgid "Bashkir" -msgstr "" +msgstr "Bashkir" #. name for bal msgid "Baluchi" -msgstr "" +msgstr "Baluchi" #. name for bam msgid "Bambara" -msgstr "" +msgstr "Bambara" #. name for ban msgid "Balinese" -msgstr "" +msgstr "Balinese" #. name for bao msgid "Waimaha" -msgstr "" +msgstr "Waimaha" #. name for bap msgid "Bantawa" -msgstr "" +msgstr "Bantawa" #. name for bar msgid "Bavarian" -msgstr "" +msgstr "Bavarian" #. name for bas msgid "Basa (Cameroon)" -msgstr "" +msgstr "Basa (Cameroon)" #. name for bau msgid "Bada (Nigeria)" -msgstr "" +msgstr "Bada (Nigeria)" #. name for bav msgid "Vengo" -msgstr "" +msgstr "Vengo" #. name for baw msgid "Bambili-Bambui" -msgstr "" +msgstr "Bambili-Bambui" #. name for bax msgid "Bamun" -msgstr "" +msgstr "Bamun" #. name for bay msgid "Batuley" -msgstr "" +msgstr "Batuley" #. name for baz msgid "Tunen" -msgstr "" +msgstr "Tunen" #. name for bba msgid "Baatonum" -msgstr "" +msgstr "Baatonum" #. name for bbb msgid "Barai" -msgstr "" +msgstr "Barai" #. name for bbc msgid "Batak Toba" -msgstr "" +msgstr "Batak Toba" #. name for bbd msgid "Bau" -msgstr "" +msgstr "Bau" #. name for bbe msgid "Bangba" -msgstr "" +msgstr "Bangba" #. name for bbf msgid "Baibai" -msgstr "" +msgstr "Baibai" #. name for bbg msgid "Barama" -msgstr "" +msgstr "Barama" #. name for bbh msgid "Bugan" -msgstr "" +msgstr "Bugan" #. name for bbi msgid "Barombi" -msgstr "" +msgstr "Barombi" #. name for bbj msgid "Ghomálá'" -msgstr "" +msgstr "Ghomálá'" #. name for bbk msgid "Babanki" -msgstr "" +msgstr "Babanki" #. name for bbl msgid "Bats" -msgstr "" +msgstr "Bats" #. name for bbm msgid "Babango" -msgstr "" +msgstr "Babango" #. name for bbn msgid "Uneapa" -msgstr "" +msgstr "Uneapa" #. name for bbo msgid "Bobo Madaré; Northern" -msgstr "" +msgstr "Bobo Madaré; Northern" #. name for bbp msgid "Banda; West Central" -msgstr "" +msgstr "Banda; West Central" #. name for bbq msgid "Bamali" -msgstr "" +msgstr "Bamali" #. name for bbr msgid "Girawa" -msgstr "" +msgstr "Girawa" #. name for bbs msgid "Bakpinka" -msgstr "" +msgstr "Bakpinka" #. name for bbt msgid "Mburku" -msgstr "" +msgstr "Mburku" #. name for bbu msgid "Kulung (Nigeria)" -msgstr "" +msgstr "Kulung (Nigeria)" #. name for bbv msgid "Karnai" -msgstr "" +msgstr "Karnai" #. name for bbw msgid "Baba" -msgstr "" +msgstr "Baba" #. name for bbx msgid "Bubia" -msgstr "" +msgstr "Bubia" #. name for bby msgid "Befang" -msgstr "" +msgstr "Befang" #. name for bbz msgid "Creole Arabic; Babalia" -msgstr "" +msgstr "Creole Arabic; Babalia" #. name for bca msgid "Bai; Central" -msgstr "" +msgstr "Bai; Central" #. name for bcb msgid "Bainouk-Samik" -msgstr "" +msgstr "Bainouk-Samik" #. name for bcc msgid "Balochi; Southern" -msgstr "" +msgstr "Balochi; Southern" #. name for bcd msgid "Babar; North" -msgstr "" +msgstr "Babar; North" #. name for bce msgid "Bamenyam" -msgstr "" +msgstr "Bamenyam" #. name for bcf msgid "Bamu" -msgstr "" +msgstr "Bamu" #. name for bcg msgid "Baga Binari" -msgstr "" +msgstr "Baga Binari" #. name for bch msgid "Bariai" -msgstr "" +msgstr "Bariai" #. name for bci msgid "Baoulé" -msgstr "" +msgstr "Baoulé" #. name for bcj msgid "Bardi" -msgstr "" +msgstr "Bardi" #. name for bck msgid "Bunaba" -msgstr "" +msgstr "Bunaba" #. name for bcl msgid "Bicolano; Central" -msgstr "" +msgstr "Bicolano; Central" #. name for bcm msgid "Bannoni" -msgstr "" +msgstr "Bannoni" #. name for bcn msgid "Bali (Nigeria)" -msgstr "" +msgstr "Bali (Nigeria)" #. name for bco msgid "Kaluli" -msgstr "" +msgstr "Kaluli" #. name for bcp msgid "Bali (Democratic Republic of Congo)" -msgstr "" +msgstr "Bali (Democratic Republic of Congo)" #. name for bcq msgid "Bench" -msgstr "" +msgstr "Bench" #. name for bcr msgid "Babine" -msgstr "" +msgstr "Babine" #. name for bcs msgid "Kohumono" -msgstr "" +msgstr "Kohumono" #. name for bct msgid "Bendi" -msgstr "" +msgstr "Bendi" #. name for bcu msgid "Awad Bing" -msgstr "" +msgstr "Awad Bing" #. name for bcv msgid "Shoo-Minda-Nye" -msgstr "" +msgstr "Shoo-Minda-Nye" #. name for bcw msgid "Bana" -msgstr "" +msgstr "Bana" #. name for bcy msgid "Bacama" -msgstr "" +msgstr "Bacama" #. name for bcz msgid "Bainouk-Gunyaamolo" -msgstr "" +msgstr "Bainouk-Gunyaamolo" #. name for bda msgid "Bayot" -msgstr "" +msgstr "Bayot" #. name for bdb msgid "Basap" -msgstr "" +msgstr "Basap" #. name for bdc msgid "Emberá-Baudó" -msgstr "" +msgstr "Emberá-Baudó" #. name for bdd msgid "Bunama" -msgstr "" +msgstr "Bunama" #. name for bde msgid "Bade" -msgstr "" +msgstr "Bade" #. name for bdf msgid "Biage" -msgstr "" +msgstr "Biage" #. name for bdg msgid "Bonggi" -msgstr "" +msgstr "Bonggi" #. name for bdh msgid "Baka (Sudan)" -msgstr "" +msgstr "Baka (Sudan)" #. name for bdi msgid "Burun" -msgstr "" +msgstr "Burun" #. name for bdj msgid "Bai" -msgstr "" +msgstr "Bai" #. name for bdk msgid "Budukh" -msgstr "" +msgstr "Budukh" #. name for bdl msgid "Bajau; Indonesian" -msgstr "" +msgstr "Bajau; Indonesian" #. name for bdm msgid "Buduma" -msgstr "" +msgstr "Buduma" #. name for bdn msgid "Baldemu" -msgstr "" +msgstr "Baldemu" #. name for bdo msgid "Morom" -msgstr "" +msgstr "Morom" #. name for bdp msgid "Bende" -msgstr "" +msgstr "Bende" #. name for bdq msgid "Bahnar" -msgstr "" +msgstr "Bahnar" #. name for bdr msgid "Bajau; West Coast" -msgstr "" +msgstr "Bajau; West Coast" #. name for bds msgid "Burunge" -msgstr "" +msgstr "Burunge" #. name for bdt msgid "Bokoto" -msgstr "" +msgstr "Bokoto" #. name for bdu msgid "Oroko" -msgstr "" +msgstr "Oroko" #. name for bdv msgid "Bodo Parja" -msgstr "" +msgstr "Bodo Parja" #. name for bdw msgid "Baham" -msgstr "" +msgstr "Baham" #. name for bdx msgid "Budong-Budong" -msgstr "" +msgstr "Budong-Budong" #. name for bdy msgid "Bandjalang" -msgstr "" +msgstr "Bandjalang" #. name for bdz msgid "Badeshi" -msgstr "" +msgstr "Badeshi" #. name for bea msgid "Beaver" -msgstr "" +msgstr "Beaver" #. name for beb msgid "Bebele" -msgstr "" +msgstr "Bebele" #. name for bec msgid "Iceve-Maci" -msgstr "" +msgstr "Iceve-Maci" #. name for bed msgid "Bedoanas" -msgstr "" +msgstr "Bedoanas" #. name for bee msgid "Byangsi" -msgstr "" +msgstr "Byangsi" #. name for bef msgid "Benabena" -msgstr "" +msgstr "Benabena" #. name for beg msgid "Belait" -msgstr "" +msgstr "Belait" #. name for beh msgid "Biali" -msgstr "" +msgstr "Biali" #. name for bei msgid "Bekati'" -msgstr "" +msgstr "Bekati'" #. name for bej msgid "Beja" -msgstr "" +msgstr "Beja" #. name for bek msgid "Bebeli" -msgstr "" +msgstr "Bebeli" #. name for bel msgid "Belarusian" -msgstr "" +msgstr "Belarusian" #. name for bem msgid "Bemba (Zambia)" -msgstr "" +msgstr "Bemba (Zambia)" #. name for ben msgid "Bengali" -msgstr "" +msgstr "Bengali" #. name for beo msgid "Beami" -msgstr "" +msgstr "Beami" #. name for bep msgid "Besoa" -msgstr "" +msgstr "Besoa" #. name for beq msgid "Beembe" -msgstr "" +msgstr "Beembe" #. name for bes msgid "Besme" -msgstr "" +msgstr "Besme" #. name for bet msgid "Béte; Guiberoua" -msgstr "" +msgstr "Béte; Guiberoua" #. name for beu msgid "Blagar" -msgstr "" +msgstr "Blagar" #. name for bev msgid "Bété; Daloa" -msgstr "" +msgstr "Bété; Daloa" #. name for bew msgid "Betawi" -msgstr "" +msgstr "Betawi" #. name for bex msgid "Jur Modo" -msgstr "" +msgstr "Jur Modo" #. name for bey msgid "Beli (Papua New Guinea)" -msgstr "" +msgstr "Beli (Papua New Guinea)" #. name for bez msgid "Bena (Tanzania)" -msgstr "" +msgstr "Bena (Tanzania)" #. name for bfa msgid "Bari" -msgstr "" +msgstr "Bari" #. name for bfb msgid "Bareli; Pauri" -msgstr "" +msgstr "Bareli; Pauri" #. name for bfc msgid "Bai; Northern" -msgstr "" +msgstr "Bai; Northern" #. name for bfd msgid "Bafut" -msgstr "" +msgstr "Bafut" #. name for bfe msgid "Betaf" -msgstr "" +msgstr "Betaf" #. name for bff msgid "Bofi" -msgstr "" +msgstr "Bofi" #. name for bfg msgid "Kayan; Busang" -msgstr "" +msgstr "Kayan; Busang" #. name for bfh msgid "Blafe" -msgstr "" +msgstr "Blafe" #. name for bfi msgid "British Sign Language" -msgstr "" +msgstr "British Sign Language" #. name for bfj msgid "Bafanji" -msgstr "" +msgstr "Bafanji" #. name for bfk msgid "Ban Khor Sign Language" -msgstr "" +msgstr "Ban Khor Sign Language" #. name for bfl msgid "Banda-Ndélé" -msgstr "" +msgstr "Banda-Ndélé" #. name for bfm msgid "Mmen" -msgstr "" +msgstr "Mmen" #. name for bfn msgid "Bunak" -msgstr "" +msgstr "Bunak" #. name for bfo msgid "Birifor; Malba" -msgstr "" +msgstr "Birifor; Malba" #. name for bfp msgid "Beba" -msgstr "" +msgstr "Beba" #. name for bfq msgid "Badaga" -msgstr "" +msgstr "Badaga" #. name for bfr msgid "Bazigar" -msgstr "" +msgstr "Bazigar" #. name for bfs msgid "Bai; Southern" -msgstr "" +msgstr "Bai; Southern" #. name for bft msgid "Balti" -msgstr "" +msgstr "Balti" #. name for bfu msgid "Gahri" -msgstr "" +msgstr "Gahri" #. name for bfw msgid "Bondo" -msgstr "" +msgstr "Bondo" #. name for bfx msgid "Bantayanon" -msgstr "" +msgstr "Bantayanon" #. name for bfy msgid "Bagheli" -msgstr "" +msgstr "Bagheli" #. name for bfz msgid "Pahari; Mahasu" -msgstr "" +msgstr "Pahari; Mahasu" #. name for bga msgid "Gwamhi-Wuri" -msgstr "" +msgstr "Gwamhi-Wuri" #. name for bgb msgid "Bobongko" -msgstr "" +msgstr "Bobongko" #. name for bgc msgid "Haryanvi" -msgstr "" +msgstr "Haryanvi" #. name for bgd msgid "Bareli; Rathwi" -msgstr "" +msgstr "Bareli; Rathwi" #. name for bge msgid "Bauria" -msgstr "" +msgstr "Bauria" #. name for bgf msgid "Bangandu" -msgstr "" +msgstr "Bangandu" #. name for bgg msgid "Bugun" -msgstr "" +msgstr "Bugun" #. name for bgi msgid "Giangan" -msgstr "" +msgstr "Giangan" #. name for bgj msgid "Bangolan" -msgstr "" +msgstr "Bangolan" #. name for bgk msgid "Bit" -msgstr "" +msgstr "Bit" #. name for bgl msgid "Bo (Laos)" -msgstr "" +msgstr "Bo (Laos)" #. name for bgm msgid "Baga Mboteni" -msgstr "" +msgstr "Baga Mboteni" #. name for bgn msgid "Balochi; Western" -msgstr "" +msgstr "Balochi; Western" #. name for bgo msgid "Baga Koga" -msgstr "" +msgstr "Baga Koga" #. name for bgp msgid "Balochi; Eastern" -msgstr "" +msgstr "Balochi; Eastern" #. name for bgq msgid "Bagri" -msgstr "" +msgstr "Bagri" #. name for bgr msgid "Chin; Bawm" -msgstr "" +msgstr "Chin; Bawm" #. name for bgs msgid "Tagabawa" -msgstr "" +msgstr "Tagabawa" #. name for bgt msgid "Bughotu" -msgstr "" +msgstr "Bughotu" #. name for bgu msgid "Mbongno" -msgstr "" +msgstr "Mbongno" #. name for bgv msgid "Warkay-Bipim" -msgstr "" +msgstr "Warkay-Bipim" #. name for bgw msgid "Bhatri" -msgstr "" +msgstr "Bhatri" #. name for bgx msgid "Turkish; Balkan Gagauz" -msgstr "" +msgstr "Turkish; Balkan Gagauz" #. name for bgy msgid "Benggoi" -msgstr "" +msgstr "Benggoi" #. name for bgz msgid "Banggai" -msgstr "" +msgstr "Banggai" #. name for bha msgid "Bharia" -msgstr "" +msgstr "Bharia" #. name for bhb msgid "Bhili" -msgstr "" +msgstr "Bhili" #. name for bhc msgid "Biga" -msgstr "" +msgstr "Biga" #. name for bhd msgid "Bhadrawahi" -msgstr "" +msgstr "Bhadrawahi" #. name for bhe msgid "Bhaya" -msgstr "" +msgstr "Bhaya" #. name for bhf msgid "Odiai" -msgstr "" +msgstr "Odiai" #. name for bhg msgid "Binandere" -msgstr "" +msgstr "Binandere" #. name for bhh msgid "Bukharic" -msgstr "" +msgstr "Bukharic" #. name for bhi msgid "Bhilali" -msgstr "" +msgstr "Bhilali" #. name for bhj msgid "Bahing" -msgstr "" +msgstr "Bahing" #. name for bhl msgid "Bimin" -msgstr "" +msgstr "Bimin" #. name for bhm msgid "Bathari" -msgstr "" +msgstr "Bathari" #. name for bhn msgid "Neo-Aramaic; Bohtan" -msgstr "" +msgstr "Neo-Aramaic; Bohtan" #. name for bho msgid "Bhojpuri" -msgstr "" +msgstr "Bhojpuri" #. name for bhp msgid "Bima" -msgstr "" +msgstr "Bima" #. name for bhq msgid "Tukang Besi South" -msgstr "" +msgstr "Tukang Besi South" #. name for bhr msgid "Malagasy; Bara" -msgstr "" +msgstr "Malagasy; Bara" #. name for bhs msgid "Buwal" -msgstr "" +msgstr "Buwal" #. name for bht msgid "Bhattiyali" -msgstr "" +msgstr "Bhattiyali" #. name for bhu msgid "Bhunjia" -msgstr "" +msgstr "Bhunjia" #. name for bhv msgid "Bahau" -msgstr "" +msgstr "Bahau" #. name for bhw msgid "Biak" -msgstr "" +msgstr "Biak" #. name for bhx msgid "Bhalay" -msgstr "" +msgstr "Bhalay" #. name for bhy msgid "Bhele" -msgstr "" +msgstr "Bhele" #. name for bhz msgid "Bada (Indonesia)" -msgstr "" +msgstr "Bada (Indonesia)" #. name for bia msgid "Badimaya" -msgstr "" +msgstr "Badimaya" #. name for bib msgid "Bissa" -msgstr "" +msgstr "Bissa" #. name for bic msgid "Bikaru" -msgstr "" +msgstr "Bikaru" #. name for bid msgid "Bidiyo" -msgstr "" +msgstr "Bidiyo" #. name for bie msgid "Bepour" -msgstr "" +msgstr "Bepour" #. name for bif msgid "Biafada" -msgstr "" +msgstr "Biafada" #. name for big msgid "Biangai" -msgstr "" +msgstr "Biangai" #. name for bij msgid "Vaghat-Ya-Bijim-Legeri" -msgstr "" +msgstr "Vaghat-Ya-Bijim-Legeri" #. name for bik msgid "Bikol" -msgstr "" +msgstr "Bikol" #. name for bil msgid "Bile" -msgstr "" +msgstr "Bile" #. name for bim msgid "Bimoba" -msgstr "" +msgstr "Bimoba" #. name for bin msgid "Bini" -msgstr "" +msgstr "Bini" #. name for bio msgid "Nai" -msgstr "" +msgstr "Nai" #. name for bip msgid "Bila" -msgstr "" +msgstr "Bila" #. name for biq msgid "Bipi" -msgstr "" +msgstr "Bipi" #. name for bir msgid "Bisorio" -msgstr "" +msgstr "Bisorio" #. name for bis msgid "Bislama" -msgstr "" +msgstr "Bislama" #. name for bit msgid "Berinomo" -msgstr "" +msgstr "Berinomo" #. name for biu msgid "Biete" -msgstr "" +msgstr "Biete" #. name for biv msgid "Birifor; Southern" -msgstr "" +msgstr "Birifor; Southern" #. name for biw msgid "Kol (Cameroon)" -msgstr "" +msgstr "Kol (Cameroon)" #. name for bix msgid "Bijori" -msgstr "" +msgstr "Bijori" #. name for biy msgid "Birhor" -msgstr "" +msgstr "Birhor" #. name for biz msgid "Baloi" -msgstr "" +msgstr "Baloi" #. name for bja msgid "Budza" -msgstr "" +msgstr "Budza" #. name for bjb msgid "Banggarla" -msgstr "" +msgstr "Banggarla" #. name for bjc msgid "Bariji" -msgstr "" +msgstr "Bariji" #. name for bjd msgid "Bandjigali" -msgstr "" +msgstr "Bandjigali" #. name for bje msgid "Mien; Biao-Jiao" -msgstr "" +msgstr "Mien; Biao-Jiao" #. name for bjf msgid "Neo-Aramaic; Barzani Jewish" -msgstr "" +msgstr "Neo-Aramaic; Barzani Jewish" #. name for bjg msgid "Bidyogo" -msgstr "" +msgstr "Bidyogo" #. name for bjh msgid "Bahinemo" -msgstr "" +msgstr "Bahinemo" #. name for bji msgid "Burji" -msgstr "" +msgstr "Burji" #. name for bjj msgid "Kanauji" -msgstr "" +msgstr "Kanauji" #. name for bjk msgid "Barok" -msgstr "" +msgstr "Barok" #. name for bjl msgid "Bulu (Papua New Guinea)" -msgstr "" +msgstr "Bulu (Papua New Guinea)" #. name for bjm msgid "Bajelani" -msgstr "" +msgstr "Bajelani" #. name for bjn msgid "Banjar" -msgstr "" +msgstr "Banjar" #. name for bjo msgid "Banda; Mid-Southern" -msgstr "" +msgstr "Banda; Mid-Southern" #. name for bjr msgid "Binumarien" -msgstr "" +msgstr "Binumarien" #. name for bjs msgid "Bajan" -msgstr "" +msgstr "Bajan" #. name for bjt msgid "Balanta-Ganja" -msgstr "" +msgstr "Balanta-Ganja" #. name for bju msgid "Busuu" -msgstr "" +msgstr "Busuu" #. name for bjv msgid "Bedjond" -msgstr "" +msgstr "Bedjond" #. name for bjw msgid "Bakwé" -msgstr "" +msgstr "Bakwé" #. name for bjx msgid "Itneg; Banao" -msgstr "" +msgstr "Itneg; Banao" #. name for bjy msgid "Bayali" -msgstr "" +msgstr "Bayali" #. name for bjz msgid "Baruga" -msgstr "" +msgstr "Baruga" #. name for bka msgid "Kyak" -msgstr "" +msgstr "Kyak" #. name for bkc msgid "Baka (Cameroon)" -msgstr "" +msgstr "Baka (Cameroon)" #. name for bkd msgid "Binukid" -msgstr "" +msgstr "Binukid" #. name for bkf msgid "Beeke" -msgstr "" +msgstr "Beeke" #. name for bkg msgid "Buraka" -msgstr "" +msgstr "Buraka" #. name for bkh msgid "Bakoko" -msgstr "" +msgstr "Bakoko" #. name for bki msgid "Baki" -msgstr "" +msgstr "Baki" #. name for bkj msgid "Pande" -msgstr "" +msgstr "Pande" #. name for bkk msgid "Brokskat" -msgstr "" +msgstr "Brokskat" #. name for bkl msgid "Berik" -msgstr "" +msgstr "Berik" #. name for bkm msgid "Kom (Cameroon)" -msgstr "" +msgstr "Kom (Cameroon)" #. name for bkn msgid "Bukitan" -msgstr "" +msgstr "Bukitan" #. name for bko msgid "Kwa'" -msgstr "" +msgstr "Kwa'" #. name for bkp msgid "Boko (Democratic Republic of Congo)" -msgstr "" +msgstr "Boko (Democratic Republic of Congo)" #. name for bkq msgid "Bakairí" -msgstr "" +msgstr "Bakairí" #. name for bkr msgid "Bakumpai" -msgstr "" +msgstr "Bakumpai" #. name for bks msgid "Sorsoganon; Northern" -msgstr "" +msgstr "Sorsoganon; Northern" #. name for bkt msgid "Boloki" -msgstr "" +msgstr "Boloki" #. name for bku msgid "Buhid" -msgstr "" +msgstr "Buhid" #. name for bkv msgid "Bekwarra" -msgstr "" +msgstr "Bekwarra" #. name for bkw msgid "Bekwel" -msgstr "" +msgstr "Bekwel" #. name for bkx msgid "Baikeno" -msgstr "" +msgstr "Baikeno" #. name for bky msgid "Bokyi" -msgstr "" +msgstr "Bokyi" #. name for bkz msgid "Bungku" -msgstr "" +msgstr "Bungku" #. name for bla msgid "Siksika" -msgstr "" +msgstr "Siksika" #. name for blb msgid "Bilua" -msgstr "" +msgstr "Bilua" #. name for blc msgid "Bella Coola" -msgstr "" +msgstr "Bella Coola" #. name for bld msgid "Bolango" -msgstr "" +msgstr "Bolango" #. name for ble msgid "Balanta-Kentohe" -msgstr "" +msgstr "Balanta-Kentohe" #. name for blf msgid "Buol" -msgstr "" +msgstr "Buol" #. name for blg msgid "Balau" -msgstr "" +msgstr "Balau" #. name for blh msgid "Kuwaa" -msgstr "" +msgstr "Kuwaa" #. name for bli msgid "Bolia" -msgstr "" +msgstr "Bolia" #. name for blj msgid "Bolongan" -msgstr "" +msgstr "Bolongan" #. name for blk msgid "Karen; Pa'o" -msgstr "" +msgstr "Karen; Pa'o" #. name for bll msgid "Biloxi" -msgstr "" +msgstr "Biloxi" #. name for blm msgid "Beli (Sudan)" -msgstr "" +msgstr "Beli (Sudan)" #. name for bln msgid "Bicolano; Southern Catanduanes" -msgstr "" +msgstr "Bicolano; Southern Catanduanes" #. name for blo msgid "Anii" -msgstr "" +msgstr "Anii" #. name for blp msgid "Blablanga" -msgstr "" +msgstr "Blablanga" #. name for blq msgid "Baluan-Pam" -msgstr "" +msgstr "Baluan-Pam" #. name for blr msgid "Blang" -msgstr "" +msgstr "Blang" #. name for bls msgid "Balaesang" -msgstr "" +msgstr "Balaesang" #. name for blt msgid "Tai Dam" -msgstr "" +msgstr "Tai Dam" #. name for blv msgid "Bolo" -msgstr "" +msgstr "Bolo" #. name for blw msgid "Balangao" -msgstr "" +msgstr "Balangao" #. name for blx msgid "Ayta; Mag-Indi" -msgstr "" +msgstr "Ayta; Mag-Indi" #. name for bly msgid "Notre" -msgstr "" +msgstr "Notre" #. name for blz msgid "Balantak" -msgstr "" +msgstr "Balantak" #. name for bma msgid "Lame" -msgstr "" +msgstr "Lame" #. name for bmb msgid "Bembe" -msgstr "" +msgstr "Bembe" #. name for bmc msgid "Biem" -msgstr "" +msgstr "Biem" #. name for bmd msgid "Manduri; Baga" -msgstr "" +msgstr "Manduri; Baga" #. name for bme msgid "Limassa" -msgstr "" +msgstr "Limassa" #. name for bmf msgid "Bom" -msgstr "" +msgstr "Bom" #. name for bmg msgid "Bamwe" -msgstr "" +msgstr "Bamwe" #. name for bmh msgid "Kein" -msgstr "" +msgstr "Kein" #. name for bmi msgid "Bagirmi" -msgstr "" +msgstr "Bagirmi" #. name for bmj msgid "Bote-Majhi" -msgstr "" +msgstr "Bote-Majhi" #. name for bmk msgid "Ghayavi" -msgstr "" +msgstr "Ghayavi" #. name for bml msgid "Bomboli" -msgstr "" +msgstr "Bomboli" #. name for bmm msgid "Malagasy; Northern Betsimisaraka" -msgstr "" +msgstr "Malagasy; Northern Betsimisaraka" #. name for bmn msgid "Bina (Papua New Guinea)" -msgstr "" +msgstr "Bina (Papua New Guinea)" #. name for bmo msgid "Bambalang" -msgstr "" +msgstr "Bambalang" #. name for bmp msgid "Bulgebi" -msgstr "" +msgstr "Bulgebi" #. name for bmq msgid "Bomu" -msgstr "" +msgstr "Bomu" #. name for bmr msgid "Muinane" -msgstr "" +msgstr "Muinane" #. name for bms msgid "Kanuri; Bilma" -msgstr "" +msgstr "Kanuri; Bilma" #. name for bmt msgid "Biao Mon" -msgstr "" +msgstr "Biao Mon" #. name for bmu msgid "Somba-Siawari" -msgstr "" +msgstr "Somba-Siawari" #. name for bmv msgid "Bum" -msgstr "" +msgstr "Bum" #. name for bmw msgid "Bomwali" -msgstr "" +msgstr "Bomwali" #. name for bmx msgid "Baimak" -msgstr "" +msgstr "Baimak" #. name for bmy msgid "Bemba (Democratic Republic of Congo)" -msgstr "" +msgstr "Bemba (Democratic Republic of Congo)" #. name for bmz msgid "Baramu" -msgstr "" +msgstr "Baramu" #. name for bna msgid "Bonerate" -msgstr "" +msgstr "Bonerate" #. name for bnb msgid "Bookan" -msgstr "" +msgstr "Bookan" #. name for bnc msgid "Bontok" -msgstr "" +msgstr "Bontok" #. name for bnd msgid "Banda (Indonesia)" -msgstr "" +msgstr "Banda (Indonesia)" #. name for bne msgid "Bintauna" -msgstr "" +msgstr "Bintauna" #. name for bnf msgid "Masiwang" -msgstr "" +msgstr "Masiwang" #. name for bng msgid "Benga" -msgstr "" +msgstr "Benga" #. name for bni msgid "Bangi" -msgstr "" +msgstr "Bangi" #. name for bnj msgid "Tawbuid; Eastern" -msgstr "" +msgstr "Tawbuid; Eastern" #. name for bnk msgid "Bierebo" -msgstr "" +msgstr "Bierebo" #. name for bnl msgid "Boon" -msgstr "" +msgstr "Boon" #. name for bnm msgid "Batanga" -msgstr "" +msgstr "Batanga" #. name for bnn msgid "Bunun" -msgstr "" +msgstr "Bunun" #. name for bno msgid "Bantoanon" -msgstr "" +msgstr "Bantoanon" #. name for bnp msgid "Bola" -msgstr "" +msgstr "Bola" #. name for bnq msgid "Bantik" -msgstr "" +msgstr "Bantik" #. name for bnr msgid "Butmas-Tur" -msgstr "" +msgstr "Butmas-Tur" #. name for bns msgid "Bundeli" -msgstr "" +msgstr "Bundeli" #. name for bnu msgid "Bentong" -msgstr "" +msgstr "Bentong" #. name for bnv msgid "Bonerif" -msgstr "" +msgstr "Bonerif" #. name for bnw msgid "Bisis" -msgstr "" +msgstr "Bisis" #. name for bnx msgid "Bangubangu" -msgstr "" +msgstr "Bangubangu" #. name for bny msgid "Bintulu" -msgstr "" +msgstr "Bintulu" #. name for bnz msgid "Beezen" -msgstr "" +msgstr "Beezen" #. name for boa msgid "Bora" -msgstr "" +msgstr "Bora" #. name for bob msgid "Aweer" -msgstr "" +msgstr "Aweer" #. name for bod msgid "Tibetan" -msgstr "" +msgstr "Tibetan" #. name for boe msgid "Mundabli" -msgstr "" +msgstr "Mundabli" #. name for bof msgid "Bolon" -msgstr "" +msgstr "Bolon" #. name for bog msgid "Bamako Sign Language" -msgstr "" +msgstr "Bamako Sign Language" #. name for boh msgid "Boma" -msgstr "" +msgstr "Boma" #. name for boi msgid "Barbareño" -msgstr "" +msgstr "Barbareño" #. name for boj msgid "Anjam" -msgstr "" +msgstr "Anjam" #. name for bok msgid "Bonjo" -msgstr "" +msgstr "Bonjo" #. name for bol msgid "Bole" -msgstr "" +msgstr "Bole" #. name for bom msgid "Berom" -msgstr "" +msgstr "Berom" #. name for bon msgid "Bine" -msgstr "" +msgstr "Bine" #. name for boo msgid "Bozo; Tiemacèwè" -msgstr "" +msgstr "Bozo; Tiemacèwè" #. name for bop msgid "Bonkiman" -msgstr "" +msgstr "Bonkiman" #. name for boq msgid "Bogaya" -msgstr "" +msgstr "Bogaya" #. name for bor msgid "Borôro" -msgstr "" +msgstr "Borôro" #. name for bos msgid "Bosnian" -msgstr "" +msgstr "Bosnian" #. name for bot msgid "Bongo" -msgstr "" +msgstr "Bongo" #. name for bou msgid "Bondei" -msgstr "" +msgstr "Bondei" #. name for bov msgid "Tuwuli" -msgstr "" +msgstr "Tuwuli" #. name for bow msgid "Rema" -msgstr "" +msgstr "Rema" #. name for box msgid "Buamu" -msgstr "" +msgstr "Buamu" #. name for boy msgid "Bodo (Central African Republic)" -msgstr "" +msgstr "Bodo (Central African Republic)" #. name for boz msgid "Bozo; Tiéyaxo" -msgstr "" +msgstr "Bozo; Tiéyaxo" #. name for bpa msgid "Dakaka" -msgstr "" +msgstr "Dakaka" #. name for bpb msgid "Barbacoas" -msgstr "" +msgstr "Barbacoas" #. name for bpd msgid "Banda-Banda" -msgstr "" +msgstr "Banda-Banda" #. name for bpg msgid "Bonggo" -msgstr "" +msgstr "Bonggo" #. name for bph msgid "Botlikh" -msgstr "" +msgstr "Botlikh" #. name for bpi msgid "Bagupi" -msgstr "" +msgstr "Bagupi" #. name for bpj msgid "Binji" -msgstr "" +msgstr "Binji" #. name for bpk msgid "Orowe" -msgstr "" +msgstr "Orowe" #. name for bpl msgid "Broome Pearling Lugger Pidgin" -msgstr "" +msgstr "Broome Pearling Lugger Pidgin" #. name for bpm msgid "Biyom" -msgstr "" +msgstr "Biyom" #. name for bpn msgid "Dzao Min" -msgstr "" +msgstr "Dzao Min" #. name for bpo msgid "Anasi" -msgstr "" +msgstr "Anasi" #. name for bpp msgid "Kaure" -msgstr "" +msgstr "Kaure" #. name for bpq msgid "Malay; Banda" -msgstr "" +msgstr "Malay; Banda" #. name for bpr msgid "Blaan; Koronadal" -msgstr "" +msgstr "Blaan; Koronadal" #. name for bps msgid "Blaan; Sarangani" -msgstr "" +msgstr "Blaan; Sarangani" #. name for bpt msgid "Barrow Point" -msgstr "" +msgstr "Barrow Point" #. name for bpu msgid "Bongu" -msgstr "" +msgstr "Bongu" #. name for bpv msgid "Marind; Bian" -msgstr "" +msgstr "Marind; Bian" #. name for bpw msgid "Bo (Papua New Guinea)" -msgstr "" +msgstr "Bo (Papua New Guinea)" #. name for bpx msgid "Bareli; Palya" -msgstr "" +msgstr "Bareli; Palya" #. name for bpy msgid "Bishnupriya" -msgstr "" +msgstr "Bishnupriya" #. name for bpz msgid "Bilba" -msgstr "" +msgstr "Bilba" #. name for bqa msgid "Tchumbuli" -msgstr "" +msgstr "Tchumbuli" #. name for bqb msgid "Bagusa" -msgstr "" +msgstr "Bagusa" #. name for bqc msgid "Boko (Benin)" -msgstr "" +msgstr "Boko (Benin)" #. name for bqd msgid "Bung" -msgstr "" +msgstr "Bung" #. name for bqf msgid "Baga Kaloum" -msgstr "" +msgstr "Baga Kaloum" #. name for bqg msgid "Bago-Kusuntu" -msgstr "" +msgstr "Bago-Kusuntu" #. name for bqh msgid "Baima" -msgstr "" +msgstr "Baima" #. name for bqi msgid "Bakhtiari" -msgstr "" +msgstr "Bakhtiari" #. name for bqj msgid "Bandial" -msgstr "" +msgstr "Bandial" #. name for bqk msgid "Banda-Mbrès" -msgstr "" +msgstr "Banda-Mbrès" #. name for bql msgid "Bilakura" -msgstr "" +msgstr "Bilakura" #. name for bqm msgid "Wumboko" -msgstr "" +msgstr "Wumboko" #. name for bqn msgid "Bulgarian Sign Language" -msgstr "" +msgstr "Bulgarian Sign Language" #. name for bqo msgid "Balo" -msgstr "" +msgstr "Balo" #. name for bqp msgid "Busa" -msgstr "" +msgstr "Busa" #. name for bqq msgid "Biritai" -msgstr "" +msgstr "Biritai" #. name for bqr msgid "Burusu" -msgstr "" +msgstr "Burusu" #. name for bqs msgid "Bosngun" -msgstr "" +msgstr "Bosngun" #. name for bqt msgid "Bamukumbit" -msgstr "" +msgstr "Bamukumbit" #. name for bqu msgid "Boguru" -msgstr "" +msgstr "Boguru" #. name for bqv msgid "Begbere-Ejar" -msgstr "" +msgstr "Begbere-Ejar" #. name for bqw msgid "Buru (Nigeria)" -msgstr "" +msgstr "Buru (Nigeria)" #. name for bqx msgid "Baangi" -msgstr "" +msgstr "Baangi" #. name for bqy msgid "Bengkala Sign Language" -msgstr "" +msgstr "Bengkala Sign Language" #. name for bqz msgid "Bakaka" -msgstr "" +msgstr "Bakaka" #. name for bra msgid "Braj" -msgstr "" +msgstr "Braj" #. name for brb msgid "Lave" -msgstr "" +msgstr "Lave" #. name for brc msgid "Creole Dutch; Berbice" -msgstr "" +msgstr "Creole Dutch; Berbice" #. name for brd msgid "Baraamu" -msgstr "" +msgstr "Baraamu" #. name for bre msgid "Breton" -msgstr "" +msgstr "Breton" #. name for brf msgid "Bera" -msgstr "" +msgstr "Bera" #. name for brg msgid "Baure" -msgstr "" +msgstr "Baure" #. name for brh msgid "Brahui" -msgstr "" +msgstr "Brahui" #. name for bri msgid "Mokpwe" -msgstr "" +msgstr "Mokpwe" #. name for brj msgid "Bieria" -msgstr "" +msgstr "Bieria" #. name for brk msgid "Birked" -msgstr "" +msgstr "Birked" #. name for brl msgid "Birwa" -msgstr "" +msgstr "Birwa" #. name for brm msgid "Barambu" -msgstr "" +msgstr "Barambu" #. name for brn msgid "Boruca" -msgstr "" +msgstr "Boruca" #. name for bro msgid "Brokkat" -msgstr "" +msgstr "Brokkat" #. name for brp msgid "Barapasi" -msgstr "" +msgstr "Barapasi" #. name for brq msgid "Breri" -msgstr "" +msgstr "Breri" #. name for brr msgid "Birao" -msgstr "" +msgstr "Birao" #. name for brs msgid "Baras" -msgstr "" +msgstr "Baras" #. name for brt msgid "Bitare" -msgstr "" +msgstr "Bitare" #. name for bru msgid "Bru; Eastern" -msgstr "" +msgstr "Bru; Eastern" #. name for brv msgid "Bru; Western" -msgstr "" +msgstr "Bru; Western" #. name for brw msgid "Bellari" -msgstr "" +msgstr "Bellari" #. name for brx msgid "Bodo (India)" -msgstr "" +msgstr "Bodo (India)" #. name for bry msgid "Burui" -msgstr "" +msgstr "Burui" #. name for brz msgid "Bilbil" -msgstr "" +msgstr "Bilbil" #. name for bsa msgid "Abinomn" -msgstr "" +msgstr "Abinomn" #. name for bsb msgid "Bisaya; Brunei" -msgstr "" +msgstr "Bisaya; Brunei" #. name for bsc msgid "Bassari" -msgstr "" +msgstr "Bassari" #. name for bse msgid "Wushi" -msgstr "" +msgstr "Wushi" #. name for bsf msgid "Bauchi" -msgstr "" +msgstr "Bauchi" #. name for bsg msgid "Bashkardi" -msgstr "" +msgstr "Bashkardi" #. name for bsh msgid "Kati" -msgstr "" +msgstr "Kati" #. name for bsi msgid "Bassossi" -msgstr "" +msgstr "Bassossi" #. name for bsj msgid "Bangwinji" -msgstr "" +msgstr "Bangwinji" #. name for bsk msgid "Burushaski" -msgstr "" +msgstr "Burushaski" #. name for bsl msgid "Basa-Gumna" -msgstr "" +msgstr "Basa-Gumna" #. name for bsm msgid "Busami" -msgstr "" +msgstr "Busami" #. name for bsn msgid "Barasana-Eduria" -msgstr "" +msgstr "Barasana-Eduria" #. name for bso msgid "Buso" -msgstr "" +msgstr "Buso" #. name for bsp msgid "Baga Sitemu" -msgstr "" +msgstr "Baga Sitemu" #. name for bsq msgid "Bassa" -msgstr "" +msgstr "Bassa" #. name for bsr msgid "Bassa-Kontagora" -msgstr "" +msgstr "Bassa-Kontagora" #. name for bss msgid "Akoose" -msgstr "" +msgstr "Akoose" #. name for bst msgid "Basketo" -msgstr "" +msgstr "Basketo" #. name for bsu msgid "Bahonsuai" -msgstr "" +msgstr "Bahonsuai" #. name for bsv msgid "Baga Sobané" -msgstr "" +msgstr "Baga Sobané" #. name for bsw msgid "Baiso" -msgstr "" +msgstr "Baiso" #. name for bsx msgid "Yangkam" -msgstr "" +msgstr "Yangkam" #. name for bsy msgid "Bisaya; Sabah" -msgstr "" +msgstr "Bisaya; Sabah" #. name for bta msgid "Bata" -msgstr "" +msgstr "Bata" #. name for btc msgid "Bati (Cameroon)" -msgstr "" +msgstr "Bati (Cameroon)" #. name for btd msgid "Batak Dairi" -msgstr "" +msgstr "Batak Dairi" #. name for bte msgid "Gamo-Ningi" -msgstr "" +msgstr "Gamo-Ningi" #. name for btf msgid "Birgit" -msgstr "" +msgstr "Birgit" #. name for btg msgid "Bété; Gagnoa" -msgstr "" +msgstr "Bété; Gagnoa" #. name for bth msgid "Bidayuh; Biatah" -msgstr "" +msgstr "Bidayuh; Biatah" #. name for bti msgid "Burate" -msgstr "" +msgstr "Burate" #. name for btj msgid "Malay; Bacanese" -msgstr "" +msgstr "Malay; Bacanese" #. name for btl msgid "Bhatola" -msgstr "" +msgstr "Bhatola" #. name for btm msgid "Batak Mandailing" -msgstr "" +msgstr "Batak Mandailing" #. name for btn msgid "Ratagnon" -msgstr "" +msgstr "Ratagnon" #. name for bto msgid "Bikol; Rinconada" -msgstr "" +msgstr "Bikol; Rinconada" #. name for btp msgid "Budibud" -msgstr "" +msgstr "Budibud" #. name for btq msgid "Batek" -msgstr "" +msgstr "Batek" #. name for btr msgid "Baetora" -msgstr "" +msgstr "Baetora" #. name for bts msgid "Batak Simalungun" -msgstr "" +msgstr "Batak Simalungun" #. name for btt msgid "Bete-Bendi" -msgstr "" +msgstr "Bete-Bendi" #. name for btu msgid "Batu" -msgstr "" +msgstr "Batu" #. name for btv msgid "Bateri" -msgstr "" +msgstr "Bateri" #. name for btw msgid "Butuanon" -msgstr "" +msgstr "Butuanon" #. name for btx msgid "Batak Karo" -msgstr "" +msgstr "Batak Karo" #. name for bty msgid "Bobot" -msgstr "" +msgstr "Bobot" #. name for btz msgid "Batak Alas-Kluet" -msgstr "" +msgstr "Batak Alas-Kluet" #. name for bua msgid "Buriat" -msgstr "" +msgstr "Buriat" #. name for bub msgid "Bua" -msgstr "" +msgstr "Bua" #. name for buc msgid "Bushi" -msgstr "" +msgstr "Bushi" #. name for bud msgid "Ntcham" -msgstr "" +msgstr "Ntcham" #. name for bue msgid "Beothuk" -msgstr "" +msgstr "Beothuk" #. name for buf msgid "Bushoong" -msgstr "" +msgstr "Bushoong" #. name for bug msgid "Buginese" -msgstr "" +msgstr "Buginese" #. name for buh msgid "Bunu; Younuo" -msgstr "" +msgstr "Bunu; Younuo" #. name for bui msgid "Bongili" -msgstr "" +msgstr "Bongili" #. name for buj msgid "Basa-Gurmana" -msgstr "" +msgstr "Basa-Gurmana" #. name for buk msgid "Bugawac" -msgstr "" +msgstr "Bugawac" #. name for bul msgid "Bulgarian" -msgstr "" +msgstr "Bulgarian" #. name for bum msgid "Bulu (Cameroon)" -msgstr "" +msgstr "Bulu (Cameroon)" #. name for bun msgid "Sherbro" -msgstr "" +msgstr "Sherbro" #. name for buo msgid "Terei" -msgstr "" +msgstr "Terei" #. name for bup msgid "Busoa" -msgstr "" +msgstr "Busoa" #. name for buq msgid "Brem" -msgstr "" +msgstr "Brem" #. name for bus msgid "Bokobaru" -msgstr "" +msgstr "Bokobaru" #. name for but msgid "Bungain" -msgstr "" +msgstr "Bungain" #. name for buu msgid "Budu" -msgstr "" +msgstr "Budu" #. name for buv msgid "Bun" -msgstr "" +msgstr "Bun" #. name for buw msgid "Bubi" -msgstr "" +msgstr "Bubi" #. name for bux msgid "Boghom" -msgstr "" +msgstr "Boghom" #. name for buy msgid "Bullom So" -msgstr "" +msgstr "Bullom So" #. name for buz msgid "Bukwen" -msgstr "" +msgstr "Bukwen" #. name for bva msgid "Barein" -msgstr "" +msgstr "Barein" #. name for bvb msgid "Bube" -msgstr "" +msgstr "Bube" #. name for bvc msgid "Baelelea" -msgstr "" +msgstr "Baelelea" #. name for bvd msgid "Baeggu" -msgstr "" +msgstr "Baeggu" #. name for bve msgid "Malay; Berau" -msgstr "" +msgstr "Malay; Berau" #. name for bvf msgid "Boor" -msgstr "" +msgstr "Boor" #. name for bvg msgid "Bonkeng" -msgstr "" +msgstr "Bonkeng" #. name for bvh msgid "Bure" -msgstr "" +msgstr "Bure" #. name for bvi msgid "Belanda Viri" -msgstr "" +msgstr "Belanda Viri" #. name for bvj msgid "Baan" -msgstr "" +msgstr "Baan" #. name for bvk msgid "Bukat" -msgstr "" +msgstr "Bukat" #. name for bvl msgid "Bolivian Sign Language" -msgstr "" +msgstr "Bolivian Sign Language" #. name for bvm msgid "Bamunka" -msgstr "" +msgstr "Bamunka" #. name for bvn msgid "Buna" -msgstr "" +msgstr "Buna" #. name for bvo msgid "Bolgo" -msgstr "" +msgstr "Bolgo" #. name for bvq msgid "Birri" -msgstr "" +msgstr "Birri" #. name for bvr msgid "Burarra" -msgstr "" +msgstr "Burarra" #. name for bvt msgid "Bati (Indonesia)" -msgstr "" +msgstr "Bati (Indonesia)" #. name for bvu msgid "Malay; Bukit" -msgstr "" +msgstr "Malay; Bukit" #. name for bvv msgid "Baniva" -msgstr "" +msgstr "Baniva" #. name for bvw msgid "Boga" -msgstr "" +msgstr "Boga" #. name for bvx msgid "Dibole" -msgstr "" +msgstr "Dibole" #. name for bvy msgid "Baybayanon" -msgstr "" +msgstr "Baybayanon" #. name for bvz msgid "Bauzi" -msgstr "" +msgstr "Bauzi" #. name for bwa msgid "Bwatoo" -msgstr "" +msgstr "Bwatoo" #. name for bwb msgid "Namosi-Naitasiri-Serua" -msgstr "" +msgstr "Namosi-Naitasiri-Serua" #. name for bwc msgid "Bwile" -msgstr "" +msgstr "Bwile" #. name for bwd msgid "Bwaidoka" -msgstr "" +msgstr "Bwaidoka" #. name for bwe msgid "Karen; Bwe" -msgstr "" +msgstr "Karen; Bwe" #. name for bwf msgid "Boselewa" -msgstr "" +msgstr "Boselewa" #. name for bwg msgid "Barwe" -msgstr "" +msgstr "Barwe" #. name for bwh msgid "Bishuo" -msgstr "" +msgstr "Bishuo" #. name for bwi msgid "Baniwa" -msgstr "" +msgstr "Baniwa" #. name for bwj msgid "Bwamu; Láá Láá" -msgstr "" +msgstr "Bwamu; Láá Láá" #. name for bwk msgid "Bauwaki" -msgstr "" +msgstr "Bauwaki" #. name for bwl msgid "Bwela" -msgstr "" +msgstr "Bwela" #. name for bwm msgid "Biwat" -msgstr "" +msgstr "Biwat" #. name for bwn msgid "Bunu; Wunai" -msgstr "" +msgstr "Bunu; Wunai" #. name for bwo msgid "Boro (Ethiopia)" -msgstr "" +msgstr "Boro (Ethiopia)" #. name for bwp msgid "Mandobo Bawah" -msgstr "" +msgstr "Mandobo Bawah" #. name for bwq msgid "Bobo Madaré; Southern" -msgstr "" +msgstr "Bobo Madaré; Southern" #. name for bwr msgid "Bura-Pabir" -msgstr "" +msgstr "Bura-Pabir" #. name for bws msgid "Bomboma" -msgstr "" +msgstr "Bomboma" #. name for bwt msgid "Bafaw-Balong" -msgstr "" +msgstr "Bafaw-Balong" #. name for bwu msgid "Buli (Ghana)" -msgstr "" +msgstr "Buli (Ghana)" #. name for bww msgid "Bwa" -msgstr "" +msgstr "Bwa" #. name for bwx msgid "Bunu; Bu-Nao" -msgstr "" +msgstr "Bunu; Bu-Nao" #. name for bwy msgid "Bwamu; Cwi" -msgstr "" +msgstr "Bwamu; Cwi" #. name for bwz msgid "Bwisi" -msgstr "" +msgstr "Bwisi" #. name for bxa msgid "Bauro" -msgstr "" +msgstr "Bauro" #. name for bxb msgid "Bor; Belanda" -msgstr "" +msgstr "Bor; Belanda" #. name for bxc msgid "Molengue" -msgstr "" +msgstr "Molengue" #. name for bxd msgid "Pela" -msgstr "" +msgstr "Pela" #. name for bxe msgid "Birale" -msgstr "" +msgstr "Birale" #. name for bxf msgid "Bilur" -msgstr "" +msgstr "Bilur" #. name for bxg msgid "Bangala" -msgstr "" +msgstr "Bangala" #. name for bxh msgid "Buhutu" -msgstr "" +msgstr "Buhutu" #. name for bxi msgid "Pirlatapa" -msgstr "" +msgstr "Pirlatapa" #. name for bxj msgid "Bayungu" -msgstr "" +msgstr "Bayungu" #. name for bxk msgid "Bukusu" -msgstr "" +msgstr "Bukusu" #. name for bxl msgid "Jalkunan" -msgstr "" +msgstr "Jalkunan" #. name for bxm msgid "Buriat; Mongolia" -msgstr "" +msgstr "Buriat; Mongolia" #. name for bxn msgid "Burduna" -msgstr "" +msgstr "Burduna" #. name for bxo msgid "Barikanchi" -msgstr "" +msgstr "Barikanchi" #. name for bxp msgid "Bebil" -msgstr "" +msgstr "Bebil" #. name for bxq msgid "Beele" -msgstr "" +msgstr "Beele" #. name for bxr msgid "Buriat; Russia" -msgstr "" +msgstr "Buriat; Russia" #. name for bxs msgid "Busam" -msgstr "" +msgstr "Busam" #. name for bxu msgid "Buriat; China" -msgstr "" +msgstr "Buriat; China" #. name for bxv msgid "Berakou" -msgstr "" +msgstr "Berakou" #. name for bxw msgid "Bankagooma" -msgstr "" +msgstr "Bankagooma" #. name for bxx msgid "Borna (Democratic Republic of Congo)" -msgstr "" +msgstr "Borna (Democratic Republic of Congo)" #. name for bxz msgid "Binahari" -msgstr "" +msgstr "Binahari" #. name for bya msgid "Batak" -msgstr "" +msgstr "Batak" #. name for byb msgid "Bikya" -msgstr "" +msgstr "Bikya" #. name for byc msgid "Ubaghara" -msgstr "" +msgstr "Ubaghara" #. name for byd msgid "Benyadu'" -msgstr "" +msgstr "Benyadu'" #. name for bye msgid "Pouye" -msgstr "" +msgstr "Pouye" #. name for byf msgid "Bete" -msgstr "" +msgstr "Bete" #. name for byg msgid "Baygo" -msgstr "" +msgstr "Baygo" #. name for byh msgid "Bhujel" -msgstr "" +msgstr "Bhujel" #. name for byi msgid "Buyu" -msgstr "" +msgstr "Buyu" #. name for byj msgid "Bina (Nigeria)" -msgstr "" +msgstr "Bina (Nigeria)" #. name for byk msgid "Biao" -msgstr "" +msgstr "Biao" #. name for byl msgid "Bayono" -msgstr "" +msgstr "Bayono" #. name for bym msgid "Bidyara" -msgstr "" +msgstr "Bidyara" #. name for byn msgid "Bilin" -msgstr "" +msgstr "Bilin" #. name for byo msgid "Biyo" -msgstr "" +msgstr "Biyo" #. name for byp msgid "Bumaji" -msgstr "" +msgstr "Bumaji" #. name for byq msgid "Basay" -msgstr "" +msgstr "Basay" #. name for byr msgid "Baruya" -msgstr "" +msgstr "Baruya" #. name for bys msgid "Burak" -msgstr "" +msgstr "Burak" #. name for byt msgid "Berti" -msgstr "" +msgstr "Berti" #. name for byv msgid "Medumba" -msgstr "" +msgstr "Medumba" #. name for byw msgid "Belhariya" -msgstr "" +msgstr "Belhariya" #. name for byx msgid "Qaqet" -msgstr "" +msgstr "Qaqet" #. name for byy msgid "Buya" -msgstr "" +msgstr "Buya" #. name for byz msgid "Banaro" -msgstr "" +msgstr "Banaro" #. name for bza msgid "Bandi" -msgstr "" +msgstr "Bandi" #. name for bzb msgid "Andio" -msgstr "" +msgstr "Andio" #. name for bzc msgid "Malagasy; Southern Betsimisaraka" -msgstr "" +msgstr "Malagasy; Southern Betsimisaraka" #. name for bzd msgid "Bribri" -msgstr "" +msgstr "Bribri" #. name for bze msgid "Bozo; Jenaama" -msgstr "" +msgstr "Bozo; Jenaama" #. name for bzf msgid "Boikin" -msgstr "" +msgstr "Boikin" #. name for bzg msgid "Babuza" -msgstr "" +msgstr "Babuza" #. name for bzh msgid "Buang; Mapos" -msgstr "" +msgstr "Buang; Mapos" #. name for bzi msgid "Bisu" -msgstr "" +msgstr "Bisu" #. name for bzj msgid "Kriol English; Belize" -msgstr "" +msgstr "Kriol English; Belize" #. name for bzk msgid "Creole English; Nicaragua" -msgstr "" +msgstr "Creole English; Nicaragua" #. name for bzl msgid "Boano (Sulawesi)" -msgstr "" +msgstr "Boano (Sulawesi)" #. name for bzm msgid "Bolondo" -msgstr "" +msgstr "Bolondo" #. name for bzn msgid "Boano (Maluku)" -msgstr "" +msgstr "Boano (Maluku)" #. name for bzo msgid "Bozaba" -msgstr "" +msgstr "Bozaba" #. name for bzp msgid "Kemberano" -msgstr "" +msgstr "Kemberano" #. name for bzq msgid "Buli (Indonesia)" -msgstr "" +msgstr "Buli (Indonesia)" #. name for bzr msgid "Biri" -msgstr "" +msgstr "Biri" #. name for bzs msgid "Brazilian Sign Language" -msgstr "" +msgstr "Brazilian Sign Language" #. name for bzt msgid "Brithenig" -msgstr "" +msgstr "Brithenig" #. name for bzu msgid "Burmeso" -msgstr "" +msgstr "Burmeso" #. name for bzv msgid "Bebe" -msgstr "" +msgstr "Bebe" #. name for bzw msgid "Basa (Nigeria)" -msgstr "" +msgstr "Basa (Nigeria)" #. name for bzx msgid "Bozo; Kɛlɛngaxo" -msgstr "" +msgstr "Bozo; Kɛlɛngaxo" #. name for bzy msgid "Obanliku" -msgstr "" +msgstr "Obanliku" #. name for bzz msgid "Evant" -msgstr "" +msgstr "Evant" #. name for caa msgid "Chortí" -msgstr "" +msgstr "Chortí" #. name for cab msgid "Garifuna" -msgstr "" +msgstr "Garifuna" #. name for cac msgid "Chuj" -msgstr "" +msgstr "Chuj" #. name for cad msgid "Caddo" -msgstr "" +msgstr "Caddo" #. name for cae msgid "Lehar" -msgstr "" +msgstr "Lehar" #. name for caf msgid "Carrier; Southern" -msgstr "" +msgstr "Carrier; Southern" #. name for cag msgid "Nivaclé" -msgstr "" +msgstr "Nivaclé" #. name for cah msgid "Cahuarano" -msgstr "" +msgstr "Cahuarano" #. name for caj msgid "Chané" -msgstr "" +msgstr "Chané" #. name for cak msgid "Kaqchikel" -msgstr "" +msgstr "Kaqchikel" #. name for cal msgid "Carolinian" -msgstr "" +msgstr "Carolinian" #. name for cam msgid "Cemuhî" -msgstr "" +msgstr "Cemuhî" #. name for can msgid "Chambri" -msgstr "" +msgstr "Chambri" #. name for cao msgid "Chácobo" -msgstr "" +msgstr "Chácobo" #. name for cap msgid "Chipaya" -msgstr "" +msgstr "Chipaya" #. name for caq msgid "Nicobarese; Car" -msgstr "" +msgstr "Nicobarese; Car" #. name for car msgid "Carib; Galibi" -msgstr "" +msgstr "Carib; Galibi" #. name for cas msgid "Tsimané" -msgstr "" +msgstr "Tsimané" #. name for cat msgid "Catalan" -msgstr "" +msgstr "Catalan" #. name for cav msgid "Cavineña" -msgstr "" +msgstr "Cavineña" #. name for caw msgid "Callawalla" -msgstr "" +msgstr "Callawalla" #. name for cax msgid "Chiquitano" -msgstr "" +msgstr "Chiquitano" #. name for cay msgid "Cayuga" -msgstr "" +msgstr "Cayuga" #. name for caz msgid "Canichana" -msgstr "" +msgstr "Canichana" #. name for cbb msgid "Cabiyarí" -msgstr "" +msgstr "Cabiyarí" #. name for cbc msgid "Carapana" -msgstr "" +msgstr "Carapana" #. name for cbd msgid "Carijona" -msgstr "" +msgstr "Carijona" #. name for cbe msgid "Chipiajes" -msgstr "" +msgstr "Chipiajes" #. name for cbg msgid "Chimila" -msgstr "" +msgstr "Chimila" #. name for cbh msgid "Cagua" -msgstr "" +msgstr "Cagua" #. name for cbi msgid "Chachi" -msgstr "" +msgstr "Chachi" #. name for cbj msgid "Ede Cabe" -msgstr "" +msgstr "Ede Cabe" #. name for cbk msgid "Chavacano" -msgstr "" +msgstr "Chavacano" #. name for cbl msgid "Chin; Bualkhaw" -msgstr "" +msgstr "Chin; Bualkhaw" #. name for cbn msgid "Nyahkur" -msgstr "" +msgstr "Nyahkur" #. name for cbo msgid "Izora" -msgstr "" +msgstr "Izora" #. name for cbr msgid "Cashibo-Cacataibo" -msgstr "" +msgstr "Cashibo-Cacataibo" #. name for cbs msgid "Cashinahua" -msgstr "" +msgstr "Cashinahua" #. name for cbt msgid "Chayahuita" -msgstr "" +msgstr "Chayahuita" #. name for cbu msgid "Candoshi-Shapra" -msgstr "" +msgstr "Candoshi-Shapra" #. name for cbv msgid "Cacua" -msgstr "" +msgstr "Cacua" #. name for cbw msgid "Kinabalian" -msgstr "" +msgstr "Kinabalian" #. name for cby msgid "Carabayo" -msgstr "" +msgstr "Carabayo" #. name for cca msgid "Cauca" -msgstr "" +msgstr "Cauca" #. name for ccc msgid "Chamicuro" -msgstr "" +msgstr "Chamicuro" #. name for ccd msgid "Creole; Cafundo" -msgstr "" +msgstr "Creole; Cafundo" #. name for cce msgid "Chopi" -msgstr "" +msgstr "Chopi" #. name for ccg msgid "Daka; Samba" -msgstr "" +msgstr "Daka; Samba" #. name for cch msgid "Atsam" -msgstr "" +msgstr "Atsam" #. name for ccj msgid "Kasanga" -msgstr "" +msgstr "Kasanga" #. name for ccl msgid "Cutchi-Swahili" -msgstr "" +msgstr "Cutchi-Swahili" #. name for ccm msgid "Creole Malay; Malaccan" -msgstr "" +msgstr "Creole Malay; Malaccan" #. name for cco msgid "Chinantec; Comaltepec" -msgstr "" +msgstr "Chinantec; Comaltepec" #. name for ccp msgid "Chakma" -msgstr "" +msgstr "Chakma" #. name for ccq msgid "Chaungtha" -msgstr "" +msgstr "Chaungtha" #. name for ccr msgid "Cacaopera" -msgstr "" +msgstr "Cacaopera" #. name for cda msgid "Choni" -msgstr "" +msgstr "Choni" #. name for cde msgid "Chenchu" -msgstr "" +msgstr "Chenchu" #. name for cdf msgid "Chiru" -msgstr "" +msgstr "Chiru" #. name for cdg msgid "Chamari" -msgstr "" +msgstr "Chamari" #. name for cdh msgid "Chambeali" -msgstr "" +msgstr "Chambeali" #. name for cdi msgid "Chodri" -msgstr "" +msgstr "Chodri" #. name for cdj msgid "Churahi" -msgstr "" +msgstr "Churahi" #. name for cdm msgid "Chepang" -msgstr "" +msgstr "Chepang" #. name for cdn msgid "Chaudangsi" -msgstr "" +msgstr "Chaudangsi" #. name for cdo msgid "Chinese; Min Dong" -msgstr "" +msgstr "Chinese; Min Dong" #. name for cdr msgid "Cinda-Regi-Tiyal" -msgstr "" +msgstr "Cinda-Regi-Tiyal" #. name for cds msgid "Chadian Sign Language" -msgstr "" +msgstr "Chadian Sign Language" #. name for cdy msgid "Chadong" -msgstr "" +msgstr "Chadong" #. name for cdz msgid "Koda" -msgstr "" +msgstr "Koda" #. name for cea msgid "Chehalis; Lower" -msgstr "" +msgstr "Chehalis; Lower" #. name for ceb msgid "Cebuano" -msgstr "" +msgstr "Cebuano" #. name for ceg msgid "Chamacoco" -msgstr "" +msgstr "Chamacoco" #. name for cen msgid "Cen" -msgstr "" +msgstr "Cen" #. name for ces msgid "Czech" -msgstr "" +msgstr "Czech" #. name for cet msgid "Centúúm" -msgstr "" +msgstr "Centúúm" #. name for cfa msgid "Dijim-Bwilim" -msgstr "" +msgstr "Dijim-Bwilim" #. name for cfd msgid "Cara" -msgstr "" +msgstr "Cara" #. name for cfg msgid "Como Karim" -msgstr "" +msgstr "Como Karim" #. name for cfm msgid "Chin; Falam" -msgstr "" +msgstr "Chin; Falam" #. name for cga msgid "Changriwa" -msgstr "" +msgstr "Changriwa" #. name for cgc msgid "Kagayanen" -msgstr "" +msgstr "Kagayanen" #. name for cgg msgid "Chiga" -msgstr "" +msgstr "Chiga" #. name for cgk msgid "Chocangacakha" -msgstr "" +msgstr "Chocangacakha" #. name for cha msgid "Chamorro" -msgstr "" +msgstr "Chamorro" #. name for chb msgid "Chibcha" -msgstr "" +msgstr "Chibcha" #. name for chc msgid "Catawba" -msgstr "" +msgstr "Catawba" #. name for chd msgid "Chontal; Highland Oaxaca" -msgstr "" +msgstr "Chontal; Highland Oaxaca" #. name for che msgid "Chechen" -msgstr "" +msgstr "Chechen" #. name for chf msgid "Chontal; Tabasco" -msgstr "" +msgstr "Chontal; Tabasco" #. name for chg msgid "Chagatai" -msgstr "" +msgstr "Chagatai" #. name for chh msgid "Chinook" -msgstr "" +msgstr "Chinook" #. name for chj msgid "Chinantec; Ojitlán" -msgstr "" +msgstr "Chinantec; Ojitlán" #. name for chk msgid "Chuukese" -msgstr "" +msgstr "Chuukese" #. name for chl msgid "Cahuilla" -msgstr "" +msgstr "Cahuilla" #. name for chm msgid "Mari (Russia)" -msgstr "" +msgstr "Mari (Russia)" #. name for chn msgid "Chinook jargon" -msgstr "" +msgstr "Chinook jargon" #. name for cho msgid "Choctaw" -msgstr "" +msgstr "Choctaw" #. name for chp msgid "Chipewyan" -msgstr "" +msgstr "Chipewyan" #. name for chq msgid "Chinantec; Quiotepec" -msgstr "" +msgstr "Chinantec; Quiotepec" #. name for chr msgid "Cherokee" -msgstr "" +msgstr "Cherokee" #. name for cht msgid "Cholón" -msgstr "" +msgstr "Cholón" #. name for chu msgid "Slavonic; Old" -msgstr "" +msgstr "Slavonic; Old" #. name for chv msgid "Chuvash" -msgstr "" +msgstr "Chuvash" #. name for chw msgid "Chuwabu" -msgstr "" +msgstr "Chuwabu" #. name for chx msgid "Chantyal" -msgstr "" +msgstr "Chantyal" #. name for chy msgid "Cheyenne" -msgstr "" +msgstr "Cheyenne" #. name for chz msgid "Chinantec; Ozumacín" -msgstr "" +msgstr "Chinantec; Ozumacín" #. name for cia msgid "Cia-Cia" -msgstr "" +msgstr "Cia-Cia" #. name for cib msgid "Gbe; Ci" -msgstr "" +msgstr "Gbe; Ci" #. name for cic msgid "Chickasaw" -msgstr "" +msgstr "Chickasaw" #. name for cid msgid "Chimariko" -msgstr "" +msgstr "Chimariko" #. name for cie msgid "Cineni" -msgstr "" +msgstr "Cineni" #. name for cih msgid "Chinali" -msgstr "" +msgstr "Chinali" #. name for cik msgid "Kinnauri; Chitkuli" -msgstr "" +msgstr "Kinnauri; Chitkuli" #. name for cim msgid "Cimbrian" -msgstr "" +msgstr "Cimbrian" #. name for cin msgid "Cinta Larga" -msgstr "" +msgstr "Cinta Larga" #. name for cip msgid "Chiapanec" -msgstr "" +msgstr "Chiapanec" #. name for cir msgid "Tiri" -msgstr "" +msgstr "Tiri" #. name for ciw msgid "Chippewa" -msgstr "" +msgstr "Chippewa" #. name for ciy msgid "Chaima" -msgstr "" +msgstr "Chaima" #. name for cja msgid "Cham; Western" -msgstr "" +msgstr "Cham; Western" #. name for cje msgid "Chru" -msgstr "" +msgstr "Chru" #. name for cjh msgid "Chehalis; Upper" -msgstr "" +msgstr "Chehalis; Upper" #. name for cji msgid "Chamalal" -msgstr "" +msgstr "Chamalal" #. name for cjk msgid "Chokwe" -msgstr "" +msgstr "Chokwe" #. name for cjm msgid "Cham; Eastern" -msgstr "" +msgstr "Cham; Eastern" #. name for cjn msgid "Chenapian" -msgstr "" +msgstr "Chenapian" #. name for cjo msgid "Ashéninka Pajonal" -msgstr "" +msgstr "Ashéninka Pajonal" #. name for cjp msgid "Cabécar" -msgstr "" +msgstr "Cabécar" #. name for cjs msgid "Shor" -msgstr "" +msgstr "Shor" #. name for cjv msgid "Chuave" -msgstr "" +msgstr "Chuave" #. name for cjy msgid "Chinese; Jinyu" -msgstr "" +msgstr "Chinese; Jinyu" #. name for cka msgid "Chin; Khumi Awa" -msgstr "" +msgstr "Chin; Khumi Awa" #. name for ckb msgid "Kurdish; Central" -msgstr "" +msgstr "Kurdish; Central" #. name for ckh msgid "Chak" -msgstr "" +msgstr "Chak" #. name for ckl msgid "Cibak" -msgstr "" +msgstr "Cibak" #. name for cko msgid "Anufo" -msgstr "" +msgstr "Anufo" #. name for ckq msgid "Kajakse" -msgstr "" +msgstr "Kajakse" #. name for ckr msgid "Kairak" -msgstr "" +msgstr "Kairak" #. name for cks msgid "Tayo" -msgstr "" +msgstr "Tayo" #. name for ckt msgid "Chukot" -msgstr "" +msgstr "Chukot" #. name for cku msgid "Koasati" -msgstr "" +msgstr "Koasati" #. name for ckv msgid "Kavalan" -msgstr "" +msgstr "Kavalan" #. name for ckx msgid "Caka" -msgstr "" +msgstr "Caka" #. name for cky msgid "Cakfem-Mushere" -msgstr "" +msgstr "Cakfem-Mushere" #. name for ckz msgid "Cakchiquel-Quiché Mixed Language" -msgstr "" +msgstr "Cakchiquel-Quiché Mixed Language" #. name for cla msgid "Ron" -msgstr "" +msgstr "Ron" #. name for clc msgid "Chilcotin" -msgstr "" +msgstr "Chilcotin" #. name for cld msgid "Neo-Aramaic; Chaldean" -msgstr "" +msgstr "Neo-Aramaic; Chaldean" #. name for cle msgid "Chinantec; Lealao" -msgstr "" +msgstr "Chinantec; Lealao" #. name for clh msgid "Chilisso" -msgstr "" +msgstr "Chilisso" #. name for cli msgid "Chakali" -msgstr "" +msgstr "Chakali" #. name for clk msgid "Idu-Mishmi" -msgstr "" +msgstr "Idu-Mishmi" #. name for cll msgid "Chala" -msgstr "" +msgstr "Chala" #. name for clm msgid "Clallam" -msgstr "" +msgstr "Clallam" #. name for clo msgid "Chontal; Lowland Oaxaca" -msgstr "" +msgstr "Chontal; Lowland Oaxaca" #. name for clu msgid "Caluyanun" -msgstr "" +msgstr "Caluyanun" #. name for clw msgid "Chulym" -msgstr "" +msgstr "Chulym" #. name for cly msgid "Chatino; Eastern Highland" -msgstr "" +msgstr "Chatino; Eastern Highland" #. name for cma msgid "Maa" -msgstr "" +msgstr "Maa" #. name for cme msgid "Cerma" -msgstr "" +msgstr "Cerma" #. name for cmg msgid "Mongolian; Classical" -msgstr "" +msgstr "Mongolian; Classical" #. name for cmi msgid "Emberá-Chamí" -msgstr "" +msgstr "Emberá-Chamí" #. name for cml msgid "Campalagian" -msgstr "" +msgstr "Campalagian" #. name for cmm msgid "Michigamea" -msgstr "" +msgstr "Michigamea" #. name for cmn msgid "Chinese; Mandarin" -msgstr "" +msgstr "Chinese; Mandarin" #. name for cmo msgid "Mnong; Central" -msgstr "" +msgstr "Mnong; Central" #. name for cmr msgid "Chin; Mro" -msgstr "" +msgstr "Chin; Mro" #. name for cms msgid "Messapic" -msgstr "" +msgstr "Messapic" #. name for cmt msgid "Camtho" -msgstr "" +msgstr "Camtho" #. name for cna msgid "Changthang" -msgstr "" +msgstr "Changthang" #. name for cnb msgid "Chin; Chinbon" -msgstr "" +msgstr "Chin; Chinbon" #. name for cnc msgid "Côông" -msgstr "" +msgstr "Côông" #. name for cng msgid "Qiang; Northern" -msgstr "" +msgstr "Qiang; Northern" #. name for cnh msgid "Chin; Haka" -msgstr "" +msgstr "Chin; Haka" #. name for cni msgid "Asháninka" -msgstr "" +msgstr "Asháninka" #. name for cnk msgid "Chin; Khumi" -msgstr "" +msgstr "Chin; Khumi" #. name for cnl msgid "Chinantec; Lalana" -msgstr "" +msgstr "Chinantec; Lalana" #. name for cno msgid "Con" -msgstr "" +msgstr "Con" #. name for cns msgid "Asmat; Central" -msgstr "" +msgstr "Asmat; Central" #. name for cnt msgid "Chinantec; Tepetotutla" -msgstr "" +msgstr "Chinantec; Tepetotutla" #. name for cnu msgid "Chenoua" -msgstr "" +msgstr "Chenoua" #. name for cnw msgid "Chin; Ngawn" -msgstr "" +msgstr "Chin; Ngawn" #. name for cnx msgid "Cornish; Middle" -msgstr "" +msgstr "Cornish; Middle" #. name for coa msgid "Malay; Cocos Islands" -msgstr "" +msgstr "Malay; Cocos Islands" #. name for cob msgid "Chicomuceltec" -msgstr "" +msgstr "Chicomuceltec" #. name for coc msgid "Cocopa" -msgstr "" +msgstr "Cocopa" #. name for cod msgid "Cocama-Cocamilla" -msgstr "" +msgstr "Cocama-Cocamilla" #. name for coe msgid "Koreguaje" -msgstr "" +msgstr "Koreguaje" #. name for cof msgid "Colorado" -msgstr "" +msgstr "Colorado" #. name for cog msgid "Chong" -msgstr "" +msgstr "Chong" #. name for coh msgid "Chonyi-Dzihana-Kauma" -msgstr "" +msgstr "Chonyi-Dzihana-Kauma" #. name for coj msgid "Cochimi" -msgstr "" +msgstr "Cochimi" #. name for cok msgid "Cora; Santa Teresa" -msgstr "" +msgstr "Cora; Santa Teresa" #. name for col msgid "Columbia-Wenatchi" -msgstr "" +msgstr "Columbia-Wenatchi" #. name for com msgid "Comanche" -msgstr "" +msgstr "Comanche" #. name for con msgid "Cofán" -msgstr "" +msgstr "Cofán" #. name for coo msgid "Comox" -msgstr "" +msgstr "Comox" #. name for cop msgid "Coptic" -msgstr "" +msgstr "Coptic" #. name for coq msgid "Coquille" -msgstr "" +msgstr "Coquille" #. name for cor msgid "Cornish" -msgstr "" +msgstr "Cornish" #. name for cos msgid "Corsican" -msgstr "" +msgstr "Corsican" #. name for cot msgid "Caquinte" -msgstr "" +msgstr "Caquinte" #. name for cou msgid "Wamey" -msgstr "" +msgstr "Wamey" #. name for cov msgid "Cao Miao" -msgstr "" +msgstr "Cao Miao" #. name for cow msgid "Cowlitz" -msgstr "" +msgstr "Cowlitz" #. name for cox msgid "Nanti" -msgstr "" +msgstr "Nanti" #. name for coy msgid "Coyaima" -msgstr "" +msgstr "Coyaima" #. name for coz msgid "Chochotec" -msgstr "" +msgstr "Chochotec" #. name for cpa msgid "Chinantec; Palantla" -msgstr "" +msgstr "Chinantec; Palantla" #. name for cpb msgid "Ashéninka; Ucayali-Yurúa" -msgstr "" +msgstr "Ashéninka; Ucayali-Yurúa" #. name for cpc msgid "Ajyíninka Apurucayali" -msgstr "" +msgstr "Ajyíninka Apurucayali" #. name for cpg msgid "Greek; Cappadocian" -msgstr "" +msgstr "Greek; Cappadocian" #. name for cpi msgid "Pidgin English; Chinese" -msgstr "" +msgstr "Pidgin English; Chinese" #. name for cpn msgid "Cherepon" -msgstr "" +msgstr "Cherepon" #. name for cps msgid "Capiznon" -msgstr "" +msgstr "Capiznon" #. name for cpu msgid "Ashéninka; Pichis" -msgstr "" +msgstr "Ashéninka; Pichis" #. name for cpx msgid "Chinese; Pu-Xian" -msgstr "" +msgstr "Chinese; Pu-Xian" #. name for cpy msgid "Ashéninka; South Ucayali" -msgstr "" +msgstr "Ashéninka; South Ucayali" #. name for cqd msgid "Miao; Chuanqiandian Cluster" -msgstr "" +msgstr "Miao; Chuanqiandian Cluster" #. name for cqu msgid "Quechua; Chilean" -msgstr "" +msgstr "Quechua; Chilean" #. name for cra msgid "Chara" -msgstr "" +msgstr "Chara" #. name for crb msgid "Carib; Island" -msgstr "" +msgstr "Carib; Island" #. name for crc msgid "Lonwolwol" -msgstr "" +msgstr "Lonwolwol" #. name for crd msgid "Coeur d'Alene" -msgstr "" +msgstr "Coeur d'Alene" #. name for cre msgid "Cree" -msgstr "" +msgstr "Cree" #. name for crf msgid "Caramanta" -msgstr "" +msgstr "Caramanta" #. name for crg msgid "Michif" -msgstr "" +msgstr "Michif" #. name for crh msgid "Turkish; Crimean" -msgstr "" +msgstr "Turkish; Crimean" #. name for cri msgid "Sãotomense" -msgstr "" +msgstr "Sãotomense" #. name for crj msgid "Cree; Southern East" -msgstr "" +msgstr "Cree; Southern East" #. name for crk msgid "Cree; Plains" -msgstr "" +msgstr "Cree; Plains" #. name for crl msgid "Cree; Northern East" -msgstr "" +msgstr "Cree; Northern East" #. name for crm msgid "Cree; Moose" -msgstr "" +msgstr "Cree; Moose" #. name for crn msgid "Cora; El Nayar" -msgstr "" +msgstr "Cora; El Nayar" #. name for cro msgid "Crow" -msgstr "" +msgstr "Crow" #. name for crq msgid "Chorote; Iyo'wujwa" -msgstr "" +msgstr "Chorote; Iyo'wujwa" #. name for crr msgid "Algonquian; Carolina" -msgstr "" +msgstr "Algonquian; Carolina" #. name for crs msgid "Creole French; Seselwa" -msgstr "" +msgstr "Creole French; Seselwa" #. name for crt msgid "Chorote; Iyojwa'ja" -msgstr "" +msgstr "Chorote; Iyojwa'ja" #. name for crv msgid "Chaura" -msgstr "" +msgstr "Chaura" #. name for crw msgid "Chrau" -msgstr "" +msgstr "Chrau" #. name for crx msgid "Carrier" -msgstr "" +msgstr "Carrier" #. name for cry msgid "Cori" -msgstr "" +msgstr "Cori" #. name for crz msgid "Cruzeño" -msgstr "" +msgstr "Cruzeño" #. name for csa msgid "Chinantec; Chiltepec" -msgstr "" +msgstr "Chinantec; Chiltepec" #. name for csb msgid "Kashubian" -msgstr "" +msgstr "Kashubian" #. name for csc msgid "Catalan Sign Language" -msgstr "" +msgstr "Catalan Sign Language" #. name for csd msgid "Chiangmai Sign Language" -msgstr "" +msgstr "Chiangmai Sign Language" #. name for cse msgid "Czech Sign Language" -msgstr "" +msgstr "Czech Sign Language" #. name for csf msgid "Cuba Sign Language" -msgstr "" +msgstr "Cuba Sign Language" #. name for csg msgid "Chilean Sign Language" -msgstr "" +msgstr "Chilean Sign Language" #. name for csh msgid "Chin; Asho" -msgstr "" +msgstr "Chin; Asho" #. name for csi msgid "Miwok; Coast" -msgstr "" +msgstr "Miwok; Coast" #. name for csk msgid "Jola-Kasa" -msgstr "" +msgstr "Jola-Kasa" #. name for csl msgid "Chinese Sign Language" -msgstr "" +msgstr "Chinese Sign Language" #. name for csm msgid "Miwok; Central Sierra" -msgstr "" +msgstr "Miwok; Central Sierra" #. name for csn msgid "Colombian Sign Language" -msgstr "" +msgstr "Colombian Sign Language" #. name for cso msgid "Chinantec; Sochiapan" -msgstr "" +msgstr "Chinantec; Sochiapan" #. name for csq msgid "Croatia Sign Language" -msgstr "" +msgstr "Croatia Sign Language" #. name for csr msgid "Costa Rican Sign Language" -msgstr "" +msgstr "Costa Rican Sign Language" #. name for css msgid "Ohlone; Southern" -msgstr "" +msgstr "Ohlone; Southern" #. name for cst msgid "Ohlone; Northern" -msgstr "" +msgstr "Ohlone; Northern" #. name for csw msgid "Cree; Swampy" -msgstr "" +msgstr "Cree; Swampy" #. name for csy msgid "Chin; Siyin" -msgstr "" +msgstr "Chin; Siyin" #. name for csz msgid "Coos" -msgstr "" +msgstr "Coos" #. name for cta msgid "Chatino; Tataltepec" -msgstr "" +msgstr "Chatino; Tataltepec" #. name for ctc msgid "Chetco" -msgstr "" +msgstr "Chetco" #. name for ctd msgid "Chin; Tedim" -msgstr "" +msgstr "Chin; Tedim" #. name for cte msgid "Chinantec; Tepinapa" -msgstr "" +msgstr "Chinantec; Tepinapa" #. name for ctg msgid "Chittagonian" -msgstr "" +msgstr "Chittagonian" #. name for ctl msgid "Chinantec; Tlacoatzintepec" -msgstr "" +msgstr "Chinantec; Tlacoatzintepec" #. name for ctm msgid "Chitimacha" -msgstr "" +msgstr "Chitimacha" #. name for ctn msgid "Chhintange" -msgstr "" +msgstr "Chhintange" #. name for cto msgid "Emberá-Catío" -msgstr "" +msgstr "Emberá-Catío" #. name for ctp msgid "Chatino; Western Highland" -msgstr "" +msgstr "Chatino; Western Highland" #. name for cts msgid "Bicolano; Northern Catanduanes" -msgstr "" +msgstr "Bicolano; Northern Catanduanes" #. name for ctt msgid "Chetti; Wayanad" -msgstr "" +msgstr "Chetti; Wayanad" #. name for ctu msgid "Chol" -msgstr "" +msgstr "Chol" #. name for ctz msgid "Chatino; Zacatepec" -msgstr "" +msgstr "Chatino; Zacatepec" #. name for cua msgid "Cua" -msgstr "" +msgstr "Cua" #. name for cub msgid "Cubeo" -msgstr "" +msgstr "Cubeo" #. name for cuc msgid "Chinantec; Usila" -msgstr "" +msgstr "Chinantec; Usila" #. name for cug msgid "Cung" -msgstr "" +msgstr "Cung" #. name for cuh msgid "Chuka" -msgstr "" +msgstr "Chuka" #. name for cui msgid "Cuiba" -msgstr "" +msgstr "Cuiba" #. name for cuj msgid "Mashco Piro" -msgstr "" +msgstr "Mashco Piro" #. name for cuk msgid "Kuna; San Blas" -msgstr "" +msgstr "Kuna; San Blas" #. name for cul msgid "Culina" -msgstr "" +msgstr "Culina" #. name for cum msgid "Cumeral" -msgstr "" +msgstr "Cumeral" #. name for cuo msgid "Cumanagoto" -msgstr "" +msgstr "Cumanagoto" #. name for cup msgid "Cupeño" -msgstr "" +msgstr "Cupeño" #. name for cuq msgid "Cun" -msgstr "" +msgstr "Cun" #. name for cur msgid "Chhulung" -msgstr "" +msgstr "Chhulung" #. name for cut msgid "Cuicatec; Teutila" -msgstr "" +msgstr "Cuicatec; Teutila" #. name for cuu msgid "Tai Ya" -msgstr "" +msgstr "Tai Ya" #. name for cuv msgid "Cuvok" -msgstr "" +msgstr "Cuvok" #. name for cuw msgid "Chukwa" -msgstr "" +msgstr "Chukwa" #. name for cux msgid "Cuicatec; Tepeuxila" -msgstr "" +msgstr "Cuicatec; Tepeuxila" #. name for cvg msgid "Chug" -msgstr "" +msgstr "Chug" #. name for cvn msgid "Chinantec; Valle Nacional" -msgstr "" +msgstr "Chinantec; Valle Nacional" #. name for cwa msgid "Kabwa" -msgstr "" +msgstr "Kabwa" #. name for cwb msgid "Maindo" -msgstr "" +msgstr "Maindo" #. name for cwd msgid "Cree; Woods" -msgstr "" +msgstr "Cree; Woods" #. name for cwe msgid "Kwere" -msgstr "" +msgstr "Kwere" #. name for cwg msgid "Chewong" -msgstr "" +msgstr "Chewong" #. name for cwt msgid "Kuwaataay" -msgstr "" +msgstr "Kuwaataay" #. name for cya msgid "Chatino; Nopala" -msgstr "" +msgstr "Chatino; Nopala" #. name for cyb msgid "Cayubaba" -msgstr "" +msgstr "Cayubaba" #. name for cym msgid "Welsh" -msgstr "" +msgstr "Welsh" #. name for cyo msgid "Cuyonon" -msgstr "" +msgstr "Cuyonon" #. name for czh msgid "Chinese; Huizhou" -msgstr "" +msgstr "Chinese; Huizhou" #. name for czk msgid "Knaanic" -msgstr "" +msgstr "Knaanic" #. name for czn msgid "Chatino; Zenzontepec" -msgstr "" +msgstr "Chatino; Zenzontepec" #. name for czo msgid "Chinese; Min Zhong" -msgstr "" +msgstr "Chinese; Min Zhong" #. name for czt msgid "Chin; Zotung" -msgstr "" +msgstr "Chin; Zotung" #. name for daa msgid "Dangaléat" -msgstr "" +msgstr "Dangaléat" #. name for dac msgid "Dambi" -msgstr "" +msgstr "Dambi" #. name for dad msgid "Marik" -msgstr "" +msgstr "Marik" #. name for dae msgid "Duupa" -msgstr "" +msgstr "Duupa" #. name for daf msgid "Dan" -msgstr "" +msgstr "Dan" #. name for dag msgid "Dagbani" -msgstr "" +msgstr "Dagbani" #. name for dah msgid "Gwahatike" -msgstr "" +msgstr "Gwahatike" #. name for dai msgid "Day" -msgstr "" +msgstr "Day" #. name for daj msgid "Daju; Dar Fur" -msgstr "" +msgstr "Daju; Dar Fur" #. name for dak msgid "Dakota" -msgstr "" +msgstr "Dakota" #. name for dal msgid "Dahalo" -msgstr "" +msgstr "Dahalo" #. name for dam msgid "Damakawa" -msgstr "" +msgstr "Damakawa" #. name for dan msgid "Danish" -msgstr "" +msgstr "Danish" #. name for dao msgid "Chin; Daai" -msgstr "" +msgstr "Chin; Daai" #. name for dap msgid "Nisi (India)" -msgstr "" +msgstr "Nisi (India)" #. name for daq msgid "Maria; Dandami" -msgstr "" +msgstr "Maria; Dandami" #. name for dar msgid "Dargwa" -msgstr "" +msgstr "Dargwa" #. name for das msgid "Daho-Doo" -msgstr "" +msgstr "Daho-Doo" #. name for dau msgid "Daju; Dar Sila" -msgstr "" +msgstr "Daju; Dar Sila" #. name for dav msgid "Taita" -msgstr "" +msgstr "Taita" #. name for daw msgid "Davawenyo" -msgstr "" +msgstr "Davawenyo" #. name for dax msgid "Dayi" -msgstr "" +msgstr "Dayi" #. name for daz msgid "Dao" -msgstr "" +msgstr "Dao" #. name for dba msgid "Bangi Me" -msgstr "" +msgstr "Bangi Me" #. name for dbb msgid "Deno" -msgstr "" +msgstr "Deno" #. name for dbd msgid "Dadiya" -msgstr "" +msgstr "Dadiya" #. name for dbe msgid "Dabe" -msgstr "" +msgstr "Dabe" #. name for dbf msgid "Edopi" -msgstr "" +msgstr "Edopi" #. name for dbg msgid "Dogon; Dogul Dom" -msgstr "" +msgstr "Dogon; Dogul Dom" #. name for dbi msgid "Doka" -msgstr "" +msgstr "Doka" #. name for dbj msgid "Ida'an" -msgstr "" +msgstr "Ida'an" #. name for dbl msgid "Dyirbal" -msgstr "" +msgstr "Dyirbal" #. name for dbm msgid "Duguri" -msgstr "" +msgstr "Duguri" #. name for dbn msgid "Duriankere" -msgstr "" +msgstr "Duriankere" #. name for dbo msgid "Dulbu" -msgstr "" +msgstr "Dulbu" #. name for dbp msgid "Duwai" -msgstr "" +msgstr "Duwai" #. name for dbq msgid "Daba" -msgstr "" +msgstr "Daba" #. name for dbr msgid "Dabarre" -msgstr "" +msgstr "Dabarre" #. name for dbu msgid "Dogon; Bondum Dom" -msgstr "" +msgstr "Dogon; Bondum Dom" #. name for dbv msgid "Dungu" -msgstr "" +msgstr "Dungu" #. name for dby msgid "Dibiyaso" -msgstr "" +msgstr "Dibiyaso" #. name for dcc msgid "Deccan" -msgstr "" +msgstr "Deccan" #. name for dcr msgid "Negerhollands" -msgstr "" +msgstr "Negerhollands" #. name for ddd msgid "Dongotono" -msgstr "" +msgstr "Dongotono" #. name for dde msgid "Doondo" -msgstr "" +msgstr "Doondo" #. name for ddg msgid "Fataluku" -msgstr "" +msgstr "Fataluku" #. name for ddi msgid "Goodenough; West" -msgstr "" +msgstr "Goodenough; West" #. name for ddj msgid "Jaru" -msgstr "" +msgstr "Jaru" #. name for ddn msgid "Dendi (Benin)" -msgstr "" +msgstr "Dendi (Benin)" #. name for ddo msgid "Dido" -msgstr "" +msgstr "Dido" #. name for dds msgid "Dogon; Donno So" -msgstr "" +msgstr "Dogon; Donno So" #. name for ddw msgid "Dawera-Daweloor" -msgstr "" +msgstr "Dawera-Daweloor" #. name for dec msgid "Dagik" -msgstr "" +msgstr "Dagik" #. name for ded msgid "Dedua" -msgstr "" +msgstr "Dedua" #. name for dee msgid "Dewoin" -msgstr "" +msgstr "Dewoin" #. name for def msgid "Dezfuli" -msgstr "" +msgstr "Dezfuli" #. name for deg msgid "Degema" -msgstr "" +msgstr "Degema" #. name for deh msgid "Dehwari" -msgstr "" +msgstr "Dehwari" #. name for dei msgid "Demisa" -msgstr "" +msgstr "Demisa" #. name for dek msgid "Dek" -msgstr "" +msgstr "Dek" #. name for del msgid "Delaware" -msgstr "" +msgstr "Delaware" #. name for dem msgid "Dem" -msgstr "" +msgstr "Dem" #. name for den msgid "Slave (Athapascan)" -msgstr "" +msgstr "Slave (Athapascan)" #. name for dep msgid "Delaware; Pidgin" -msgstr "" +msgstr "Delaware; Pidgin" #. name for deq msgid "Dendi (Central African Republic)" -msgstr "" +msgstr "Dendi (Central African Republic)" #. name for der msgid "Deori" -msgstr "" +msgstr "Deori" #. name for des msgid "Desano" -msgstr "" +msgstr "Desano" #. name for deu msgid "German" -msgstr "" +msgstr "German" #. name for dev msgid "Domung" -msgstr "" +msgstr "Domung" #. name for dez msgid "Dengese" -msgstr "" +msgstr "Dengese" #. name for dga msgid "Dagaare; Southern" -msgstr "" +msgstr "Dagaare; Southern" #. name for dgb msgid "Dogon; Bunoge" -msgstr "" +msgstr "Dogon; Bunoge" #. name for dgc msgid "Agta; Casiguran Dumagat" -msgstr "" +msgstr "Agta; Casiguran Dumagat" #. name for dgd msgid "Dagaari Dioula" -msgstr "" +msgstr "Dagaari Dioula" #. name for dge msgid "Degenan" -msgstr "" +msgstr "Degenan" #. name for dgg msgid "Doga" -msgstr "" +msgstr "Doga" #. name for dgh msgid "Dghwede" -msgstr "" +msgstr "Dghwede" #. name for dgi msgid "Dagara; Northern" -msgstr "" +msgstr "Dagara; Northern" #. name for dgk msgid "Dagba" -msgstr "" +msgstr "Dagba" #. name for dgn msgid "Dagoman" -msgstr "" +msgstr "Dagoman" #. name for dgo msgid "Dogri (individual language)" -msgstr "" +msgstr "Dogri (individual language)" #. name for dgr msgid "Dogrib" -msgstr "" +msgstr "Dogrib" #. name for dgs msgid "Dogoso" -msgstr "" +msgstr "Dogoso" #. name for dgu msgid "Degaru" -msgstr "" +msgstr "Degaru" #. name for dgx msgid "Doghoro" -msgstr "" +msgstr "Doghoro" #. name for dgz msgid "Daga" -msgstr "" +msgstr "Daga" #. name for dhd msgid "Dhundari" -msgstr "" +msgstr "Dhundari" #. name for dhg msgid "Dhangu" -msgstr "" +msgstr "Dhangu" #. name for dhi msgid "Dhimal" -msgstr "" +msgstr "Dhimal" #. name for dhl msgid "Dhalandji" -msgstr "" +msgstr "Dhalandji" #. name for dhm msgid "Zemba" -msgstr "" +msgstr "Zemba" #. name for dhn msgid "Dhanki" -msgstr "" +msgstr "Dhanki" #. name for dho msgid "Dhodia" -msgstr "" +msgstr "Dhodia" #. name for dhr msgid "Dhargari" -msgstr "" +msgstr "Dhargari" #. name for dhs msgid "Dhaiso" -msgstr "" +msgstr "Dhaiso" #. name for dhu msgid "Dhurga" -msgstr "" +msgstr "Dhurga" #. name for dhv msgid "Dehu" -msgstr "" +msgstr "Dehu" #. name for dhw msgid "Dhanwar (Nepal)" -msgstr "" +msgstr "Dhanwar (Nepal)" #. name for dia msgid "Dia" -msgstr "" +msgstr "Dia" #. name for dib msgid "Dinka; South Central" -msgstr "" +msgstr "Dinka; South Central" #. name for dic msgid "Dida; Lakota" -msgstr "" +msgstr "Dida; Lakota" #. name for did msgid "Didinga" -msgstr "" +msgstr "Didinga" #. name for dif msgid "Dieri" -msgstr "" +msgstr "Dieri" #. name for dig msgid "Digo" -msgstr "" +msgstr "Digo" #. name for dih msgid "Kumiai" -msgstr "" +msgstr "Kumiai" #. name for dii msgid "Dimbong" -msgstr "" +msgstr "Dimbong" #. name for dij msgid "Dai" -msgstr "" +msgstr "Dai" #. name for dik msgid "Dinka; Southwestern" -msgstr "" +msgstr "Dinka; Southwestern" #. name for dil msgid "Dilling" -msgstr "" +msgstr "Dilling" #. name for dim msgid "Dime" -msgstr "" +msgstr "Dime" #. name for din msgid "Dinka" -msgstr "" +msgstr "Dinka" #. name for dio msgid "Dibo" -msgstr "" +msgstr "Dibo" #. name for dip msgid "Dinka; Northeastern" -msgstr "" +msgstr "Dinka; Northeastern" #. name for diq msgid "Dimli (individual language)" -msgstr "" +msgstr "Dimli (individual language)" #. name for dir msgid "Dirim" -msgstr "" +msgstr "Dirim" #. name for dis msgid "Dimasa" -msgstr "" +msgstr "Dimasa" #. name for dit msgid "Dirari" -msgstr "" +msgstr "Dirari" #. name for diu msgid "Diriku" -msgstr "" +msgstr "Diriku" #. name for div msgid "Dhivehi" -msgstr "" +msgstr "Dhivehi" #. name for diw msgid "Dinka; Northwestern" -msgstr "" +msgstr "Dinka; Northwestern" #. name for dix msgid "Dixon Reef" -msgstr "" +msgstr "Dixon Reef" #. name for diy msgid "Diuwe" -msgstr "" +msgstr "Diuwe" #. name for diz msgid "Ding" -msgstr "" +msgstr "Ding" #. name for djb msgid "Djinba" -msgstr "" +msgstr "Djinba" #. name for djc msgid "Daju; Dar Daju" -msgstr "" +msgstr "Daju; Dar Daju" #. name for djd msgid "Djamindjung" -msgstr "" +msgstr "Djamindjung" #. name for dje msgid "Zarma" -msgstr "" +msgstr "Zarma" #. name for djf msgid "Djangun" -msgstr "" +msgstr "Djangun" #. name for dji msgid "Djinang" -msgstr "" +msgstr "Djinang" #. name for djj msgid "Djeebbana" -msgstr "" +msgstr "Djeebbana" #. name for djk msgid "Eastern Maroon Creole" -msgstr "" +msgstr "Eastern Maroon Creole" #. name for djl msgid "Djiwarli" -msgstr "" +msgstr "Djiwarli" #. name for djm msgid "Dogon; Jamsay" -msgstr "" +msgstr "Dogon; Jamsay" #. name for djn msgid "Djauan" -msgstr "" +msgstr "Djauan" #. name for djo msgid "Jangkang" -msgstr "" +msgstr "Jangkang" #. name for djr msgid "Djambarrpuyngu" -msgstr "" +msgstr "Djambarrpuyngu" #. name for dju msgid "Kapriman" -msgstr "" +msgstr "Kapriman" #. name for djw msgid "Djawi" -msgstr "" +msgstr "Djawi" #. name for dka msgid "Dakpakha" -msgstr "" +msgstr "Dakpakha" #. name for dkk msgid "Dakka" -msgstr "" +msgstr "Dakka" #. name for dkr msgid "Kuijau" -msgstr "" +msgstr "Kuijau" #. name for dks msgid "Dinka; Southeastern" -msgstr "" +msgstr "Dinka; Southeastern" #. name for dkx msgid "Mazagway" -msgstr "" +msgstr "Mazagway" #. name for dlg msgid "Dolgan" -msgstr "" +msgstr "Dolgan" #. name for dlm msgid "Dalmatian" -msgstr "" +msgstr "Dalmatian" #. name for dln msgid "Darlong" -msgstr "" +msgstr "Darlong" #. name for dma msgid "Duma" -msgstr "" +msgstr "Duma" #. name for dmb msgid "Dogon; Mombo" -msgstr "" +msgstr "Dogon; Mombo" #. name for dmc msgid "Dimir" -msgstr "" +msgstr "Dimir" #. name for dme msgid "Dugwor" -msgstr "" +msgstr "Dugwor" #. name for dmg msgid "Kinabatangan; Upper" -msgstr "" +msgstr "Kinabatangan; Upper" #. name for dmk msgid "Domaaki" -msgstr "" +msgstr "Domaaki" #. name for dml msgid "Dameli" -msgstr "" +msgstr "Dameli" #. name for dmm msgid "Dama" -msgstr "" +msgstr "Dama" #. name for dmo msgid "Kemezung" -msgstr "" +msgstr "Kemezung" #. name for dmr msgid "Damar; East" -msgstr "" +msgstr "Damar; East" #. name for dms msgid "Dampelas" -msgstr "" +msgstr "Dampelas" #. name for dmu msgid "Dubu" -msgstr "" +msgstr "Dubu" #. name for dmv msgid "Dumpas" -msgstr "" +msgstr "Dumpas" #. name for dmx msgid "Dema" -msgstr "" +msgstr "Dema" #. name for dmy msgid "Demta" -msgstr "" +msgstr "Demta" #. name for dna msgid "Dani; Upper Grand Valley" -msgstr "" +msgstr "Dani; Upper Grand Valley" #. name for dnd msgid "Daonda" -msgstr "" +msgstr "Daonda" #. name for dne msgid "Ndendeule" -msgstr "" +msgstr "Ndendeule" #. name for dng msgid "Dungan" -msgstr "" +msgstr "Dungan" #. name for dni msgid "Dani; Lower Grand Valley" -msgstr "" +msgstr "Dani; Lower Grand Valley" #. name for dnk msgid "Dengka" -msgstr "" +msgstr "Dengka" #. name for dnn msgid "Dzùùngoo" -msgstr "" +msgstr "Dzùùngoo" #. name for dnr msgid "Danaru" -msgstr "" +msgstr "Danaru" #. name for dnt msgid "Dani; Mid Grand Valley" -msgstr "" +msgstr "Dani; Mid Grand Valley" #. name for dnu msgid "Danau" -msgstr "" +msgstr "Danau" #. name for dnw msgid "Dani; Western" -msgstr "" +msgstr "Dani; Western" #. name for dny msgid "Dení" -msgstr "" +msgstr "Dení" #. name for doa msgid "Dom" -msgstr "" +msgstr "Dom" #. name for dob msgid "Dobu" -msgstr "" +msgstr "Dobu" #. name for doc msgid "Dong; Northern" -msgstr "" +msgstr "Dong; Northern" #. name for doe msgid "Doe" -msgstr "" +msgstr "Doe" #. name for dof msgid "Domu" -msgstr "" +msgstr "Domu" #. name for doh msgid "Dong" -msgstr "" +msgstr "Dong" #. name for doi msgid "Dogri (macrolanguage)" -msgstr "" +msgstr "Dogri (macrolanguage)" #. name for dok msgid "Dondo" -msgstr "" +msgstr "Dondo" #. name for dol msgid "Doso" -msgstr "" +msgstr "Doso" #. name for don msgid "Toura (Papua New Guinea)" -msgstr "" +msgstr "Toura (Papua New Guinea)" #. name for doo msgid "Dongo" -msgstr "" +msgstr "Dongo" #. name for dop msgid "Lukpa" -msgstr "" +msgstr "Lukpa" #. name for doq msgid "Dominican Sign Language" -msgstr "" +msgstr "Dominican Sign Language" #. name for dor msgid "Dori'o" -msgstr "" +msgstr "Dori'o" #. name for dos msgid "Dogosé" -msgstr "" +msgstr "Dogosé" #. name for dot msgid "Dass" -msgstr "" +msgstr "Dass" #. name for dov msgid "Dombe" -msgstr "" +msgstr "Dombe" #. name for dow msgid "Doyayo" -msgstr "" +msgstr "Doyayo" #. name for dox msgid "Bussa" -msgstr "" +msgstr "Bussa" #. name for doy msgid "Dompo" -msgstr "" +msgstr "Dompo" #. name for doz msgid "Dorze" -msgstr "" +msgstr "Dorze" #. name for dpp msgid "Papar" -msgstr "" +msgstr "Papar" #. name for drb msgid "Dair" -msgstr "" +msgstr "Dair" #. name for drc msgid "Minderico" -msgstr "" +msgstr "Minderico" #. name for drd msgid "Darmiya" -msgstr "" +msgstr "Darmiya" #. name for dre msgid "Dolpo" -msgstr "" +msgstr "Dolpo" #. name for drg msgid "Rungus" -msgstr "" +msgstr "Rungus" #. name for dri msgid "C'lela" -msgstr "" +msgstr "C'lela" #. name for drl msgid "Darling" -msgstr "" +msgstr "Darling" #. name for drn msgid "Damar; West" -msgstr "" +msgstr "Damar; West" #. name for dro msgid "Melanau; Daro-Matu" -msgstr "" +msgstr "Melanau; Daro-Matu" #. name for drq msgid "Dura" -msgstr "" +msgstr "Dura" #. name for drr msgid "Dororo" -msgstr "" +msgstr "Dororo" #. name for drs msgid "Gedeo" -msgstr "" +msgstr "Gedeo" #. name for drt msgid "Drents" -msgstr "" +msgstr "Drents" #. name for dru msgid "Rukai" -msgstr "" +msgstr "Rukai" #. name for dry msgid "Darai" -msgstr "" +msgstr "Darai" #. name for dsb msgid "Sorbian; Lower" -msgstr "" +msgstr "Sorbian; Lower" #. name for dse msgid "Dutch Sign Language" -msgstr "" +msgstr "Dutch Sign Language" #. name for dsh msgid "Daasanach" -msgstr "" +msgstr "Daasanach" #. name for dsi msgid "Disa" -msgstr "" +msgstr "Disa" #. name for dsl msgid "Danish Sign Language" -msgstr "" +msgstr "Danish Sign Language" #. name for dsn msgid "Dusner" -msgstr "" +msgstr "Dusner" #. name for dso msgid "Desiya" -msgstr "" +msgstr "Desiya" #. name for dsq msgid "Tadaksahak" -msgstr "" +msgstr "Tadaksahak" #. name for dta msgid "Daur" -msgstr "" +msgstr "Daur" #. name for dtb msgid "Kadazan; Labuk-Kinabatangan" -msgstr "" +msgstr "Kadazan; Labuk-Kinabatangan" #. name for dtd msgid "Ditidaht" -msgstr "" +msgstr "Ditidaht" #. name for dti msgid "Dogon; Ana Tinga" -msgstr "" +msgstr "Dogon; Ana Tinga" #. name for dtk msgid "Dogon; Tene Kan" -msgstr "" +msgstr "Dogon; Tene Kan" #. name for dtm msgid "Dogon; Tomo Kan" -msgstr "" +msgstr "Dogon; Tomo Kan" #. name for dtp msgid "Dusun; Central" -msgstr "" +msgstr "Dusun; Central" #. name for dtr msgid "Lotud" -msgstr "" +msgstr "Lotud" #. name for dts msgid "Dogon; Toro So" -msgstr "" +msgstr "Dogon; Toro So" #. name for dtt msgid "Dogon; Toro Tegu" -msgstr "" +msgstr "Dogon; Toro Tegu" #. name for dtu msgid "Dogon; Tebul Ure" -msgstr "" +msgstr "Dogon; Tebul Ure" #. name for dua msgid "Duala" -msgstr "" +msgstr "Duala" #. name for dub msgid "Dubli" -msgstr "" +msgstr "Dubli" #. name for duc msgid "Duna" -msgstr "" +msgstr "Duna" #. name for dud msgid "Hun-Saare" -msgstr "" +msgstr "Hun-Saare" #. name for due msgid "Agta; Umiray Dumaget" -msgstr "" +msgstr "Agta; Umiray Dumaget" #. name for duf msgid "Dumbea" -msgstr "" +msgstr "Dumbea" #. name for dug msgid "Duruma" -msgstr "" +msgstr "Duruma" #. name for duh msgid "Dungra Bhil" -msgstr "" +msgstr "Dungra Bhil" #. name for dui msgid "Dumun" -msgstr "" +msgstr "Dumun" #. name for duj msgid "Dhuwal" -msgstr "" +msgstr "Dhuwal" #. name for duk msgid "Uyajitaya" -msgstr "" +msgstr "Uyajitaya" #. name for dul msgid "Agta; Alabat Island" -msgstr "" +msgstr "Agta; Alabat Island" #. name for dum msgid "Dutch; Middle (ca. 1050-1350)" -msgstr "" +msgstr "Dutch; Middle (ca. 1050-1350)" #. name for dun msgid "Dusun Deyah" -msgstr "" +msgstr "Dusun Deyah" #. name for duo msgid "Agta; Dupaninan" -msgstr "" +msgstr "Agta; Dupaninan" #. name for dup msgid "Duano" -msgstr "" +msgstr "Duano" #. name for duq msgid "Dusun Malang" -msgstr "" +msgstr "Dusun Malang" #. name for dur msgid "Dii" -msgstr "" +msgstr "Dii" #. name for dus msgid "Dumi" -msgstr "" +msgstr "Dumi" #. name for duu msgid "Drung" -msgstr "" +msgstr "Drung" #. name for duv msgid "Duvle" -msgstr "" +msgstr "Duvle" #. name for duw msgid "Dusun Witu" -msgstr "" +msgstr "Dusun Witu" #. name for dux msgid "Duungooma" -msgstr "" +msgstr "Duungooma" #. name for duy msgid "Agta; Dicamay" -msgstr "" +msgstr "Agta; Dicamay" #. name for duz msgid "Duli" -msgstr "" +msgstr "Duli" #. name for dva msgid "Duau" -msgstr "" +msgstr "Duau" #. name for dwa msgid "Diri" -msgstr "" +msgstr "Diri" #. name for dwl msgid "Dogon; Walo Kumbe" -msgstr "" +msgstr "Dogon; Walo Kumbe" #. name for dwr msgid "Dawro" -msgstr "" +msgstr "Dawro" #. name for dws msgid "Dutton World Speedwords" -msgstr "" +msgstr "Dutton World Speedwords" #. name for dww msgid "Dawawa" -msgstr "" +msgstr "Dawawa" #. name for dya msgid "Dyan" -msgstr "" +msgstr "Dyan" #. name for dyb msgid "Dyaberdyaber" -msgstr "" +msgstr "Dyaberdyaber" #. name for dyd msgid "Dyugun" -msgstr "" +msgstr "Dyugun" #. name for dyg msgid "Agta; Villa Viciosa" -msgstr "" +msgstr "Agta; Villa Viciosa" #. name for dyi msgid "Senoufo; Djimini" -msgstr "" +msgstr "Senoufo; Djimini" #. name for dym msgid "Dogon; Yanda Dom" -msgstr "" +msgstr "Dogon; Yanda Dom" #. name for dyn msgid "Dyangadi" -msgstr "" +msgstr "Dyangadi" #. name for dyo msgid "Jola-Fonyi" -msgstr "" +msgstr "Jola-Fonyi" #. name for dyu msgid "Dyula" -msgstr "" +msgstr "Dyula" #. name for dyy msgid "Dyaabugay" -msgstr "" +msgstr "Dyaabugay" #. name for dza msgid "Tunzu" -msgstr "" +msgstr "Tunzu" #. name for dzd msgid "Daza" -msgstr "" +msgstr "Daza" #. name for dzg msgid "Dazaga" -msgstr "" +msgstr "Dazaga" #. name for dzl msgid "Dzalakha" -msgstr "" +msgstr "Dzalakha" #. name for dzn msgid "Dzando" -msgstr "" +msgstr "Dzando" #. name for dzo msgid "Dzongkha" -msgstr "" +msgstr "Dzongkha" #. name for ebg msgid "Ebughu" -msgstr "" +msgstr "Ebughu" #. name for ebk msgid "Bontok; Eastern" -msgstr "" +msgstr "Bontok; Eastern" #. name for ebo msgid "Teke-Ebo" -msgstr "" +msgstr "Teke-Ebo" #. name for ebr msgid "Ebrié" -msgstr "" +msgstr "Ebrié" #. name for ebu msgid "Embu" -msgstr "" +msgstr "Embu" #. name for ecr msgid "Eteocretan" -msgstr "" +msgstr "Eteocretan" #. name for ecs msgid "Ecuadorian Sign Language" -msgstr "" +msgstr "Ecuadorian Sign Language" #. name for ecy msgid "Eteocypriot" -msgstr "" +msgstr "Eteocypriot" #. name for eee msgid "E" -msgstr "" +msgstr "E" #. name for efa msgid "Efai" -msgstr "" +msgstr "Efai" #. name for efe msgid "Efe" -msgstr "" +msgstr "Efe" #. name for efi msgid "Efik" -msgstr "" +msgstr "Efik" #. name for ega msgid "Ega" -msgstr "" +msgstr "Ega" #. name for egl msgid "Emilian" -msgstr "" +msgstr "Emilian" #. name for ego msgid "Eggon" -msgstr "" +msgstr "Eggon" #. name for egy msgid "Egyptian (Ancient)" -msgstr "" +msgstr "Egyptian (Ancient)" #. name for ehu msgid "Ehueun" -msgstr "" +msgstr "Ehueun" #. name for eip msgid "Eipomek" -msgstr "" +msgstr "Eipomek" #. name for eit msgid "Eitiep" -msgstr "" +msgstr "Eitiep" #. name for eiv msgid "Askopan" -msgstr "" +msgstr "Askopan" #. name for eja msgid "Ejamat" -msgstr "" +msgstr "Ejamat" #. name for eka msgid "Ekajuk" -msgstr "" +msgstr "Ekajuk" #. name for eke msgid "Ekit" -msgstr "" +msgstr "Ekit" #. name for ekg msgid "Ekari" -msgstr "" +msgstr "Ekari" #. name for eki msgid "Eki" diff --git a/setup/iso_639/nl.po b/setup/iso_639/nl.po index 09c011a93a..170c434536 100644 --- a/setup/iso_639/nl.po +++ b/setup/iso_639/nl.po @@ -12,14 +12,14 @@ msgstr "" "Report-Msgid-Bugs-To: Debian iso-codes team \n" "POT-Creation-Date: 2011-11-25 14:01+0000\n" -"PO-Revision-Date: 2011-11-03 23:08+0000\n" +"PO-Revision-Date: 2012-02-01 20:12+0000\n" "Last-Translator: drMerry \n" "Language-Team: Dutch \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Launchpad-Export-Date: 2011-11-26 05:12+0000\n" -"X-Generator: Launchpad (build 14381)\n" +"X-Launchpad-Export-Date: 2012-02-02 05:57+0000\n" +"X-Generator: Launchpad (build 14738)\n" "Language: nl\n" #. name for aaa @@ -17956,7 +17956,7 @@ msgstr "" #. name for nds msgid "German; Low" -msgstr "" +msgstr "Duits; Laag" #. name for ndt msgid "Ndunga" @@ -30424,7 +30424,7 @@ msgstr "" #. name for zlm msgid "Malay (individual language)" -msgstr "" +msgstr "Maleis (aparte taal)" #. name for zln msgid "Zhuang; Lianshan" diff --git a/setup/translations.py b/setup/translations.py index 5cfae07711..05056c0b27 100644 --- a/setup/translations.py +++ b/setup/translations.py @@ -151,7 +151,7 @@ class Translations(POT): # {{{ self.info('\tCopying ISO 639 translations') subprocess.check_call(['msgfmt', '-o', dest, iso639]) elif locale not in ('en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', - 'ltg', 'nds', 'te', 'yi', 'fo', 'sq', 'ast', 'ml'): + 'ltg', 'nds', 'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku'): self.warn('No ISO 639 translations for locale:', locale) self.write_stats() diff --git a/src/calibre/constants.py b/src/calibre/constants.py index a5f41f5404..5269ea52ba 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 8, 35) +numeric_version = (0, 8, 38) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " @@ -161,4 +161,32 @@ def get_version(): v += '*' return v +def get_unicode_windows_env_var(name): + import ctypes + name = unicode(name) + n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0) + if n == 0: + return None + buf = ctypes.create_unicode_buffer(u'\0'*n) + ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n) + return buf.value +def get_windows_username(): + ''' + Return the user name of the currently loggen in user as a unicode string. + Note that usernames on windows are case insensitive, the case of the value + returned depends on what the user typed into the login box at login time. + ''' + import ctypes + try: + advapi32 = ctypes.windll.advapi32 + GetUserName = getattr(advapi32, u'GetUserNameW') + except AttributeError: + pass + else: + buf = ctypes.create_unicode_buffer(257) + n = ctypes.c_int(257) + if GetUserName(buf, ctypes.byref(n)): + return buf.value + + return get_unicode_windows_env_var(u'USERNAME') diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 4cd43d9973..855d105e15 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -5,13 +5,14 @@ __copyright__ = '2008, Kovid Goyal ' import os, glob, functools, re from calibre import guess_type -from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \ - MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase, StoreBase +from calibre.customize import (FileTypePlugin, MetadataReaderPlugin, + MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase, StoreBase) from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata -from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ebooks.html.to_zip import HTML2ZIP +plugins = [] + # To archive plugins {{{ class PML2PMLZ(FileTypePlugin): @@ -86,6 +87,8 @@ class TXT2TXTZ(FileTypePlugin): return list(set(images)) def run(self, path_to_ebook): + from calibre.ebooks.metadata.opf2 import metadata_to_opf + with open(path_to_ebook, 'rb') as ebf: txt = ebf.read() base_dir = os.path.dirname(path_to_ebook) @@ -117,6 +120,7 @@ class TXT2TXTZ(FileTypePlugin): # No images so just import the TXT file. return path_to_ebook +plugins += [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract,] # }}} # Metadata reader plugins {{{ @@ -399,6 +403,10 @@ class ZipMetadataReader(MetadataReaderPlugin): def get_metadata(self, stream, ftype): from calibre.ebooks.metadata.zip import get_metadata return get_metadata(stream) + +plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ + x.__name__.endswith('MetadataReader')] + # }}} # Metadata writer plugins {{{ @@ -499,107 +507,51 @@ class TXTZMetadataWriter(MetadataWriterPlugin): from calibre.ebooks.metadata.extz import set_metadata set_metadata(stream, mi) -# }}} - -from calibre.ebooks.comic.input import ComicInput -from calibre.ebooks.djvu.input import DJVUInput -from calibre.ebooks.epub.input import EPUBInput -from calibre.ebooks.fb2.input import FB2Input -from calibre.ebooks.html.input import HTMLInput -from calibre.ebooks.htmlz.input import HTMLZInput -from calibre.ebooks.lit.input import LITInput -from calibre.ebooks.mobi.input import MOBIInput -from calibre.ebooks.odt.input import ODTInput -from calibre.ebooks.pdb.input import PDBInput -from calibre.ebooks.azw4.input import AZW4Input -from calibre.ebooks.pdf.input import PDFInput -from calibre.ebooks.pml.input import PMLInput -from calibre.ebooks.rb.input import RBInput -from calibre.web.feeds.input import RecipeInput -from calibre.ebooks.rtf.input import RTFInput -from calibre.ebooks.tcr.input import TCRInput -from calibre.ebooks.txt.input import TXTInput -from calibre.ebooks.lrf.input import LRFInput -from calibre.ebooks.chm.input import CHMInput -from calibre.ebooks.snb.input import SNBInput - -from calibre.ebooks.epub.output import EPUBOutput -from calibre.ebooks.fb2.output import FB2Output -from calibre.ebooks.lit.output import LITOutput -from calibre.ebooks.lrf.output import LRFOutput -from calibre.ebooks.mobi.output import MOBIOutput -from calibre.ebooks.oeb.output import OEBOutput -from calibre.ebooks.pdb.output import PDBOutput -from calibre.ebooks.pdf.output import PDFOutput -from calibre.ebooks.pml.output import PMLOutput -from calibre.ebooks.rb.output import RBOutput -from calibre.ebooks.rtf.output import RTFOutput -from calibre.ebooks.tcr.output import TCROutput -from calibre.ebooks.txt.output import TXTOutput -from calibre.ebooks.txt.output import TXTZOutput -from calibre.ebooks.html.output import HTMLOutput -from calibre.ebooks.htmlz.output import HTMLZOutput -from calibre.ebooks.snb.output import SNBOutput - -from calibre.customize.profiles import input_profiles, output_profiles - -from calibre.devices.apple.driver import ITUNES -from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA -from calibre.devices.blackberry.driver import BLACKBERRY, PLAYBOOK -from calibre.devices.cybook.driver import CYBOOK, ORIZON -from calibre.devices.eb600.driver import (EB600, COOL_ER, SHINEBOOK, - POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK, - BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602, - POCKETBOOK701, POCKETBOOK360P, PI2) -from calibre.devices.iliad.driver import ILIAD -from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800 -from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI -from calibre.devices.kindle.driver import (KINDLE, KINDLE2, KINDLE_DX, - KINDLE_FIRE) -from calibre.devices.nook.driver import NOOK, NOOK_COLOR -from calibre.devices.prs505.driver import PRS505 -from calibre.devices.prst1.driver import PRST1 -from calibre.devices.user_defined.driver import USER_DEFINED -from calibre.devices.android.driver import ANDROID, S60, WEBOS -from calibre.devices.nokia.driver import N770, N810, E71X, E52 -from calibre.devices.eslick.driver import ESLICK, EBK52 -from calibre.devices.nuut2.driver import NUUT2 -from calibre.devices.iriver.driver import IRIVER_STORY -from calibre.devices.binatone.driver import README -from calibre.devices.hanvon.driver import (N516, EB511, ALEX, AZBOOKA, THEBOOK, - LIBREAIR, ODYSSEY) -from calibre.devices.edge.driver import EDGE -from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS, - SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER) -from calibre.devices.sne.driver import SNE -from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL, - GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, - TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK, COBY, EX124G) -from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG -from calibre.devices.kobo.driver import KOBO -from calibre.devices.bambook.driver import BAMBOOK -from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX - -from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX -from calibre.ebooks.epub.fix.unmanifested import Unmanifested -from calibre.ebooks.epub.fix.epubcheck import Epubcheck - -plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested, - Epubcheck, ] - -# New metadata download plugins {{{ -from calibre.ebooks.metadata.sources.google import GoogleBooks -from calibre.ebooks.metadata.sources.amazon import Amazon -from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary -from calibre.ebooks.metadata.sources.isbndb import ISBNDB -from calibre.ebooks.metadata.sources.overdrive import OverDrive -from calibre.ebooks.metadata.sources.douban import Douban -from calibre.ebooks.metadata.sources.ozon import Ozon - -plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive, Douban, Ozon] +plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ + x.__name__.endswith('MetadataWriter')] # }}} +# Conversion plugins {{{ +from calibre.ebooks.conversion.plugins.comic_input import ComicInput +from calibre.ebooks.conversion.plugins.djvu_input import DJVUInput +from calibre.ebooks.conversion.plugins.epub_input import EPUBInput +from calibre.ebooks.conversion.plugins.fb2_input import FB2Input +from calibre.ebooks.conversion.plugins.html_input import HTMLInput +from calibre.ebooks.conversion.plugins.htmlz_input import HTMLZInput +from calibre.ebooks.conversion.plugins.lit_input import LITInput +from calibre.ebooks.conversion.plugins.mobi_input import MOBIInput +from calibre.ebooks.conversion.plugins.odt_input import ODTInput +from calibre.ebooks.conversion.plugins.pdb_input import PDBInput +from calibre.ebooks.conversion.plugins.azw4_input import AZW4Input +from calibre.ebooks.conversion.plugins.pdf_input import PDFInput +from calibre.ebooks.conversion.plugins.pml_input import PMLInput +from calibre.ebooks.conversion.plugins.rb_input import RBInput +from calibre.ebooks.conversion.plugins.recipe_input import RecipeInput +from calibre.ebooks.conversion.plugins.rtf_input import RTFInput +from calibre.ebooks.conversion.plugins.tcr_input import TCRInput +from calibre.ebooks.conversion.plugins.txt_input import TXTInput +from calibre.ebooks.conversion.plugins.lrf_input import LRFInput +from calibre.ebooks.conversion.plugins.chm_input import CHMInput +from calibre.ebooks.conversion.plugins.snb_input import SNBInput + +from calibre.ebooks.conversion.plugins.epub_output import EPUBOutput +from calibre.ebooks.conversion.plugins.fb2_output import FB2Output +from calibre.ebooks.conversion.plugins.lit_output import LITOutput +from calibre.ebooks.conversion.plugins.lrf_output import LRFOutput +from calibre.ebooks.conversion.plugins.mobi_output import MOBIOutput +from calibre.ebooks.conversion.plugins.oeb_output import OEBOutput +from calibre.ebooks.conversion.plugins.pdb_output import PDBOutput +from calibre.ebooks.conversion.plugins.pdf_output import PDFOutput +from calibre.ebooks.conversion.plugins.pml_output import PMLOutput +from calibre.ebooks.conversion.plugins.rb_output import RBOutput +from calibre.ebooks.conversion.plugins.rtf_output import RTFOutput +from calibre.ebooks.conversion.plugins.tcr_output import TCROutput +from calibre.ebooks.conversion.plugins.txt_output import TXTOutput, TXTZOutput +from calibre.ebooks.conversion.plugins.html_output import HTMLOutput +from calibre.ebooks.conversion.plugins.htmlz_output import HTMLZOutput +from calibre.ebooks.conversion.plugins.snb_output import SNBOutput + plugins += [ ComicInput, DJVUInput, @@ -642,6 +594,66 @@ plugins += [ HTMLZOutput, SNBOutput, ] +# }}} + +# Catalog plugins {{{ +from calibre.library.catalogs.csv_xml import CSV_XML +from calibre.library.catalogs.bibtex import BIBTEX +from calibre.library.catalogs.epub_mobi import EPUB_MOBI +plugins += [CSV_XML, BIBTEX, EPUB_MOBI] +# }}} + +# EPUB Fix plugins {{{ +from calibre.ebooks.epub.fix.unmanifested import Unmanifested +from calibre.ebooks.epub.fix.epubcheck import Epubcheck +plugins += [Unmanifested, Epubcheck] +# }}} + +# Profiles {{{ +from calibre.customize.profiles import input_profiles, output_profiles +plugins += input_profiles + output_profiles +# }}} + +# Device driver plugins {{{ +from calibre.devices.apple.driver import ITUNES +from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA +from calibre.devices.blackberry.driver import BLACKBERRY, PLAYBOOK +from calibre.devices.cybook.driver import CYBOOK, ORIZON +from calibre.devices.eb600.driver import (EB600, COOL_ER, SHINEBOOK, + POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK, + BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602, + POCKETBOOK701, POCKETBOOK360P, PI2) +from calibre.devices.iliad.driver import ILIAD +from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800 +from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI +from calibre.devices.kindle.driver import (KINDLE, KINDLE2, KINDLE_DX, + KINDLE_FIRE) +from calibre.devices.nook.driver import NOOK, NOOK_COLOR +from calibre.devices.prs505.driver import PRS505 +from calibre.devices.prst1.driver import PRST1 +from calibre.devices.user_defined.driver import USER_DEFINED +from calibre.devices.android.driver import ANDROID, S60, WEBOS +from calibre.devices.nokia.driver import N770, N810, E71X, E52 +from calibre.devices.eslick.driver import ESLICK, EBK52 +from calibre.devices.nuut2.driver import NUUT2 +from calibre.devices.iriver.driver import IRIVER_STORY +from calibre.devices.binatone.driver import README +from calibre.devices.hanvon.driver import (N516, EB511, ALEX, AZBOOKA, THEBOOK, + LIBREAIR, ODYSSEY) +from calibre.devices.edge.driver import EDGE +from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS, + SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER) +from calibre.devices.sne.driver import SNE +from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL, + GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, + TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK, COBY, EX124G) +from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG +from calibre.devices.kobo.driver import KOBO +from calibre.devices.bambook.driver import BAMBOOK +from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX + + + # Order here matters. The first matched device is the one used. plugins += [ HANLINV3, @@ -716,11 +728,20 @@ plugins += [ BOEYE_BDX, USER_DEFINED, ] -plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ - x.__name__.endswith('MetadataReader')] -plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ - x.__name__.endswith('MetadataWriter')] -plugins += input_profiles + output_profiles +# }}} + +# New metadata download plugins {{{ +from calibre.ebooks.metadata.sources.google import GoogleBooks +from calibre.ebooks.metadata.sources.amazon import Amazon +from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary +from calibre.ebooks.metadata.sources.isbndb import ISBNDB +from calibre.ebooks.metadata.sources.overdrive import OverDrive +from calibre.ebooks.metadata.sources.douban import Douban +from calibre.ebooks.metadata.sources.ozon import Ozon + +plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive, Douban, Ozon] + +# }}} # Interface Actions {{{ @@ -1508,6 +1529,7 @@ class StoreVirtualoStore(StoreBase): headquarters = 'PL' formats = ['EPUB', 'MOBI', 'PDF'] + affiliate = True class StoreWaterstonesUKStore(StoreBase): name = 'Waterstones UK' @@ -1622,3 +1644,34 @@ plugins += [ ] # }}} + +if __name__ == '__main__': + # Test load speed + import subprocess, textwrap + try: + subprocess.check_call(['python', '-c', textwrap.dedent( + ''' + from __future__ import print_function + import time, sys, init_calibre + st = time.time() + import calibre.customize.builtins + t = time.time() - st + ret = 0 + + for x in ('lxml', 'calibre.ebooks.BeautifulSoup', 'uuid', + 'calibre.utils.terminfo', 'calibre.utils.magick', 'PIL', 'Image', + 'sqlite3', 'mechanize', 'httplib', 'xml'): + if x in sys.modules: + ret = 1 + print (x, 'has been loaded by a plugin') + if ret: + print ('\\nA good way to track down what is loading something is to run' + ' python -c "import init_calibre; import calibre.customize.builtins"') + print() + print ('Time taken to import all plugins: %.2f'%t) + sys.exit(ret) + + ''')]) + except subprocess.CalledProcessError: + raise SystemExit(1) + diff --git a/src/calibre/customize/profiles.py b/src/calibre/customize/profiles.py index 8bb0e55f5e..f6ed6ce3ec 100644 --- a/src/calibre/customize/profiles.py +++ b/src/calibre/customize/profiles.py @@ -5,7 +5,6 @@ __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' from itertools import izip -from xml.sax.saxutils import escape from calibre.customize import Plugin as _Plugin @@ -268,6 +267,7 @@ class OutputProfile(Plugin): @classmethod def tags_to_string(cls, tags): + from xml.sax.saxutils import escape return escape(', '.join(tags)) class iPadOutput(OutputProfile): diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index e309533235..b365eb1346 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -447,11 +447,14 @@ def plugin_for_catalog_format(fmt): # }}} -def device_plugins(): # {{{ +def device_plugins(include_disabled=False): # {{{ for plugin in _initialized_plugins: if isinstance(plugin, DevicePlugin): - if not is_disabled(plugin): + if include_disabled or not is_disabled(plugin): if platform in plugin.supported_platforms: + if getattr(plugin, 'plugin_needs_delayed_initialization', + False): + plugin.do_delayed_plugin_initialization() yield plugin # }}} @@ -496,7 +499,7 @@ def initialize_plugin(plugin, path_to_zip_file): def has_external_plugins(): return bool(config['plugins']) -def initialize_plugins(): +def initialize_plugins(perf=False): global _initialized_plugins _initialized_plugins = [] conflicts = [name for name in config['plugins'] if name in @@ -504,6 +507,11 @@ def initialize_plugins(): for p in conflicts: remove_plugin(p) external_plugins = config['plugins'] + ostdout, ostderr = sys.stdout, sys.stderr + if perf: + from collections import defaultdict + import time + times = defaultdict(lambda:0) for zfp in list(external_plugins) + builtin_plugins: try: if not isinstance(zfp, type): @@ -516,12 +524,22 @@ def initialize_plugins(): plugin = load_plugin(zfp) if not isinstance(zfp, type) else zfp except PluginNotFound: continue + if perf: + st = time.time() plugin = initialize_plugin(plugin, None if isinstance(zfp, type) else zfp) + if perf: + times[plugin.name] = time.time() - st _initialized_plugins.append(plugin) except: print 'Failed to initialize plugin:', repr(zfp) if DEBUG: traceback.print_exc() + # Prevent a custom plugin from overriding stdout/stderr as this breaks + # ipython + sys.stdout, sys.stderr = ostdout, ostderr + if perf: + for x in sorted(times, key=lambda x:times[x]): + print ('%50s: %.3f'%(x, times[x])) _initialized_plugins.sort(cmp=lambda x,y:cmp(x.priority, y.priority), reverse=True) reread_filetype_plugins() reread_metadata_plugins() diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index d9a7578f6a..97e494b2dd 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -38,6 +38,8 @@ class ANDROID(USBMS): 0xca4 : [0x100, 0x0227, 0x0226, 0x222], 0xca9 : [0x100, 0x0227, 0x0226, 0x222], 0xcac : [0x100, 0x0227, 0x0226, 0x222], + 0xccf : [0x100, 0x0227, 0x0226, 0x222], + 0x2910 : [0x222], }, # Eken @@ -51,6 +53,7 @@ class ANDROID(USBMS): 0x70c6 : [0x226], 0x4316 : [0x216], 0x42d6 : [0x216], + 0x42d7 : [0x216], }, # Freescale 0x15a2 : { @@ -162,7 +165,7 @@ class ANDROID(USBMS): 'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS', 'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA', 'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON', - 'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO'] + 'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', @@ -175,13 +178,14 @@ class ANDROID(USBMS): 'GT-S5830_CARD', 'GT-S5570_CARD', 'MB870', 'MID7015A', 'ALPANDIGITAL', 'ANDROID_MID', 'VTAB1008', 'EMX51_BBG_ANDROI', 'UMS', '.K080', 'P990', 'LTE', 'MB853', 'GT-S5660_CARD', 'A107', - 'GT-I9003_CARD', 'XT912', 'FILE-CD_GADGET'] + 'GT-I9003_CARD', 'XT912', 'FILE-CD_GADGET', 'RK29_SDK', 'MB855', + 'XT910'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', '__UMS_COMPOSITE', 'SGH-I997_CARD', 'MB870', 'ALPANDIGITAL', 'ANDROID_MID', 'P990_SD_CARD', '.K080', 'LTE_CARD', 'MB853', - 'A1-07___C0541A4F', 'XT912'] + 'A1-07___C0541A4F', 'XT912', 'MB855', 'XT910'] OSX_MAIN_MEM = 'Android Device Main Memory' @@ -220,6 +224,20 @@ class ANDROID(USBMS): drives['main'] = letter_a return drives + @classmethod + def configure_for_kindle_app(cls): + proxy = cls._configProxy() + proxy['format_map'] = ['mobi', 'azw', 'azw1', 'azw4', 'pdf'] + proxy['use_subdirs'] = False + proxy['extra_customization'] = ','.join(['kindle']+cls.EBOOK_DIR_MAIN) + + @classmethod + def configure_for_generic_epub_app(cls): + proxy = cls._configProxy() + del proxy['format_map'] + del proxy['use_subdirs'] + del proxy['extra_customization'] + class S60(USBMS): name = 'S60 driver' diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index b6d258ad81..524a62224f 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -8,27 +8,39 @@ __docformat__ = 'restructuredtext en' import cStringIO, ctypes, datetime, os, re, shutil, sys, tempfile, time from calibre.constants import __appname__, __version__, DEBUG -from calibre import fit_image, confirm_config_name +from calibre import fit_image, confirm_config_name, strftime as _strftime from calibre.constants import isosx, iswindows from calibre.devices.errors import OpenFeedback, UserFeedback from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.devices.interface import DevicePlugin -from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.metadata import authors_to_string, MetaInformation, title_sort from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.metadata.epub import set_metadata -from calibre.library.server.utils import strftime from calibre.utils.config import config_dir, dynamic, prefs from calibre.utils.date import now, parse_date -from calibre.utils.logging import Log from calibre.utils.zipfile import ZipFile +def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None): + + if not hasattr(dt, 'timetuple'): + dt = now() + dt = dt.timetuple() + try: + return _strftime(fmt, dt) + except: + return _strftime(fmt, now().timetuple()) + +_log = None +def logger(): + global _log + if _log is None: + from calibre.utils.logging import ThreadSafeLog + _log = ThreadSafeLog() + return _log class AppleOpenFeedback(OpenFeedback): def __init__(self, plugin): OpenFeedback.__init__(self, u'') - self.log = plugin.log self.plugin = plugin def custom_dialog(self, parent): @@ -78,19 +90,18 @@ class AppleOpenFeedback(OpenFeedback): self.finished.connect(self.do_it) def do_it(self, return_code): + from calibre.utils.logging import default_log if return_code == self.Accepted: - self.cd.log.info(" Apple driver ENABLED") + default_log.info(" Apple driver ENABLED") dynamic[confirm_config_name(self.cd.plugin.DISPLAY_DISABLE_DIALOG)] = False else: from calibre.customize.ui import disable_plugin - self.cd.log.info(" Apple driver DISABLED") + default_log.info(" Apple driver DISABLED") disable_plugin(self.cd.plugin) return Dialog(parent, self) -from PIL import Image as PILImage -from lxml import etree if isosx: try: @@ -297,7 +308,6 @@ class ITUNES(DriverBase): iTunes= None iTunes_local_storage = None library_orphans = None - log = Log() manual_sync_mode = False path_template = 'iTunes/%s - %s.%s' plugboards = None @@ -323,7 +333,7 @@ class ITUNES(DriverBase): L{books}(oncard='cardb')). ''' if DEBUG: - self.log.info("ITUNES.add_books_to_metadata()") + logger().info("ITUNES.add_books_to_metadata()") task_count = float(len(self.update_list)) @@ -337,10 +347,10 @@ class ITUNES(DriverBase): for (j,p_book) in enumerate(self.update_list): if False: if isosx: - self.log.info(" looking for '%s' by %s uuid:%s" % + logger().info(" looking for '%s' by %s uuid:%s" % (p_book['title'],p_book['author'], p_book['uuid'])) elif iswindows: - self.log.info(" looking for '%s' by %s (%s)" % + logger().info(" looking for '%s' by %s (%s)" % (p_book['title'],p_book['author'], p_book['uuid'])) # Purge the booklist, self.cached_books @@ -350,10 +360,10 @@ class ITUNES(DriverBase): booklists[0].pop(i) if False: if isosx: - self.log.info(" removing old %s %s from booklists[0]" % + logger().info(" removing old %s %s from booklists[0]" % (p_book['title'], str(p_book['lib_book'])[-9:])) elif iswindows: - self.log.info(" removing old '%s' from booklists[0]" % + logger().info(" removing old '%s' from booklists[0]" % (p_book['title'])) # If >1 matching uuid, remove old title @@ -383,7 +393,7 @@ class ITUNES(DriverBase): # for new_book in metadata[0]: for new_book in locations[0]: if DEBUG: - self.log.info(" adding '%s' by '%s' to booklists[0]" % + logger().info(" adding '%s' by '%s' to booklists[0]" % (new_book.title, new_book.author)) booklists[0].append(new_book) @@ -408,15 +418,15 @@ class ITUNES(DriverBase): """ if not oncard: if DEBUG: - self.log.info("ITUNES:books():") + logger().info("ITUNES:books():") if self.settings().extra_customization[self.CACHE_COVERS]: - self.log.info(" Cover fetching/caching enabled") + logger().info(" Cover fetching/caching enabled") else: - self.log.info(" Cover fetching/caching disabled") + logger().info(" Cover fetching/caching disabled") # Fetch a list of books from iPod device connected to iTunes if 'iPod' in self.sources: - booklist = BookList(self.log) + booklist = BookList(logger()) cached_books = {} if isosx: @@ -507,7 +517,7 @@ class ITUNES(DriverBase): self._dump_cached_books('returning from books()',indent=2) return booklist else: - return BookList(self.log) + return BookList(logger()) def can_handle(self, device_info, debug=False): ''' @@ -544,7 +554,7 @@ class ITUNES(DriverBase): # We need to know if iTunes sees the iPad # It may have been ejected if DEBUG: - self.log.info("ITUNES.can_handle()") + logger().info("ITUNES.can_handle()") self._launch_iTunes() self.sources = self._get_sources() @@ -557,15 +567,15 @@ class ITUNES(DriverBase): attempts -= 1 time.sleep(0.5) if DEBUG: - self.log.warning(" waiting for connected iPad, attempt #%d" % (10 - attempts)) + logger().warning(" waiting for connected iPad, attempt #%d" % (10 - attempts)) else: if DEBUG: - self.log.info(' found connected iPad') + logger().info(' found connected iPad') break else: # iTunes running, but not connected iPad if DEBUG: - self.log.info(' self.ejected = True') + logger().info(' self.ejected = True') self.ejected = True return False @@ -599,26 +609,26 @@ class ITUNES(DriverBase): sys.stdout.write('.') sys.stdout.flush() if DEBUG: - self.log.info('ITUNES.can_handle_windows:\n confirming connected iPad') + logger().info('ITUNES.can_handle_windows:\n confirming connected iPad') self.ejected = False self._discover_manual_sync_mode() return True else: if DEBUG: - self.log.info("ITUNES.can_handle_windows():\n device ejected") + logger().info("ITUNES.can_handle_windows():\n device ejected") self.ejected = True return False except: # iTunes connection failed, probably not running anymore - self.log.error("ITUNES.can_handle_windows():\n lost connection to iTunes") + logger().error("ITUNES.can_handle_windows():\n lost connection to iTunes") return False finally: pythoncom.CoUninitialize() else: if DEBUG: - self.log.info("ITUNES:can_handle_windows():\n Launching iTunes") + logger().info("ITUNES:can_handle_windows():\n Launching iTunes") try: pythoncom.CoInitialize() @@ -633,19 +643,19 @@ class ITUNES(DriverBase): attempts -= 1 time.sleep(0.5) if DEBUG: - self.log.warning(" waiting for connected iPad, attempt #%d" % (10 - attempts)) + logger().warning(" waiting for connected iPad, attempt #%d" % (10 - attempts)) else: if DEBUG: - self.log.info(' found connected iPad in iTunes') + logger().info(' found connected iPad in iTunes') break else: # iTunes running, but not connected iPad if DEBUG: - self.log.info(' iDevice has been ejected') + logger().info(' iDevice has been ejected') self.ejected = True return False - self.log.info(' found connected iPad in sources') + logger().info(' found connected iPad in sources') self._discover_manual_sync_mode(wait=1.0) finally: @@ -688,11 +698,11 @@ class ITUNES(DriverBase): self.problem_msg = _("Some books not found in iTunes database.\n" "Delete using the iBooks app.\n" "Click 'Show Details' for a list.") - self.log.info("ITUNES:delete_books()") + logger().info("ITUNES:delete_books()") for path in paths: if self.cached_books[path]['lib_book']: if DEBUG: - self.log.info(" Deleting '%s' from iTunes library" % (path)) + logger().info(" Deleting '%s' from iTunes library" % (path)) if isosx: self._remove_from_iTunes(self.cached_books[path]) @@ -712,7 +722,7 @@ class ITUNES(DriverBase): self.update_needed = True self.update_msg = "Deleted books from device" else: - self.log.info(" skipping sync phase, manual_sync_mode: True") + logger().info(" skipping sync phase, manual_sync_mode: True") else: if self.manual_sync_mode: metadata = MetaInformation(self.cached_books[path]['title'], @@ -739,7 +749,7 @@ class ITUNES(DriverBase): are pending GUI jobs that need to communicate with the device. ''' if DEBUG: - self.log.info("ITUNES:eject(): ejecting '%s'" % self.sources['iPod']) + logger().info("ITUNES:eject(): ejecting '%s'" % self.sources['iPod']) if isosx: self.iTunes.eject(self.sources['iPod']) elif iswindows: @@ -768,7 +778,7 @@ class ITUNES(DriverBase): In Windows, a sync-in-progress blocks this call until sync is complete """ if DEBUG: - self.log.info("ITUNES:free_space()") + logger().info("ITUNES:free_space()") free_space = 0 if isosx: @@ -790,7 +800,7 @@ class ITUNES(DriverBase): pythoncom.CoUninitialize() break except: - self.log.error(' waiting for free_space() call to go through') + logger().error(' waiting for free_space() call to go through') return (free_space,-1,-1) @@ -800,7 +810,7 @@ class ITUNES(DriverBase): @return: (device name, device version, software version on device, mime type) """ if DEBUG: - self.log.info("ITUNES:get_device_information()") + logger().info("ITUNES:get_device_information()") return (self.sources['iPod'],'hw v1.0','sw v1.0', 'mime type normally goes here') @@ -810,7 +820,7 @@ class ITUNES(DriverBase): @param outfile: file object like C{sys.stdout} or the result of an C{open} call ''' if DEBUG: - self.log.info("ITUNES.get_file(): exporting '%s'" % path) + logger().info("ITUNES.get_file(): exporting '%s'" % path) outfile.write(open(self.cached_books[path]['lib_book'].location().path).read()) @@ -830,7 +840,7 @@ class ITUNES(DriverBase): ''' if DEBUG: - self.log.info("ITUNES.open(connected_device: %s)" % repr(connected_device)) + logger().info("ITUNES.open(connected_device: %s)" % repr(connected_device)) # Display a dialog recommending using 'Connect to iTunes' if user hasn't # previously disabled the dialog @@ -838,33 +848,33 @@ class ITUNES(DriverBase): raise AppleOpenFeedback(self) else: if DEBUG: - self.log.warning(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE) + logger().warning(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE) # Confirm/create thumbs archive if not os.path.exists(self.cache_dir): if DEBUG: - self.log.info(" creating thumb cache at '%s'" % self.cache_dir) + logger().info(" creating thumb cache at '%s'" % self.cache_dir) os.makedirs(self.cache_dir) if not os.path.exists(self.archive_path): - self.log.info(" creating zip archive") + logger().info(" creating zip archive") zfw = ZipFile(self.archive_path, mode='w') zfw.writestr("iTunes Thumbs Archive",'') zfw.close() else: if DEBUG: - self.log.info(" existing thumb cache at '%s'" % self.archive_path) + logger().info(" existing thumb cache at '%s'" % self.archive_path) # If enabled in config options, create/confirm an iTunes storage folder if not self.settings().extra_customization[self.USE_ITUNES_STORAGE]: self.iTunes_local_storage = os.path.join(config_dir,'iTunes storage') if not os.path.exists(self.iTunes_local_storage): if DEBUG: - self.log(" creating iTunes_local_storage at '%s'" % self.iTunes_local_storage) + logger()(" creating iTunes_local_storage at '%s'" % self.iTunes_local_storage) os.mkdir(self.iTunes_local_storage) else: if DEBUG: - self.log(" existing iTunes_local_storage at '%s'" % self.iTunes_local_storage) + logger()(" existing iTunes_local_storage at '%s'" % self.iTunes_local_storage) def remove_books_from_metadata(self, paths, booklists): ''' @@ -879,11 +889,11 @@ class ITUNES(DriverBase): as uuids are different ''' if DEBUG: - self.log.info("ITUNES.remove_books_from_metadata()") + logger().info("ITUNES.remove_books_from_metadata()") for path in paths: if DEBUG: self._dump_cached_book(self.cached_books[path], indent=2) - self.log.info(" looking for '%s' by '%s' uuid:%s" % + logger().info(" looking for '%s' by '%s' uuid:%s" % (self.cached_books[path]['title'], self.cached_books[path]['author'], self.cached_books[path]['uuid'])) @@ -891,19 +901,19 @@ class ITUNES(DriverBase): # Purge the booklist, self.cached_books, thumb cache for i,bl_book in enumerate(booklists[0]): if False: - self.log.info(" evaluating '%s' by '%s' uuid:%s" % + logger().info(" evaluating '%s' by '%s' uuid:%s" % (bl_book.title, bl_book.author,bl_book.uuid)) found = False if bl_book.uuid == self.cached_books[path]['uuid']: if False: - self.log.info(" matched with uuid") + logger().info(" matched with uuid") booklists[0].pop(i) found = True elif bl_book.title == self.cached_books[path]['title'] and \ bl_book.author[0] == self.cached_books[path]['author']: if False: - self.log.info(" matched with title + author") + logger().info(" matched with title + author") booklists[0].pop(i) found = True @@ -924,17 +934,17 @@ class ITUNES(DriverBase): thumb = None if thumb: if DEBUG: - self.log.info(" deleting '%s' from cover cache" % (thumb_path)) + logger().info(" deleting '%s' from cover cache" % (thumb_path)) zf.delete(thumb_path) else: if DEBUG: - self.log.info(" '%s' not found in cover cache" % thumb_path) + logger().info(" '%s' not found in cover cache" % thumb_path) zf.close() break else: if DEBUG: - self.log.error(" unable to find '%s' by '%s' (%s)" % + logger().error(" unable to find '%s' by '%s' (%s)" % (bl_book.title, bl_book.author,bl_book.uuid)) if False: @@ -953,7 +963,7 @@ class ITUNES(DriverBase): :detected_device: Device information from the device scanner """ if DEBUG: - self.log.info("ITUNES.reset()") + logger().info("ITUNES.reset()") if report_progress: self.set_progress_reporter(report_progress) @@ -965,7 +975,7 @@ class ITUNES(DriverBase): task does not have any progress information ''' if DEBUG: - self.log.info("ITUNES.set_progress_reporter()") + logger().info("ITUNES.set_progress_reporter()") self.report_progress = report_progress @@ -973,8 +983,8 @@ class ITUNES(DriverBase): # This method is called with the plugboard that matches the format # declared in use_plugboard_ext and a device name of ITUNES if DEBUG: - self.log.info("ITUNES.set_plugboard()") - #self.log.info(' plugboard: %s' % plugboards) + logger().info("ITUNES.set_plugboard()") + #logger().info(' plugboard: %s' % plugboards) self.plugboards = plugboards self.plugboard_func = pb_func @@ -987,11 +997,11 @@ class ITUNES(DriverBase): ''' if DEBUG: - self.log.info("ITUNES.sync_booklists()") + logger().info("ITUNES.sync_booklists()") if self.update_needed: if DEBUG: - self.log.info(' calling _update_device') + logger().info(' calling _update_device') self._update_device(msg=self.update_msg, wait=False) self.update_needed = False @@ -1014,7 +1024,7 @@ class ITUNES(DriverBase): particular device doesn't have any of these locations it should return 0. """ if DEBUG: - self.log.info("ITUNES:total_space()") + logger().info("ITUNES:total_space()") capacity = 0 if isosx: if 'iPod' in self.sources: @@ -1052,7 +1062,7 @@ class ITUNES(DriverBase): "Click 'Show Details' for a list.") if DEBUG: - self.log.info("ITUNES.upload_books()") + logger().info("ITUNES.upload_books()") if isosx: for (i,fpath) in enumerate(files): @@ -1069,8 +1079,8 @@ class ITUNES(DriverBase): # Add new_book to self.cached_books if DEBUG: - self.log.info("ITUNES.upload_books()") - self.log.info(" adding '%s' by '%s' uuid:%s to self.cached_books" % + logger().info("ITUNES.upload_books()") + logger().info(" adding '%s' by '%s' uuid:%s to self.cached_books" % (metadata[i].title, authors_to_string(metadata[i].authors), metadata[i].uuid)) @@ -1113,8 +1123,8 @@ class ITUNES(DriverBase): # Add new_book to self.cached_books if DEBUG: - self.log.info("ITUNES.upload_books()") - self.log.info(" adding '%s' by '%s' uuid:%s to self.cached_books" % + logger().info("ITUNES.upload_books()") + logger().info(" adding '%s' by '%s' uuid:%s to self.cached_books" % (metadata[i].title, authors_to_string(metadata[i].authors), metadata[i].uuid)) @@ -1151,7 +1161,7 @@ class ITUNES(DriverBase): ''' assumes pythoncom wrapper for windows ''' - self.log.info(" ITUNES._add_device_book()") + logger().info(" ITUNES._add_device_book()") if isosx: if 'iPod' in self.sources: connected_device = self.sources['iPod'] @@ -1161,12 +1171,12 @@ class ITUNES(DriverBase): break else: if DEBUG: - self.log.error(" Device|Books playlist not found") + logger().error(" Device|Books playlist not found") # Add the passed book to the Device|Books playlist added = pl.add(appscript.mactypes.File(fpath),to=pl) if False: - self.log.info(" '%s' added to Device|Books" % metadata.title) + logger().info(" '%s' added to Device|Books" % metadata.title) self._wait_for_writable_metadata(added) return added @@ -1183,7 +1193,7 @@ class ITUNES(DriverBase): break else: if DEBUG: - self.log.info(" no Books playlist found") + logger().info(" no Books playlist found") # Add the passed book to the Device|Books playlist if pl: @@ -1245,7 +1255,7 @@ class ITUNES(DriverBase): windows assumes pythoncom wrapper ''' if DEBUG: - self.log.info(" ITUNES._add_library_book()") + logger().info(" ITUNES._add_library_book()") if isosx: added = self.iTunes.add(appscript.mactypes.File(file)) @@ -1256,9 +1266,9 @@ class ITUNES(DriverBase): fa = FileArray(file_s) op_status = lib.AddFiles(fa) if DEBUG: - self.log.info(" file added to Library|Books") + logger().info(" file added to Library|Books") - self.log.info(" iTunes adding '%s'" % file) + logger().info(" iTunes adding '%s'" % file) if DEBUG: sys.stdout.write(" iTunes copying '%s' ..." % metadata.title) @@ -1312,7 +1322,7 @@ class ITUNES(DriverBase): fp = cached_book['lib_book'].Location ''' if DEBUG: - self.log.info(" ITUNES._add_new_copy()") + logger().info(" ITUNES._add_new_copy()") if fpath.rpartition('.')[2].lower() == 'epub': self._update_epub_metadata(fpath, metadata) @@ -1333,7 +1343,7 @@ class ITUNES(DriverBase): db_added = self._add_device_book(fpath, metadata) lb_added = self._add_library_book(fpath, metadata) if not lb_added and DEBUG: - self.log.warn(" failed to add '%s' to iTunes, iTunes Media folder inaccessible" % metadata.title) + logger().warn(" failed to add '%s' to iTunes, iTunes Media folder inaccessible" % metadata.title) else: lb_added = self._add_library_book(fpath, metadata) if not lb_added: @@ -1348,8 +1358,10 @@ class ITUNES(DriverBase): assumes pythoncom wrapper for db_added as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation ''' + from PIL import Image as PILImage + if DEBUG: - self.log.info(" ITUNES._cover_to_thumb()") + logger().info(" ITUNES._cover_to_thumb()") thumb = None if metadata.cover: @@ -1366,7 +1378,7 @@ class ITUNES(DriverBase): scaled, nwidth, nheight = fit_image(width, height, self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT) if scaled: if DEBUG: - self.log.info(" cover scaled from %sx%s to %sx%s" % + logger().info(" cover scaled from %sx%s to %sx%s" % (width,height,nwidth,nheight)) img = img.resize((nwidth, nheight), PILImage.ANTIALIAS) cd = cStringIO.StringIO() @@ -1378,7 +1390,7 @@ class ITUNES(DriverBase): cover_data = cd.read() except: self.problem_titles.append("'%s' by %s" % (metadata.title, authors_to_string(metadata.authors))) - self.log.error(" error scaling '%s' for '%s'" % (metadata.cover,metadata.title)) + logger().error(" error scaling '%s' for '%s'" % (metadata.cover,metadata.title)) import traceback traceback.print_exc() @@ -1396,17 +1408,17 @@ class ITUNES(DriverBase): lb_added.artworks[1].data_.set(cover_data) except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " adding artwork to '%s' in the iTunes Library" % metadata.title) pass if db_added: try: db_added.artworks[1].data_.set(cover_data) - self.log.info(" writing '%s' cover to iDevice" % metadata.title) + logger().info(" writing '%s' cover to iDevice" % metadata.title) except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " adding artwork to '%s' on the iDevice" % metadata.title) #import traceback #traceback.print_exc() @@ -1428,7 +1440,7 @@ class ITUNES(DriverBase): lb_added.AddArtworkFromFile(tc) except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " when adding artwork to '%s' in the iTunes Library" % metadata.title) pass @@ -1440,7 +1452,7 @@ class ITUNES(DriverBase): elif format == 'pdf': if DEBUG: - self.log.info(" unable to set PDF cover via automation interface") + logger().info(" unable to set PDF cover via automation interface") try: # Resize for thumb @@ -1455,13 +1467,13 @@ class ITUNES(DriverBase): # Refresh the thumbnail cache if DEBUG: - self.log.info( " refreshing cached thumb for '%s'" % metadata.title) + logger().info( " refreshing cached thumb for '%s'" % metadata.title) zfw = ZipFile(self.archive_path, mode='a') thumb_path = path.rpartition('.')[0] + '.jpg' zfw.writestr(thumb_path, thumb) except: self.problem_titles.append("'%s' by %s" % (metadata.title, authors_to_string(metadata.authors))) - self.log.error(" error converting '%s' to thumb for '%s'" % (metadata.cover,metadata.title)) + logger().error(" error converting '%s' to thumb for '%s'" % (metadata.cover,metadata.title)) finally: try: zfw.close() @@ -1469,14 +1481,14 @@ class ITUNES(DriverBase): pass else: if DEBUG: - self.log.info(" no cover defined in metadata for '%s'" % metadata.title) + logger().info(" no cover defined in metadata for '%s'" % metadata.title) return thumb def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb, format): ''' ''' if DEBUG: - self.log.info(" ITUNES._create_new_book()") + logger().info(" ITUNES._create_new_book()") this_book = Book(metadata.title, authors_to_string(metadata.authors)) this_book.datetime = time.gmtime() @@ -1525,7 +1537,7 @@ class ITUNES(DriverBase): wait is passed when launching iTunes, as it seems to need a moment to come to its senses ''' if DEBUG: - self.log.info(" ITUNES._discover_manual_sync_mode()") + logger().info(" ITUNES._discover_manual_sync_mode()") if wait: time.sleep(wait) if isosx: @@ -1537,12 +1549,12 @@ class ITUNES(DriverBase): dev_books = pl.file_tracks() break else: - self.log.error(" book_playlist not found") + logger().error(" book_playlist not found") if dev_books is not None and len(dev_books): first_book = dev_books[0] if False: - self.log.info(" determing manual mode by modifying '%s' by %s" % (first_book.name(), first_book.artist())) + logger().info(" determing manual mode by modifying '%s' by %s" % (first_book.name(), first_book.artist())) try: first_book.bpm.set(0) self.manual_sync_mode = True @@ -1550,7 +1562,7 @@ class ITUNES(DriverBase): self.manual_sync_mode = False else: if DEBUG: - self.log.info(" adding tracer to empty Books|Playlist") + logger().info(" adding tracer to empty Books|Playlist") try: added = pl.add(appscript.mactypes.File(P('tracer.epub')),to=pl) time.sleep(0.5) @@ -1573,7 +1585,7 @@ class ITUNES(DriverBase): if dev_books is not None and dev_books.Count: first_book = dev_books.Item(1) #if DEBUG: - #self.log.info(" determing manual mode by modifying '%s' by %s" % (first_book.Name, first_book.Artist)) + #logger().info(" determing manual mode by modifying '%s' by %s" % (first_book.Name, first_book.Artist)) try: first_book.BPM = 0 self.manual_sync_mode = True @@ -1581,7 +1593,7 @@ class ITUNES(DriverBase): self.manual_sync_mode = False else: if DEBUG: - self.log.info(" sending tracer to empty Books|Playlist") + logger().info(" sending tracer to empty Books|Playlist") fpath = P('tracer.epub') mi = MetaInformation('Tracer',['calibre']) try: @@ -1592,24 +1604,24 @@ class ITUNES(DriverBase): except: self.manual_sync_mode = False - self.log.info(" iTunes.manual_sync_mode: %s" % self.manual_sync_mode) + logger().info(" iTunes.manual_sync_mode: %s" % self.manual_sync_mode) def _dump_booklist(self, booklist, header=None,indent=0): ''' ''' if header: msg = '\n%sbooklist %s:' % (' '*indent,header) - self.log.info(msg) - self.log.info('%s%s' % (' '*indent,'-' * len(msg))) + logger().info(msg) + logger().info('%s%s' % (' '*indent,'-' * len(msg))) for book in booklist: if isosx: - self.log.info("%s%-40.40s %-30.30s %-10.10s %s" % + logger().info("%s%-40.40s %-30.30s %-10.10s %s" % (' '*indent,book.title, book.author, str(book.library_id)[-9:], book.uuid)) elif iswindows: - self.log.info("%s%-40.40s %-30.30s" % + logger().info("%s%-40.40s %-30.30s" % (' '*indent,book.title, book.author)) - self.log.info() + logger().info() def _dump_cached_book(self, cached_book, header=None,indent=0): ''' @@ -1617,16 +1629,16 @@ class ITUNES(DriverBase): if isosx: if header: msg = '%s%s' % (' '*indent,header) - self.log.info(msg) - self.log.info( "%s%s" % (' '*indent, '-' * len(msg))) - self.log.info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % + logger().info(msg) + logger().info( "%s%s" % (' '*indent, '-' * len(msg))) + logger().info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % (' '*indent, 'title', 'author', 'lib_book', 'dev_book', 'uuid')) - self.log.info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % + logger().info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % (' '*indent, cached_book['title'], cached_book['author'], @@ -1636,10 +1648,10 @@ class ITUNES(DriverBase): elif iswindows: if header: msg = '%s%s' % (' '*indent,header) - self.log.info(msg) - self.log.info( "%s%s" % (' '*indent, '-' * len(msg))) + logger().info(msg) + logger().info( "%s%s" % (' '*indent, '-' * len(msg))) - self.log.info("%s%-40.40s %-30.30s %s" % + logger().info("%s%-40.40s %-30.30s %s" % (' '*indent, cached_book['title'], cached_book['author'], @@ -1650,11 +1662,11 @@ class ITUNES(DriverBase): ''' if header: msg = '\n%sself.cached_books %s:' % (' '*indent,header) - self.log.info(msg) - self.log.info( "%s%s" % (' '*indent,'-' * len(msg))) + logger().info(msg) + logger().info( "%s%s" % (' '*indent,'-' * len(msg))) if isosx: for cb in self.cached_books.keys(): - self.log.info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % + logger().info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % (' '*indent, self.cached_books[cb]['title'], self.cached_books[cb]['author'], @@ -1663,19 +1675,21 @@ class ITUNES(DriverBase): self.cached_books[cb]['uuid'])) elif iswindows: for cb in self.cached_books.keys(): - self.log.info("%s%-40.40s %-30.30s %-4.4s %s" % + logger().info("%s%-40.40s %-30.30s %-4.4s %s" % (' '*indent, self.cached_books[cb]['title'], self.cached_books[cb]['author'], self.cached_books[cb]['format'], self.cached_books[cb]['uuid'])) - self.log.info() + logger().info() def _dump_epub_metadata(self, fpath): ''' ''' - self.log.info(" ITUNES.__get_epub_metadata()") + from calibre.ebooks.BeautifulSoup import BeautifulSoup + + logger().info(" ITUNES.__get_epub_metadata()") title = None author = None timestamp = None @@ -1695,11 +1709,11 @@ class ITUNES(DriverBase): if not title or not author: if DEBUG: - self.log.error(" couldn't extract title/author from %s in %s" % (opf,fpath)) - self.log.error(" title: %s author: %s timestamp: %s" % (title, author, timestamp)) + logger().error(" couldn't extract title/author from %s in %s" % (opf,fpath)) + logger().error(" title: %s author: %s timestamp: %s" % (title, author, timestamp)) else: if DEBUG: - self.log.error(" can't find .opf in %s" % fpath) + logger().error(" can't find .opf in %s" % fpath) zf.close() return (title, author, timestamp) @@ -1720,20 +1734,20 @@ class ITUNES(DriverBase): ''' ''' if DEBUG: - self.log.info("\n library_books:") + logger().info("\n library_books:") for book in library_books: - self.log.info(" %s" % book) - self.log.info() + logger().info(" %s" % book) + logger().info() def _dump_update_list(self,header=None,indent=0): if header and self.update_list: msg = '\n%sself.update_list %s' % (' '*indent,header) - self.log.info(msg) - self.log.info( "%s%s" % (' '*indent,'-' * len(msg))) + logger().info(msg) + logger().info( "%s%s" % (' '*indent,'-' * len(msg))) if isosx: for ub in self.update_list: - self.log.info("%s%-40.40s %-30.30s %-10.10s %s" % + logger().info("%s%-40.40s %-30.30s %-10.10s %s" % (' '*indent, ub['title'], ub['author'], @@ -1741,7 +1755,7 @@ class ITUNES(DriverBase): ub['uuid'])) elif iswindows: for ub in self.update_list: - self.log.info("%s%-40.40s %-30.30s" % + logger().info("%s%-40.40s %-30.30s" % (' '*indent, ub['title'], ub['author'])) @@ -1753,42 +1767,42 @@ class ITUNES(DriverBase): if iswindows: dev_books = self._get_device_books_playlist() if DEBUG: - self.log.info(" ITUNES._find_device_book()") - self.log.info(" searching for '%s' by '%s' (%s)" % + logger().info(" ITUNES._find_device_book()") + logger().info(" searching for '%s' by '%s' (%s)" % (search['title'], search['author'],search['uuid'])) attempts = 9 while attempts: # Try by uuid - only one hit if 'uuid' in search and search['uuid']: if DEBUG: - self.log.info(" searching by uuid '%s' ..." % search['uuid']) + logger().info(" searching by uuid '%s' ..." % search['uuid']) hits = dev_books.Search(search['uuid'],self.SearchField.index('All')) if hits: hit = hits[0] - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit # Try by author - there could be multiple hits if search['author']: if DEBUG: - self.log.info(" searching by author '%s' ..." % search['author']) + logger().info(" searching by author '%s' ..." % search['author']) hits = dev_books.Search(search['author'],self.SearchField.index('Artists')) if hits: for hit in hits: if hit.Name == search['title']: if DEBUG: - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit # Search by title if no author available if DEBUG: - self.log.info(" searching by title '%s' ..." % search['title']) + logger().info(" searching by title '%s' ..." % search['title']) hits = dev_books.Search(search['title'],self.SearchField.index('All')) if hits: for hit in hits: if hit.Name == search['title']: if DEBUG: - self.log.info(" found '%s'" % (hit.Name)) + logger().info(" found '%s'" % (hit.Name)) return hit # PDF just sent, title not updated yet, look for export pattern @@ -1797,24 +1811,24 @@ class ITUNES(DriverBase): title = re.sub(r'[^0-9a-zA-Z ]', '_', search['title']) author = re.sub(r'[^0-9a-zA-Z ]', '_', search['author']) if DEBUG: - self.log.info(" searching by name: '%s - %s'" % (title,author)) + logger().info(" searching by name: '%s - %s'" % (title,author)) hits = dev_books.Search('%s - %s' % (title,author), self.SearchField.index('All')) if hits: hit = hits[0] - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit else: if DEBUG: - self.log.info(" no PDF hits") + logger().info(" no PDF hits") attempts -= 1 time.sleep(0.5) if DEBUG: - self.log.warning(" attempt #%d" % (10 - attempts)) + logger().warning(" attempt #%d" % (10 - attempts)) if DEBUG: - self.log.error(" no hits") + logger().error(" no hits") return None def _find_library_book(self, search): @@ -1823,13 +1837,13 @@ class ITUNES(DriverBase): ''' if iswindows: if DEBUG: - self.log.info(" ITUNES._find_library_book()") + logger().info(" ITUNES._find_library_book()") ''' if 'uuid' in search: - self.log.info(" looking for '%s' by %s (%s)" % + logger().info(" looking for '%s' by %s (%s)" % (search['title'], search['author'], search['uuid'])) else: - self.log.info(" looking for '%s' by %s" % + logger().info(" looking for '%s' by %s" % (search['title'], search['author'])) ''' @@ -1837,11 +1851,11 @@ class ITUNES(DriverBase): if source.Kind == self.Sources.index('Library'): lib = source if DEBUG: - self.log.info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind])) + logger().info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind])) break else: if DEBUG: - self.log.info(" Library source not found") + logger().info(" Library source not found") if lib is not None: lib_books = None @@ -1849,12 +1863,12 @@ class ITUNES(DriverBase): if pl.Kind == self.PlaylistKind.index('User') and \ pl.SpecialKind == self.PlaylistSpecialKind.index('Books'): if DEBUG: - self.log.info(" Books playlist: '%s'" % (pl.Name)) + logger().info(" Books playlist: '%s'" % (pl.Name)) lib_books = pl break else: if DEBUG: - self.log.error(" no Books playlist found") + logger().error(" no Books playlist found") attempts = 9 @@ -1862,35 +1876,35 @@ class ITUNES(DriverBase): # Find book whose Album field = search['uuid'] if 'uuid' in search and search['uuid']: if DEBUG: - self.log.info(" searching by uuid '%s' ..." % search['uuid']) + logger().info(" searching by uuid '%s' ..." % search['uuid']) hits = lib_books.Search(search['uuid'],self.SearchField.index('All')) if hits: hit = hits[0] if DEBUG: - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit # Search by author if known if search['author']: if DEBUG: - self.log.info(" searching by author '%s' ..." % search['author']) + logger().info(" searching by author '%s' ..." % search['author']) hits = lib_books.Search(search['author'],self.SearchField.index('Artists')) if hits: for hit in hits: if hit.Name == search['title']: if DEBUG: - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit # Search by title if no author available if DEBUG: - self.log.info(" searching by title '%s' ..." % search['title']) + logger().info(" searching by title '%s' ..." % search['title']) hits = lib_books.Search(search['title'],self.SearchField.index('All')) if hits: for hit in hits: if hit.Name == search['title']: if DEBUG: - self.log.info(" found '%s'" % (hit.Name)) + logger().info(" found '%s'" % (hit.Name)) return hit # PDF just sent, title not updated yet, look for export pattern @@ -1899,24 +1913,24 @@ class ITUNES(DriverBase): title = re.sub(r'[^0-9a-zA-Z ]', '_', search['title']) author = re.sub(r'[^0-9a-zA-Z ]', '_', search['author']) if DEBUG: - self.log.info(" searching by name: %s - %s" % (title,author)) + logger().info(" searching by name: %s - %s" % (title,author)) hits = lib_books.Search('%s - %s' % (title,author), self.SearchField.index('All')) if hits: hit = hits[0] - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit else: if DEBUG: - self.log.info(" no PDF hits") + logger().info(" no PDF hits") attempts -= 1 time.sleep(0.5) if DEBUG: - self.log.warning(" attempt #%d" % (10 - attempts)) + logger().warning(" attempt #%d" % (10 - attempts)) if DEBUG: - self.log.error(" search for '%s' yielded no hits" % search['title']) + logger().error(" search for '%s' yielded no hits" % search['title']) return None def _generate_thumbnail(self, book_path, book): @@ -1926,6 +1940,7 @@ class ITUNES(DriverBase): cache_dir = os.path.join(config_dir, 'caches', 'itunes') as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation ''' + from PIL import Image as PILImage if not self.settings().extra_customization[self.CACHE_COVERS]: thumb_data = None @@ -1942,18 +1957,18 @@ class ITUNES(DriverBase): thumb_data = zfr.read(thumb_path) if thumb_data == 'None': if False: - self.log.info(" ITUNES._generate_thumbnail()\n returning None from cover cache for '%s'" % title) + logger().info(" ITUNES._generate_thumbnail()\n returning None from cover cache for '%s'" % title) zfr.close() return None except: zfw = ZipFile(self.archive_path, mode='a') else: if False: - self.log.info(" returning thumb from cache for '%s'" % title) + logger().info(" returning thumb from cache for '%s'" % title) return thumb_data if DEBUG: - self.log.info(" ITUNES._generate_thumbnail('%s'):" % title) + logger().info(" ITUNES._generate_thumbnail('%s'):" % title) if isosx: # Fetch the artwork from iTunes @@ -1962,7 +1977,7 @@ class ITUNES(DriverBase): except: # If no artwork, write an empty marker to cache if DEBUG: - self.log.error(" error fetching iTunes artwork for '%s'" % title) + logger().error(" error fetching iTunes artwork for '%s'" % title) zfw.writestr(thumb_path, 'None') zfw.close() return None @@ -1979,12 +1994,12 @@ class ITUNES(DriverBase): thumb_data = thumb.getvalue() thumb.close() if False: - self.log.info(" generated thumb for '%s', caching" % title) + logger().info(" generated thumb for '%s', caching" % title) # Cache the tagged thumb zfw.writestr(thumb_path, thumb_data) except: if DEBUG: - self.log.error(" error generating thumb for '%s', caching empty marker" % book.name()) + logger().error(" error generating thumb for '%s', caching empty marker" % book.name()) self._dump_hex(data[:32]) thumb_data = None # Cache the empty cover @@ -1999,7 +2014,7 @@ class ITUNES(DriverBase): elif iswindows: if not book.Artwork.Count: if DEBUG: - self.log.info(" no artwork available for '%s'" % book.Name) + logger().info(" no artwork available for '%s'" % book.Name) zfw.writestr(thumb_path, 'None') zfw.close() return None @@ -2019,12 +2034,12 @@ class ITUNES(DriverBase): os.remove(tmp_thumb) thumb.close() if False: - self.log.info(" generated thumb for '%s', caching" % book.Name) + logger().info(" generated thumb for '%s', caching" % book.Name) # Cache the tagged thumb zfw.writestr(thumb_path, thumb_data) except: if DEBUG: - self.log.error(" error generating thumb for '%s', caching empty marker" % book.Name) + logger().error(" error generating thumb for '%s', caching empty marker" % book.Name) thumb_data = None # Cache the empty cover zfw.writestr(thumb_path,'None') @@ -2047,9 +2062,9 @@ class ITUNES(DriverBase): for file in myZipList: exploded_file_size += file.file_size if False: - self.log.info(" ITUNES._get_device_book_size()") - self.log.info(" %d items in archive" % len(myZipList)) - self.log.info(" compressed: %d exploded: %d" % (compressed_size, exploded_file_size)) + logger().info(" ITUNES._get_device_book_size()") + logger().info(" %d items in archive" % len(myZipList)) + logger().info(" compressed: %d exploded: %d" % (compressed_size, exploded_file_size)) myZip.close() return exploded_file_size @@ -2058,7 +2073,7 @@ class ITUNES(DriverBase): Assumes pythoncom wrapper for Windows ''' if DEBUG: - self.log.info("\n ITUNES._get_device_books()") + logger().info("\n ITUNES._get_device_books()") device_books = [] if isosx: @@ -2069,24 +2084,24 @@ class ITUNES(DriverBase): for pl in device.playlists(): if pl.special_kind() == appscript.k.Books: if DEBUG: - self.log.info(" Book playlist: '%s'" % (pl.name())) + logger().info(" Book playlist: '%s'" % (pl.name())) dev_books = pl.file_tracks() break else: - self.log.error(" book_playlist not found") + logger().error(" book_playlist not found") for book in dev_books: # This may need additional entries for international iTunes users if book.kind() in self.Audiobooks: if DEBUG: - self.log.info(" ignoring '%s' of type '%s'" % (book.name(), book.kind())) + logger().info(" ignoring '%s' of type '%s'" % (book.name(), book.kind())) else: if DEBUG: - self.log.info(" %-30.30s %-30.30s %-40.40s [%s]" % + logger().info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.name(), book.artist(), book.album(), book.kind())) device_books.append(book) if DEBUG: - self.log.info() + logger().info() elif iswindows: if 'iPod' in self.sources: @@ -2100,24 +2115,24 @@ class ITUNES(DriverBase): if pl.Kind == self.PlaylistKind.index('User') and \ pl.SpecialKind == self.PlaylistSpecialKind.index('Books'): if DEBUG: - self.log.info(" Books playlist: '%s'" % (pl.Name)) + logger().info(" Books playlist: '%s'" % (pl.Name)) dev_books = pl.Tracks break else: if DEBUG: - self.log.info(" no Books playlist found") + logger().info(" no Books playlist found") for book in dev_books: # This may need additional entries for international iTunes users if book.KindAsString in self.Audiobooks: if DEBUG: - self.log.info(" ignoring '%s' of type '%s'" % (book.Name, book.KindAsString)) + logger().info(" ignoring '%s' of type '%s'" % (book.Name, book.KindAsString)) else: if DEBUG: - self.log.info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) + logger().info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) device_books.append(book) if DEBUG: - self.log.info() + logger().info() finally: pythoncom.CoUninitialize() @@ -2140,7 +2155,7 @@ class ITUNES(DriverBase): break else: if DEBUG: - self.log.error(" no iPad|Books playlist found") + logger().error(" no iPad|Books playlist found") return pl def _get_library_books(self): @@ -2149,7 +2164,7 @@ class ITUNES(DriverBase): Windows assumes pythoncom wrapper ''' if DEBUG: - self.log.info("\n ITUNES._get_library_books()") + logger().info("\n ITUNES._get_library_books()") library_books = {} library_orphans = {} @@ -2160,11 +2175,11 @@ class ITUNES(DriverBase): if source.kind() == appscript.k.library: lib = source if DEBUG: - self.log.info(" Library source: '%s'" % (lib.name())) + logger().info(" Library source: '%s'" % (lib.name())) break else: if DEBUG: - self.log.error(' Library source not found') + logger().error(' Library source not found') if lib is not None: lib_books = None @@ -2172,18 +2187,18 @@ class ITUNES(DriverBase): for pl in lib.playlists(): if pl.special_kind() == appscript.k.Books: if DEBUG: - self.log.info(" Books playlist: '%s'" % (pl.name())) + logger().info(" Books playlist: '%s'" % (pl.name())) break else: if DEBUG: - self.log.info(" no Library|Books playlist found") + logger().info(" no Library|Books playlist found") lib_books = pl.file_tracks() for book in lib_books: # This may need additional entries for international iTunes users if book.kind() in self.Audiobooks: if DEBUG: - self.log.info(" ignoring '%s' of type '%s'" % (book.name(), book.kind())) + logger().info(" ignoring '%s' of type '%s'" % (book.name(), book.kind())) else: # Collect calibre orphans - remnants of recipe uploads format = 'pdf' if book.kind().startswith('PDF') else 'epub' @@ -2193,31 +2208,31 @@ class ITUNES(DriverBase): if book.location() == appscript.k.missing_value: library_orphans[path] = book if False: - self.log.info(" found iTunes PTF '%s' in Library|Books" % book.name()) + logger().info(" found iTunes PTF '%s' in Library|Books" % book.name()) except: if DEBUG: - self.log.error(" iTunes returned an error returning .location() with %s" % book.name()) + logger().error(" iTunes returned an error returning .location() with %s" % book.name()) library_books[path] = book if DEBUG: - self.log.info(" %-30.30s %-30.30s %-40.40s [%s]" % + logger().info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.name(), book.artist(), book.album(), book.kind())) else: if DEBUG: - self.log.info(' no Library playlists') + logger().info(' no Library playlists') else: if DEBUG: - self.log.info(' no Library found') + logger().info(' no Library found') elif iswindows: lib = None for source in self.iTunes.sources: if source.Kind == self.Sources.index('Library'): lib = source - self.log.info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind])) + logger().info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind])) break else: - self.log.error(" Library source not found") + logger().error(" Library source not found") if lib is not None: lib_books = None @@ -2226,22 +2241,22 @@ class ITUNES(DriverBase): if pl.Kind == self.PlaylistKind.index('User') and \ pl.SpecialKind == self.PlaylistSpecialKind.index('Books'): if DEBUG: - self.log.info(" Books playlist: '%s'" % (pl.Name)) + logger().info(" Books playlist: '%s'" % (pl.Name)) lib_books = pl.Tracks break else: if DEBUG: - self.log.error(" no Library|Books playlist found") + logger().error(" no Library|Books playlist found") else: if DEBUG: - self.log.error(" no Library playlists found") + logger().error(" no Library playlists found") try: for book in lib_books: # This may need additional entries for international iTunes users if book.KindAsString in self.Audiobooks: if DEBUG: - self.log.info(" ignoring %-30.30s of type '%s'" % (book.Name, book.KindAsString)) + logger().info(" ignoring %-30.30s of type '%s'" % (book.Name, book.KindAsString)) else: format = 'pdf' if book.KindAsString.startswith('PDF') else 'epub' path = self.path_template % (book.Name, book.Artist,format) @@ -2251,14 +2266,14 @@ class ITUNES(DriverBase): if not book.Location: library_orphans[path] = book if False: - self.log.info(" found iTunes PTF '%s' in Library|Books" % book.Name) + logger().info(" found iTunes PTF '%s' in Library|Books" % book.Name) library_books[path] = book if DEBUG: - self.log.info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) + logger().info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) except: if DEBUG: - self.log.info(" no books in library") + logger().info(" no books in library") self.library_orphans = library_orphans return library_books @@ -2303,7 +2318,7 @@ class ITUNES(DriverBase): # If more than one connected iDevice, remove all from list to prevent driver initialization if kinds.count('iPod') > 1: if DEBUG: - self.log.error(" %d connected iPod devices detected, calibre supports a single connected iDevice" % kinds.count('iPod')) + logger().error(" %d connected iPod devices detected, calibre supports a single connected iDevice" % kinds.count('iPod')) while kinds.count('iPod'): index = kinds.index('iPod') kinds.pop(index) @@ -2323,7 +2338,7 @@ class ITUNES(DriverBase): ''' ''' if DEBUG: - self.log.info(" ITUNES:_launch_iTunes():\n Instantiating iTunes") + logger().info(" ITUNES:_launch_iTunes():\n Instantiating iTunes") if isosx: ''' @@ -2333,7 +2348,7 @@ class ITUNES(DriverBase): running_apps = appscript.app('System Events') if not 'iTunes' in running_apps.processes.name(): if DEBUG: - self.log.info( "ITUNES:_launch_iTunes(): Launching iTunes" ) + logger().info( "ITUNES:_launch_iTunes(): Launching iTunes" ) try: self.iTunes = iTunes= appscript.app('iTunes', hide=True) except: @@ -2355,16 +2370,16 @@ class ITUNES(DriverBase): if os.path.exists(media_dir): self.iTunes_media = media_dir else: - self.log.error(" could not confirm valid iTunes.media_dir from %s" % 'com.apple.itunes') - self.log.error(" media_dir: %s" % media_dir) + logger().error(" could not confirm valid iTunes.media_dir from %s" % 'com.apple.itunes') + logger().error(" media_dir: %s" % media_dir) ''' if DEBUG: - self.log.info(" %s %s" % (__appname__, __version__)) - self.log.info(" [OSX %s - %s (%s), driver version %d.%d.%d]" % + logger().info(" %s %s" % (__appname__, __version__)) + logger().info(" [OSX %s - %s (%s), driver version %d.%d.%d]" % (self.iTunes.name(), self.iTunes.version(), self.initial_status, self.version[0],self.version[1],self.version[2])) - self.log.info(" calibre_library_path: %s" % self.calibre_library_path) + logger().info(" calibre_library_path: %s" % self.calibre_library_path) if iswindows: ''' @@ -2417,19 +2432,19 @@ class ITUNES(DriverBase): if os.path.exists(media_dir): self.iTunes_media = media_dir elif hasattr(string,'parent'): - self.log.error(" could not extract valid iTunes.media_dir from %s" % self.iTunes.LibraryXMLPath) - self.log.error(" %s" % string.parent.prettify()) - self.log.error(" '%s' not found" % media_dir) + logger().error(" could not extract valid iTunes.media_dir from %s" % self.iTunes.LibraryXMLPath) + logger().error(" %s" % string.parent.prettify()) + logger().error(" '%s' not found" % media_dir) else: - self.log.error(" no media dir found: string: %s" % string) + logger().error(" no media dir found: string: %s" % string) ''' if DEBUG: - self.log.info(" %s %s" % (__appname__, __version__)) - self.log.info(" [Windows %s - %s (%s), driver version %d.%d.%d]" % + logger().info(" %s %s" % (__appname__, __version__)) + logger().info(" [Windows %s - %s (%s), driver version %d.%d.%d]" % (self.iTunes.Windows[0].name, self.iTunes.Version, self.initial_status, self.version[0],self.version[1],self.version[2])) - self.log.info(" calibre_library_path: %s" % self.calibre_library_path) + logger().info(" calibre_library_path: %s" % self.calibre_library_path) def _purge_orphans(self,library_books, cached_books): ''' @@ -2438,16 +2453,16 @@ class ITUNES(DriverBase): This occurs when the user deletes a book in iBooks while disconnected ''' if DEBUG: - self.log.info(" ITUNES._purge_orphans()") + logger().info(" ITUNES._purge_orphans()") #self._dump_library_books(library_books) - #self.log.info(" cached_books:\n %s" % "\n ".join(cached_books.keys())) + #logger().info(" cached_books:\n %s" % "\n ".join(cached_books.keys())) for book in library_books: if isosx: if book not in cached_books and \ str(library_books[book].description()).startswith(self.description_prefix): if DEBUG: - self.log.info(" '%s' not found on iDevice, removing from iTunes" % book) + logger().info(" '%s' not found on iDevice, removing from iTunes" % book) btr = { 'title':library_books[book].name(), 'author':library_books[book].artist(), 'lib_book':library_books[book]} @@ -2456,19 +2471,19 @@ class ITUNES(DriverBase): if book not in cached_books and \ library_books[book].Description.startswith(self.description_prefix): if DEBUG: - self.log.info(" '%s' not found on iDevice, removing from iTunes" % book) + logger().info(" '%s' not found on iDevice, removing from iTunes" % book) btr = { 'title':library_books[book].Name, 'author':library_books[book].Artist, 'lib_book':library_books[book]} self._remove_from_iTunes(btr) if DEBUG: - self.log.info() + logger().info() def _remove_existing_copy(self, path, metadata): ''' ''' if DEBUG: - self.log.info(" ITUNES._remove_existing_copy()") + logger().info(" ITUNES._remove_existing_copy()") if self.manual_sync_mode: # Delete existing from Device|Books, add to self.update_list @@ -2480,16 +2495,16 @@ class ITUNES(DriverBase): self.update_list.append(self.cached_books[book]) if DEBUG: - self.log.info( " deleting device book '%s'" % (metadata.title)) + logger().info( " deleting device book '%s'" % (metadata.title)) self._remove_from_device(self.cached_books[book]) if DEBUG: - self.log.info(" deleting library book '%s'" % metadata.title) + logger().info(" deleting library book '%s'" % metadata.title) self._remove_from_iTunes(self.cached_books[book]) break else: if DEBUG: - self.log.info(" '%s' not in cached_books" % metadata.title) + logger().info(" '%s' not in cached_books" % metadata.title) else: # Delete existing from Library|Books, add to self.update_list # for deletion from booklist[0] during add_books_to_metadata @@ -2499,35 +2514,35 @@ class ITUNES(DriverBase): self.cached_books[book]['author'] == authors_to_string(metadata.authors)): self.update_list.append(self.cached_books[book]) if DEBUG: - self.log.info( " deleting library book '%s'" % metadata.title) + logger().info( " deleting library book '%s'" % metadata.title) self._remove_from_iTunes(self.cached_books[book]) break else: if DEBUG: - self.log.info(" '%s' not found in cached_books" % metadata.title) + logger().info(" '%s' not found in cached_books" % metadata.title) def _remove_from_device(self, cached_book): ''' Windows assumes pythoncom wrapper ''' if DEBUG: - self.log.info(" ITUNES._remove_from_device()") + logger().info(" ITUNES._remove_from_device()") if isosx: if DEBUG: - self.log.info(" deleting '%s' from iDevice" % cached_book['title']) + logger().info(" deleting '%s' from iDevice" % cached_book['title']) try: cached_book['dev_book'].delete() except: - self.log.error(" error deleting '%s'" % cached_book['title']) + logger().error(" error deleting '%s'" % cached_book['title']) elif iswindows: hit = self._find_device_book(cached_book) if hit: if DEBUG: - self.log.info(" deleting '%s' from iDevice" % cached_book['title']) + logger().info(" deleting '%s' from iDevice" % cached_book['title']) hit.Delete() else: if DEBUG: - self.log.warning(" unable to remove '%s' by '%s' (%s) from device" % + logger().warning(" unable to remove '%s' by '%s' (%s) from device" % (cached_book['title'],cached_book['author'],cached_book['uuid'])) def _remove_from_iTunes(self, cached_book): @@ -2535,34 +2550,34 @@ class ITUNES(DriverBase): iTunes does not delete books from storage when removing from database via automation ''' if DEBUG: - self.log.info(" ITUNES._remove_from_iTunes():") + logger().info(" ITUNES._remove_from_iTunes():") if isosx: ''' Manually remove the book from iTunes storage ''' try: fp = cached_book['lib_book'].location().path if DEBUG: - self.log.info(" processing %s" % fp) + logger().info(" processing %s" % fp) if fp.startswith(prefs['library_path']): - self.log.info(" '%s' stored in calibre database, not removed" % cached_book['title']) + logger().info(" '%s' stored in calibre database, not removed" % cached_book['title']) elif not self.settings().extra_customization[self.USE_ITUNES_STORAGE] and \ fp.startswith(self.iTunes_local_storage) and \ os.path.exists(fp): # Delete the copy in iTunes_local_storage os.remove(fp) if DEBUG: - self.log(" removing from iTunes_local_storage") + logger()(" removing from iTunes_local_storage") else: # Delete from iTunes Media folder if os.path.exists(fp): os.remove(fp) if DEBUG: - self.log.info(" deleting from iTunes storage") + logger().info(" deleting from iTunes storage") author_storage_path = os.path.split(fp)[0] try: os.rmdir(author_storage_path) if DEBUG: - self.log.info(" removing empty author directory") + logger().info(" removing empty author directory") except: author_files = os.listdir(author_storage_path) if '.DS_Store' in author_files: @@ -2570,23 +2585,23 @@ class ITUNES(DriverBase): if not author_files: os.rmdir(author_storage_path) if DEBUG: - self.log.info(" removing empty author directory") + logger().info(" removing empty author directory") else: - self.log.info(" '%s' does not exist at storage location" % cached_book['title']) + logger().info(" '%s' does not exist at storage location" % cached_book['title']) except: # We get here if there was an error with .location().path if DEBUG: - self.log.info(" '%s' not found in iTunes storage" % cached_book['title']) + logger().info(" '%s' not found in iTunes storage" % cached_book['title']) # Delete the book from the iTunes database try: self.iTunes.delete(cached_book['lib_book']) if DEBUG: - self.log.info(" removing from iTunes database") + logger().info(" removing from iTunes database") except: if DEBUG: - self.log.info(" unable to remove from iTunes database") + logger().info(" unable to remove from iTunes database") elif iswindows: ''' @@ -2604,43 +2619,43 @@ class ITUNES(DriverBase): if book: if DEBUG: - self.log.info(" processing %s" % fp) + logger().info(" processing %s" % fp) if fp.startswith(prefs['library_path']): - self.log.info(" '%s' stored in calibre database, not removed" % cached_book['title']) + logger().info(" '%s' stored in calibre database, not removed" % cached_book['title']) elif not self.settings().extra_customization[self.USE_ITUNES_STORAGE] and \ fp.startswith(self.iTunes_local_storage) and \ os.path.exists(fp): # Delete the copy in iTunes_local_storage os.remove(fp) if DEBUG: - self.log(" removing from iTunes_local_storage") + logger()(" removing from iTunes_local_storage") else: # Delete from iTunes Media folder if os.path.exists(fp): os.remove(fp) if DEBUG: - self.log.info(" deleting from iTunes storage") + logger().info(" deleting from iTunes storage") author_storage_path = os.path.split(fp)[0] try: os.rmdir(author_storage_path) if DEBUG: - self.log.info(" removing empty author directory") + logger().info(" removing empty author directory") except: pass else: - self.log.info(" '%s' does not exist at storage location" % cached_book['title']) + logger().info(" '%s' does not exist at storage location" % cached_book['title']) else: if DEBUG: - self.log.info(" '%s' not found in iTunes storage" % cached_book['title']) + logger().info(" '%s' not found in iTunes storage" % cached_book['title']) # Delete the book from the iTunes database try: book.Delete() if DEBUG: - self.log.info(" removing from iTunes database") + logger().info(" removing from iTunes database") except: if DEBUG: - self.log.info(" unable to remove from iTunes database") + logger().info(" unable to remove from iTunes database") def title_sorter(self, title): return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip() @@ -2648,8 +2663,11 @@ class ITUNES(DriverBase): def _update_epub_metadata(self, fpath, metadata): ''' ''' + from calibre.ebooks.metadata.epub import set_metadata + from lxml import etree + if DEBUG: - self.log.info(" ITUNES._update_epub_metadata()") + logger().info(" ITUNES._update_epub_metadata()") # Fetch plugboard updates metadata_x = self._xform_metadata_via_plugboard(metadata, 'epub') @@ -2677,17 +2695,17 @@ class ITUNES(DriverBase): metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour, old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo) if DEBUG: - self.log.info(" existing timestamp: %s" % metadata.timestamp) + logger().info(" existing timestamp: %s" % metadata.timestamp) else: metadata.timestamp = now() if DEBUG: - self.log.info(" add timestamp: %s" % metadata.timestamp) + logger().info(" add timestamp: %s" % metadata.timestamp) else: metadata.timestamp = now() if DEBUG: - self.log.warning(" missing block in OPF file") - self.log.info(" add timestamp: %s" % metadata.timestamp) + logger().warning(" missing block in OPF file") + logger().info(" add timestamp: %s" % metadata.timestamp) zf_opf.close() @@ -2717,7 +2735,7 @@ class ITUNES(DriverBase): Trigger a sync, wait for completion ''' if DEBUG: - self.log.info(" ITUNES:_update_device():\n %s" % msg) + logger().info(" ITUNES:_update_device():\n %s" % msg) if isosx: self.iTunes.update() @@ -2763,7 +2781,7 @@ class ITUNES(DriverBase): ''' ''' if DEBUG: - self.log.info(" ITUNES._update_iTunes_metadata()") + logger().info(" ITUNES._update_iTunes_metadata()") STRIP_TAGS = re.compile(r'<[^<]*?/?>') @@ -2815,8 +2833,8 @@ class ITUNES(DriverBase): # If title_sort applied in plugboard, that overrides using series/index as title_sort if metadata_x.series and self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY]: if DEBUG: - self.log.info(" ITUNES._update_iTunes_metadata()") - self.log.info(" using Series name '%s' as Genre" % metadata_x.series) + logger().info(" ITUNES._update_iTunes_metadata()") + logger().info(" using Series name '%s' as Genre" % metadata_x.series) # Format the index as a sort key index = metadata_x.series_index @@ -2840,7 +2858,7 @@ class ITUNES(DriverBase): break if db_added: - self.log.warning(" waiting for db_added to become writeable ") + logger().warning(" waiting for db_added to become writeable ") time.sleep(1.0) # If no title_sort plugboard tweak, create sort_name from series/index if metadata.title_sort == metadata_x.title_sort: @@ -2860,7 +2878,7 @@ class ITUNES(DriverBase): elif metadata_x.tags is not None: if DEBUG: - self.log.info(" %susing Tag as Genre" % + logger().info(" %susing Tag as Genre" % "no Series name available, " if self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY] else '') for tag in metadata_x.tags: if self._is_alpha(tag[0]): @@ -2883,7 +2901,7 @@ class ITUNES(DriverBase): lb_added.Year = metadata_x.pubdate.year if db_added: - self.log.warning(" waiting for db_added to become writeable ") + logger().warning(" waiting for db_added to become writeable ") time.sleep(1.0) db_added.Name = metadata_x.title db_added.Album = metadata_x.title @@ -2910,7 +2928,7 @@ class ITUNES(DriverBase): db_added.AlbumRating = (metadata_x.rating*10) except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " setting AlbumRating on iDevice") # Set Genre from first alpha tag, overwrite with series if available @@ -2919,7 +2937,7 @@ class ITUNES(DriverBase): if metadata_x.series and self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY]: if DEBUG: - self.log.info(" using Series name as Genre") + logger().info(" using Series name as Genre") # Format the index as a sort key index = metadata_x.series_index integer = int(index) @@ -2935,13 +2953,13 @@ class ITUNES(DriverBase): lb_added.TrackNumber = metadata_x.series_index except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " setting TrackNumber in iTunes") try: lb_added.EpisodeNumber = metadata_x.series_index except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " setting EpisodeNumber in iTunes") # If no plugboard transform applied to tags, change the Genre/Category to Series @@ -2963,13 +2981,13 @@ class ITUNES(DriverBase): db_added.TrackNumber = metadata_x.series_index except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " setting TrackNumber on iDevice") try: db_added.EpisodeNumber = metadata_x.series_index except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " setting EpisodeNumber on iDevice") # If no plugboard transform applied to tags, change the Genre/Category to Series @@ -2983,7 +3001,7 @@ class ITUNES(DriverBase): elif metadata_x.tags is not None: if DEBUG: - self.log.info(" using Tag as Genre") + logger().info(" using Tag as Genre") for tag in metadata_x.tags: if self._is_alpha(tag[0]): if lb_added: @@ -2997,8 +3015,8 @@ class ITUNES(DriverBase): Ensure iDevice metadata is writable. Direct connect mode only ''' if DEBUG: - self.log.info(" ITUNES._wait_for_writable_metadata()") - self.log.warning(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE) + logger().info(" ITUNES._wait_for_writable_metadata()") + logger().warning(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE) attempts = 9 while attempts: @@ -3012,40 +3030,40 @@ class ITUNES(DriverBase): attempts -= 1 time.sleep(delay) if DEBUG: - self.log.warning(" waiting %.1f seconds for iDevice metadata to become writable (attempt #%d)" % + logger().warning(" waiting %.1f seconds for iDevice metadata to become writable (attempt #%d)" % (delay, (10 - attempts))) else: if DEBUG: - self.log.error(" failed to write device metadata") + logger().error(" failed to write device metadata") def _xform_metadata_via_plugboard(self, book, format): ''' Transform book metadata from plugboard templates ''' if DEBUG: - self.log.info(" ITUNES._xform_metadata_via_plugboard()") + logger().info(" ITUNES._xform_metadata_via_plugboard()") if self.plugboard_func: pb = self.plugboard_func(self.DEVICE_PLUGBOARD_NAME, format, self.plugboards) newmi = book.deepcopy_metadata() newmi.template_to_attribute(book, pb) if pb is not None and DEBUG: - #self.log.info(" transforming %s using %s:" % (format, pb)) - self.log.info(" title: '%s' %s" % (book.title, ">>> '%s'" % + #logger().info(" transforming %s using %s:" % (format, pb)) + logger().info(" title: '%s' %s" % (book.title, ">>> '%s'" % newmi.title if book.title != newmi.title else '')) - self.log.info(" title_sort: %s %s" % (book.title_sort, ">>> %s" % + logger().info(" title_sort: %s %s" % (book.title_sort, ">>> %s" % newmi.title_sort if book.title_sort != newmi.title_sort else '')) - self.log.info(" authors: %s %s" % (book.authors, ">>> %s" % + logger().info(" authors: %s %s" % (book.authors, ">>> %s" % newmi.authors if book.authors != newmi.authors else '')) - self.log.info(" author_sort: %s %s" % (book.author_sort, ">>> %s" % + logger().info(" author_sort: %s %s" % (book.author_sort, ">>> %s" % newmi.author_sort if book.author_sort != newmi.author_sort else '')) - self.log.info(" language: %s %s" % (book.language, ">>> %s" % + logger().info(" language: %s %s" % (book.language, ">>> %s" % newmi.language if book.language != newmi.language else '')) - self.log.info(" publisher: %s %s" % (book.publisher, ">>> %s" % + logger().info(" publisher: %s %s" % (book.publisher, ">>> %s" % newmi.publisher if book.publisher != newmi.publisher else '')) - self.log.info(" tags: %s %s" % (book.tags, ">>> %s" % + logger().info(" tags: %s %s" % (book.tags, ">>> %s" % newmi.tags if book.tags != newmi.tags else '')) else: if DEBUG: - self.log(" matching plugboard not found") + logger()(" matching plugboard not found") else: newmi = book @@ -3068,7 +3086,7 @@ class ITUNES_ASYNC(ITUNES): def __init__(self,path): if DEBUG: - self.log.info("ITUNES_ASYNC:__init__()") + logger().info("ITUNES_ASYNC:__init__()") if isosx and appscript is None: self.connected = False @@ -3110,15 +3128,15 @@ class ITUNES_ASYNC(ITUNES): """ if not oncard: if DEBUG: - self.log.info("ITUNES_ASYNC:books()") + logger().info("ITUNES_ASYNC:books()") if self.settings().extra_customization[self.CACHE_COVERS]: - self.log.info(" Cover fetching/caching enabled") + logger().info(" Cover fetching/caching enabled") else: - self.log.info(" Cover fetching/caching disabled") + logger().info(" Cover fetching/caching disabled") # Fetch a list of books from iTunes - booklist = BookList(self.log) + booklist = BookList(logger()) cached_books = {} if isosx: @@ -3214,7 +3232,7 @@ class ITUNES_ASYNC(ITUNES): return booklist else: - return BookList(self.log) + return BookList(logger()) def eject(self): ''' @@ -3222,7 +3240,7 @@ class ITUNES_ASYNC(ITUNES): are pending GUI jobs that need to communicate with the device. ''' if DEBUG: - self.log.info("ITUNES_ASYNC:eject()") + logger().info("ITUNES_ASYNC:eject()") self.iTunes = None self.connected = False @@ -3237,7 +3255,7 @@ class ITUNES_ASYNC(ITUNES): particular device doesn't have any of these locations it should return -1. """ if DEBUG: - self.log.info("ITUNES_ASYNC:free_space()") + logger().info("ITUNES_ASYNC:free_space()") free_space = 0 if isosx: s = os.statvfs(os.sep) @@ -3254,7 +3272,7 @@ class ITUNES_ASYNC(ITUNES): @return: (device name, device version, software version on device, mime type) """ if DEBUG: - self.log.info("ITUNES_ASYNC:get_device_information()") + logger().info("ITUNES_ASYNC:get_device_information()") return ('iTunes','hw v1.0','sw v1.0', 'mime type normally goes here') @@ -3277,33 +3295,33 @@ class ITUNES_ASYNC(ITUNES): we need to talk to iTunes to discover if there's a connected iPod ''' if DEBUG: - self.log.info("ITUNES_ASYNC.open(connected_device: %s)" % repr(connected_device)) + logger().info("ITUNES_ASYNC.open(connected_device: %s)" % repr(connected_device)) # Confirm/create thumbs archive if not os.path.exists(self.cache_dir): if DEBUG: - self.log.info(" creating thumb cache '%s'" % self.cache_dir) + logger().info(" creating thumb cache '%s'" % self.cache_dir) os.makedirs(self.cache_dir) if not os.path.exists(self.archive_path): - self.log.info(" creating zip archive") + logger().info(" creating zip archive") zfw = ZipFile(self.archive_path, mode='w') zfw.writestr("iTunes Thumbs Archive",'') zfw.close() else: if DEBUG: - self.log.info(" existing thumb cache at '%s'" % self.archive_path) + logger().info(" existing thumb cache at '%s'" % self.archive_path) # If enabled in config options, create/confirm an iTunes storage folder if not self.settings().extra_customization[self.USE_ITUNES_STORAGE]: self.iTunes_local_storage = os.path.join(config_dir,'iTunes storage') if not os.path.exists(self.iTunes_local_storage): if DEBUG: - self.log(" creating iTunes_local_storage at '%s'" % self.iTunes_local_storage) + logger()(" creating iTunes_local_storage at '%s'" % self.iTunes_local_storage) os.mkdir(self.iTunes_local_storage) else: if DEBUG: - self.log(" existing iTunes_local_storage at '%s'" % self.iTunes_local_storage) + logger()(" existing iTunes_local_storage at '%s'" % self.iTunes_local_storage) def sync_booklists(self, booklists, end_session=True): ''' @@ -3314,7 +3332,7 @@ class ITUNES_ASYNC(ITUNES): ''' if DEBUG: - self.log.info("ITUNES_ASYNC.sync_booklists()") + logger().info("ITUNES_ASYNC.sync_booklists()") # Inform user of any problem books if self.problem_titles: @@ -3328,7 +3346,7 @@ class ITUNES_ASYNC(ITUNES): ''' ''' if DEBUG: - self.log.info("ITUNES_ASYNC:unmount_device()") + logger().info("ITUNES_ASYNC:unmount_device()") self.connected = False class BookList(list): diff --git a/src/calibre/devices/bambook/libbambookcore.py b/src/calibre/devices/bambook/libbambookcore.py index e77ac1da7b..a1c6046df0 100644 --- a/src/calibre/devices/bambook/libbambookcore.py +++ b/src/calibre/devices/bambook/libbambookcore.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' Sanda library wrapper ''' -import ctypes, uuid, hashlib, os, sys +import ctypes, hashlib, os, sys from threading import Event, Lock from calibre.constants import iswindows from calibre import load_library @@ -350,6 +350,7 @@ class Bambook: return None def SendFile(self, fileName, guid = None): + import uuid if self.handle: taskID = job.NewJob() if guid: diff --git a/src/calibre/devices/cybook/t2b.py b/src/calibre/devices/cybook/t2b.py index 7aaeeb63d7..fc0c772bf7 100644 --- a/src/calibre/devices/cybook/t2b.py +++ b/src/calibre/devices/cybook/t2b.py @@ -5,7 +5,6 @@ Write a t2b file to disk. ''' import StringIO -from PIL import Image DEFAULT_T2B_DATA = '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00\x0f\xff\xff\xff\xf0\xff\x0f\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf8\x00\x00\xff\xff\xff\xf0\xff\x0f\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe0\xff\xf0\xff\xff\xff\xf0\xff\xff\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc3\xff\xff\xff\xff\xff\xf0\xff\xff\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x07\xff\xff\xfc\x00?\xf0\xff\x0f\xc3\x00?\xf0\xc0\xfe\x00?\xff\xff\xff\xff\xff\xff\xff\x0f\xff\xff\xf0<\x0f\xf0\xff\x0f\xc0,\x0f\xf0\x0e\xf0,\x0f\xff\xff\xff\xff\xff\xff\xff\x0f\xff\xff\xff\xff\xc3\xf0\xff\x0f\xc0\xff\x0f\xf0\xff\xf0\xff\xc7\xff\xff\xff\xff\xff\xff\xff\x0f\xff\xff\xff\xff\xc3\xf0\xff\x0f\xc3\xff\xc3\xf0\xff\xc3\xff\xc3\xff\xff\xff\xff\xff\xff\xff\x0f\xff\xff\xff\x00\x03\xf0\xff\x0f\xc3\xff\xc3\xf0\xff\xc3\xff\xc3\xff\xff\xff\xff\xff\xff\xff\x0f\xff\xff\xf0\x1f\xc3\xf0\xff\x0f\xc3\xff\xc3\xf0\xff\xc0\x00\x03\xff\xff\xff\xff\xff\xff\xff\x0b\xff\xff\xf0\xff\xc3\xf0\xff\x0f\xc3\xff\xc3\xf0\xff\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc3\xff\xff\xf3\xff\xc3\xf0\xff\x0f\xc3\xff\xc3\xf0\xff\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc0\xff\xfc\xf0\xff\x03\xf0\xff\x0f\xc0\xff\x0f\xf0\xff\xf0\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf0\x0f\x00\xf08\x03\xf0\xff\x0f\xc0,\x0f\xf0\xff\xf0\x1f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x00\x0f\xfc\x00\xc3\xf0\xff\x0f\xc3\x00?\xf0\xff\xff\x00\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf0\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf0\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x03\xfe\x94\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc0\x00\x00\x00\x0f\xff\xff\xff\xff\xff\xff\xfc\x7f\xfe\x94\xff\xff\xff\xff\xff\xff\xff\xff\xfc\x0f\xff\xfe\xa9@\xff\xff\xff\xff\xff\xff\xfc?\xfe\xa4\xff\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xe9P\xff\xff\xff\xff\xff\xff\xfe/\xfe\xa8\xff\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xf9T\xff\xff\xff\xff\xf0@\x00+\xfa\xa8?\xff\xff\xff\xff\xff\xff\xff\xfc\xbf\xff\xff\xf9T\xff\xff\xff\xff\xcb\xe4}*\xaa\xaa?\xff\xff\xff\xff\xff\xff\xff\xfc\xbf\xff\xff\xe9T\xff\xff\xff\xff\xc7\xe4\xfd\x1a\xaa\xaa?\xff\xff\xff\xff\xff\xff\xff\xfc\xaf\xea\xaa\xa6\xa4\xff@\x00\x0f\xc3\xe8\xfe\x1a\xaa\xaa?\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4\x00\x7f\xfe\x90\x03\xe8\xfe\n\xaa\xaa?\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4?\xff\xff\xa5C\xe8\xfe\x06\xaa\xaa?\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4?\xff\xff\xeaC\xe8\xbe\x06\xaa\xaa\x0f\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4/\xff\xff\xea\x82\xe8j\x06\xaa\xaa\x0f\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4/\xff\xff\xaa\x82\xe8*F\xaa\xaa\x8f\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4+\xff\xfe\xaa\x82\xe8*\x86\xaa\xaa\x8f\xff\xff\x80\xff\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8*\x86\xaa\xaa\x8f\xf0\x00T?\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8*\x81\xaa\xaa\x8c\x03\xff\x95?\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8*\x81\xaa\xaa\x80\xbf\xff\x95?\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8*\x81\xaa\xaa\x9b\xff\xff\x95\x0f\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8\x1a\x81\xaa\xaa\x9a\xff\xfe\x95\x0f\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8\n\x81\xaa\xaa\xa6\xbf\xfeUO\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xa8\n\x91j\xaa\xa5\xaa\xa9ZO\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xa8\n\xa0j\xaa\xa5Z\x95ZO\xff\xff\xff\xfcj\x95UV\xa4*\xfa\xaa\xaa\x82\xa9\n\xa0j\xaa\xa5UUZC\xff\xff\xff\xfcj\x95UV\xa4*\xfa\xaa\xaa\x82\xaa\n\xa0j\xaa\xa4UUZS\xff\xff\xff\xfcZ\x95UV\xa4*\xfa\xaa\xaa\x82\xaa\n\xa0j\xaa\xa4UUZS\xff\xff\xff\xfcZ\x95UU\xa4*\xfa\xaa\xaa\x82\xaa\n\xa0j\xaa\xa8UUVS\xff\xff\xff\xfcZ\x95UU\xa4*\xea\xaa\xaa\x82\xaa\x06\xa0Z\xaa\xa8UUV\x93\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x81\xaa\x02\xa0\x1a\xaa\xa8UUV\x90\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x80\xaa\x02\xa0\x1a\xaa\xa8\x15UU\x94\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x80\xaa"\xa0\x1a\xaa\xa8\x15UU\x94\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x80\xaa2\xa4\x16\xaa\xa8\x15UU\x94\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x80\xaa2\xa8\x16\xa6\xa9\x15UU\x94\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x80\xaa2\xa8\x16\xa6\xa9\x05UUT?\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x84\xaa2\xa8\x16\xaa\xaa\x05UUU?\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x88\xaa2\xa8\x06\xaa\xaa\x05UUU?\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa1\xa8\xc5\xaa\xaa\x05UUU?\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa0\xa8E\xa9\xaa\x05UUU/\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa<\xa8\x05\xa9\xaaAUUU\x0f\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa<\xa8\x05\xa9\xaaAUUUO\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa<\xa9\x05\xaa\xaaAUUUO\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x1c\xaa\x01\xaa\xaa\x81UUUO\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x0c\xaa\x01\xaa\xaa\x81UUUO\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x0c\xaa1j\xaa\x80UUUC\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x0cj1jj\x90UUUS\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x0c*1jj\x90UUUS\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaaL*1jj\xa0UUUS\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x8f* j\xaa\xa0\x15UUS\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x8f*@j\xaa\xa0\x15UUP\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x8f*\x8cZ\xaa\xa1\x15UUT\xff\xff\xfcZ\x95UU\xa4j\xaa\xaa\xaa\x8c\xaa\x8f*\x8cZ\x9a\xa0\x15UUT\xff\xff\xfcZ\x95UU\xa4j\xaa\xaa\xaa\x8c\xaa\x8f*\x8cZ\x9a\xa0\x15UUT\xff\xff\xfcZ\x95UU\xa4j\xaa\xaa\xaa\x8c\xaa\x8f\x1a\x8cZ\x9a\xa4\x15UUT?\xff\xfcZ\x95UU\x94j\xaa\xaa\xaa\x8cj\x8f\n\x8cVj\xa4\x05UU\xa4?\xff\xfcVUUU\xa4j\xaa\xaa\xaa\x8cj\x8fJ\x8c\x16\xaa\xa8\xc5UZ\xa5?\xff\xfcUUUV\xa4j\xaa\xaa\xaa\x8cj\x8f\xca\x8f\x16\xaa\xa8\xc5V\xaa\xa5?\xff\xfcUj\xaa\xaa\xa4j\xaa\xaa\xaa\x8cj\x8f\xca\x8f\x1a\xaa\xa8\x05Z\xaaU?\xff\xfcV\xaa\xaa\xaa\xa5j\xaa\xaa\xaa\x8e*\x8f\xca\x83\x1a\xaa\xa4\x01eUU?\xff\xfcZ\xaa\xaa\xaa\xa5j\xaa\xaa\xaa\x8f*\x8f\xca\x83\x1a\xa5U\x01U\x00\x00\x0f\xff\xfcUUUUUZ\xaa\xaa\xaaO%\x8f\xc6\x93\x15\x00\x001@\x0f\xff\xff\xff\xfcP\x00\x00\x00\x15\x00\x00\x00\x00\x0f\x00\x07\xc0\x03\x00\xff\xff0\x1f\xff\xff\xff\xff\xfc\x00\xff\xff\xf8\x00?\xff\xff\xff\x0f?\xc7\xc3\xf7\x0f\xff\xff\xf1\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xf4\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' @@ -18,7 +17,7 @@ def reduce_color(c): return 2 else: return 3 - + def i2b(n): return "".join([str((n >> y) & 1) for y in range(1, -1, -1)]) @@ -27,12 +26,13 @@ def write_t2b(t2bfile, coverdata=None): t2bfile is a file handle ready to write binary data to disk. coverdata is a string representation of a JPEG file. ''' + from PIL import Image if coverdata != None: coverdata = StringIO.StringIO(coverdata) cover = Image.open(coverdata).convert("L") cover.thumbnail((96, 144), Image.ANTIALIAS) t2bcover = Image.new('L', (96, 144), 'white') - + x, y = cover.size t2bcover.paste(cover, ((96-x)/2, (144-y)/2)) diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py index 09df8cd6d8..c8ee3a2e77 100644 --- a/src/calibre/devices/folder_device/driver.py +++ b/src/calibre/devices/folder_device/driver.py @@ -97,3 +97,13 @@ class FOLDER_DEVICE(USBMS): @classmethod def settings(self): return FOLDER_DEVICE_FOR_CONFIG._config().parse() + + @classmethod + def config_widget(cls): + return FOLDER_DEVICE_FOR_CONFIG.config_widget() + + @classmethod + def save_settings(cls, config_widget): + return FOLDER_DEVICE_FOR_CONFIG.save_settings(config_widget) + + diff --git a/src/calibre/devices/kindle/apnx.py b/src/calibre/devices/kindle/apnx.py index 178c1091f3..a051c84be6 100644 --- a/src/calibre/devices/kindle/apnx.py +++ b/src/calibre/devices/kindle/apnx.py @@ -9,7 +9,6 @@ Generates and writes an APNX page mapping file. ''' import struct -import uuid from calibre.ebooks.mobi.reader import MobiReader from calibre.ebooks.pdb.header import PdbHeaderReader @@ -51,6 +50,7 @@ class APNXBuilder(object): apnxf.write(apnx) def generate_apnx(self, pages): + import uuid apnx = '' content_vals = { diff --git a/src/calibre/devices/kindle/driver.py b/src/calibre/devices/kindle/driver.py index 3c69245cf9..1b10ce3050 100644 --- a/src/calibre/devices/kindle/driver.py +++ b/src/calibre/devices/kindle/driver.py @@ -10,10 +10,8 @@ Device driver for Amazon's Kindle import datetime, os, re, sys, json, hashlib -from calibre.devices.kindle.apnx import APNXBuilder from calibre.devices.kindle.bookmark import Bookmark from calibre.devices.usbms.driver import USBMS -from calibre.ebooks.metadata import MetaInformation from calibre import strftime ''' @@ -152,6 +150,7 @@ class KINDLE(USBMS): path_map, book_ext = resolve_bookmark_paths(storage, path_map) bookmarked_books = {} + for id in path_map: bookmark_ext = path_map[id].rpartition('.')[2] myBookmark = Bookmark(path_map[id], id, book_ext[id], bookmark_ext) @@ -236,6 +235,8 @@ class KINDLE(USBMS): def add_annotation_to_library(self, db, db_id, annotation): from calibre.ebooks.BeautifulSoup import Tag + from calibre.ebooks.metadata import MetaInformation + bm = annotation ignore_tags = set(['Catalog', 'Clippings']) @@ -363,6 +364,8 @@ class KINDLE2(KINDLE): ''' Hijacking this function to write the apnx file. ''' + from calibre.devices.kindle.apnx import APNXBuilder + opts = self.settings() if not opts.extra_customization[self.OPT_APNX]: return diff --git a/src/calibre/devices/kobo/bookmark.py b/src/calibre/devices/kobo/bookmark.py index 8e199f77a6..afb392403d 100644 --- a/src/calibre/devices/kobo/bookmark.py +++ b/src/calibre/devices/kobo/bookmark.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' import os from contextlib import closing -import sqlite3 as sqlite class Bookmark(): # {{{ ''' @@ -32,7 +31,7 @@ class Bookmark(): # {{{ def get_bookmark_data(self): ''' Return the timestamp and last_read_location ''' - + import sqlite3 as sqlite user_notes = {} self.timestamp = os.path.getmtime(self.path) with closing(sqlite.connect(self.db_path)) as connection: diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 0bc578155d..f68ea8feff 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -6,7 +6,6 @@ __copyright__ = '2010, Timothy Legge and Kovid Goyal >sys.stderr, "Execution failed:", e + return ans + + + linux_scanner = None if islinux: linux_scanner = LinuxScanner() +freebsd_scanner = None + +if isfreebsd: + freebsd_scanner = FreeBSDScanner() + + class DeviceScanner(object): def __init__(self, *args): if isosx and osx_scanner is None: raise RuntimeError('The Python extension usbobserver must be available on OS X.') - self.scanner = win_scanner if iswindows else osx_scanner if isosx else linux_scanner + self.scanner = win_scanner if iswindows else osx_scanner if isosx else freebsd_scanner if isfreebsd else linux_scanner self.devices = [] def scan(self): diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index f62dc44149..6765c3e9c0 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -591,26 +591,7 @@ class Device(DeviceConfig, DevicePlugin): mp = self.node_mountpoint(node) if mp is not None: return mp, 0 - if type == 'main': - label = self.MAIN_MEMORY_VOLUME_LABEL - if type == 'carda': - label = self.STORAGE_CARD_VOLUME_LABEL - if type == 'cardb': - label = self.STORAGE_CARD2_VOLUME_LABEL - if not label: - label = self.STORAGE_CARD_VOLUME_LABEL + ' 2' - if not label: - label = 'E-book Reader (%s)'%type - extra = 0 - while True: - q = ' (%d)'%extra if extra else '' - if not os.path.exists('/media/'+label+q): - break - extra += 1 - if extra: - label += ' (%d)'%extra - - def do_mount(node, label): + def do_mount(node): try: from calibre.devices.udisks import mount mount(node) @@ -621,8 +602,7 @@ class Device(DeviceConfig, DevicePlugin): traceback.print_exc() return 1 - - ret = do_mount(node, label) + ret = do_mount(node) if ret != 0: return None, ret return self.node_mountpoint(node)+'/', 0 @@ -697,19 +677,21 @@ class Device(DeviceConfig, DevicePlugin): self._card_a_prefix = self._card_b_prefix self._card_b_prefix = None + # ------------------------------------------------------ # # open for FreeBSD -# find the device node or nodes that match the S/N we already have from the scanner -# and attempt to mount each one -# 1. get list of disk devices from sysctl -# 2. compare that list with the one from camcontrol -# 3. and see if it has a matching s/n -# 6. find any partitions/slices associated with each node -# 7. attempt to mount, using calibre-mount-helper, each one -# 8. when finished, we have a list of mount points and associated device nodes +# find the device node or nodes that match the S/N we already have from the scanner +# and attempt to mount each one +# 1. get list of devices in /dev with matching s/n etc. +# 2. get list of volumes associated with each +# 3. attempt to mount each one using Hal +# 4. when finished, we have a list of mount points and associated dbus nodes # def open_freebsd(self): + import dbus + # There should be some way to access the -v arg... + verbose = False # this gives us access to the S/N, etc. of the reader that the scanner has found # and the match routines for some of that data, like s/n, vendor ID, etc. @@ -719,128 +701,146 @@ class Device(DeviceConfig, DevicePlugin): raise DeviceError("Device has no S/N. Can't continue") return False - devs={} - di=0 - ndevs=4 # number of possible devices per reader (main, carda, cardb, launcher) + vols=[] - #get list of disk devices - p=subprocess.Popen(["sysctl", "kern.disks"], stdout=subprocess.PIPE) - kdsks=subprocess.Popen(["sed", "s/kern.disks: //"], stdin=p.stdout, stdout=subprocess.PIPE).communicate()[0] - p.stdout.close() - #print kdsks - for dvc in kdsks.split(): - # for each one that's also in the list of cam devices ... - p=subprocess.Popen(["camcontrol", "devlist"], stdout=subprocess.PIPE) - devmatch=subprocess.Popen(["grep", dvc], stdin=p.stdout, stdout=subprocess.PIPE).communicate()[0] - p.stdout.close() - if devmatch: - #print "Checking ", devmatch - # ... see if we can get a S/N from the actual device node - sn=subprocess.Popen(["camcontrol", "inquiry", dvc, "-S"], stdout=subprocess.PIPE).communicate()[0] - sn=sn[0:-1] # drop the trailing newline - #print "S/N = ", sn - if sn and d.match_serial(sn): - # we have a matching s/n, record this device node - #print "match found: ", dvc - devs[di]=dvc - di += 1 + bus = dbus.SystemBus() + manager = dbus.Interface(bus.get_object('org.freedesktop.Hal', + '/org/freedesktop/Hal/Manager'), 'org.freedesktop.Hal.Manager') + paths = manager.FindDeviceStringMatch('usb.serial',d.serial) + for path in paths: + objif = dbus.Interface(bus.get_object('org.freedesktop.Hal', path), 'org.freedesktop.Hal.Device') + # Extra paranoia... + try: + if d.idVendor == objif.GetProperty('usb.vendor_id') and \ + d.idProduct == objif.GetProperty('usb.product_id') and \ + d.manufacturer == objif.GetProperty('usb.vendor') and \ + d.product == objif.GetProperty('usb.product') and \ + d.serial == objif.GetProperty('usb.serial'): + dpaths = manager.FindDeviceStringMatch('storage.originating_device', path) + for dpath in dpaths: + #devif = dbus.Interface(bus.get_object('org.freedesktop.Hal', dpath), 'org.freedesktop.Hal.Device') + try: + vpaths = manager.FindDeviceStringMatch('block.storage_device', dpath) + for vpath in vpaths: + try: + vdevif = dbus.Interface(bus.get_object('org.freedesktop.Hal', vpath), 'org.freedesktop.Hal.Device') + if not vdevif.GetProperty('block.is_volume'): + continue + if vdevif.GetProperty('volume.fsusage') != 'filesystem': + continue + volif = dbus.Interface(bus.get_object('org.freedesktop.Hal', vpath), 'org.freedesktop.Hal.Device.Volume') + pdevif = dbus.Interface(bus.get_object('org.freedesktop.Hal', vdevif.GetProperty('info.parent')), 'org.freedesktop.Hal.Device') + vol = {'node': pdevif.GetProperty('block.device'), + 'dev': vdevif, + 'vol': volif, + 'label': vdevif.GetProperty('volume.label')} + vols.append(vol) + except dbus.exceptions.DBusException, e: + print e + continue + except dbus.exceptions.DBusException, e: + print e + continue + except dbus.exceptions.DBusException, e: + continue - # sort the list of devices - for i in range(1,ndevs+1): - for j in reversed(range(1,i)): - if devs[j-1] > devs[j]: - x=devs[j-1] - devs[j-1]=devs[j] - devs[j]=x - #print devs + def ocmp(x,y): + if x['node'] < y['node']: + return -1 + if x['node'] > y['node']: + return 1 + return 0 + + vols.sort(cmp=ocmp) + + if verbose: + print "FBSD: ", vols - # now we need to see if any of these have slices/partitions mtd=0 - label="READER" # could use something more unique, like S/N or productID... - cmd = '/usr/local/bin/calibre-mount-helper' - cmd = [cmd, 'mount'] - for i in range(0,ndevs): - cmd2="ls /dev/"+devs[i]+"*" - p=subprocess.Popen(cmd2, shell=True, stdout=subprocess.PIPE) - devs[i]=subprocess.Popen(["cut", "-d", "/", "-f" "3"], stdin=p.stdout, stdout=subprocess.PIPE).communicate()[0] - p.stdout.close() - # try all the nodes to see what we can mount - for dev in devs[i].split(): - mp='/media/'+label+'-'+dev - mmp = mp - if mmp.endswith('/'): - mmp = mmp[:-1] - #print "trying ", dev, "on", mp + for vol in vols: + mp = '' + if vol['dev'].GetProperty('volume.is_mounted'): + mp = vol['dev'].GetProperty('volume.mount_point') + else: try: - p = subprocess.Popen(cmd + ["/dev/"+dev, mmp]) - except OSError: - raise DeviceError(_('Could not find mount helper: %s.')%cmd[0]) - while p.poll() is None: - time.sleep(0.1) + vol['vol'].Mount('Calibre-'+vol['label'], + vol['dev'].GetProperty('volume.fstype'), []) + loops = 0 + while not vol['dev'].GetProperty('volume.is_mounted'): + time.sleep(1) + loops += 1 + if loops > 100: + print "ERROR: Timeout waiting for mount to complete" + continue + mp = vol['dev'].GetProperty('volume.mount_point') + except dbus.exceptions.DBusException, e: + print "Failed to mount ", e + continue - if p.returncode == 0: - #print " mounted", dev - if i == 0: - self._main_prefix = mp - self._main_dev = "/dev/"+dev - #print "main = ", self._main_dev, self._main_prefix - if i == 1: - self._card_a_prefix = mp - self._card_a_dev = "/dev/"+dev - #print "card a = ", self._card_a_dev, self._card_a_prefix - if i == 2: - self._card_b_prefix = mp - self._card_b_dev = "/dev/"+dev - #print "card b = ", self._card_b_dev, self._card_b_prefix + # Mount Point becomes Mount Path + mp += '/' - mtd += 1 - break + if verbose: + print "FBSD: mounted", vol['label'], "on", mp + if mtd == 0: + self._main_prefix = mp + self._main_vol = vol['vol'] + if verbose: + print "FBSD: main = ", self._main_prefix + if mtd == 1: + self._card_a_prefix = mp + self._card_a_vol = vol['vol'] + if verbose: + print "FBSD: card a = ", self._card_a_prefix + if mtd == 2: + self._card_b_prefix = mp + self._card_b_vol = vol['vol'] + if verbose: + print "FBSD: card b = ", self._card_b_prefix + # Note that mtd is used as a bool... not incrementing is fine. + break + mtd += 1 if mtd > 0: return True - else : - return False + raise DeviceError(_('Unable to mount the device')) + # # ------------------------------------------------------ # -# this one is pretty simple: -# just umount each of the previously -# mounted filesystems, using the mount helper +# this one is pretty simple: +# just umount each of the previously +# mounted filesystems, using the stored volume object # def eject_freebsd(self): - cmd = '/usr/local/bin/calibre-mount-helper' - cmd = [cmd, 'eject'] + import dbus + # There should be some way to access the -v arg... + verbose = False if self._main_prefix: - #print "umount main:", cmd, self._main_dev, self._main_prefix + if verbose: + print "FBSD: umount main:", self._main_prefix try: - p = subprocess.Popen(cmd + [self._main_dev, self._main_prefix]) - except OSError: - raise DeviceError( - _('Could not find mount helper: %s.')%cmd[0]) - while p.poll() is None: - time.sleep(0.1) + self._main_vol.Unmount([]) + except dbus.exceptions.DBusException, e: + print 'Unable to eject ', e if self._card_a_prefix: - #print "umount card a:", cmd, self._card_a_dev, self._card_a_prefix + if verbose: + print "FBSD: umount card a:", self._card_a_prefix try: - p = subprocess.Popen(cmd + [self._card_a_dev, self._card_a_prefix]) - except OSError: - raise DeviceError( - _('Could not find mount helper: %s.')%cmd[0]) - while p.poll() is None: - time.sleep(0.1) + self._card_a_vol.Unmount([]) + except dbus.exceptions.DBusException, e: + print 'Unable to eject ', e if self._card_b_prefix: - #print "umount card b:", cmd, self._card_b_dev, self._card_b_prefix + if verbose: + print "FBSD: umount card b:", self._card_b_prefix try: - p = subprocess.Popen(cmd + [self._card_b_dev, self._card_b_prefix]) - except OSError: - raise DeviceError( - _('Could not find mount helper: %s.')%cmd[0]) - while p.poll() is None: - time.sleep(0.1) + self._card_b_vol.Unmount([]) + except dbus.exceptions.DBusException, e: + print 'Unable to eject ', e self._main_prefix = None self._card_a_prefix = None @@ -859,11 +859,10 @@ class Device(DeviceConfig, DevicePlugin): time.sleep(7) self.open_linux() if isfreebsd: - self._main_dev = self._card_a_dev = self._card_b_dev = None + self._main_vol = self._card_a_vol = self._card_b_vol = None try: self.open_freebsd() except DeviceError: - subprocess.Popen(["camcontrol", "rescan", "all"]) time.sleep(2) self.open_freebsd() if iswindows: diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 70b30e98a6..b061bafc03 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -10,7 +10,7 @@ driver. It is intended to be subclassed with the relevant parts implemented for a particular device. ''' -import os, re, time, json, uuid, functools, shutil +import os, re, time, json, functools, shutil from itertools import cycle from calibre.constants import numeric_version @@ -58,6 +58,7 @@ class USBMS(CLI, Device): SCAN_FROM_ROOT = False def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None): + import uuid if not isinstance(dinfo, dict): dinfo = {} if dinfo.get('device_store_uuid', None) is None: diff --git a/src/calibre/devices/user_defined/driver.py b/src/calibre/devices/user_defined/driver.py index d613a09508..6c4e1f77d9 100644 --- a/src/calibre/devices/user_defined/driver.py +++ b/src/calibre/devices/user_defined/driver.py @@ -90,6 +90,10 @@ class USER_DEFINED(USBMS): OPT_CARD_A_FOLDER = 9 def initialize(self): + self.plugin_needs_delayed_initialization = True + USBMS.initialize(self) + + def do_delayed_plugin_initialization(self): try: e = self.settings().extra_customization self.VENDOR_ID = int(e[self.OPT_USB_VENDOR_ID], 16) @@ -107,4 +111,6 @@ class USER_DEFINED(USBMS): except: import traceback traceback.print_exc() - USBMS.initialize(self) + self.plugin_needs_delayed_initialization = False + + diff --git a/src/calibre/ebooks/chardet.py b/src/calibre/ebooks/chardet.py index 598fc673a1..864d09108b 100644 --- a/src/calibre/ebooks/chardet.py +++ b/src/calibre/ebooks/chardet.py @@ -8,7 +8,6 @@ __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' import re, codecs -from chardet import detect ENCODING_PATS = [ re.compile(r'<\?[^<>]+encoding\s*=\s*[\'"](.*?)[\'"][^<>]*>', @@ -34,8 +33,13 @@ def substitute_entites(raw): _CHARSET_ALIASES = { "macintosh" : "mac-roman", "x-sjis" : "shift-jis" } +def detect(*args, **kwargs): + from chardet import detect + return detect(*args, **kwargs) + def force_encoding(raw, verbose, assume_utf8=False): from calibre.constants import preferred_encoding + try: chardet = detect(raw[:1024*50]) except: diff --git a/src/calibre/ebooks/comic/input.py b/src/calibre/ebooks/comic/input.py index 9fcfc559aa..221bece092 100755 --- a/src/calibre/ebooks/comic/input.py +++ b/src/calibre/ebooks/comic/input.py @@ -7,11 +7,10 @@ __docformat__ = 'restructuredtext en' Based on ideas from comiclrf created by FangornUK. ''' -import os, shutil, traceback, textwrap, time, codecs +import os, traceback, time from Queue import Empty -from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from calibre import extract, CurrentDir, prints, walk +from calibre import extract, prints, walk from calibre.constants import filesystem_encoding from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.ipc.server import Server @@ -273,245 +272,4 @@ def process_pages(pages, opts, update, tdir): return ans, failures -class ComicInput(InputFormatPlugin): - - name = 'Comic Input' - author = 'Kovid Goyal' - description = 'Optimize comic files (.cbz, .cbr, .cbc) for viewing on portable devices' - file_types = set(['cbz', 'cbr', 'cbc']) - is_image_collection = True - core_usage = -1 - - options = set([ - OptionRecommendation(name='colors', recommended_value=256, - help=_('Number of colors for grayscale image conversion. Default: ' - '%default. Values of less than 256 may result in blurred text ' - 'on your device if you are creating your comics in EPUB format.')), - OptionRecommendation(name='dont_normalize', recommended_value=False, - help=_('Disable normalize (improve contrast) color range ' - 'for pictures. Default: False')), - OptionRecommendation(name='keep_aspect_ratio', recommended_value=False, - help=_('Maintain picture aspect ratio. Default is to fill the screen.')), - OptionRecommendation(name='dont_sharpen', recommended_value=False, - help=_('Disable sharpening.')), - OptionRecommendation(name='disable_trim', recommended_value=False, - help=_('Disable trimming of comic pages. For some comics, ' - 'trimming might remove content as well as borders.')), - OptionRecommendation(name='landscape', recommended_value=False, - help=_("Don't split landscape images into two portrait images")), - OptionRecommendation(name='wide', recommended_value=False, - help=_("Keep aspect ratio and scale image using screen height as " - "image width for viewing in landscape mode.")), - OptionRecommendation(name='right2left', recommended_value=False, - help=_('Used for right-to-left publications like manga. ' - 'Causes landscape pages to be split into portrait pages ' - 'from right to left.')), - OptionRecommendation(name='despeckle', recommended_value=False, - help=_('Enable Despeckle. Reduces speckle noise. ' - 'May greatly increase processing time.')), - OptionRecommendation(name='no_sort', recommended_value=False, - help=_("Don't sort the files found in the comic " - "alphabetically by name. Instead use the order they were " - "added to the comic.")), - OptionRecommendation(name='output_format', choices=['png', 'jpg'], - recommended_value='png', help=_('The format that images in the created ebook ' - 'are converted to. You can experiment to see which format gives ' - 'you optimal size and look on your device.')), - OptionRecommendation(name='no_process', recommended_value=False, - help=_("Apply no processing to the image")), - OptionRecommendation(name='dont_grayscale', recommended_value=False, - help=_('Do not convert the image to grayscale (black and white)')), - OptionRecommendation(name='comic_image_size', recommended_value=None, - help=_('Specify the image size as widthxheight pixels. Normally,' - ' an image size is automatically calculated from the output ' - 'profile, this option overrides it.')), - OptionRecommendation(name='dont_add_comic_pages_to_toc', recommended_value=False, - help=_('When converting a CBC do not add links to each page to' - ' the TOC. Note this only applies if the TOC has more than one' - ' section')), - ]) - - recommendations = set([ - ('margin_left', 0, OptionRecommendation.HIGH), - ('margin_top', 0, OptionRecommendation.HIGH), - ('margin_right', 0, OptionRecommendation.HIGH), - ('margin_bottom', 0, OptionRecommendation.HIGH), - ('insert_blank_line', False, OptionRecommendation.HIGH), - ('remove_paragraph_spacing', False, OptionRecommendation.HIGH), - ('change_justification', 'left', OptionRecommendation.HIGH), - ('dont_split_on_pagebreaks', True, OptionRecommendation.HIGH), - ('chapter', None, OptionRecommendation.HIGH), - ('page_breaks_brefore', None, OptionRecommendation.HIGH), - ('use_auto_toc', False, OptionRecommendation.HIGH), - ('page_breaks_before', None, OptionRecommendation.HIGH), - ('disable_font_rescaling', True, OptionRecommendation.HIGH), - ('linearize_tables', False, OptionRecommendation.HIGH), - ]) - - def get_comics_from_collection(self, stream): - from calibre.libunzip import extract as zipextract - tdir = PersistentTemporaryDirectory('_comic_collection') - zipextract(stream, tdir) - comics = [] - with CurrentDir(tdir): - if not os.path.exists('comics.txt'): - raise ValueError(( - '%s is not a valid comic collection' - ' no comics.txt was found in the file') - %stream.name) - raw = open('comics.txt', 'rb').read() - if raw.startswith(codecs.BOM_UTF16_BE): - raw = raw.decode('utf-16-be')[1:] - elif raw.startswith(codecs.BOM_UTF16_LE): - raw = raw.decode('utf-16-le')[1:] - elif raw.startswith(codecs.BOM_UTF8): - raw = raw.decode('utf-8')[1:] - else: - raw = raw.decode('utf-8') - for line in raw.splitlines(): - line = line.strip() - if not line: - continue - fname, title = line.partition(':')[0], line.partition(':')[-1] - fname = fname.replace('#', '_') - fname = os.path.join(tdir, *fname.split('/')) - if not title: - title = os.path.basename(fname).rpartition('.')[0] - if os.access(fname, os.R_OK): - comics.append([title, fname]) - if not comics: - raise ValueError('%s has no comics'%stream.name) - return comics - - def get_pages(self, comic, tdir2): - tdir = extract_comic(comic) - new_pages = find_pages(tdir, sort_on_mtime=self.opts.no_sort, - verbose=self.opts.verbose) - thumbnail = None - if not new_pages: - raise ValueError('Could not find any pages in the comic: %s' - %comic) - if self.opts.no_process: - n2 = [] - for page in new_pages: - n2.append(os.path.join(tdir2, os.path.basename(page))) - shutil.copyfile(page, n2[-1]) - new_pages = n2 - else: - new_pages, failures = process_pages(new_pages, self.opts, - self.report_progress, tdir2) - if failures: - self.log.warning('Could not process the following pages ' - '(run with --verbose to see why):') - for f in failures: - self.log.warning('\t', f) - if not new_pages: - raise ValueError('Could not find any valid pages in comic: %s' - % comic) - thumbnail = os.path.join(tdir2, - 'thumbnail.'+self.opts.output_format.lower()) - if not os.access(thumbnail, os.R_OK): - thumbnail = None - return new_pages - - def get_images(self): - return self._images - - def convert(self, stream, opts, file_ext, log, accelerators): - from calibre.ebooks.metadata import MetaInformation - from calibre.ebooks.metadata.opf2 import OPFCreator - from calibre.ebooks.metadata.toc import TOC - - self.opts, self.log= opts, log - if file_ext == 'cbc': - comics_ = self.get_comics_from_collection(stream) - else: - comics_ = [['Comic', os.path.abspath(stream.name)]] - stream.close() - comics = [] - for i, x in enumerate(comics_): - title, fname = x - cdir = 'comic_%d'%(i+1) if len(comics_) > 1 else '.' - cdir = os.path.abspath(cdir) - if not os.path.exists(cdir): - os.makedirs(cdir) - pages = self.get_pages(fname, cdir) - if not pages: continue - wrappers = self.create_wrappers(pages) - comics.append((title, pages, wrappers)) - - if not comics: - raise ValueError('No comic pages found in %s'%stream.name) - - mi = MetaInformation(os.path.basename(stream.name).rpartition('.')[0], - [_('Unknown')]) - opf = OPFCreator(os.path.abspath('.'), mi) - entries = [] - - def href(x): - if len(comics) == 1: return os.path.basename(x) - return '/'.join(x.split(os.sep)[-2:]) - - for comic in comics: - pages, wrappers = comic[1:] - entries += [(w, None) for w in map(href, wrappers)] + \ - [(x, None) for x in map(href, pages)] - opf.create_manifest(entries) - spine = [] - for comic in comics: - spine.extend(map(href, comic[2])) - self._images = [] - for comic in comics: - self._images.extend(comic[1]) - opf.create_spine(spine) - toc = TOC() - if len(comics) == 1: - wrappers = comics[0][2] - for i, x in enumerate(wrappers): - toc.add_item(href(x), None, _('Page')+' %d'%(i+1), - play_order=i) - else: - po = 0 - for comic in comics: - po += 1 - wrappers = comic[2] - stoc = toc.add_item(href(wrappers[0]), - None, comic[0], play_order=po) - if not opts.dont_add_comic_pages_to_toc: - for i, x in enumerate(wrappers): - stoc.add_item(href(x), None, - _('Page')+' %d'%(i+1), play_order=po) - po += 1 - opf.set_toc(toc) - m, n = open('metadata.opf', 'wb'), open('toc.ncx', 'wb') - opf.render(m, n, 'toc.ncx') - return os.path.abspath('metadata.opf') - - def create_wrappers(self, pages): - from calibre.ebooks.oeb.base import XHTML_NS - wrappers = [] - WRAPPER = textwrap.dedent('''\ - - - Page #%d - - - -
- comic page #%d -
- - - ''') - dir = os.path.dirname(pages[0]) - for i, page in enumerate(pages): - wrapper = WRAPPER%(XHTML_NS, i+1, os.path.basename(page), i+1) - page = os.path.join(dir, 'page_%d.xhtml'%(i+1)) - open(page, 'wb').write(wrapper) - wrappers.append(page) - return wrappers diff --git a/src/calibre/ebooks/conversion/plugins/__init__.py b/src/calibre/ebooks/conversion/plugins/__init__.py new file mode 100644 index 0000000000..dd9615356c --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/__init__.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + + diff --git a/src/calibre/ebooks/azw4/input.py b/src/calibre/ebooks/conversion/plugins/azw4_input.py similarity index 84% rename from src/calibre/ebooks/azw4/input.py rename to src/calibre/ebooks/conversion/plugins/azw4_input.py index 1ac7657342..6d2b2a917e 100644 --- a/src/calibre/ebooks/azw4/input.py +++ b/src/calibre/ebooks/conversion/plugins/azw4_input.py @@ -7,8 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.pdb.header import PdbHeaderReader -from calibre.ebooks.azw4.reader import Reader class AZW4Input(InputFormatPlugin): @@ -19,6 +17,9 @@ class AZW4Input(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.pdb.header import PdbHeaderReader + from calibre.ebooks.azw4.reader import Reader + header = PdbHeaderReader(stream) reader = Reader(header, stream, log, options) opf = reader.extract_content(os.getcwd()) diff --git a/src/calibre/ebooks/chm/input.py b/src/calibre/ebooks/conversion/plugins/chm_input.py similarity index 98% rename from src/calibre/ebooks/chm/input.py rename to src/calibre/ebooks/conversion/plugins/chm_input.py index f36685bd91..a674735f1d 100644 --- a/src/calibre/ebooks/chm/input.py +++ b/src/calibre/ebooks/conversion/plugins/chm_input.py @@ -3,9 +3,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ,' \ ' and Alex Bramley .' -import os, uuid - -from lxml import html +import os from calibre.customize.conversion import InputFormatPlugin from calibre.ptempfile import TemporaryDirectory @@ -77,7 +75,7 @@ class CHMInput(InputFormatPlugin): def _create_oebbook_html(self, htmlpath, basedir, opts, log, mi): # use HTMLInput plugin to generate book - from calibre.ebooks.html.input import HTMLInput + from calibre.customize.builtins import HTMLInput opts.breadth_first = True htmlinput = HTMLInput(None) oeb = htmlinput.create_oebbook(htmlpath, basedir, opts, log, mi) @@ -85,6 +83,8 @@ class CHMInput(InputFormatPlugin): def _create_oebbook(self, hhcpath, basedir, opts, log, mi): + import uuid + from lxml import html from calibre.ebooks.conversion.plumber import create_oebbook from calibre.ebooks.oeb.base import DirContainer oeb = create_oebbook(log, None, opts, @@ -142,6 +142,7 @@ class CHMInput(InputFormatPlugin): return oeb def _create_html_root(self, hhcpath, log): + from lxml import html hhcdata = self._read_file(hhcpath) hhcroot = html.fromstring(hhcdata) chapters = self._process_nodes(hhcroot) diff --git a/src/calibre/ebooks/conversion/plugins/comic_input.py b/src/calibre/ebooks/conversion/plugins/comic_input.py new file mode 100644 index 0000000000..77ae7d8086 --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/comic_input.py @@ -0,0 +1,259 @@ +from __future__ import with_statement +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + +''' +Based on ideas from comiclrf created by FangornUK. +''' + +import shutil, textwrap, codecs, os + +from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation +from calibre import CurrentDir +from calibre.ptempfile import PersistentTemporaryDirectory + +class ComicInput(InputFormatPlugin): + + name = 'Comic Input' + author = 'Kovid Goyal' + description = 'Optimize comic files (.cbz, .cbr, .cbc) for viewing on portable devices' + file_types = set(['cbz', 'cbr', 'cbc']) + is_image_collection = True + core_usage = -1 + + options = set([ + OptionRecommendation(name='colors', recommended_value=256, + help=_('Number of colors for grayscale image conversion. Default: ' + '%default. Values of less than 256 may result in blurred text ' + 'on your device if you are creating your comics in EPUB format.')), + OptionRecommendation(name='dont_normalize', recommended_value=False, + help=_('Disable normalize (improve contrast) color range ' + 'for pictures. Default: False')), + OptionRecommendation(name='keep_aspect_ratio', recommended_value=False, + help=_('Maintain picture aspect ratio. Default is to fill the screen.')), + OptionRecommendation(name='dont_sharpen', recommended_value=False, + help=_('Disable sharpening.')), + OptionRecommendation(name='disable_trim', recommended_value=False, + help=_('Disable trimming of comic pages. For some comics, ' + 'trimming might remove content as well as borders.')), + OptionRecommendation(name='landscape', recommended_value=False, + help=_("Don't split landscape images into two portrait images")), + OptionRecommendation(name='wide', recommended_value=False, + help=_("Keep aspect ratio and scale image using screen height as " + "image width for viewing in landscape mode.")), + OptionRecommendation(name='right2left', recommended_value=False, + help=_('Used for right-to-left publications like manga. ' + 'Causes landscape pages to be split into portrait pages ' + 'from right to left.')), + OptionRecommendation(name='despeckle', recommended_value=False, + help=_('Enable Despeckle. Reduces speckle noise. ' + 'May greatly increase processing time.')), + OptionRecommendation(name='no_sort', recommended_value=False, + help=_("Don't sort the files found in the comic " + "alphabetically by name. Instead use the order they were " + "added to the comic.")), + OptionRecommendation(name='output_format', choices=['png', 'jpg'], + recommended_value='png', help=_('The format that images in the created ebook ' + 'are converted to. You can experiment to see which format gives ' + 'you optimal size and look on your device.')), + OptionRecommendation(name='no_process', recommended_value=False, + help=_("Apply no processing to the image")), + OptionRecommendation(name='dont_grayscale', recommended_value=False, + help=_('Do not convert the image to grayscale (black and white)')), + OptionRecommendation(name='comic_image_size', recommended_value=None, + help=_('Specify the image size as widthxheight pixels. Normally,' + ' an image size is automatically calculated from the output ' + 'profile, this option overrides it.')), + OptionRecommendation(name='dont_add_comic_pages_to_toc', recommended_value=False, + help=_('When converting a CBC do not add links to each page to' + ' the TOC. Note this only applies if the TOC has more than one' + ' section')), + ]) + + recommendations = set([ + ('margin_left', 0, OptionRecommendation.HIGH), + ('margin_top', 0, OptionRecommendation.HIGH), + ('margin_right', 0, OptionRecommendation.HIGH), + ('margin_bottom', 0, OptionRecommendation.HIGH), + ('insert_blank_line', False, OptionRecommendation.HIGH), + ('remove_paragraph_spacing', False, OptionRecommendation.HIGH), + ('change_justification', 'left', OptionRecommendation.HIGH), + ('dont_split_on_pagebreaks', True, OptionRecommendation.HIGH), + ('chapter', None, OptionRecommendation.HIGH), + ('page_breaks_brefore', None, OptionRecommendation.HIGH), + ('use_auto_toc', False, OptionRecommendation.HIGH), + ('page_breaks_before', None, OptionRecommendation.HIGH), + ('disable_font_rescaling', True, OptionRecommendation.HIGH), + ('linearize_tables', False, OptionRecommendation.HIGH), + ]) + + def get_comics_from_collection(self, stream): + from calibre.libunzip import extract as zipextract + tdir = PersistentTemporaryDirectory('_comic_collection') + zipextract(stream, tdir) + comics = [] + with CurrentDir(tdir): + if not os.path.exists('comics.txt'): + raise ValueError(( + '%s is not a valid comic collection' + ' no comics.txt was found in the file') + %stream.name) + raw = open('comics.txt', 'rb').read() + if raw.startswith(codecs.BOM_UTF16_BE): + raw = raw.decode('utf-16-be')[1:] + elif raw.startswith(codecs.BOM_UTF16_LE): + raw = raw.decode('utf-16-le')[1:] + elif raw.startswith(codecs.BOM_UTF8): + raw = raw.decode('utf-8')[1:] + else: + raw = raw.decode('utf-8') + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + fname, title = line.partition(':')[0], line.partition(':')[-1] + fname = fname.replace('#', '_') + fname = os.path.join(tdir, *fname.split('/')) + if not title: + title = os.path.basename(fname).rpartition('.')[0] + if os.access(fname, os.R_OK): + comics.append([title, fname]) + if not comics: + raise ValueError('%s has no comics'%stream.name) + return comics + + def get_pages(self, comic, tdir2): + from calibre.ebooks.comic.input import (extract_comic, process_pages, + find_pages) + tdir = extract_comic(comic) + new_pages = find_pages(tdir, sort_on_mtime=self.opts.no_sort, + verbose=self.opts.verbose) + thumbnail = None + if not new_pages: + raise ValueError('Could not find any pages in the comic: %s' + %comic) + if self.opts.no_process: + n2 = [] + for page in new_pages: + n2.append(os.path.join(tdir2, os.path.basename(page))) + shutil.copyfile(page, n2[-1]) + new_pages = n2 + else: + new_pages, failures = process_pages(new_pages, self.opts, + self.report_progress, tdir2) + if failures: + self.log.warning('Could not process the following pages ' + '(run with --verbose to see why):') + for f in failures: + self.log.warning('\t', f) + if not new_pages: + raise ValueError('Could not find any valid pages in comic: %s' + % comic) + thumbnail = os.path.join(tdir2, + 'thumbnail.'+self.opts.output_format.lower()) + if not os.access(thumbnail, os.R_OK): + thumbnail = None + return new_pages + + def get_images(self): + return self._images + + def convert(self, stream, opts, file_ext, log, accelerators): + from calibre.ebooks.metadata import MetaInformation + from calibre.ebooks.metadata.opf2 import OPFCreator + from calibre.ebooks.metadata.toc import TOC + + self.opts, self.log= opts, log + if file_ext == 'cbc': + comics_ = self.get_comics_from_collection(stream) + else: + comics_ = [['Comic', os.path.abspath(stream.name)]] + stream.close() + comics = [] + for i, x in enumerate(comics_): + title, fname = x + cdir = 'comic_%d'%(i+1) if len(comics_) > 1 else '.' + cdir = os.path.abspath(cdir) + if not os.path.exists(cdir): + os.makedirs(cdir) + pages = self.get_pages(fname, cdir) + if not pages: continue + wrappers = self.create_wrappers(pages) + comics.append((title, pages, wrappers)) + + if not comics: + raise ValueError('No comic pages found in %s'%stream.name) + + mi = MetaInformation(os.path.basename(stream.name).rpartition('.')[0], + [_('Unknown')]) + opf = OPFCreator(os.path.abspath('.'), mi) + entries = [] + + def href(x): + if len(comics) == 1: return os.path.basename(x) + return '/'.join(x.split(os.sep)[-2:]) + + for comic in comics: + pages, wrappers = comic[1:] + entries += [(w, None) for w in map(href, wrappers)] + \ + [(x, None) for x in map(href, pages)] + opf.create_manifest(entries) + spine = [] + for comic in comics: + spine.extend(map(href, comic[2])) + self._images = [] + for comic in comics: + self._images.extend(comic[1]) + opf.create_spine(spine) + toc = TOC() + if len(comics) == 1: + wrappers = comics[0][2] + for i, x in enumerate(wrappers): + toc.add_item(href(x), None, _('Page')+' %d'%(i+1), + play_order=i) + else: + po = 0 + for comic in comics: + po += 1 + wrappers = comic[2] + stoc = toc.add_item(href(wrappers[0]), + None, comic[0], play_order=po) + if not opts.dont_add_comic_pages_to_toc: + for i, x in enumerate(wrappers): + stoc.add_item(href(x), None, + _('Page')+' %d'%(i+1), play_order=po) + po += 1 + opf.set_toc(toc) + m, n = open('metadata.opf', 'wb'), open('toc.ncx', 'wb') + opf.render(m, n, 'toc.ncx') + return os.path.abspath('metadata.opf') + + def create_wrappers(self, pages): + from calibre.ebooks.oeb.base import XHTML_NS + wrappers = [] + WRAPPER = textwrap.dedent('''\ + + + Page #%d + + + +
+ comic page #%d +
+ + + ''') + dir = os.path.dirname(pages[0]) + for i, page in enumerate(pages): + wrapper = WRAPPER%(XHTML_NS, i+1, os.path.basename(page), i+1) + page = os.path.join(dir, 'page_%d.xhtml'%(i+1)) + open(page, 'wb').write(wrapper) + wrappers.append(page) + return wrappers + diff --git a/src/calibre/ebooks/djvu/input.py b/src/calibre/ebooks/conversion/plugins/djvu_input.py similarity index 98% rename from src/calibre/ebooks/djvu/input.py rename to src/calibre/ebooks/conversion/plugins/djvu_input.py index 70dbf97f5d..936ef1a702 100644 --- a/src/calibre/ebooks/djvu/input.py +++ b/src/calibre/ebooks/conversion/plugins/djvu_input.py @@ -12,7 +12,6 @@ from subprocess import Popen, PIPE from cStringIO import StringIO from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from calibre.ebooks.txt.processor import convert_basic class DJVUInput(InputFormatPlugin): @@ -28,6 +27,8 @@ class DJVUInput(InputFormatPlugin): ]) def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.txt.processor import convert_basic + stdout = StringIO() ppdjvu = True # using djvutxt is MUCH faster, should make it an option diff --git a/src/calibre/ebooks/epub/input.py b/src/calibre/ebooks/conversion/plugins/epub_input.py similarity index 98% rename from src/calibre/ebooks/epub/input.py rename to src/calibre/ebooks/conversion/plugins/epub_input.py index c2cfedd7d4..47356dbd1f 100644 --- a/src/calibre/ebooks/epub/input.py +++ b/src/calibre/ebooks/conversion/plugins/epub_input.py @@ -3,11 +3,9 @@ __license__ = 'GPL 3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, uuid +import os from itertools import cycle -from lxml import etree - from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation class EPUBInput(InputFormatPlugin): @@ -30,6 +28,8 @@ class EPUBInput(InputFormatPlugin): f.write(raw[1024:]) def process_encryption(self, encfile, opf, log): + from lxml import etree + import uuid key = None for item in opf.identifier_iter(): scheme = None @@ -65,6 +65,7 @@ class EPUBInput(InputFormatPlugin): return False def rationalize_cover(self, opf, log): + from lxml import etree guide_cover, guide_elem = None, None for guide_elem in opf.iterguide(): if guide_elem.get('type', '').lower() == 'cover': @@ -110,6 +111,7 @@ class EPUBInput(InputFormatPlugin): renderer) def find_opf(self): + from lxml import etree def attr(n, attr): for k, v in n.attrib.items(): if k.endswith(attr): diff --git a/src/calibre/ebooks/epub/output.py b/src/calibre/ebooks/conversion/plugins/epub_output.py similarity index 99% rename from src/calibre/ebooks/epub/output.py rename to src/calibre/ebooks/conversion/plugins/epub_output.py index 2bdfb0d934..44249e49a2 100644 --- a/src/calibre/ebooks/epub/output.py +++ b/src/calibre/ebooks/conversion/plugins/epub_output.py @@ -8,14 +8,12 @@ __docformat__ = 'restructuredtext en' import os, shutil, re -from calibre.customize.conversion import OutputFormatPlugin +from calibre.customize.conversion import (OutputFormatPlugin, + OptionRecommendation) from calibre.ptempfile import TemporaryDirectory from calibre import CurrentDir -from calibre.customize.conversion import OptionRecommendation from calibre.constants import filesystem_encoding -from lxml import etree - block_level_tags = ( 'address', 'body', @@ -289,6 +287,7 @@ class EPUBOutput(OutputFormatPlugin): # }}} def condense_ncx(self, ncx_path): + from lxml import etree if not self.opts.pretty_print: tree = etree.parse(ncx_path) for tag in tree.getroot().iter(tag=etree.Element): diff --git a/src/calibre/ebooks/fb2/input.py b/src/calibre/ebooks/conversion/plugins/fb2_input.py similarity index 99% rename from src/calibre/ebooks/fb2/input.py rename to src/calibre/ebooks/conversion/plugins/fb2_input.py index 147e940eb4..747f8f19d8 100644 --- a/src/calibre/ebooks/fb2/input.py +++ b/src/calibre/ebooks/conversion/plugins/fb2_input.py @@ -6,7 +6,6 @@ Convert .fb2 files to .lrf """ import os, re from base64 import b64decode -from lxml import etree from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation from calibre import guess_type @@ -38,6 +37,7 @@ class FB2Input(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from lxml import etree from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.oeb.base import XLINK_NS, XHTML_NS, RECOVER_PARSER diff --git a/src/calibre/ebooks/fb2/output.py b/src/calibre/ebooks/conversion/plugins/fb2_output.py similarity index 99% rename from src/calibre/ebooks/fb2/output.py rename to src/calibre/ebooks/conversion/plugins/fb2_output.py index 2042902724..d7db2a0a33 100644 --- a/src/calibre/ebooks/fb2/output.py +++ b/src/calibre/ebooks/conversion/plugins/fb2_output.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation -from calibre.ebooks.fb2.fb2ml import FB2MLizer class FB2Output(OutputFormatPlugin): @@ -162,6 +161,7 @@ class FB2Output(OutputFormatPlugin): def convert(self, oeb_book, output_path, input_plugin, opts, log): from calibre.ebooks.oeb.transforms.jacket import linearize_jacket from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer, Unavailable + from calibre.ebooks.fb2.fb2ml import FB2MLizer try: rasterizer = SVGRasterizer() diff --git a/src/calibre/ebooks/conversion/plugins/html_input.py b/src/calibre/ebooks/conversion/plugins/html_input.py new file mode 100644 index 0000000000..cfd2ebf8cf --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/html_input.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import re, tempfile, os +from functools import partial +from itertools import izip +from urllib import quote + +from calibre.constants import islinux, isbsd +from calibre.customize.conversion import (InputFormatPlugin, + OptionRecommendation) +from calibre.utils.localization import get_lang +from calibre.utils.filenames import ascii_filename + + +class HTMLInput(InputFormatPlugin): + + name = 'HTML Input' + author = 'Kovid Goyal' + description = 'Convert HTML and OPF files to an OEB' + file_types = set(['opf', 'html', 'htm', 'xhtml', 'xhtm', 'shtm', 'shtml']) + + options = set([ + OptionRecommendation(name='breadth_first', + recommended_value=False, level=OptionRecommendation.LOW, + help=_('Traverse links in HTML files breadth first. Normally, ' + 'they are traversed depth first.' + ) + ), + + OptionRecommendation(name='max_levels', + recommended_value=5, level=OptionRecommendation.LOW, + help=_('Maximum levels of recursion when following links in ' + 'HTML files. Must be non-negative. 0 implies that no ' + 'links in the root HTML file are followed. Default is ' + '%default.' + ) + ), + + OptionRecommendation(name='dont_package', + recommended_value=False, level=OptionRecommendation.LOW, + help=_('Normally this input plugin re-arranges all the input ' + 'files into a standard folder hierarchy. Only use this option ' + 'if you know what you are doing as it can result in various ' + 'nasty side effects in the rest of the conversion pipeline.' + ) + ), + + ]) + + def convert(self, stream, opts, file_ext, log, + accelerators): + self._is_case_sensitive = None + basedir = os.getcwd() + self.opts = opts + + fname = None + if hasattr(stream, 'name'): + basedir = os.path.dirname(stream.name) + fname = os.path.basename(stream.name) + + if file_ext != 'opf': + if opts.dont_package: + raise ValueError('The --dont-package option is not supported for an HTML input file') + from calibre.ebooks.metadata.html import get_metadata + mi = get_metadata(stream) + if fname: + from calibre.ebooks.metadata.meta import metadata_from_filename + fmi = metadata_from_filename(fname) + fmi.smart_update(mi) + mi = fmi + oeb = self.create_oebbook(stream.name, basedir, opts, log, mi) + return oeb + + from calibre.ebooks.conversion.plumber import create_oebbook + return create_oebbook(log, stream.name, opts, + encoding=opts.input_encoding) + + def is_case_sensitive(self, path): + if getattr(self, '_is_case_sensitive', None) is not None: + return self._is_case_sensitive + if not path or not os.path.exists(path): + return islinux or isbsd + self._is_case_sensitive = not (os.path.exists(path.lower()) \ + and os.path.exists(path.upper())) + return self._is_case_sensitive + + def create_oebbook(self, htmlpath, basedir, opts, log, mi): + import uuid + from calibre.ebooks.conversion.plumber import create_oebbook + from calibre.ebooks.oeb.base import (DirContainer, + rewrite_links, urlnormalize, urldefrag, BINARY_MIME, OEB_STYLES, + xpath) + from calibre import guess_type + from calibre.ebooks.oeb.transforms.metadata import \ + meta_info_to_oeb_metadata + from calibre.ebooks.html.input import get_filelist + import cssutils, logging + cssutils.log.setLevel(logging.WARN) + self.OEB_STYLES = OEB_STYLES + oeb = create_oebbook(log, None, opts, self, + encoding=opts.input_encoding, populate=False) + self.oeb = oeb + + metadata = oeb.metadata + meta_info_to_oeb_metadata(mi, metadata, log) + if not metadata.language: + oeb.logger.warn(u'Language not specified') + metadata.add('language', get_lang().replace('_', '-')) + if not metadata.creator: + oeb.logger.warn('Creator not specified') + metadata.add('creator', self.oeb.translate(__('Unknown'))) + if not metadata.title: + oeb.logger.warn('Title not specified') + metadata.add('title', self.oeb.translate(__('Unknown'))) + bookid = str(uuid.uuid4()) + metadata.add('identifier', bookid, id='uuid_id', scheme='uuid') + for ident in metadata.identifier: + if 'id' in ident.attrib: + self.oeb.uid = metadata.identifier[0] + break + + filelist = get_filelist(htmlpath, basedir, opts, log) + filelist = [f for f in filelist if not f.is_binary] + htmlfile_map = {} + for f in filelist: + path = f.path + oeb.container = DirContainer(os.path.dirname(path), log, + ignore_opf=True) + bname = os.path.basename(path) + id, href = oeb.manifest.generate(id='html', + href=ascii_filename(bname)) + htmlfile_map[path] = href + item = oeb.manifest.add(id, href, 'text/html') + item.html_input_href = bname + oeb.spine.add(item, True) + + self.added_resources = {} + self.log = log + self.log('Normalizing filename cases') + for path, href in htmlfile_map.items(): + if not self.is_case_sensitive(path): + path = path.lower() + self.added_resources[path] = href + self.urlnormalize, self.DirContainer = urlnormalize, DirContainer + self.urldefrag = urldefrag + self.guess_type, self.BINARY_MIME = guess_type, BINARY_MIME + + self.log('Rewriting HTML links') + for f in filelist: + path = f.path + dpath = os.path.dirname(path) + oeb.container = DirContainer(dpath, log, ignore_opf=True) + item = oeb.manifest.hrefs[htmlfile_map[path]] + rewrite_links(item.data, partial(self.resource_adder, base=dpath)) + + for item in oeb.manifest.values(): + if item.media_type in self.OEB_STYLES: + dpath = None + for path, href in self.added_resources.items(): + if href == item.href: + dpath = os.path.dirname(path) + break + cssutils.replaceUrls(item.data, + partial(self.resource_adder, base=dpath)) + + toc = self.oeb.toc + self.oeb.auto_generated_toc = True + titles = [] + headers = [] + for item in self.oeb.spine: + if not item.linear: continue + html = item.data + title = ''.join(xpath(html, '/h:html/h:head/h:title/text()')) + title = re.sub(r'\s+', ' ', title.strip()) + if title: + titles.append(title) + headers.append('(unlabled)') + for tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'strong'): + expr = '/h:html/h:body//h:%s[position()=1]/text()' + header = ''.join(xpath(html, expr % tag)) + header = re.sub(r'\s+', ' ', header.strip()) + if header: + headers[-1] = header + break + use = titles + if len(titles) > len(set(titles)): + use = headers + for title, item in izip(use, self.oeb.spine): + if not item.linear: continue + toc.add(title, item.href) + + oeb.container = DirContainer(os.getcwdu(), oeb.log, ignore_opf=True) + return oeb + + def link_to_local_path(self, link_, base=None): + from calibre.ebooks.html.input import Link + if not isinstance(link_, unicode): + try: + link_ = link_.decode('utf-8', 'error') + except: + self.log.warn('Failed to decode link %r. Ignoring'%link_) + return None, None + try: + l = Link(link_, base if base else os.getcwdu()) + except: + self.log.exception('Failed to process link: %r'%link_) + return None, None + if l.path is None: + # Not a local resource + return None, None + link = l.path.replace('/', os.sep).strip() + frag = l.fragment + if not link: + return None, None + return link, frag + + def resource_adder(self, link_, base=None): + link, frag = self.link_to_local_path(link_, base=base) + if link is None: + return link_ + try: + if base and not os.path.isabs(link): + link = os.path.join(base, link) + link = os.path.abspath(link) + except: + return link_ + if not os.access(link, os.R_OK): + return link_ + if os.path.isdir(link): + self.log.warn(link_, 'is a link to a directory. Ignoring.') + return link_ + if not self.is_case_sensitive(tempfile.gettempdir()): + link = link.lower() + if link not in self.added_resources: + bhref = os.path.basename(link) + id, href = self.oeb.manifest.generate(id='added', + href=bhref) + guessed = self.guess_type(href)[0] + media_type = guessed or self.BINARY_MIME + if media_type == 'text/plain': + self.log.warn('Ignoring link to text file %r'%link_) + return None + + self.oeb.log.debug('Added', link) + self.oeb.container = self.DirContainer(os.path.dirname(link), + self.oeb.log, ignore_opf=True) + # Load into memory + item = self.oeb.manifest.add(id, href, media_type) + # bhref refers to an already existing file. The read() method of + # DirContainer will call unquote on it before trying to read the + # file, therefore we quote it here. + if isinstance(bhref, unicode): + bhref = bhref.encode('utf-8') + item.html_input_href = quote(bhref).decode('utf-8') + if guessed in self.OEB_STYLES: + item.override_css_fetch = partial( + self.css_import_handler, os.path.dirname(link)) + item.data + self.added_resources[link] = href + + nlink = self.added_resources[link] + if frag: + nlink = '#'.join((nlink, frag)) + return nlink + + def css_import_handler(self, base, href): + link, frag = self.link_to_local_path(href, base=base) + if link is None or not os.access(link, os.R_OK) or os.path.isdir(link): + return (None, None) + try: + raw = open(link, 'rb').read().decode('utf-8', 'replace') + raw = self.oeb.css_preprocessor(raw, add_namespace=True) + except: + self.log.exception('Failed to read CSS file: %r'%link) + return (None, None) + return (None, raw) diff --git a/src/calibre/ebooks/html/output.py b/src/calibre/ebooks/conversion/plugins/html_output.py similarity index 96% rename from src/calibre/ebooks/html/output.py rename to src/calibre/ebooks/conversion/plugins/html_output.py index fe7b4cf274..3821ba41a4 100644 --- a/src/calibre/ebooks/html/output.py +++ b/src/calibre/ebooks/conversion/plugins/html_output.py @@ -4,22 +4,11 @@ __copyright__ = '2010, Fabian Grassl ' __docformat__ = 'restructuredtext en' import os, re, shutil - -from calibre.utils import zipfile - from os.path import dirname, abspath, relpath, exists, basename -from lxml import etree -from templite import Templite - from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation from calibre import CurrentDir from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.utils.zipfile import ZipFile - -from urllib import unquote - -from calibre.ebooks.html.meta import EasyMeta class HTMLOutput(OutputFormatPlugin): @@ -50,6 +39,9 @@ class HTMLOutput(OutputFormatPlugin): ''' Generate table of contents ''' + from lxml import etree + from urllib import unquote + from calibre.ebooks.oeb.base import element with CurrentDir(output_dir): def build_node(current_node, parent=None): @@ -72,11 +64,18 @@ class HTMLOutput(OutputFormatPlugin): return wrap def generate_html_toc(self, oeb_book, ref_url, output_dir): + from lxml import etree + root = self.generate_toc(oeb_book, ref_url, output_dir) return etree.tostring(root, pretty_print=True, encoding='utf-8', xml_declaration=False) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from lxml import etree + from calibre.utils import zipfile + from templite import Templite + from urllib import unquote + from calibre.ebooks.html.meta import EasyMeta # read template files if opts.template_html_index is not None: @@ -192,7 +191,7 @@ class HTMLOutput(OutputFormatPlugin): f.write(t) item.unload_data_from_memory(memory=path) - zfile = ZipFile(output_path, "w") + zfile = zipfile.ZipFile(output_path, "w") zfile.add_dir(output_dir, basename(output_dir)) zfile.write(output_file, basename(output_file), zipfile.ZIP_DEFLATED) diff --git a/src/calibre/ebooks/htmlz/input.py b/src/calibre/ebooks/conversion/plugins/htmlz_input.py similarity index 96% rename from src/calibre/ebooks/htmlz/input.py rename to src/calibre/ebooks/conversion/plugins/htmlz_input.py index f0f45f72fe..e9fbb1d7c2 100644 --- a/src/calibre/ebooks/htmlz/input.py +++ b/src/calibre/ebooks/conversion/plugins/htmlz_input.py @@ -10,9 +10,6 @@ import os from calibre import guess_type from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.chardet import xml_to_unicode -from calibre.ebooks.metadata.opf2 import OPF -from calibre.utils.zipfile import ZipFile class HTMLZInput(InputFormatPlugin): @@ -23,6 +20,10 @@ class HTMLZInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.chardet import xml_to_unicode + from calibre.ebooks.metadata.opf2 import OPF + from calibre.utils.zipfile import ZipFile + self.log = log html = u'' top_levels = [] diff --git a/src/calibre/ebooks/htmlz/output.py b/src/calibre/ebooks/conversion/plugins/htmlz_output.py similarity index 96% rename from src/calibre/ebooks/htmlz/output.py rename to src/calibre/ebooks/conversion/plugins/htmlz_output.py index a1ef57af2c..f35dbc4dad 100644 --- a/src/calibre/ebooks/htmlz/output.py +++ b/src/calibre/ebooks/conversion/plugins/htmlz_output.py @@ -9,13 +9,10 @@ __docformat__ = 'restructuredtext en' import os from cStringIO import StringIO -from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf from calibre.ptempfile import TemporaryDirectory -from calibre.utils.zipfile import ZipFile class HTMLZOutput(OutputFormatPlugin): @@ -43,7 +40,10 @@ class HTMLZOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from lxml import etree from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME + from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf + from calibre.utils.zipfile import ZipFile # HTML if opts.htmlz_css_type == 'inline': @@ -81,7 +81,7 @@ class HTMLZOutput(OutputFormatPlugin): fname = os.path.join(tdir, 'images', images[item.href]) with open(fname, 'wb') as img: img.write(data) - + # Cover cover_path = None try: diff --git a/src/calibre/ebooks/lit/input.py b/src/calibre/ebooks/conversion/plugins/lit_input.py similarity index 100% rename from src/calibre/ebooks/lit/input.py rename to src/calibre/ebooks/conversion/plugins/lit_input.py diff --git a/src/calibre/ebooks/lit/output.py b/src/calibre/ebooks/conversion/plugins/lit_output.py similarity index 100% rename from src/calibre/ebooks/lit/output.py rename to src/calibre/ebooks/conversion/plugins/lit_output.py diff --git a/src/calibre/ebooks/conversion/plugins/lrf_input.py b/src/calibre/ebooks/conversion/plugins/lrf_input.py new file mode 100644 index 0000000000..63af39e1e0 --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/lrf_input.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, sys +from calibre.customize.conversion import InputFormatPlugin + +class LRFInput(InputFormatPlugin): + + name = 'LRF Input' + author = 'Kovid Goyal' + description = 'Convert LRF files to HTML' + file_types = set(['lrf']) + + def convert(self, stream, options, file_ext, log, + accelerators): + from lxml import etree + from calibre.ebooks.lrf.input import (MediaType, Styles, TextBlock, + Canvas, ImageBlock, RuledLine) + self.log = log + self.log('Generating XML') + from calibre.ebooks.lrf.lrfparser import LRFDocument + d = LRFDocument(stream) + d.parse() + xml = d.to_xml(write_files=True) + if options.verbose > 2: + open('lrs.xml', 'wb').write(xml.encode('utf-8')) + parser = etree.XMLParser(no_network=True, huge_tree=True) + try: + doc = etree.fromstring(xml, parser=parser) + except: + self.log.warn('Failed to parse XML. Trying to recover') + parser = etree.XMLParser(no_network=True, huge_tree=True, + recover=True) + doc = etree.fromstring(xml, parser=parser) + + + char_button_map = {} + for x in doc.xpath('//CharButton[@refobj]'): + ro = x.get('refobj') + jump_button = doc.xpath('//*[@objid="%s"]'%ro) + if jump_button: + jump_to = jump_button[0].xpath('descendant::JumpTo[@refpage and @refobj]') + if jump_to: + char_button_map[ro] = '%s.xhtml#%s'%(jump_to[0].get('refpage'), + jump_to[0].get('refobj')) + plot_map = {} + for x in doc.xpath('//Plot[@refobj]'): + ro = x.get('refobj') + image = doc.xpath('//Image[@objid="%s" and @refstream]'%ro) + if image: + imgstr = doc.xpath('//ImageStream[@objid="%s" and @file]'% + image[0].get('refstream')) + if imgstr: + plot_map[ro] = imgstr[0].get('file') + + self.log('Converting XML to HTML...') + styledoc = etree.fromstring(P('templates/lrf.xsl', data=True)) + media_type = MediaType() + styles = Styles() + text_block = TextBlock(styles, char_button_map, plot_map, log) + canvas = Canvas(doc, styles, text_block, log) + image_block = ImageBlock(canvas) + ruled_line = RuledLine() + extensions = { + ('calibre', 'media-type') : media_type, + ('calibre', 'text-block') : text_block, + ('calibre', 'ruled-line') : ruled_line, + ('calibre', 'styles') : styles, + ('calibre', 'canvas') : canvas, + ('calibre', 'image-block'): image_block, + } + transform = etree.XSLT(styledoc, extensions=extensions) + try: + result = transform(doc) + except RuntimeError: + sys.setrecursionlimit(5000) + result = transform(doc) + + with open('content.opf', 'wb') as f: + f.write(result) + styles.write() + return os.path.abspath('content.opf') diff --git a/src/calibre/ebooks/lrf/output.py b/src/calibre/ebooks/conversion/plugins/lrf_output.py similarity index 100% rename from src/calibre/ebooks/lrf/output.py rename to src/calibre/ebooks/conversion/plugins/lrf_output.py diff --git a/src/calibre/ebooks/mobi/input.py b/src/calibre/ebooks/conversion/plugins/mobi_input.py similarity index 100% rename from src/calibre/ebooks/mobi/input.py rename to src/calibre/ebooks/conversion/plugins/mobi_input.py diff --git a/src/calibre/ebooks/mobi/output.py b/src/calibre/ebooks/conversion/plugins/mobi_output.py similarity index 100% rename from src/calibre/ebooks/mobi/output.py rename to src/calibre/ebooks/conversion/plugins/mobi_output.py diff --git a/src/calibre/ebooks/conversion/plugins/odt_input.py b/src/calibre/ebooks/conversion/plugins/odt_input.py new file mode 100644 index 0000000000..5e92ea5163 --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/odt_input.py @@ -0,0 +1,25 @@ +from __future__ import with_statement +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + +''' +Convert an ODT file into a Open Ebook +''' + +from calibre.customize.conversion import InputFormatPlugin + +class ODTInput(InputFormatPlugin): + + name = 'ODT Input' + author = 'Kovid Goyal' + description = 'Convert ODT (OpenOffice) files to HTML' + file_types = set(['odt']) + + + def convert(self, stream, options, file_ext, log, + accelerators): + from calibre.ebooks.odt.input import Extract + return Extract()(stream, '.', log) + + diff --git a/src/calibre/ebooks/oeb/output.py b/src/calibre/ebooks/conversion/plugins/oeb_output.py similarity index 96% rename from src/calibre/ebooks/oeb/output.py rename to src/calibre/ebooks/conversion/plugins/oeb_output.py index 38ac2495fd..b69e095d0f 100644 --- a/src/calibre/ebooks/oeb/output.py +++ b/src/calibre/ebooks/conversion/plugins/oeb_output.py @@ -5,13 +5,10 @@ __docformat__ = 'restructuredtext en' import os, re -from lxml import etree -from calibre.customize.conversion import OutputFormatPlugin +from calibre.customize.conversion import (OutputFormatPlugin, + OptionRecommendation) from calibre import CurrentDir -from calibre.customize.conversion import OptionRecommendation - -from urllib import unquote class OEBOutput(OutputFormatPlugin): @@ -23,6 +20,9 @@ class OEBOutput(OutputFormatPlugin): def convert(self, oeb_book, output_path, input_plugin, opts, log): + from urllib import unquote + from lxml import etree + self.log, self.opts = log, opts if not os.path.exists(output_path): os.makedirs(output_path) diff --git a/src/calibre/ebooks/pdb/input.py b/src/calibre/ebooks/conversion/plugins/pdb_input.py similarity index 87% rename from src/calibre/ebooks/pdb/input.py rename to src/calibre/ebooks/conversion/plugins/pdb_input.py index cd861216af..69984ab268 100644 --- a/src/calibre/ebooks/pdb/input.py +++ b/src/calibre/ebooks/conversion/plugins/pdb_input.py @@ -7,8 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.pdb.header import PdbHeaderReader -from calibre.ebooks.pdb import PDBError, IDENTITY_TO_NAME, get_reader class PDBInput(InputFormatPlugin): @@ -19,6 +17,9 @@ class PDBInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.pdb.header import PdbHeaderReader + from calibre.ebooks.pdb import PDBError, IDENTITY_TO_NAME, get_reader + header = PdbHeaderReader(stream) Reader = get_reader(header.ident) diff --git a/src/calibre/ebooks/pdb/output.py b/src/calibre/ebooks/conversion/plugins/pdb_output.py similarity index 91% rename from src/calibre/ebooks/pdb/output.py rename to src/calibre/ebooks/conversion/plugins/pdb_output.py index 7bca4e5c5d..b80f9958ef 100644 --- a/src/calibre/ebooks/pdb/output.py +++ b/src/calibre/ebooks/conversion/plugins/pdb_output.py @@ -8,7 +8,7 @@ import os from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.pdb import PDBError, get_writer, FORMAT_WRITERS +from calibre.ebooks.pdb import PDBError, get_writer, ALL_FORMAT_WRITERS class PDBOutput(OutputFormatPlugin): @@ -19,9 +19,9 @@ class PDBOutput(OutputFormatPlugin): options = set([ OptionRecommendation(name='format', recommended_value='doc', level=OptionRecommendation.LOW, - short_switch='f', choices=FORMAT_WRITERS.keys(), + short_switch='f', choices=list(ALL_FORMAT_WRITERS), help=(_('Format to use inside the pdb container. Choices are:')+\ - ' %s' % FORMAT_WRITERS.keys())), + ' %s' % list(ALL_FORMAT_WRITERS))), OptionRecommendation(name='pdb_output_encoding', recommended_value='cp1252', level=OptionRecommendation.LOW, help=_('Specify the character encoding of the output document. ' \ diff --git a/src/calibre/ebooks/pdf/input.py b/src/calibre/ebooks/conversion/plugins/pdf_input.py similarity index 92% rename from src/calibre/ebooks/pdf/input.py rename to src/calibre/ebooks/conversion/plugins/pdf_input.py index 51f44ba502..be0150834b 100644 --- a/src/calibre/ebooks/pdf/input.py +++ b/src/calibre/ebooks/conversion/plugins/pdf_input.py @@ -7,10 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from calibre.ebooks.pdf.pdftohtml import pdftohtml -from calibre.ebooks.metadata.opf2 import OPFCreator -from calibre.constants import plugins -pdfreflow, pdfreflow_err = plugins['pdfreflow'] class PDFInput(InputFormatPlugin): @@ -31,6 +27,9 @@ class PDFInput(InputFormatPlugin): ]) def convert_new(self, stream, accelerators): + from calibre.constants import plugins + pdfreflow, pdfreflow_err = plugins['pdfreflow'] + from calibre.ebooks.pdf.reflow import PDFDocument from calibre.utils.cleantext import clean_ascii_chars if pdfreflow_err: @@ -43,6 +42,9 @@ class PDFInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.metadata.opf2 import OPFCreator + from calibre.ebooks.pdf.pdftohtml import pdftohtml + log.debug('Converting file to html...') # The main html file will be named index.html self.opts, self.log = options, log diff --git a/src/calibre/ebooks/pdf/output.py b/src/calibre/ebooks/conversion/plugins/pdf_output.py similarity index 86% rename from src/calibre/ebooks/pdf/output.py rename to src/calibre/ebooks/conversion/plugins/pdf_output.py index 14dd27368c..4422265976 100644 --- a/src/calibre/ebooks/pdf/output.py +++ b/src/calibre/ebooks/conversion/plugins/pdf_output.py @@ -13,10 +13,50 @@ import os from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.metadata.opf2 import OPF from calibre.ptempfile import TemporaryDirectory -from calibre.ebooks.pdf.pageoptions import UNITS, PAPER_SIZES, \ - ORIENTATIONS + +UNITS = [ + 'millimeter', + 'point', + 'inch' , + 'pica' , + 'didot', + 'cicero', + 'devicepixel', + ] + +PAPER_SIZES = ['b2', + 'a9', + 'executive', + 'tabloid', + 'b4', + 'b5', + 'b6', + 'b7', + 'b0', + 'b1', + 'letter', + 'b3', + 'a7', + 'a8', + 'b8', + 'b9', + 'a3', + 'a1', + 'folio', + 'c5e', + 'dle', + 'a0', + 'ledger', + 'legal', + 'a6', + 'a2', + 'b10', + 'a5', + 'comm10e', + 'a4'] + +ORIENTATIONS = ['portrait', 'landscape'] class PDFOutput(OutputFormatPlugin): @@ -26,23 +66,23 @@ class PDFOutput(OutputFormatPlugin): options = set([ OptionRecommendation(name='unit', recommended_value='inch', - level=OptionRecommendation.LOW, short_switch='u', choices=UNITS.keys(), + level=OptionRecommendation.LOW, short_switch='u', choices=UNITS, help=_('The unit of measure. Default is inch. Choices ' 'are %s ' - 'Note: This does not override the unit for margins!') % UNITS.keys()), + 'Note: This does not override the unit for margins!') % UNITS), OptionRecommendation(name='paper_size', recommended_value='letter', - level=OptionRecommendation.LOW, choices=PAPER_SIZES.keys(), + level=OptionRecommendation.LOW, choices=PAPER_SIZES, help=_('The size of the paper. This size will be overridden when a ' 'non default output profile is used. Default is letter. Choices ' - 'are %s') % PAPER_SIZES.keys()), + 'are %s') % PAPER_SIZES), OptionRecommendation(name='custom_size', recommended_value=None, help=_('Custom size of the document. Use the form widthxheight ' 'EG. `123x321` to specify the width and height. ' 'This overrides any specified paper-size.')), OptionRecommendation(name='orientation', recommended_value='portrait', - level=OptionRecommendation.LOW, choices=ORIENTATIONS.keys(), + level=OptionRecommendation.LOW, choices=ORIENTATIONS, help=_('The orientation of the page. Default is portrait. Choices ' - 'are %s') % ORIENTATIONS.keys()), + 'are %s') % ORIENTATIONS), OptionRecommendation(name='preserve_cover_aspect_ratio', recommended_value=False, help=_('Preserve the aspect ratio of the cover, instead' @@ -105,6 +145,8 @@ class PDFOutput(OutputFormatPlugin): def convert_text(self, oeb_book): from calibre.ebooks.pdf.writer import PDFWriter + from calibre.ebooks.metadata.opf2 import OPF + self.log.debug('Serializing oeb input to disk for processing...') self.get_cover_data() diff --git a/src/calibre/ebooks/pml/input.py b/src/calibre/ebooks/conversion/plugins/pml_input.py similarity index 96% rename from src/calibre/ebooks/pml/input.py rename to src/calibre/ebooks/conversion/plugins/pml_input.py index 4d59668b12..1351a5c492 100644 --- a/src/calibre/ebooks/pml/input.py +++ b/src/calibre/ebooks/conversion/plugins/pml_input.py @@ -11,9 +11,6 @@ import shutil from calibre.customize.conversion import InputFormatPlugin from calibre.ptempfile import TemporaryDirectory from calibre.utils.zipfile import ZipFile -from calibre.ebooks.pml.pmlconverter import PML_HTMLizer -from calibre.ebooks.metadata.toc import TOC -from calibre.ebooks.metadata.opf2 import OPFCreator class PMLInput(InputFormatPlugin): @@ -24,6 +21,8 @@ class PMLInput(InputFormatPlugin): file_types = set(['pml', 'pmlz']) def process_pml(self, pml_path, html_path, close_all=False): + from calibre.ebooks.pml.pmlconverter import PML_HTMLizer + pclose = False hclose = False @@ -85,6 +84,9 @@ class PMLInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.metadata.toc import TOC + from calibre.ebooks.metadata.opf2 import OPFCreator + self.options = options self.log = log pages, images = [], [] diff --git a/src/calibre/ebooks/pml/output.py b/src/calibre/ebooks/conversion/plugins/pml_output.py similarity index 88% rename from src/calibre/ebooks/pml/output.py rename to src/calibre/ebooks/conversion/plugins/pml_output.py index 63d8a8b220..b406537a98 100644 --- a/src/calibre/ebooks/pml/output.py +++ b/src/calibre/ebooks/conversion/plugins/pml_output.py @@ -4,21 +4,11 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' -import os +import os, cStringIO -try: - from PIL import Image - Image -except ImportError: - import Image - -import cStringIO - -from calibre.customize.conversion import OutputFormatPlugin -from calibre.customize.conversion import OptionRecommendation +from calibre.customize.conversion import (OutputFormatPlugin, + OptionRecommendation) from calibre.ptempfile import TemporaryDirectory -from calibre.utils.zipfile import ZipFile -from calibre.ebooks.pml.pmlml import PMLMLizer class PMLOutput(OutputFormatPlugin): @@ -43,6 +33,9 @@ class PMLOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.pml.pmlml import PMLMLizer + from calibre.utils.zipfile import ZipFile + with TemporaryDirectory('_pmlz_output') as tdir: pmlmlizer = PMLMLizer(log) pml = unicode(pmlmlizer.extract_content(oeb_book, opts)) @@ -59,6 +52,13 @@ class PMLOutput(OutputFormatPlugin): pmlz.add_dir(tdir) def write_images(self, manifest, image_hrefs, out_dir, opts): + try: + from PIL import Image + Image + except ImportError: + import Image + + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES for item in manifest: if item.media_type in OEB_RASTER_IMAGES and item.href in image_hrefs.keys(): diff --git a/src/calibre/ebooks/rb/input.py b/src/calibre/ebooks/conversion/plugins/rb_input.py similarity index 91% rename from src/calibre/ebooks/rb/input.py rename to src/calibre/ebooks/conversion/plugins/rb_input.py index 8b05c1d42e..6a6ca3205a 100644 --- a/src/calibre/ebooks/rb/input.py +++ b/src/calibre/ebooks/conversion/plugins/rb_input.py @@ -6,7 +6,6 @@ __docformat__ = 'restructuredtext en' import os -from calibre.ebooks.rb.reader import Reader from calibre.customize.conversion import InputFormatPlugin class RBInput(InputFormatPlugin): @@ -18,6 +17,8 @@ class RBInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.rb.reader import Reader + reader = Reader(stream, log, options.input_encoding) opf = reader.extract_content(os.getcwd()) diff --git a/src/calibre/ebooks/rb/output.py b/src/calibre/ebooks/conversion/plugins/rb_output.py similarity index 95% rename from src/calibre/ebooks/rb/output.py rename to src/calibre/ebooks/conversion/plugins/rb_output.py index a16e408b0f..992843719c 100644 --- a/src/calibre/ebooks/rb/output.py +++ b/src/calibre/ebooks/conversion/plugins/rb_output.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation -from calibre.ebooks.rb.writer import RBWriter class RBOutput(OutputFormatPlugin): @@ -22,6 +21,8 @@ class RBOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.rb.writer import RBWriter + close = False if not hasattr(output_path, 'write'): close = True diff --git a/src/calibre/web/feeds/input.py b/src/calibre/ebooks/conversion/plugins/recipe_input.py similarity index 100% rename from src/calibre/web/feeds/input.py rename to src/calibre/ebooks/conversion/plugins/recipe_input.py diff --git a/src/calibre/ebooks/conversion/plugins/rtf_input.py b/src/calibre/ebooks/conversion/plugins/rtf_input.py new file mode 100644 index 0000000000..d85c067242 --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/rtf_input.py @@ -0,0 +1,298 @@ +from __future__ import with_statement +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal ' + +import os, glob, re, textwrap + +from calibre.customize.conversion import InputFormatPlugin + +border_style_map = { + 'single' : 'solid', + 'double-thickness-border' : 'double', + 'shadowed-border': 'outset', + 'double-border': 'double', + 'dotted-border': 'dotted', + 'dashed': 'dashed', + 'hairline': 'solid', + 'inset': 'inset', + 'dash-small': 'dashed', + 'dot-dash': 'dotted', + 'dot-dot-dash': 'dotted', + 'outset': 'outset', + 'tripple': 'double', + 'triple': 'double', + 'thick-thin-small': 'solid', + 'thin-thick-small': 'solid', + 'thin-thick-thin-small': 'solid', + 'thick-thin-medium': 'solid', + 'thin-thick-medium': 'solid', + 'thin-thick-thin-medium': 'solid', + 'thick-thin-large': 'solid', + 'thin-thick-thin-large': 'solid', + 'wavy': 'ridge', + 'double-wavy': 'ridge', + 'striped': 'ridge', + 'emboss': 'inset', + 'engrave': 'inset', + 'frame': 'ridge', +} + + +class RTFInput(InputFormatPlugin): + + name = 'RTF Input' + author = 'Kovid Goyal' + description = 'Convert RTF files to HTML' + file_types = set(['rtf']) + + def generate_xml(self, stream): + from calibre.ebooks.rtf2xml.ParseRtf import ParseRtf + ofile = 'dataxml.xml' + run_lev, debug_dir, indent_out = 1, None, 0 + if getattr(self.opts, 'debug_pipeline', None) is not None: + try: + os.mkdir('rtfdebug') + debug_dir = 'rtfdebug' + run_lev = 4 + indent_out = 1 + self.log('Running RTFParser in debug mode') + except: + self.log.warn('Impossible to run RTFParser in debug mode') + parser = ParseRtf( + in_file = stream, + out_file = ofile, + # Convert symbol fonts to unicode equivalents. Default + # is 1 + convert_symbol = 1, + + # Convert Zapf fonts to unicode equivalents. Default + # is 1. + convert_zapf = 1, + + # Convert Wingding fonts to unicode equivalents. + # Default is 1. + convert_wingdings = 1, + + # Convert RTF caps to real caps. + # Default is 1. + convert_caps = 1, + + # Indent resulting XML. + # Default is 0 (no indent). + indent = indent_out, + + # Form lists from RTF. Default is 1. + form_lists = 1, + + # Convert headings to sections. Default is 0. + headings_to_sections = 1, + + # Group paragraphs with the same style name. Default is 1. + group_styles = 1, + + # Group borders. Default is 1. + group_borders = 1, + + # Write or do not write paragraphs. Default is 0. + empty_paragraphs = 1, + + #debug + deb_dir = debug_dir, + run_level = run_lev, + ) + parser.parse_rtf() + with open(ofile, 'rb') as f: + return f.read() + + def extract_images(self, picts): + import imghdr + self.log('Extracting images...') + + with open(picts, 'rb') as f: + raw = f.read() + picts = filter(len, re.findall(r'\{\\pict([^}]+)\}', raw)) + hex = re.compile(r'[^a-fA-F0-9]') + encs = [hex.sub('', pict) for pict in picts] + + count = 0 + imap = {} + for enc in encs: + if len(enc) % 2 == 1: + enc = enc[:-1] + data = enc.decode('hex') + fmt = imghdr.what(None, data) + if fmt is None: + fmt = 'wmf' + count += 1 + name = '%04d.%s' % (count, fmt) + with open(name, 'wb') as f: + f.write(data) + imap[count] = name + # with open(name+'.hex', 'wb') as f: + # f.write(enc) + return self.convert_images(imap) + + def convert_images(self, imap): + self.default_img = None + for count, val in imap.iteritems(): + try: + imap[count] = self.convert_image(val) + except: + self.log.exception('Failed to convert', val) + return imap + + def convert_image(self, name): + if not name.endswith('.wmf'): + return name + try: + return self.rasterize_wmf(name) + except: + self.log.exception('Failed to convert WMF image %r'%name) + return self.replace_wmf(name) + + def replace_wmf(self, name): + from calibre.ebooks import calibre_cover + if self.default_img is None: + self.default_img = calibre_cover('Conversion of WMF images is not supported', + 'Use Microsoft Word or OpenOffice to save this RTF file' + ' as HTML and convert that in calibre.', title_size=36, + author_size=20) + name = name.replace('.wmf', '.jpg') + with open(name, 'wb') as f: + f.write(self.default_img) + return name + + def rasterize_wmf(self, name): + from calibre.utils.wmf.parse import wmf_unwrap + with open(name, 'rb') as f: + data = f.read() + data = wmf_unwrap(data) + name = name.replace('.wmf', '.png') + with open(name, 'wb') as f: + f.write(data) + return name + + + def write_inline_css(self, ic, border_styles): + font_size_classes = ['span.fs%d { font-size: %spt }'%(i, x) for i, x in + enumerate(ic.font_sizes)] + color_classes = ['span.col%d { color: %s }'%(i, x) for i, x in + enumerate(ic.colors) if x != 'false'] + css = textwrap.dedent(''' + span.none { + text-decoration: none; font-weight: normal; + font-style: normal; font-variant: normal + } + + span.italics { font-style: italic } + + span.bold { font-weight: bold } + + span.small-caps { font-variant: small-caps } + + span.underlined { text-decoration: underline } + + span.strike-through { text-decoration: line-through } + + ''') + css += '\n'+'\n'.join(font_size_classes) + css += '\n' +'\n'.join(color_classes) + + for cls, val in border_styles.iteritems(): + css += '\n\n.%s {\n%s\n}'%(cls, val) + + with open('styles.css', 'ab') as f: + f.write(css) + + def convert_borders(self, doc): + border_styles = [] + style_map = {} + for elem in doc.xpath(r'//*[local-name()="cell"]'): + style = ['border-style: hidden', 'border-width: 1px', + 'border-color: black'] + for x in ('bottom', 'top', 'left', 'right'): + bs = elem.get('border-cell-%s-style'%x, None) + if bs: + cbs = border_style_map.get(bs, 'solid') + style.append('border-%s-style: %s'%(x, cbs)) + bw = elem.get('border-cell-%s-line-width'%x, None) + if bw: + style.append('border-%s-width: %spt'%(x, bw)) + bc = elem.get('border-cell-%s-color'%x, None) + if bc: + style.append('border-%s-color: %s'%(x, bc)) + style = ';\n'.join(style) + if style not in border_styles: + border_styles.append(style) + idx = border_styles.index(style) + cls = 'border_style%d'%idx + style_map[cls] = style + elem.set('class', cls) + return style_map + + def convert(self, stream, options, file_ext, log, + accelerators): + from lxml import etree + from calibre.ebooks.metadata.meta import get_metadata + from calibre.ebooks.metadata.opf2 import OPFCreator + from calibre.ebooks.rtf2xml.ParseRtf import RtfInvalidCodeException + from calibre.ebooks.rtf.input import InlineClass + self.opts = options + self.log = log + self.log('Converting RTF to XML...') + try: + xml = self.generate_xml(stream.name) + except RtfInvalidCodeException as e: + raise ValueError(_('This RTF file has a feature calibre does not ' + 'support. Convert it to HTML first and then try it.\n%s')%e) + + d = glob.glob(os.path.join('*_rtf_pict_dir', 'picts.rtf')) + if d: + imap = {} + try: + imap = self.extract_images(d[0]) + except: + self.log.exception('Failed to extract images...') + + self.log('Parsing XML...') + parser = etree.XMLParser(recover=True, no_network=True) + doc = etree.fromstring(xml, parser=parser) + border_styles = self.convert_borders(doc) + for pict in doc.xpath('//rtf:pict[@num]', + namespaces={'rtf':'http://rtf2xml.sourceforge.net/'}): + num = int(pict.get('num')) + name = imap.get(num, None) + if name is not None: + pict.set('num', name) + + self.log('Converting XML to HTML...') + inline_class = InlineClass(self.log) + styledoc = etree.fromstring(P('templates/rtf.xsl', data=True)) + extensions = { ('calibre', 'inline-class') : inline_class } + transform = etree.XSLT(styledoc, extensions=extensions) + result = transform(doc) + html = 'index.xhtml' + with open(html, 'wb') as f: + res = transform.tostring(result) + # res = res[:100].replace('xmlns:html', 'xmlns') + res[100:] + #clean multiple \n + res = re.sub('\n+', '\n', res) + # Replace newlines inserted by the 'empty_paragraphs' option in rtf2xml with html blank lines + # res = re.sub('\s*', '', res) + # res = re.sub('(?<=\n)\n{2}', + # u'

\u00a0

\n'.encode('utf-8'), res) + f.write(res) + self.write_inline_css(inline_class, border_styles) + stream.seek(0) + mi = get_metadata(stream, 'rtf') + if not mi.title: + mi.title = _('Unknown') + if not mi.authors: + mi.authors = [_('Unknown')] + opf = OPFCreator(os.getcwd(), mi) + opf.create_manifest([('index.xhtml', None)]) + opf.create_spine(['index.xhtml']) + opf.render(open('metadata.opf', 'wb')) + return os.path.abspath('metadata.opf') + + diff --git a/src/calibre/ebooks/rtf/output.py b/src/calibre/ebooks/conversion/plugins/rtf_output.py similarity index 94% rename from src/calibre/ebooks/rtf/output.py rename to src/calibre/ebooks/conversion/plugins/rtf_output.py index 5738b7e6f4..ae9e1ea566 100644 --- a/src/calibre/ebooks/rtf/output.py +++ b/src/calibre/ebooks/conversion/plugins/rtf_output.py @@ -6,7 +6,6 @@ __docformat__ = 'restructuredtext en' import os -from calibre.ebooks.rtf.rtfml import RTFMLizer from calibre.customize.conversion import OutputFormatPlugin class RTFOutput(OutputFormatPlugin): @@ -16,6 +15,8 @@ class RTFOutput(OutputFormatPlugin): file_type = 'rtf' def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.rtf.rtfml import RTFMLizer + rtfmlitzer = RTFMLizer(log) content = rtfmlitzer.extract_content(oeb_book, opts) diff --git a/src/calibre/ebooks/snb/input.py b/src/calibre/ebooks/conversion/plugins/snb_input.py similarity index 97% rename from src/calibre/ebooks/snb/input.py rename to src/calibre/ebooks/conversion/plugins/snb_input.py index 13b1ca45f9..ae3ab0033c 100755 --- a/src/calibre/ebooks/snb/input.py +++ b/src/calibre/ebooks/conversion/plugins/snb_input.py @@ -4,13 +4,11 @@ __license__ = 'GPL 3' __copyright__ = '2010, Li Fanxi ' __docformat__ = 'restructuredtext en' -import os, uuid +import os from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.snb.snbfile import SNBFile from calibre.ptempfile import TemporaryDirectory from calibre.utils.filenames import ascii_filename -from lxml import etree HTML_TEMPLATE = u'%s\n%s\n' @@ -29,7 +27,12 @@ class SNBInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + import uuid + from lxml import etree + from calibre.ebooks.oeb.base import DirContainer + from calibre.ebooks.snb.snbfile import SNBFile + log.debug("Parsing SNB file...") snbFile = SNBFile() try: diff --git a/src/calibre/ebooks/snb/output.py b/src/calibre/ebooks/conversion/plugins/snb_output.py similarity index 98% rename from src/calibre/ebooks/snb/output.py rename to src/calibre/ebooks/conversion/plugins/snb_output.py index 07a0460c57..e9b8af0db6 100644 --- a/src/calibre/ebooks/snb/output.py +++ b/src/calibre/ebooks/conversion/plugins/snb_output.py @@ -6,12 +6,9 @@ __docformat__ = 'restructuredtext en' import os, string -from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation from calibre.ptempfile import TemporaryDirectory from calibre.constants import __appname__, __version__ -from calibre.ebooks.snb.snbfile import SNBFile -from calibre.ebooks.snb.snbml import SNBMLizer, ProcessFileName class SNBOutput(OutputFormatPlugin): @@ -49,6 +46,11 @@ class SNBOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from lxml import etree + from calibre.ebooks.snb.snbfile import SNBFile + from calibre.ebooks.snb.snbml import SNBMLizer, ProcessFileName + + self.opts = opts from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer, Unavailable try: diff --git a/src/calibre/ebooks/tcr/input.py b/src/calibre/ebooks/conversion/plugins/tcr_input.py similarity index 87% rename from src/calibre/ebooks/tcr/input.py rename to src/calibre/ebooks/conversion/plugins/tcr_input.py index 4d15fd0923..5ee34285bd 100644 --- a/src/calibre/ebooks/tcr/input.py +++ b/src/calibre/ebooks/conversion/plugins/tcr_input.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' from cStringIO import StringIO from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.compression.tcr import decompress class TCRInput(InputFormatPlugin): @@ -17,6 +16,8 @@ class TCRInput(InputFormatPlugin): file_types = set(['tcr']) def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.compression.tcr import decompress + log.info('Decompressing text...') raw_txt = decompress(stream) @@ -28,7 +29,7 @@ class TCRInput(InputFormatPlugin): txt_plugin = plugin_for_input_format('txt') for opt in txt_plugin.options: if not hasattr(self.options, opt.option.name): - setattr(self.options, opt.option.name, opt.recommended_value) + setattr(options, opt.option.name, opt.recommended_value) stream.seek(0) return txt_plugin.convert(stream, options, diff --git a/src/calibre/ebooks/tcr/output.py b/src/calibre/ebooks/conversion/plugins/tcr_output.py similarity index 93% rename from src/calibre/ebooks/tcr/output.py rename to src/calibre/ebooks/conversion/plugins/tcr_output.py index 97c9cae26c..f4dbcce57b 100644 --- a/src/calibre/ebooks/tcr/output.py +++ b/src/calibre/ebooks/conversion/plugins/tcr_output.py @@ -8,8 +8,6 @@ import os from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.txt.txtml import TXTMLizer -from calibre.ebooks.compression.tcr import compress class TCROutput(OutputFormatPlugin): @@ -25,6 +23,9 @@ class TCROutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.txt.txtml import TXTMLizer + from calibre.ebooks.compression.tcr import compress + close = False if not hasattr(output_path, 'write'): close = True diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/conversion/plugins/txt_input.py similarity index 94% rename from src/calibre/ebooks/txt/input.py rename to src/calibre/ebooks/conversion/plugins/txt_input.py index 49c8a2129d..e916b30c29 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/conversion/plugins/txt_input.py @@ -8,14 +8,6 @@ import os from calibre import _ent_pat, walk, xml_entity_to_unicode from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator -from calibre.ebooks.chardet import detect -from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \ - separate_paragraphs_single_line, separate_paragraphs_print_formatted, \ - preserve_spaces, detect_paragraph_type, detect_formatting_type, \ - normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \ - separate_hard_scene_breaks -from calibre.utils.zipfile import ZipFile class TXTInput(InputFormatPlugin): @@ -61,6 +53,17 @@ class TXTInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator + from calibre.ebooks.chardet import detect + from calibre.utils.zipfile import ZipFile + from calibre.ebooks.txt.processor import (convert_basic, + convert_markdown, separate_paragraphs_single_line, + separate_paragraphs_print_formatted, preserve_spaces, + detect_paragraph_type, detect_formatting_type, + normalize_line_endings, convert_textile, remove_indents, + block_to_single_line, separate_hard_scene_breaks) + + self.log = log txt = '' log.debug('Reading text from file...') diff --git a/src/calibre/ebooks/txt/output.py b/src/calibre/ebooks/conversion/plugins/txt_output.py similarity index 93% rename from src/calibre/ebooks/txt/output.py rename to src/calibre/ebooks/conversion/plugins/txt_output.py index d9c42eb1dc..6cd4c3f801 100644 --- a/src/calibre/ebooks/txt/output.py +++ b/src/calibre/ebooks/conversion/plugins/txt_output.py @@ -7,15 +7,12 @@ __docformat__ = 'restructuredtext en' import os import shutil -from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.txt.txtml import TXTMLizer -from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines from calibre.ptempfile import TemporaryDirectory, TemporaryFile -from calibre.utils.cleantext import clean_ascii_chars -from calibre.utils.zipfile import ZipFile + +NEWLINE_TYPES = ['system', 'unix', 'old_mac', 'windows'] class TXTOutput(OutputFormatPlugin): @@ -26,11 +23,11 @@ class TXTOutput(OutputFormatPlugin): options = set([ OptionRecommendation(name='newline', recommended_value='system', level=OptionRecommendation.LOW, - short_switch='n', choices=TxtNewlines.NEWLINE_TYPES.keys(), + short_switch='n', choices=NEWLINE_TYPES, help=_('Type of newline to use. Options are %s. Default is \'system\'. ' 'Use \'old_mac\' for compatibility with Mac OS 9 and earlier. ' 'For Mac OS X use \'unix\'. \'system\' will default to the newline ' - 'type used by this OS.') % sorted(TxtNewlines.NEWLINE_TYPES.keys())), + 'type used by this OS.') % sorted(NEWLINE_TYPES)), OptionRecommendation(name='txt_output_encoding', recommended_value='utf-8', level=OptionRecommendation.LOW, help=_('Specify the character encoding of the output document. ' \ @@ -76,6 +73,11 @@ class TXTOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.txt.txtml import TXTMLizer + from calibre.utils.cleantext import clean_ascii_chars + from calibre.ebooks.txt.newlines import specified_newlines, TxtNewlines + + if opts.txt_output_formatting.lower() == 'markdown': from calibre.ebooks.txt.markdownml import MarkdownMLizer self.writer = MarkdownMLizer(log) @@ -116,6 +118,9 @@ class TXTZOutput(TXTOutput): def convert(self, oeb_book, output_path, input_plugin, opts, log): from calibre.ebooks.oeb.base import OEB_IMAGES + from calibre.utils.zipfile import ZipFile + from lxml import etree + with TemporaryDirectory('_txtz_output') as tdir: # TXT txt_name = 'index.txt' diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 8f51c3b5df..59a779b9f1 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -706,8 +706,9 @@ OptionRecommendation(name='sr3_replace', files = [f if isinstance(f, unicode) else f.decode(filesystem_encoding) for f in files] from calibre.customize.ui import available_input_formats - fmts = available_input_formats() - for x in ('htm', 'html', 'xhtm', 'xhtml'): fmts.remove(x) + fmts = set(available_input_formats()) + fmts -= {'htm', 'html', 'xhtm', 'xhtml'} + fmts -= set(ARCHIVE_FMTS) for ext in fmts: for f in files: diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index cbc8b41529..91141af1d1 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -157,7 +157,7 @@ class HeuristicProcessor(object): ITALICIZE_STYLE_PATS = [ ur'(?msu)(?<=[\s>"“\'‘])_(?P[^_]+)_', - ur'(?msu)(?<=[\s>"“\'‘])/(?P[^/\*>]+)/', + ur'(?msu)(?<=[\s>"“\'‘])/(?P[^/\*><]+)/', ur'(?msu)(?<=[\s>"“\'‘])~~(?P[^~]+)~~', ur'(?msu)(?<=[\s>"“\'‘])\*(?P[^\*]+)\*', ur'(?msu)(?<=[\s>"“\'‘])~(?P[^~]+)~', @@ -172,8 +172,11 @@ class HeuristicProcessor(object): for word in ITALICIZE_WORDS: html = re.sub(r'(?<=\s|>)' + re.escape(word) + r'(?=\s|<)', '%s' % word, html) + def sub(mo): + return '%s'%mo.group('words') + for pat in ITALICIZE_STYLE_PATS: - html = re.sub(pat, lambda mo: '%s' % mo.group('words'), html) + html = re.sub(pat, sub, html) return html diff --git a/src/calibre/ebooks/epub/fix/epubcheck.py b/src/calibre/ebooks/epub/fix/epubcheck.py index 9e812e1cf4..0029868c23 100644 --- a/src/calibre/ebooks/epub/fix/epubcheck.py +++ b/src/calibre/ebooks/epub/fix/epubcheck.py @@ -6,7 +6,6 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' from calibre.ebooks.epub.fix import ePubFixer, InvalidEpub -from calibre.utils.date import parse_date, strptime class Epubcheck(ePubFixer): @@ -35,6 +34,8 @@ class Epubcheck(ePubFixer): return 'epubcheck' def fix_pubdates(self): + from calibre.utils.date import parse_date, strptime + dirtied = False opf = self.container.opf for dcdate in opf.xpath('//dc:date', diff --git a/src/calibre/ebooks/html/__init__.py b/src/calibre/ebooks/html/__init__.py index d026256ee8..00afa6d6b6 100644 --- a/src/calibre/ebooks/html/__init__.py +++ b/src/calibre/ebooks/html/__init__.py @@ -8,12 +8,13 @@ __docformat__ = 'restructuredtext en' import re -from lxml.etree import tostring as _tostring def tostring(root, strip_comments=False, pretty_print=False): ''' Serialize processed XHTML. ''' + from lxml.etree import tostring as _tostring + root.set('xmlns', 'http://www.w3.org/1999/xhtml') root.set('{http://www.w3.org/1999/xhtml}xlink', 'http://www.w3.org/1999/xlink') for x in root.iter(): diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index d60baf8bce..6cacb34edc 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -11,19 +11,13 @@ __docformat__ = 'restructuredtext en' Input plugin for HTML or OPF ebooks. ''' -import os, re, sys, uuid, tempfile, errno as gerrno +import os, re, sys, errno as gerrno from urlparse import urlparse, urlunparse -from urllib import unquote, quote -from functools import partial -from itertools import izip +from urllib import unquote -from calibre.customize.conversion import InputFormatPlugin from calibre.ebooks.chardet import detect_xml_encoding -from calibre.customize.conversion import OptionRecommendation -from calibre.constants import islinux, isbsd, iswindows +from calibre.constants import iswindows from calibre import unicode_path, as_unicode -from calibre.utils.localization import get_lang -from calibre.utils.filenames import ascii_filename class Link(object): ''' @@ -241,260 +235,4 @@ def get_filelist(htmlfile, dir, opts, log): return filelist -class HTMLInput(InputFormatPlugin): - name = 'HTML Input' - author = 'Kovid Goyal' - description = 'Convert HTML and OPF files to an OEB' - file_types = set(['opf', 'html', 'htm', 'xhtml', 'xhtm', 'shtm', 'shtml']) - - options = set([ - OptionRecommendation(name='breadth_first', - recommended_value=False, level=OptionRecommendation.LOW, - help=_('Traverse links in HTML files breadth first. Normally, ' - 'they are traversed depth first.' - ) - ), - - OptionRecommendation(name='max_levels', - recommended_value=5, level=OptionRecommendation.LOW, - help=_('Maximum levels of recursion when following links in ' - 'HTML files. Must be non-negative. 0 implies that no ' - 'links in the root HTML file are followed. Default is ' - '%default.' - ) - ), - - OptionRecommendation(name='dont_package', - recommended_value=False, level=OptionRecommendation.LOW, - help=_('Normally this input plugin re-arranges all the input ' - 'files into a standard folder hierarchy. Only use this option ' - 'if you know what you are doing as it can result in various ' - 'nasty side effects in the rest of the conversion pipeline.' - ) - ), - - ]) - - def convert(self, stream, opts, file_ext, log, - accelerators): - self._is_case_sensitive = None - basedir = os.getcwd() - self.opts = opts - - fname = None - if hasattr(stream, 'name'): - basedir = os.path.dirname(stream.name) - fname = os.path.basename(stream.name) - - if file_ext != 'opf': - if opts.dont_package: - raise ValueError('The --dont-package option is not supported for an HTML input file') - from calibre.ebooks.metadata.html import get_metadata - mi = get_metadata(stream) - if fname: - from calibre.ebooks.metadata.meta import metadata_from_filename - fmi = metadata_from_filename(fname) - fmi.smart_update(mi) - mi = fmi - oeb = self.create_oebbook(stream.name, basedir, opts, log, mi) - return oeb - - from calibre.ebooks.conversion.plumber import create_oebbook - return create_oebbook(log, stream.name, opts, - encoding=opts.input_encoding) - - def is_case_sensitive(self, path): - if getattr(self, '_is_case_sensitive', None) is not None: - return self._is_case_sensitive - if not path or not os.path.exists(path): - return islinux or isbsd - self._is_case_sensitive = not (os.path.exists(path.lower()) \ - and os.path.exists(path.upper())) - return self._is_case_sensitive - - def create_oebbook(self, htmlpath, basedir, opts, log, mi): - from calibre.ebooks.conversion.plumber import create_oebbook - from calibre.ebooks.oeb.base import (DirContainer, - rewrite_links, urlnormalize, urldefrag, BINARY_MIME, OEB_STYLES, - xpath) - from calibre import guess_type - from calibre.ebooks.oeb.transforms.metadata import \ - meta_info_to_oeb_metadata - import cssutils, logging - cssutils.log.setLevel(logging.WARN) - self.OEB_STYLES = OEB_STYLES - oeb = create_oebbook(log, None, opts, self, - encoding=opts.input_encoding, populate=False) - self.oeb = oeb - - metadata = oeb.metadata - meta_info_to_oeb_metadata(mi, metadata, log) - if not metadata.language: - oeb.logger.warn(u'Language not specified') - metadata.add('language', get_lang().replace('_', '-')) - if not metadata.creator: - oeb.logger.warn('Creator not specified') - metadata.add('creator', self.oeb.translate(__('Unknown'))) - if not metadata.title: - oeb.logger.warn('Title not specified') - metadata.add('title', self.oeb.translate(__('Unknown'))) - bookid = str(uuid.uuid4()) - metadata.add('identifier', bookid, id='uuid_id', scheme='uuid') - for ident in metadata.identifier: - if 'id' in ident.attrib: - self.oeb.uid = metadata.identifier[0] - break - - filelist = get_filelist(htmlpath, basedir, opts, log) - filelist = [f for f in filelist if not f.is_binary] - htmlfile_map = {} - for f in filelist: - path = f.path - oeb.container = DirContainer(os.path.dirname(path), log, - ignore_opf=True) - bname = os.path.basename(path) - id, href = oeb.manifest.generate(id='html', - href=ascii_filename(bname)) - htmlfile_map[path] = href - item = oeb.manifest.add(id, href, 'text/html') - item.html_input_href = bname - oeb.spine.add(item, True) - - self.added_resources = {} - self.log = log - self.log('Normalizing filename cases') - for path, href in htmlfile_map.items(): - if not self.is_case_sensitive(path): - path = path.lower() - self.added_resources[path] = href - self.urlnormalize, self.DirContainer = urlnormalize, DirContainer - self.urldefrag = urldefrag - self.guess_type, self.BINARY_MIME = guess_type, BINARY_MIME - - self.log('Rewriting HTML links') - for f in filelist: - path = f.path - dpath = os.path.dirname(path) - oeb.container = DirContainer(dpath, log, ignore_opf=True) - item = oeb.manifest.hrefs[htmlfile_map[path]] - rewrite_links(item.data, partial(self.resource_adder, base=dpath)) - - for item in oeb.manifest.values(): - if item.media_type in self.OEB_STYLES: - dpath = None - for path, href in self.added_resources.items(): - if href == item.href: - dpath = os.path.dirname(path) - break - cssutils.replaceUrls(item.data, - partial(self.resource_adder, base=dpath)) - - toc = self.oeb.toc - self.oeb.auto_generated_toc = True - titles = [] - headers = [] - for item in self.oeb.spine: - if not item.linear: continue - html = item.data - title = ''.join(xpath(html, '/h:html/h:head/h:title/text()')) - title = re.sub(r'\s+', ' ', title.strip()) - if title: - titles.append(title) - headers.append('(unlabled)') - for tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'strong'): - expr = '/h:html/h:body//h:%s[position()=1]/text()' - header = ''.join(xpath(html, expr % tag)) - header = re.sub(r'\s+', ' ', header.strip()) - if header: - headers[-1] = header - break - use = titles - if len(titles) > len(set(titles)): - use = headers - for title, item in izip(use, self.oeb.spine): - if not item.linear: continue - toc.add(title, item.href) - - oeb.container = DirContainer(os.getcwdu(), oeb.log, ignore_opf=True) - return oeb - - def link_to_local_path(self, link_, base=None): - if not isinstance(link_, unicode): - try: - link_ = link_.decode('utf-8', 'error') - except: - self.log.warn('Failed to decode link %r. Ignoring'%link_) - return None, None - try: - l = Link(link_, base if base else os.getcwdu()) - except: - self.log.exception('Failed to process link: %r'%link_) - return None, None - if l.path is None: - # Not a local resource - return None, None - link = l.path.replace('/', os.sep).strip() - frag = l.fragment - if not link: - return None, None - return link, frag - - def resource_adder(self, link_, base=None): - link, frag = self.link_to_local_path(link_, base=base) - if link is None: - return link_ - try: - if base and not os.path.isabs(link): - link = os.path.join(base, link) - link = os.path.abspath(link) - except: - return link_ - if not os.access(link, os.R_OK): - return link_ - if os.path.isdir(link): - self.log.warn(link_, 'is a link to a directory. Ignoring.') - return link_ - if not self.is_case_sensitive(tempfile.gettempdir()): - link = link.lower() - if link not in self.added_resources: - bhref = os.path.basename(link) - id, href = self.oeb.manifest.generate(id='added', - href=bhref) - guessed = self.guess_type(href)[0] - media_type = guessed or self.BINARY_MIME - if media_type == 'text/plain': - self.log.warn('Ignoring link to text file %r'%link_) - return None - - self.oeb.log.debug('Added', link) - self.oeb.container = self.DirContainer(os.path.dirname(link), - self.oeb.log, ignore_opf=True) - # Load into memory - item = self.oeb.manifest.add(id, href, media_type) - # bhref refers to an already existing file. The read() method of - # DirContainer will call unquote on it before trying to read the - # file, therefore we quote it here. - item.html_input_href = quote(bhref) - if guessed in self.OEB_STYLES: - item.override_css_fetch = partial( - self.css_import_handler, os.path.dirname(link)) - item.data - self.added_resources[link] = href - - nlink = self.added_resources[link] - if frag: - nlink = '#'.join((nlink, frag)) - return nlink - - def css_import_handler(self, base, href): - link, frag = self.link_to_local_path(href, base=base) - if link is None or not os.access(link, os.R_OK) or os.path.isdir(link): - return (None, None) - try: - raw = open(link, 'rb').read().decode('utf-8', 'replace') - raw = self.oeb.css_preprocessor(raw, add_namespace=True) - except: - self.log.exception('Failed to read CSS file: %r'%link) - return (None, None) - return (None, raw) diff --git a/src/calibre/ebooks/lrf/__init__.py b/src/calibre/ebooks/lrf/__init__.py index e4a18a1f91..b12c0d6b34 100644 --- a/src/calibre/ebooks/lrf/__init__.py +++ b/src/calibre/ebooks/lrf/__init__.py @@ -4,7 +4,6 @@ __copyright__ = '2008, Kovid Goyal ' This package contains logic to read and write LRF files. The LRF file format is documented at U{http://www.sven.de/librie/Librie/LrfFormat}. """ -from uuid import uuid4 from calibre.ebooks.lrf.pylrs.pylrs import Book as _Book from calibre.ebooks.lrf.pylrs.pylrs import TextBlock, Header, \ @@ -60,6 +59,7 @@ def find_custom_fonts(options, logger): def Book(options, logger, font_delta=0, header=None, profile=PRS500_PROFILE, **settings): + from uuid import uuid4 ps = {} ps['topmargin'] = options.top_margin ps['evensidemargin'] = options.left_margin diff --git a/src/calibre/ebooks/lrf/input.py b/src/calibre/ebooks/lrf/input.py index 9777a8a998..e9bf42c6bd 100644 --- a/src/calibre/ebooks/lrf/input.py +++ b/src/calibre/ebooks/lrf/input.py @@ -6,12 +6,11 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, textwrap, sys, operator +import textwrap, operator from copy import deepcopy, copy from lxml import etree -from calibre.customize.conversion import InputFormatPlugin from calibre import guess_type class Canvas(etree.XSLTExtension): @@ -406,76 +405,4 @@ class Styles(etree.XSLTExtension): -class LRFInput(InputFormatPlugin): - name = 'LRF Input' - author = 'Kovid Goyal' - description = 'Convert LRF files to HTML' - file_types = set(['lrf']) - - def convert(self, stream, options, file_ext, log, - accelerators): - self.log = log - self.log('Generating XML') - from calibre.ebooks.lrf.lrfparser import LRFDocument - d = LRFDocument(stream) - d.parse() - xml = d.to_xml(write_files=True) - if options.verbose > 2: - open('lrs.xml', 'wb').write(xml.encode('utf-8')) - parser = etree.XMLParser(no_network=True, huge_tree=True) - try: - doc = etree.fromstring(xml, parser=parser) - except: - self.log.warn('Failed to parse XML. Trying to recover') - parser = etree.XMLParser(no_network=True, huge_tree=True, - recover=True) - doc = etree.fromstring(xml, parser=parser) - - - char_button_map = {} - for x in doc.xpath('//CharButton[@refobj]'): - ro = x.get('refobj') - jump_button = doc.xpath('//*[@objid="%s"]'%ro) - if jump_button: - jump_to = jump_button[0].xpath('descendant::JumpTo[@refpage and @refobj]') - if jump_to: - char_button_map[ro] = '%s.xhtml#%s'%(jump_to[0].get('refpage'), - jump_to[0].get('refobj')) - plot_map = {} - for x in doc.xpath('//Plot[@refobj]'): - ro = x.get('refobj') - image = doc.xpath('//Image[@objid="%s" and @refstream]'%ro) - if image: - imgstr = doc.xpath('//ImageStream[@objid="%s" and @file]'% - image[0].get('refstream')) - if imgstr: - plot_map[ro] = imgstr[0].get('file') - - self.log('Converting XML to HTML...') - styledoc = etree.fromstring(P('templates/lrf.xsl', data=True)) - media_type = MediaType() - styles = Styles() - text_block = TextBlock(styles, char_button_map, plot_map, log) - canvas = Canvas(doc, styles, text_block, log) - image_block = ImageBlock(canvas) - ruled_line = RuledLine() - extensions = { - ('calibre', 'media-type') : media_type, - ('calibre', 'text-block') : text_block, - ('calibre', 'ruled-line') : ruled_line, - ('calibre', 'styles') : styles, - ('calibre', 'canvas') : canvas, - ('calibre', 'image-block'): image_block, - } - transform = etree.XSLT(styledoc, extensions=extensions) - try: - result = transform(doc) - except RuntimeError: - sys.setrecursionlimit(5000) - result = transform(doc) - - with open('content.opf', 'wb') as f: - f.write(result) - styles.write() - return os.path.abspath('content.opf') diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 286bcee9d0..0312a7db6a 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -710,7 +710,7 @@ class Metadata(object): fmt('Title sort', self.title_sort) if self.authors: fmt('Author(s)', authors_to_string(self.authors) + \ - ((' [' + self.author_sort + ']') + ((' [' + self.author_sort + ']') if self.author_sort and self.author_sort != _('Unknown') else '')) if self.publisher: fmt('Publisher', self.publisher) diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py index a14e18569a..c0c3900a5d 100644 --- a/src/calibre/ebooks/metadata/book/json_codec.py +++ b/src/calibre/ebooks/metadata/book/json_codec.py @@ -12,7 +12,6 @@ from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS from calibre.constants import filesystem_encoding, preferred_encoding from calibre.library.field_metadata import FieldMetadata from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE, local_tz -from calibre.utils.magick import Image from calibre import isbytestring # Translate datetimes to and from strings. The string form is the datetime in @@ -37,6 +36,8 @@ def encode_thumbnail(thumbnail): ''' Encode the image part of a thumbnail, then return the 3 part tuple ''' + from calibre.utils.magick import Image + if thumbnail is None: return None if not isinstance(thumbnail, (tuple, list)): diff --git a/src/calibre/ebooks/metadata/epub.py b/src/calibre/ebooks/metadata/epub.py index 30fe53f1a2..477b805ba0 100644 --- a/src/calibre/ebooks/metadata/epub.py +++ b/src/calibre/ebooks/metadata/epub.py @@ -129,9 +129,57 @@ class OCFDirReader(OCFReader): def open(self, path, *args, **kwargs): return open(os.path.join(self.root, path), *args, **kwargs) -def get_cover(opf, opf_path, stream, reader=None): +def render_cover(opf, opf_path, zf, reader=None): from calibre.ebooks import render_html_svg_workaround from calibre.utils.logging import default_log + + cpage = opf.first_spine_item() + if not cpage: + return + if reader is not None and reader.encryption_meta.is_encrypted(cpage): + return + + with TemporaryDirectory('_epub_meta') as tdir: + with CurrentDir(tdir): + zf.extractall() + opf_path = opf_path.replace('/', os.sep) + cpage = os.path.join(tdir, os.path.dirname(opf_path), cpage) + if not os.path.exists(cpage): + return + + if isosx: + # On OS X trying to render a HTML cover which uses embedded + # fonts more than once in the same process causes a crash in Qt + # so be safe and remove the fonts as well as any @font-face + # rules + for f in walk('.'): + if os.path.splitext(f)[1].lower() in ('.ttf', '.otf'): + os.remove(f) + ffpat = re.compile(br'@font-face.*?{.*?}', + re.DOTALL|re.IGNORECASE) + with open(cpage, 'r+b') as f: + raw = f.read() + f.truncate(0) + raw = ffpat.sub(b'', raw) + f.write(raw) + from calibre.ebooks.chardet import xml_to_unicode + raw = xml_to_unicode(raw, + strip_encoding_pats=True, resolve_entities=True)[0] + from lxml import html + for link in html.fromstring(raw).xpath('//link'): + href = link.get('href', '') + if href: + path = os.path.join(os.path.dirname(cpage), href) + if os.path.exists(path): + with open(path, 'r+b') as f: + raw = f.read() + f.truncate(0) + raw = ffpat.sub(b'', raw) + f.write(raw) + + return render_html_svg_workaround(cpage, default_log) + +def get_cover(opf, opf_path, stream, reader=None): raster_cover = opf.raster_cover stream.seek(0) zf = ZipFile(stream) @@ -152,27 +200,7 @@ def get_cover(opf, opf_path, stream, reader=None): zf.close() return data - cpage = opf.first_spine_item() - if not cpage: - return - if reader is not None and reader.encryption_meta.is_encrypted(cpage): - return - - with TemporaryDirectory('_epub_meta') as tdir: - with CurrentDir(tdir): - zf.extractall() - if isosx: - # On OS X trying to render an HTML cover which uses embedded - # fonts more than once in the same process causes a crash in Qt - # so be safe and remove the fonts. - for f in walk('.'): - if os.path.splitext(f)[1].lower() in ('.ttf', '.otf'): - os.remove(f) - opf_path = opf_path.replace('/', os.sep) - cpage = os.path.join(tdir, os.path.dirname(opf_path), cpage) - if not os.path.exists(cpage): - return - return render_html_svg_workaround(cpage, default_log) + return render_cover(opf, opf_path, zf, reader=reader) def get_metadata(stream, extract_cover=True): """ Return metadata as a :class:`Metadata` object """ diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index 7d8855de14..73ba7e77f4 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -217,3 +217,23 @@ def opf_metadata(opfpath): import traceback traceback.print_exc() pass + +def forked_read_metadata(path, tdir): + from calibre.ebooks.metadata.opf2 import metadata_to_opf + with open(path, 'rb') as f: + fmt = os.path.splitext(path)[1][1:].lower() + f.seek(0, 2) + sz = f.tell() + with open(os.path.join(tdir, 'size.txt'), 'wb') as s: + s.write(str(sz).encode('ascii')) + f.seek(0) + mi = get_metadata(f, fmt) + if mi.cover_data and mi.cover_data[1]: + with open(os.path.join(tdir, 'cover.jpg'), 'wb') as f: + f.write(mi.cover_data[1]) + mi.cover_data = (None, None) + mi.cover = 'cover.jpg' + opf = metadata_to_opf(mi) + with open(os.path.join(tdir, 'metadata.opf'), 'wb') as f: + f.write(opf) + diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 9b8ae12b10..8d37e95dc4 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -1019,6 +1019,11 @@ class OPF(object): # {{{ mt = item.get('media-type', '') if 'xml' not in mt: return item.get('href', None) + for item in self.itermanifest(): + if item.get('href', None) == cover_id: + mt = item.get('media-type', '') + if mt.startswith('image/'): + return item.get('href', None) @dynamic_property def cover(self): diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index 5284e36cad..3d08b96c5f 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -12,19 +12,14 @@ from urllib import urlencode from threading import Thread from Queue import Queue, Empty -from lxml.html import tostring from calibre import as_unicode from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import (Source, Option, fixcase, fixauthors) -from calibre.utils.cleantext import clean_ascii_chars -from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.metadata.book.base import Metadata -from calibre.library.comments import sanitize_comments_html from calibre.utils.date import parse_date from calibre.utils.localization import canonicalize_lang -from calibre.utils.soupparser import fromstring class Worker(Thread): # Get details {{{ @@ -43,6 +38,8 @@ class Worker(Thread): # Get details {{{ self.browser = browser.clone_browser() self.cover_url = self.amazon_id = self.isbn = None self.domain = domain + from lxml.html import tostring + self.tostring = tostring months = { 'de': { @@ -176,6 +173,10 @@ class Worker(Thread): # Get details {{{ self.log.exception('get_details failed for url: %r'%self.url) def get_details(self): + from calibre.utils.cleantext import clean_ascii_chars + from calibre.utils.soupparser import fromstring + from calibre.ebooks.chardet import xml_to_unicode + try: raw = self.browser.open_novisit(self.url, timeout=self.timeout).read().strip() except Exception as e: @@ -210,7 +211,7 @@ class Worker(Thread): # Get details {{{ errmsg = root.xpath('//*[@id="errorMessage"]') if errmsg: msg = 'Failed to parse amazon details page: %r'%self.url - msg += tostring(errmsg, method='text', encoding=unicode).strip() + msg += self.tostring(errmsg, method='text', encoding=unicode).strip() self.log.error(msg) return @@ -322,10 +323,10 @@ class Worker(Thread): # Get details {{{ tdiv = root.xpath('//h1[contains(@class, "parseasinTitle")]')[0] actual_title = tdiv.xpath('descendant::*[@id="btAsinTitle"]') if actual_title: - title = tostring(actual_title[0], encoding=unicode, + title = self.tostring(actual_title[0], encoding=unicode, method='text').strip() else: - title = tostring(tdiv, encoding=unicode, method='text').strip() + title = self.tostring(tdiv, encoding=unicode, method='text').strip() return re.sub(r'[(\[].*[)\]]', '', title).strip() def parse_authors(self, root): @@ -337,7 +338,7 @@ class Worker(Thread): # Get details {{{ ''') for x in aname: x.tail = '' - authors = [tostring(x, encoding=unicode, method='text').strip() for x + authors = [self.tostring(x, encoding=unicode, method='text').strip() for x in aname] authors = [a for a in authors if a] return authors @@ -356,6 +357,8 @@ class Worker(Thread): # Get details {{{ return float(m.group(1))/float(m.group(3)) * 5 def parse_comments(self, root): + from calibre.library.comments import sanitize_comments_html + desc = root.xpath('//div[@id="productDescription"]/*[@class="content"]') if desc: desc = desc[0] @@ -365,7 +368,7 @@ class Worker(Thread): # Get details {{{ for a in desc.xpath('descendant::a[@href]'): del a.attrib['href'] a.tag = 'span' - desc = tostring(desc, method='html', encoding=unicode).strip() + desc = self.tostring(desc, method='html', encoding=unicode).strip() # Encoding bug in Amazon data U+fffd (replacement char) # in some examples it is present in place of ' @@ -602,6 +605,11 @@ class Amazon(Source): Note this method will retry without identifiers automatically if no match is found with identifiers. ''' + from lxml.html import tostring + from calibre.utils.cleantext import clean_ascii_chars + from calibre.utils.soupparser import fromstring + from calibre.ebooks.chardet import xml_to_unicode + query, domain = self.create_query(log, title=title, authors=authors, identifiers=identifiers) if query is None: @@ -767,15 +775,6 @@ if __name__ == '__main__': # tests {{{ ), - ( # This isbn not on amazon - {'identifiers':{'isbn': '8324616489'}, 'title':'Learning Python', - 'authors':['Lutz']}, - [title_test('Learning Python, 3rd Edition', - exact=True), authors_test(['Mark Lutz']) - ] - - ), - ( # Sophisticated comment formatting {'identifiers':{'isbn': '9781416580829'}}, [title_test('Angels & Demons - Movie Tie-In: A Novel', diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 9ae8902671..4c334f4e46 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -12,7 +12,6 @@ from future_builtins import map from calibre import browser, random_user_agent from calibre.customize import Plugin -from calibre.utils.logging import ThreadSafeLog, FileStream from calibre.utils.config import JSONConfig from calibre.utils.titlecase import titlecase from calibre.utils.icu import capitalize, lower, upper @@ -34,6 +33,7 @@ msprefs.defaults['fewer_tags'] = True msprefs.defaults['cover_priorities'] = {'Google':2} def create_log(ostream=None): + from calibre.utils.logging import ThreadSafeLog, FileStream log = ThreadSafeLog(level=ThreadSafeLog.DEBUG) log.outputs = [FileStream(ostream)] return log diff --git a/src/calibre/ebooks/metadata/sources/douban.py b/src/calibre/ebooks/metadata/sources/douban.py index 06e874e8ca..6857d62d4d 100644 --- a/src/calibre/ebooks/metadata/sources/douban.py +++ b/src/calibre/ebooks/metadata/sources/douban.py @@ -12,14 +12,10 @@ from urllib import urlencode from functools import partial from Queue import Queue, Empty -from lxml import etree from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.chardet import xml_to_unicode -from calibre.utils.date import parse_date, utcnow -from calibre.utils.cleantext import clean_ascii_chars from calibre import as_unicode NAMESPACES = { @@ -28,22 +24,6 @@ NAMESPACES = { 'db': 'http://www.douban.com/xmlns/', 'gd': 'http://schemas.google.com/g/2005' } -XPath = partial(etree.XPath, namespaces=NAMESPACES) -total_results = XPath('//openSearch:totalResults') -start_index = XPath('//openSearch:startIndex') -items_per_page = XPath('//openSearch:itemsPerPage') -entry = XPath('//atom:entry') -entry_id = XPath('descendant::atom:id') -title = XPath('descendant::atom:title') -description = XPath('descendant::atom:summary') -publisher = XPath("descendant::db:attribute[@name='publisher']") -isbn = XPath("descendant::db:attribute[@name='isbn13']") -date = XPath("descendant::db:attribute[@name='pubdate']") -creator = XPath("descendant::db:attribute[@name='author']") -booktag = XPath("descendant::db:tag/attribute::name") -rating = XPath("descendant::gd:rating/attribute::average") -cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href") - def get_details(browser, url, timeout): # {{{ try: if Douban.DOUBAN_API_KEY and Douban.DOUBAN_API_KEY != '': @@ -61,6 +41,25 @@ def get_details(browser, url, timeout): # {{{ # }}} def to_metadata(browser, log, entry_, timeout): # {{{ + from lxml import etree + from calibre.ebooks.chardet import xml_to_unicode + from calibre.utils.date import parse_date, utcnow + from calibre.utils.cleantext import clean_ascii_chars + + XPath = partial(etree.XPath, namespaces=NAMESPACES) + entry = XPath('//atom:entry') + entry_id = XPath('descendant::atom:id') + title = XPath('descendant::atom:title') + description = XPath('descendant::atom:summary') + publisher = XPath("descendant::db:attribute[@name='publisher']") + isbn = XPath("descendant::db:attribute[@name='isbn13']") + date = XPath("descendant::db:attribute[@name='pubdate']") + creator = XPath("descendant::db:attribute[@name='author']") + booktag = XPath("descendant::db:tag/attribute::name") + rating = XPath("descendant::gd:rating/attribute::average") + cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href") + + def get_text(extra, x): try: ans = x(extra) @@ -275,6 +274,7 @@ class Douban(Source): def get_all_details(self, br, log, entries, abort, # {{{ result_queue, timeout): + from lxml import etree for relevance, i in enumerate(entries): try: ans = to_metadata(br, log, i, timeout) @@ -298,6 +298,13 @@ class Douban(Source): def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ identifiers={}, timeout=30): + from lxml import etree + from calibre.ebooks.chardet import xml_to_unicode + from calibre.utils.cleantext import clean_ascii_chars + + XPath = partial(etree.XPath, namespaces=NAMESPACES) + entry = XPath('//atom:entry') + query = self.create_query(log, title=title, authors=authors, identifiers=identifiers) if not query: diff --git a/src/calibre/ebooks/metadata/sources/google.py b/src/calibre/ebooks/metadata/sources/google.py index f9c43d86cc..3962afcb5e 100644 --- a/src/calibre/ebooks/metadata/sources/google.py +++ b/src/calibre/ebooks/metadata/sources/google.py @@ -12,8 +12,6 @@ from urllib import urlencode from functools import partial from Queue import Queue, Empty -from lxml import etree - from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source from calibre.ebooks.metadata.book.base import Metadata @@ -29,23 +27,6 @@ NAMESPACES = { 'dc' : 'http://purl.org/dc/terms', 'gd' : 'http://schemas.google.com/g/2005' } -XPath = partial(etree.XPath, namespaces=NAMESPACES) - -total_results = XPath('//openSearch:totalResults') -start_index = XPath('//openSearch:startIndex') -items_per_page = XPath('//openSearch:itemsPerPage') -entry = XPath('//atom:entry') -entry_id = XPath('descendant::atom:id') -creator = XPath('descendant::dc:creator') -identifier = XPath('descendant::dc:identifier') -title = XPath('descendant::dc:title') -date = XPath('descendant::dc:date') -publisher = XPath('descendant::dc:publisher') -subject = XPath('descendant::dc:subject') -description = XPath('descendant::dc:description') -language = XPath('descendant::dc:language') -rating = XPath('descendant::gd:rating[@average]') - def get_details(browser, url, timeout): # {{{ try: raw = browser.open_novisit(url, timeout=timeout).read() @@ -61,6 +42,24 @@ def get_details(browser, url, timeout): # {{{ # }}} def to_metadata(browser, log, entry_, timeout): # {{{ + from lxml import etree + XPath = partial(etree.XPath, namespaces=NAMESPACES) + + # total_results = XPath('//openSearch:totalResults') + # start_index = XPath('//openSearch:startIndex') + # items_per_page = XPath('//openSearch:itemsPerPage') + entry = XPath('//atom:entry') + entry_id = XPath('descendant::atom:id') + creator = XPath('descendant::dc:creator') + identifier = XPath('descendant::dc:identifier') + title = XPath('descendant::dc:title') + date = XPath('descendant::dc:date') + publisher = XPath('descendant::dc:publisher') + subject = XPath('descendant::dc:subject') + description = XPath('descendant::dc:description') + language = XPath('descendant::dc:language') + rating = XPath('descendant::gd:rating[@average]') + def get_text(extra, x): try: @@ -266,6 +265,7 @@ class GoogleBooks(Source): def get_all_details(self, br, log, entries, abort, # {{{ result_queue, timeout): + from lxml import etree for relevance, i in enumerate(entries): try: ans = to_metadata(br, log, i, timeout) @@ -289,6 +289,10 @@ class GoogleBooks(Source): def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ identifiers={}, timeout=30): + from lxml import etree + XPath = partial(etree.XPath, namespaces=NAMESPACES) + entry = XPath('//atom:entry') + query = self.create_query(log, title=title, authors=authors, identifiers=identifiers) if not query: diff --git a/src/calibre/ebooks/metadata/sources/isbndb.py b/src/calibre/ebooks/metadata/sources/isbndb.py index 1da7f906bb..7e15ad275e 100644 --- a/src/calibre/ebooks/metadata/sources/isbndb.py +++ b/src/calibre/ebooks/metadata/sources/isbndb.py @@ -9,12 +9,9 @@ __docformat__ = 'restructuredtext en' from urllib import quote -from lxml import etree from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source, Option -from calibre.ebooks.chardet import xml_to_unicode -from calibre.utils.cleantext import clean_ascii_chars from calibre.utils.icu import lower from calibre.ebooks.metadata.book.base import Metadata @@ -122,6 +119,7 @@ class ISBNDB(Source): result_queue.put(result) def parse_feed(self, feed, seen, orig_title, orig_authors, identifiers): + from lxml import etree def tostring(x): if x is None: @@ -198,6 +196,10 @@ class ISBNDB(Source): def make_query(self, q, abort, title=None, authors=None, identifiers={}, max_pages=10, timeout=30): + from lxml import etree + from calibre.ebooks.chardet import xml_to_unicode + from calibre.utils.cleantext import clean_ascii_chars + page_num = 1 parser = etree.XMLParser(recover=True, no_network=True) br = self.browser diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 1164567ff5..bb1bbb9d42 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -9,18 +9,14 @@ __docformat__ = 'restructuredtext en' ''' Fetch metadata using Overdrive Content Reserve ''' -import re, random, mechanize, copy, json +import re, random, copy, json from threading import RLock from Queue import Queue, Empty -from lxml import html from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source, Option from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.chardet import xml_to_unicode -from calibre.library.comments import sanitize_comments_html -from calibre.utils.soupparser import fromstring ovrdrv_data_cache = {} cache_lock = RLock() @@ -80,6 +76,7 @@ class OverDrive(Source): def download_cover(self, log, result_queue, abort, # {{{ title=None, authors=None, identifiers={}, timeout=30): + import mechanize cached_url = self.get_cached_cover_url(identifiers) if cached_url is None: log.info('No cached cover found, running identify') @@ -170,6 +167,7 @@ class OverDrive(Source): this page attempts to set a cookie that Mechanize doesn't like copy the cookiejar to a separate instance and make a one-off request with the temp cookiejar ''' + import mechanize goodcookies = br._ua_handlers['_cookies'].cookiejar clean_cj = mechanize.CookieJar() cookies_to_copy = [] @@ -187,6 +185,7 @@ class OverDrive(Source): br.set_cookiejar(clean_cj) def overdrive_search(self, br, log, q, title, author): + import mechanize # re-initialize the cookiejar to so that it's clean clean_cj = mechanize.CookieJar() br.set_cookiejar(clean_cj) @@ -303,6 +302,7 @@ class OverDrive(Source): return '' def overdrive_get_record(self, br, log, q, ovrdrv_id): + import mechanize search_url = q+'SearchResults.aspx?ReserveID={'+ovrdrv_id+'}' results_url = q+'SearchResults.svc/GetResults?sEcho=1&iColumns=18&sColumns=ReserveID%2CTitle%2CSubtitle%2CEdition%2CSeries%2CPublisher%2CFormat%2CFormatID%2CCreators%2CThumbImage%2CShortDescription%2CWorldCatLink%2CExcerptLink%2CCreatorFile%2CSortTitle%2CAvailableToLibrary%2CAvailableToRetailer%2CRelevancyRank&iDisplayStart=0&iDisplayLength=10&sSearch=&bEscapeRegex=true&iSortingCols=1&iSortCol_0=17&sSortDir_0=asc' @@ -393,6 +393,11 @@ class OverDrive(Source): def get_book_detail(self, br, metadata_url, mi, ovrdrv_id, log): + from lxml import html + from calibre.ebooks.chardet import xml_to_unicode + from calibre.utils.soupparser import fromstring + from calibre.library.comments import sanitize_comments_html + try: raw = br.open_novisit(metadata_url).read() except Exception, e: diff --git a/src/calibre/ebooks/metadata/sources/ozon.py b/src/calibre/ebooks/metadata/sources/ozon.py index de45e0b8db..d40e43d582 100644 --- a/src/calibre/ebooks/metadata/sources/ozon.py +++ b/src/calibre/ebooks/metadata/sources/ozon.py @@ -6,15 +6,11 @@ __copyright__ = '2011, Roman Mukhin ' __docformat__ = 'restructuredtext en' import re -import urllib2 import datetime from urllib import quote_plus from Queue import Queue, Empty -from lxml import etree, html + from calibre import as_unicode - -from calibre.ebooks.chardet import xml_to_unicode - from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source from calibre.ebooks.metadata.book.base import Metadata @@ -43,6 +39,7 @@ class Ozon(Source): isbnRegex = re.compile(isbnPattern) def get_book_url(self, identifiers): # {{{ + import urllib2 ozon_id = identifiers.get('ozon', None) res = None if ozon_id: @@ -81,6 +78,9 @@ class Ozon(Source): def identify(self, log, result_queue, abort, title=None, authors=None, identifiers={}, timeout=30): # {{{ + from lxml import etree + from calibre.ebooks.chardet import xml_to_unicode + if not self.is_configured(): return query = self.create_query(log, title=title, authors=authors, identifiers=identifiers) @@ -283,6 +283,9 @@ class Ozon(Source): # }}} def get_book_details(self, log, metadata, timeout): # {{{ + from lxml import html, etree + from calibre.ebooks.chardet import xml_to_unicode + url = self.get_book_url(metadata.get_identifiers())[2] raw = self.browser.open_novisit(url, timeout=timeout).read() diff --git a/src/calibre/ebooks/metadata/topaz.py b/src/calibre/ebooks/metadata/topaz.py index 4db3a7b955..fab63734c5 100644 --- a/src/calibre/ebooks/metadata/topaz.py +++ b/src/calibre/ebooks/metadata/topaz.py @@ -8,6 +8,7 @@ import StringIO, sys from struct import pack from calibre.ebooks.metadata import MetaInformation +from calibre import force_unicode class StreamSlicer(object): @@ -245,7 +246,9 @@ class MetadataUpdater(object): def get_metadata(self): ''' Return MetaInformation with title, author''' self.get_original_metadata() - return MetaInformation(self.metadata['Title'], [self.metadata['Authors']]) + title = force_unicode(self.metadata['Title'], 'utf-8') + authors = force_unicode(self.metadata['Authors'], 'utf-8').split(';') + return MetaInformation(title, authors) def get_original_metadata(self): offset = self.base + self.topaz_headers['metadata']['blocks'][0]['offset'] diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index e664834260..7e69fc89d0 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -363,6 +363,11 @@ class MobiReader(object): self.log.warning('MOBI markup appears to contain random bytes. Stripping.') self.processed_html = self.remove_random_bytes(self.processed_html) root = fromstring(self.processed_html) + if len(root.xpath('body/descendant::*')) < 1: + # There are probably stray s in the markup + self.processed_html = self.processed_html.replace('', + '') + root = fromstring(self.processed_html) if root.tag != 'html': self.log.warn('File does not have opening tag') @@ -511,6 +516,17 @@ class MobiReader(object): self.processed_html = re.sub(r'(?i)(?P]*>)\s*(?P(\s*){1,})', '\g'+'\g', self.processed_html) self.processed_html = re.sub(r'(?i)(?P
(]*>\s*){1,})(?P]*>)', '\g'+'\g
', self.processed_html) self.processed_html = re.sub(r'(?i)(?P]*>)\s*(?P
(<(blockquote|div)[^>]*>\s*){1,})', '\g
'+'\g', self.processed_html) + bods = htmls = 0 + for x in re.finditer(ur'|', self.processed_html): + if x == '': bods +=1 + else: htmls += 1 + if bods > 1 and htmls > 1: + break + if bods > 1: + self.processed_html = self.processed_html.replace('', '') + if htmls > 1: + self.processed_html = self.processed_html.replace('', '') + def remove_random_bytes(self, html): diff --git a/src/calibre/ebooks/mobi/writer2/main.py b/src/calibre/ebooks/mobi/writer2/main.py index 93b3987971..a4dac33d94 100644 --- a/src/calibre/ebooks/mobi/writer2/main.py +++ b/src/calibre/ebooks/mobi/writer2/main.py @@ -494,7 +494,9 @@ class MobiWriter(object): creators = [normalize(unicode(c)) for c in items] items = ['; '.join(creators)] for item in items: - data = self.COLLAPSE_RE.sub(' ', normalize(unicode(item))) + data = normalize(unicode(item)) + if term != 'description': + data = self.COLLAPSE_RE.sub(' ', data) if term == 'identifier': if data.lower().startswith('urn:isbn:'): data = data[9:] diff --git a/src/calibre/ebooks/odt/input.py b/src/calibre/ebooks/odt/input.py index 359e3fc2ed..430d95b31f 100644 --- a/src/calibre/ebooks/odt/input.py +++ b/src/calibre/ebooks/odt/input.py @@ -12,7 +12,6 @@ from lxml import etree from odf.odf2xhtml import ODF2XHTML from calibre import CurrentDir, walk -from calibre.customize.conversion import InputFormatPlugin class Extract(ODF2XHTML): @@ -29,14 +28,38 @@ class Extract(ODF2XHTML): root = etree.fromstring(html) self.epubify_markup(root, log) self.filter_css(root, log) + self.extract_css(root) html = etree.tostring(root, encoding='utf-8', xml_declaration=True) return html + def extract_css(self, root): + ans = [] + for s in root.xpath('//*[local-name() = "style" and @type="text/css"]'): + ans.append(s.text) + s.getparent().remove(s) + + head = root.xpath('//*[local-name() = "head"]') + if head: + head = head[0] + ns = head.nsmap.get(None, '') + if ns: + ns = '{%s}'%ns + etree.SubElement(head, ns+'link', {'type':'text/css', + 'rel':'stylesheet', 'href':'odfpy.css'}) + + with open('odfpy.css', 'wb') as f: + f.write((u'\n\n'.join(ans)).encode('utf-8')) + + def epubify_markup(self, root, log): + from calibre.ebooks.oeb.base import XPath, XHTML + # Fix empty title tags + for t in XPath('//h:title')(root): + if not t.text: + t.text = u' ' # Fix

constructs as the asinine epubchecker complains # about them - from calibre.ebooks.oeb.base import XPath, XHTML pdiv = XPath('//h:p/h:div') for div in pdiv(root): div.getparent().tag = XHTML('div') @@ -146,23 +169,12 @@ class Extract(ODF2XHTML): if not mi.authors: mi.authors = [_('Unknown')] opf = OPFCreator(os.path.abspath(os.getcwdu()), mi) - opf.create_manifest([(os.path.abspath(f), None) for f in walk(os.getcwd())]) + opf.create_manifest([(os.path.abspath(f), None) for f in + walk(os.getcwdu())]) opf.create_spine([os.path.abspath('index.xhtml')]) with open('metadata.opf', 'wb') as f: opf.render(f) return os.path.abspath('metadata.opf') -class ODTInput(InputFormatPlugin): - - name = 'ODT Input' - author = 'Kovid Goyal' - description = 'Convert ODT (OpenOffice) files to HTML' - file_types = set(['odt']) - - - def convert(self, stream, options, file_ext, log, - accelerators): - return Extract()(stream, '.', log) - diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index bc01cc13cd..2778f7fc8a 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -425,15 +425,24 @@ class DirContainer(object): self.opfname = path return + def _unquote(self, path): + # urlunquote must run on a bytestring and will return a bytestring + # If it runs on a unicode object, it returns a double encoded unicode + # string: unquote(u'%C3%A4') != unquote(b'%C3%A4').decode('utf-8') + # and the latter is correct + if isinstance(path, unicode): + path = path.encode('utf-8') + return urlunquote(path).decode('utf-8') + def read(self, path): if path is None: path = self.opfname - path = os.path.join(self.rootdir, path) - with open(urlunquote(path), 'rb') as f: + path = os.path.join(self.rootdir, self._unquote(path)) + with open(path, 'rb') as f: return f.read() def write(self, path, data): - path = os.path.join(self.rootdir, urlunquote(path)) + path = os.path.join(self.rootdir, self._unquote(path)) dir = os.path.dirname(path) if not os.path.isdir(dir): os.makedirs(dir) @@ -442,7 +451,7 @@ class DirContainer(object): def exists(self, path): try: - path = os.path.join(self.rootdir, urlunquote(path)) + path = os.path.join(self.rootdir, self._unquote(path)) except ValueError: #Happens if path contains quoted special chars return False return os.path.isfile(path) @@ -1068,6 +1077,12 @@ class Manifest(object): if item in self.oeb.spine: self.oeb.spine.remove(item) + def remove_duplicate_item(self, item): + if item in self.ids: + item = self.ids[item] + del self.ids[item.id] + self.items.remove(item) + def generate(self, id=None, href=None): """Generate a new unique identifier and/or internal path for use in creating a new manifest item, using the provided :param:`id` and/or diff --git a/src/calibre/ebooks/oeb/display/cfi.coffee b/src/calibre/ebooks/oeb/display/cfi.coffee index ab4d456994..8efc6866b5 100644 --- a/src/calibre/ebooks/oeb/display/cfi.coffee +++ b/src/calibre/ebooks/oeb/display/cfi.coffee @@ -102,7 +102,7 @@ viewport_to_document = (x, y, doc=window?.document) -> # {{{ return [x, y] # }}} -# Equivalent for caretRangeFromPoint for non WebKit browsers {{{ +# Convert point to character offset {{{ range_has_point = (range, x, y) -> for rect in range.getClientRects() if (rect.left <= x <= rect.right) and (rect.top <= y <= rect.bottom) @@ -153,7 +153,7 @@ class CanonicalFragmentIdentifier ### This class is a namespace to expose CFI functions via the window.cfi - object. The three most important functions are: + object. The most important functions are: is_compatible(): Throws an error if the browser is not compatible with this script @@ -166,6 +166,8 @@ class CanonicalFragmentIdentifier ### constructor: () -> # {{{ + if not this instanceof arguments.callee + throw new Error('CFI constructor called as function') this.CREATE_RANGE_ERR = "Your browser does not support the createRange function. Update it to a newer version." this.IE_ERR = "Your browser is too old. You need Internet Explorer version 9 or newer." div = document.createElement('div') @@ -322,7 +324,7 @@ class CanonicalFragmentIdentifier point.time = r[1] - 0 # Coerce to number cfi = cfi.substr(r[0].length) - if (r = cfi.match(/^@(-?\d+(\.\d+)?),(-?\d+(\.\d+)?)/)) != null + if (r = cfi.match(/^@(-?\d+(\.\d+)?):(-?\d+(\.\d+)?)/)) != null # Spatial offset point.x = r[1] - 0 # Coerce to number point.y = r[3] - 0 # Coerce to number @@ -416,7 +418,7 @@ class CanonicalFragmentIdentifier rect = target.getBoundingClientRect() px = ((x - rect.left)*100)/target.offsetWidth py = ((y - rect.top)*100)/target.offsetHeight - tail = "#{ tail }@#{ fstr px },#{ fstr py }" + tail = "#{ tail }@#{ fstr px }:#{ fstr py }" else if name != 'audio' # Get the text offset # We use a custom function instead of caretRangeFromPoint as @@ -579,29 +581,30 @@ class CanonicalFragmentIdentifier get_cfi = (ox, oy) -> try - cfi = this.at(ox, oy) - point = this.point(cfi) + cfi = window.cfi.at(ox, oy) + point = window.cfi.point(cfi) catch err cfi = null - if point.range != null - r = point.range - rect = r.getClientRects()[0] + if cfi + if point.range != null + r = point.range + rect = r.getClientRects()[0] - x = (point.a*rect.left + (1-point.a)*rect.right) - y = (rect.top + rect.bottom)/2 - [x, y] = viewport_to_document(x, y, r.startContainer.ownerDocument) - else - node = point.node - r = node.getBoundingClientRect() - [x, y] = viewport_to_document(r.left, r.top, node.ownerDocument) - if typeof(point.x) == 'number' and node.offsetWidth - x += (point.x*node.offsetWidth)/100 - if typeof(point.y) == 'number' and node.offsetHeight - y += (point.y*node.offsetHeight)/100 + x = (point.a*rect.left + (1-point.a)*rect.right) + y = (rect.top + rect.bottom)/2 + [x, y] = viewport_to_document(x, y, r.startContainer.ownerDocument) + else + node = point.node + r = node.getBoundingClientRect() + [x, y] = viewport_to_document(r.left, r.top, node.ownerDocument) + if typeof(point.x) == 'number' and node.offsetWidth + x += (point.x*node.offsetWidth)/100 + if typeof(point.y) == 'number' and node.offsetHeight + y += (point.y*node.offsetHeight)/100 - if dist(viewport_to_document(ox, oy), [x, y]) > 50 - cfi = null + if dist(viewport_to_document(ox, oy), [x, y]) > 50 + cfi = null return cfi @@ -625,8 +628,16 @@ class CanonicalFragmentIdentifier return cfi cury += delta - # TODO: Return the CFI corresponding to the tag - null + # Use a spatial offset on the html element, since we could not find a + # normal CFI + [x, y] = window_scroll_pos() + de = document.documentElement + rect = de.getBoundingClientRect() + px = (x*100)/rect.width + py = (y*100)/rect.height + cfi = "/2@#{ fstr px }:#{ fstr py }" + + return cfi # }}} diff --git a/src/calibre/ebooks/oeb/display/test-cfi/cfi-test.coffee b/src/calibre/ebooks/oeb/display/test-cfi/cfi-test.coffee index 663e830441..bed03d6ff7 100644 --- a/src/calibre/ebooks/oeb/display/test-cfi/cfi-test.coffee +++ b/src/calibre/ebooks/oeb/display/test-cfi/cfi-test.coffee @@ -30,18 +30,23 @@ window_ypos = (pos=null) -> window.scrollTo(0, pos) mark_and_reload = (evt) -> - # Remove image in case the click was on the image itself, we want the cfi to - # be on the underlying element x = evt.clientX y = evt.clientY if evt.button == 2 return # Right mouse click, generated only in firefox - reset = document.getElementById('reset') - if document.elementFromPoint(x, y) == reset + + if document.elementFromPoint(x, y)?.getAttribute('id') in ['reset', 'viewport_mode'] return + + # Remove image in case the click was on the image itself, we want the cfi to + # be on the underlying element ms = document.getElementById("marker") - if ms - ms.parentNode?.removeChild(ms) + ms.style.display = 'none' + + if document.getElementById('viewport_mode').checked + cfi = window.cfi.at_current() + window.cfi.scroll_to(cfi) + return fn = () -> try diff --git a/src/calibre/ebooks/oeb/display/test-cfi/index.html b/src/calibre/ebooks/oeb/display/test-cfi/index.html index 8398d27791..42bcf626b1 100644 --- a/src/calibre/ebooks/oeb/display/test-cfi/index.html +++ b/src/calibre/ebooks/oeb/display/test-cfi/index.html @@ -8,6 +8,7 @@ body { font-family: sans-serif; background-color: white; + padding-bottom: 500px; } h1, h2 { color: #005a9c } @@ -48,7 +49,13 @@

Testing cfi.coffee

Click anywhere and the location will be marked with a marker, whose position is set via a CFI.

-

Reset CFI to None

+

+ Reset CFI to None +   + Test viewport location calculation: + +

A div with scrollbars

Scroll down and click on some elements. Make sure to hit both bold and not bold text as well as different points on the image

diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index 6b2cf798ea..a458df5a83 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -327,7 +327,7 @@ class OEBReader(object): manifest = self.oeb.manifest for elem in xpath(opf, '/o2:package/o2:guide/o2:reference'): href = elem.get('href') - path = urldefrag(href)[0] + path = urlnormalize(urldefrag(href)[0]) if path not in manifest.hrefs: self.logger.warn(u'Guide reference %r not found' % href) continue @@ -627,11 +627,27 @@ class OEBReader(object): return self.oeb.metadata.add('cover', cover.id) + def _manifest_remove_duplicates(self): + seen = set() + dups = set() + for item in self.oeb.manifest: + if item.href in seen: + dups.add(item.href) + seen.add(item.href) + + for href in dups: + items = [x for x in self.oeb.manifest if x.href == href] + for x in items: + if x not in self.oeb.spine: + self.oeb.log.warn('Removing duplicate manifest item with id:', x.id) + self.oeb.manifest.remove_duplicate_item(x) + def _all_from_opf(self, opf): self.oeb.version = opf.get('version', '1.2') self._metadata_from_opf(opf) self._manifest_from_opf(opf) self._spine_from_opf(opf) + self._manifest_remove_duplicates() self._guide_from_opf(opf) item = self._find_ncx(opf) self._toc_from_opf(opf, item) diff --git a/src/calibre/ebooks/oeb/transforms/structure.py b/src/calibre/ebooks/oeb/transforms/structure.py index f2a61ba6e1..dd3db1415a 100644 --- a/src/calibre/ebooks/oeb/transforms/structure.py +++ b/src/calibre/ebooks/oeb/transforms/structure.py @@ -12,7 +12,7 @@ from lxml import etree from urlparse import urlparse from collections import OrderedDict -from calibre.ebooks.oeb.base import XPNSMAP, TOC, XHTML, xml2text +from calibre.ebooks.oeb.base import XPNSMAP, TOC, XHTML, xml2text, barename from calibre.ebooks import ConversionError def XPath(x): @@ -59,6 +59,18 @@ class DetectStructure(object): pb_xpath = XPath(opts.page_breaks_before) for item in oeb.spine: for elem in pb_xpath(item.data): + try: + prev = elem.itersiblings(tag=etree.Element, + preceding=True).next() + if (barename(elem.tag) in {'h1', 'h2'} and barename( + prev.tag) in {'h1', 'h2'} and (not prev.tail or + not prev.tail.split())): + # We have two adjacent headings, do not put a page + # break on the second one + continue + except StopIteration: + pass + style = elem.get('style', '') if style: style += '; ' diff --git a/src/calibre/ebooks/pdb/__init__.py b/src/calibre/ebooks/pdb/__init__.py index c8089297db..428cbe82ab 100644 --- a/src/calibre/ebooks/pdb/__init__.py +++ b/src/calibre/ebooks/pdb/__init__.py @@ -7,31 +7,38 @@ __docformat__ = 'restructuredtext en' class PDBError(Exception): pass +FORMAT_READERS = None -from calibre.ebooks.pdb.ereader.reader import Reader as ereader_reader -from calibre.ebooks.pdb.palmdoc.reader import Reader as palmdoc_reader -from calibre.ebooks.pdb.ztxt.reader import Reader as ztxt_reader -from calibre.ebooks.pdb.pdf.reader import Reader as pdf_reader -from calibre.ebooks.pdb.plucker.reader import Reader as plucker_reader +def _import_readers(): + global FORMAT_READERS + from calibre.ebooks.pdb.ereader.reader import Reader as ereader_reader + from calibre.ebooks.pdb.palmdoc.reader import Reader as palmdoc_reader + from calibre.ebooks.pdb.ztxt.reader import Reader as ztxt_reader + from calibre.ebooks.pdb.pdf.reader import Reader as pdf_reader + from calibre.ebooks.pdb.plucker.reader import Reader as plucker_reader -FORMAT_READERS = { - 'PNPdPPrs': ereader_reader, - 'PNRdPPrs': ereader_reader, - 'zTXTGPlm': ztxt_reader, - 'TEXtREAd': palmdoc_reader, - '.pdfADBE': pdf_reader, - 'DataPlkr': plucker_reader, -} + FORMAT_READERS = { + 'PNPdPPrs': ereader_reader, + 'PNRdPPrs': ereader_reader, + 'zTXTGPlm': ztxt_reader, + 'TEXtREAd': palmdoc_reader, + '.pdfADBE': pdf_reader, + 'DataPlkr': plucker_reader, + } -from calibre.ebooks.pdb.palmdoc.writer import Writer as palmdoc_writer -from calibre.ebooks.pdb.ztxt.writer import Writer as ztxt_writer -from calibre.ebooks.pdb.ereader.writer import Writer as ereader_writer +ALL_FORMAT_WRITERS = {'doc', 'ztxt', 'ereader'} +FORMAT_WRITERS = None +def _import_writers(): + global FORMAT_WRITERS + from calibre.ebooks.pdb.palmdoc.writer import Writer as palmdoc_writer + from calibre.ebooks.pdb.ztxt.writer import Writer as ztxt_writer + from calibre.ebooks.pdb.ereader.writer import Writer as ereader_writer -FORMAT_WRITERS = { - 'doc': palmdoc_writer, - 'ztxt': ztxt_writer, - 'ereader': ereader_writer, -} + FORMAT_WRITERS = { + 'doc': palmdoc_writer, + 'ztxt': ztxt_writer, + 'ereader': ereader_writer, + } IDENTITY_TO_NAME = { 'PNPdPPrs': 'eReader', @@ -69,11 +76,17 @@ def get_reader(identity): ''' Returns None if no reader is found for the identity. ''' + global FORMAT_READERS + if FORMAT_READERS is None: + _import_readers() return FORMAT_READERS.get(identity, None) def get_writer(extension): ''' Returns None if no writer is found for extension. ''' + global FORMAT_WRITERS + if FORMAT_WRITERS is None: + _import_writers() return FORMAT_WRITERS.get(extension, None) diff --git a/src/calibre/ebooks/rb/reader.py b/src/calibre/ebooks/rb/reader.py index f97c3d78c5..e68cef41d3 100644 --- a/src/calibre/ebooks/rb/reader.py +++ b/src/calibre/ebooks/rb/reader.py @@ -65,7 +65,7 @@ class Reader(object): name = urlunquote(self.stream.read(32).strip('\x00')) size, offset, flags = self.read_i32(), self.read_i32(), self.read_i32() toc.append(RBToc.Item(name=name, size=size, offset=offset, flags=flags)) - + return toc def get_text(self, toc_item, output_dir): @@ -89,7 +89,7 @@ class Reader(object): output += self.stream.read(toc_item.size).decode('cp1252' if self.encoding is None else self.encoding, 'replace') with open(os.path.join(output_dir, toc_item.name), 'wb') as html: - html.write(output.encode('utf-8')) + html.write(output.replace('', '<TITLE> ').encode('utf-8')) def get_image(self, toc_item, output_dir): if toc_item.flags != 0: @@ -105,7 +105,7 @@ class Reader(object): self.log.debug('Extracting content from file...') html = [] images = [] - + for item in self.toc: if item.name.lower().endswith('html'): self.log.debug('HTML item %s found...' % item.name) diff --git a/src/calibre/ebooks/rtf/input.py b/src/calibre/ebooks/rtf/input.py index 5858824434..8e1a5ac775 100644 --- a/src/calibre/ebooks/rtf/input.py +++ b/src/calibre/ebooks/rtf/input.py @@ -2,42 +2,9 @@ from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' -import os, glob, re, textwrap from lxml import etree -from calibre.customize.conversion import InputFormatPlugin - -border_style_map = { - 'single' : 'solid', - 'double-thickness-border' : 'double', - 'shadowed-border': 'outset', - 'double-border': 'double', - 'dotted-border': 'dotted', - 'dashed': 'dashed', - 'hairline': 'solid', - 'inset': 'inset', - 'dash-small': 'dashed', - 'dot-dash': 'dotted', - 'dot-dot-dash': 'dotted', - 'outset': 'outset', - 'tripple': 'double', - 'triple': 'double', - 'thick-thin-small': 'solid', - 'thin-thick-small': 'solid', - 'thin-thick-thin-small': 'solid', - 'thick-thin-medium': 'solid', - 'thin-thick-medium': 'solid', - 'thin-thick-thin-medium': 'solid', - 'thick-thin-large': 'solid', - 'thin-thick-thin-large': 'solid', - 'wavy': 'ridge', - 'double-wavy': 'ridge', - 'striped': 'ridge', - 'emboss': 'inset', - 'engrave': 'inset', - 'frame': 'ridge', -} class InlineClass(etree.XSLTExtension): @@ -71,261 +38,3 @@ class InlineClass(etree.XSLTExtension): output_parent.text = ' '.join(classes) -class RTFInput(InputFormatPlugin): - - name = 'RTF Input' - author = 'Kovid Goyal' - description = 'Convert RTF files to HTML' - file_types = set(['rtf']) - - def generate_xml(self, stream): - from calibre.ebooks.rtf2xml.ParseRtf import ParseRtf - ofile = 'dataxml.xml' - run_lev, debug_dir, indent_out = 1, None, 0 - if getattr(self.opts, 'debug_pipeline', None) is not None: - try: - os.mkdir('rtfdebug') - debug_dir = 'rtfdebug' - run_lev = 4 - indent_out = 1 - self.log('Running RTFParser in debug mode') - except: - self.log.warn('Impossible to run RTFParser in debug mode') - parser = ParseRtf( - in_file = stream, - out_file = ofile, - # Convert symbol fonts to unicode equivalents. Default - # is 1 - convert_symbol = 1, - - # Convert Zapf fonts to unicode equivalents. Default - # is 1. - convert_zapf = 1, - - # Convert Wingding fonts to unicode equivalents. - # Default is 1. - convert_wingdings = 1, - - # Convert RTF caps to real caps. - # Default is 1. - convert_caps = 1, - - # Indent resulting XML. - # Default is 0 (no indent). - indent = indent_out, - - # Form lists from RTF. Default is 1. - form_lists = 1, - - # Convert headings to sections. Default is 0. - headings_to_sections = 1, - - # Group paragraphs with the same style name. Default is 1. - group_styles = 1, - - # Group borders. Default is 1. - group_borders = 1, - - # Write or do not write paragraphs. Default is 0. - empty_paragraphs = 1, - - #debug - deb_dir = debug_dir, - run_level = run_lev, - ) - parser.parse_rtf() - with open(ofile, 'rb') as f: - return f.read() - - def extract_images(self, picts): - import imghdr - self.log('Extracting images...') - - with open(picts, 'rb') as f: - raw = f.read() - picts = filter(len, re.findall(r'\{\\pict([^}]+)\}', raw)) - hex = re.compile(r'[^a-fA-F0-9]') - encs = [hex.sub('', pict) for pict in picts] - - count = 0 - imap = {} - for enc in encs: - if len(enc) % 2 == 1: - enc = enc[:-1] - data = enc.decode('hex') - fmt = imghdr.what(None, data) - if fmt is None: - fmt = 'wmf' - count += 1 - name = '%04d.%s' % (count, fmt) - with open(name, 'wb') as f: - f.write(data) - imap[count] = name - # with open(name+'.hex', 'wb') as f: - # f.write(enc) - return self.convert_images(imap) - - def convert_images(self, imap): - self.default_img = None - for count, val in imap.iteritems(): - try: - imap[count] = self.convert_image(val) - except: - self.log.exception('Failed to convert', val) - return imap - - def convert_image(self, name): - if not name.endswith('.wmf'): - return name - try: - return self.rasterize_wmf(name) - except: - self.log.exception('Failed to convert WMF image %r'%name) - return self.replace_wmf(name) - - def replace_wmf(self, name): - from calibre.ebooks import calibre_cover - if self.default_img is None: - self.default_img = calibre_cover('Conversion of WMF images is not supported', - 'Use Microsoft Word or OpenOffice to save this RTF file' - ' as HTML and convert that in calibre.', title_size=36, - author_size=20) - name = name.replace('.wmf', '.jpg') - with open(name, 'wb') as f: - f.write(self.default_img) - return name - - def rasterize_wmf(self, name): - from calibre.utils.wmf.parse import wmf_unwrap - with open(name, 'rb') as f: - data = f.read() - data = wmf_unwrap(data) - name = name.replace('.wmf', '.png') - with open(name, 'wb') as f: - f.write(data) - return name - - - def write_inline_css(self, ic, border_styles): - font_size_classes = ['span.fs%d { font-size: %spt }'%(i, x) for i, x in - enumerate(ic.font_sizes)] - color_classes = ['span.col%d { color: %s }'%(i, x) for i, x in - enumerate(ic.colors)] - css = textwrap.dedent(''' - span.none { - text-decoration: none; font-weight: normal; - font-style: normal; font-variant: normal - } - - span.italics { font-style: italic } - - span.bold { font-weight: bold } - - span.small-caps { font-variant: small-caps } - - span.underlined { text-decoration: underline } - - span.strike-through { text-decoration: line-through } - - ''') - css += '\n'+'\n'.join(font_size_classes) - css += '\n' +'\n'.join(color_classes) - - for cls, val in border_styles.iteritems(): - css += '\n\n.%s {\n%s\n}'%(cls, val) - - with open('styles.css', 'ab') as f: - f.write(css) - - def convert_borders(self, doc): - border_styles = [] - style_map = {} - for elem in doc.xpath(r'//*[local-name()="cell"]'): - style = ['border-style: hidden', 'border-width: 1px', - 'border-color: black'] - for x in ('bottom', 'top', 'left', 'right'): - bs = elem.get('border-cell-%s-style'%x, None) - if bs: - cbs = border_style_map.get(bs, 'solid') - style.append('border-%s-style: %s'%(x, cbs)) - bw = elem.get('border-cell-%s-line-width'%x, None) - if bw: - style.append('border-%s-width: %spt'%(x, bw)) - bc = elem.get('border-cell-%s-color'%x, None) - if bc: - style.append('border-%s-color: %s'%(x, bc)) - style = ';\n'.join(style) - if style not in border_styles: - border_styles.append(style) - idx = border_styles.index(style) - cls = 'border_style%d'%idx - style_map[cls] = style - elem.set('class', cls) - return style_map - - def convert(self, stream, options, file_ext, log, - accelerators): - from calibre.ebooks.metadata.meta import get_metadata - from calibre.ebooks.metadata.opf2 import OPFCreator - from calibre.ebooks.rtf2xml.ParseRtf import RtfInvalidCodeException - self.opts = options - self.log = log - self.log('Converting RTF to XML...') - try: - xml = self.generate_xml(stream.name) - except RtfInvalidCodeException as e: - raise ValueError(_('This RTF file has a feature calibre does not ' - 'support. Convert it to HTML first and then try it.\n%s')%e) - - d = glob.glob(os.path.join('*_rtf_pict_dir', 'picts.rtf')) - if d: - imap = {} - try: - imap = self.extract_images(d[0]) - except: - self.log.exception('Failed to extract images...') - - self.log('Parsing XML...') - parser = etree.XMLParser(recover=True, no_network=True) - doc = etree.fromstring(xml, parser=parser) - border_styles = self.convert_borders(doc) - for pict in doc.xpath('//rtf:pict[@num]', - namespaces={'rtf':'http://rtf2xml.sourceforge.net/'}): - num = int(pict.get('num')) - name = imap.get(num, None) - if name is not None: - pict.set('num', name) - - self.log('Converting XML to HTML...') - inline_class = InlineClass(self.log) - styledoc = etree.fromstring(P('templates/rtf.xsl', data=True)) - extensions = { ('calibre', 'inline-class') : inline_class } - transform = etree.XSLT(styledoc, extensions=extensions) - result = transform(doc) - html = 'index.xhtml' - with open(html, 'wb') as f: - res = transform.tostring(result) - # res = res[:100].replace('xmlns:html', 'xmlns') + res[100:] - #clean multiple \n - res = re.sub('\n+', '\n', res) - # Replace newlines inserted by the 'empty_paragraphs' option in rtf2xml with html blank lines - # res = re.sub('\s*<body>', '<body>', res) - # res = re.sub('(?<=\n)\n{2}', - # u'<p>\u00a0</p>\n'.encode('utf-8'), res) - f.write(res) - self.write_inline_css(inline_class, border_styles) - stream.seek(0) - mi = get_metadata(stream, 'rtf') - if not mi.title: - mi.title = _('Unknown') - if not mi.authors: - mi.authors = [_('Unknown')] - opf = OPFCreator(os.getcwd(), mi) - opf.create_manifest([('index.xhtml', None)]) - opf.create_spine(['index.xhtml']) - opf.render(open('metadata.opf', 'wb')) - return os.path.abspath('metadata.opf') - -#ebook-convert "bad.rtf" test.epub -v -d "E:\Mes eBooks\Developpement\debug" -# os.makedirs("E:\\Mes eBooks\\Developpement\\rtfdebug") -# debug_dir = "E:\\Mes eBooks\\Developpement\\rtfdebug" diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index 4cff648fa5..0880eca4ca 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -16,7 +16,7 @@ from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.ebooks.conversion.preprocess import DocAnalysis from calibre.utils.cleantext import clean_ascii_chars -HTML_TEMPLATE = u'<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/><title>%s\n%s\n' +HTML_TEMPLATE = u'%s \n%s\n' def clean_txt(txt): ''' @@ -28,7 +28,7 @@ def clean_txt(txt): # Strip whitespace from the end of the line. Also replace # all line breaks with \n. txt = '\n'.join([line.rstrip() for line in txt.splitlines()]) - + # Replace whitespace at the beginning of the line with   txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', ' ' * 4, txt) @@ -75,7 +75,7 @@ def convert_basic(txt, title='', epub_split_size_kb=0): ''' Converts plain text to html by putting all paragraphs in

tags. It condense and retains blank lines when necessary. - + Requires paragraphs to be in single line format. ''' txt = clean_txt(txt) @@ -215,7 +215,7 @@ def detect_paragraph_type(txt): def detect_formatting_type(txt): ''' Tries to determine the formatting of the document. - + markdown: Markdown formatting is used. textile: Textile formatting is used. heuristic: When none of the above formatting types are diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 2b99bcb2c2..b3e128af82 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -101,6 +101,8 @@ gprefs.defaults['preserve_date_on_ctl'] = True gprefs.defaults['cb_fullscreen'] = False gprefs.defaults['worker_max_time'] = 0 gprefs.defaults['show_files_after_save'] = True +gprefs.defaults['auto_add_path'] = None +gprefs.defaults['auto_add_check_for_duplicates'] = False # }}} NONE = QVariant() #: Null value to return from the data function of item models @@ -257,7 +259,8 @@ def extension(path): def warning_dialog(parent, title, msg, det_msg='', show=False, show_copy_button=True): from calibre.gui2.dialogs.message_box import MessageBox - d = MessageBox(MessageBox.WARNING, 'WARNING: '+title, msg, det_msg, parent=parent, + d = MessageBox(MessageBox.WARNING, _('WARNING:')+ ' ' + + title, msg, det_msg, parent=parent, show_copy_button=show_copy_button) if show: return d.exec_() @@ -266,7 +269,8 @@ def warning_dialog(parent, title, msg, det_msg='', show=False, def error_dialog(parent, title, msg, det_msg='', show=False, show_copy_button=True): from calibre.gui2.dialogs.message_box import MessageBox - d = MessageBox(MessageBox.ERROR, 'ERROR: '+title, msg, det_msg, parent=parent, + d = MessageBox(MessageBox.ERROR, _('ERROR:')+ ' ' + + title, msg, det_msg, parent=parent, show_copy_button=show_copy_button) if show: return d.exec_() diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 03a2e44c6e..bb695db841 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -37,6 +37,7 @@ def get_filters(): (_('SNB Books'), ['snb']), (_('Comics'), ['cbz', 'cbr', 'cbc']), (_('Archives'), ['zip', 'rar']), + (_('Wordprocessor files'), ['odt', 'doc', 'docx']), ] diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 7cdac3b845..972ea57cb9 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -382,7 +382,8 @@ class Adder(QObject): # {{{ if not duplicates: return self.duplicates_processed() self.pd.hide() - files = [x[0].title for x in duplicates] + files = [_('%s by %s')%(x[0].title, x[0].format_field('authors')[1]) + for x in duplicates] if question_dialog(self._parent, _('Duplicates found!'), _('Books with the same title as the following already ' 'exist in the database. Add them anyway?'), diff --git a/src/calibre/gui2/auto_add.py b/src/calibre/gui2/auto_add.py new file mode 100644 index 0000000000..6860f386d6 --- /dev/null +++ b/src/calibre/gui2/auto_add.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, tempfile, shutil, time +from threading import Thread, Event + +from PyQt4.Qt import (QFileSystemWatcher, QObject, Qt, pyqtSignal, QTimer) + +from calibre import prints +from calibre.ptempfile import PersistentTemporaryDirectory +from calibre.ebooks import BOOK_EXTENSIONS +from calibre.gui2 import question_dialog, gprefs + +class Worker(Thread): + + def __init__(self, path, callback): + Thread.__init__(self) + self.daemon = True + self.keep_running = True + self.wake_up = Event() + self.path, self.callback = path, callback + self.staging = set() + self.be = frozenset(BOOK_EXTENSIONS) + + def run(self): + self.tdir = PersistentTemporaryDirectory('_auto_adder') + while self.keep_running: + self.wake_up.wait() + self.wake_up.clear() + if not self.keep_running: + break + try: + self.auto_add() + except: + import traceback + traceback.print_exc() + + def auto_add(self): + from calibre.utils.ipc.simple_worker import fork_job, WorkerError + from calibre.ebooks.metadata.opf2 import metadata_to_opf + from calibre.ebooks.metadata.meta import metadata_from_filename + + files = [x for x in os.listdir(self.path) if + # Must not be in the process of being added to the db + x not in self.staging + # Firefox creates 0 byte placeholder files when downloading + and os.stat(os.path.join(self.path, x)).st_size > 0 + # Must be a file + and os.path.isfile(os.path.join(self.path, x)) + # Must have read and write permissions + and os.access(os.path.join(self.path, x), os.R_OK|os.W_OK) + # Must be a known ebook file type + and os.path.splitext(x)[1][1:].lower() in self.be + ] + data = {} + # Give any in progress copies time to complete + time.sleep(2) + + for fname in files: + f = os.path.join(self.path, fname) + + # Try opening the file for reading, if the OS prevents us, then at + # least on windows, it means the file is open in another + # application for writing. We will get notified by + # QFileSystemWatcher when writing is completed, so ignore for now. + try: + open(f, 'rb').close() + except: + continue + tdir = tempfile.mkdtemp(dir=self.tdir) + try: + fork_job('calibre.ebooks.metadata.meta', + 'forked_read_metadata', (f, tdir), no_output=True) + except WorkerError as e: + prints('Failed to read metadata from:', fname) + prints(e.orig_tb) + except: + import traceback + traceback.print_exc() + + # Ensure that the pre-metadata file size is present. If it isn't, + # write 0 so that the file is rescanned + szpath = os.path.join(tdir, 'size.txt') + try: + with open(szpath, 'rb') as f: + int(f.read()) + except: + with open(szpath, 'wb') as f: + f.write(b'0') + + opfpath = os.path.join(tdir, 'metadata.opf') + try: + if os.stat(opfpath).st_size < 30: + raise Exception('metadata reading failed') + except: + mi = metadata_from_filename(fname) + with open(opfpath, 'wb') as f: + f.write(metadata_to_opf(mi)) + self.staging.add(fname) + data[fname] = tdir + if data: + self.callback(data) + + +class AutoAdder(QObject): + + metadata_read = pyqtSignal(object) + + def __init__(self, path, parent): + QObject.__init__(self, parent) + if path and os.path.isdir(path) and os.access(path, os.R_OK|os.W_OK): + self.watcher = QFileSystemWatcher(self) + self.worker = Worker(path, self.metadata_read.emit) + self.watcher.directoryChanged.connect(self.dir_changed, + type=Qt.QueuedConnection) + self.metadata_read.connect(self.add_to_db, + type=Qt.QueuedConnection) + QTimer.singleShot(2000, self.initialize) + elif path: + prints(path, + 'is not a valid directory to watch for new ebooks, ignoring') + + def initialize(self): + try: + if os.listdir(self.worker.path): + self.dir_changed() + except: + pass + self.watcher.addPath(self.worker.path) + + def dir_changed(self, *args): + if os.path.isdir(self.worker.path) and os.access(self.worker.path, + os.R_OK|os.W_OK): + if not self.worker.is_alive(): + self.worker.start() + self.worker.wake_up.set() + + def stop(self): + if hasattr(self, 'worker'): + self.worker.keep_running = False + self.worker.wake_up.set() + + def wait(self): + if hasattr(self, 'worker'): + self.worker.join() + + def add_to_db(self, data): + from calibre.ebooks.metadata.opf2 import OPF + + gui = self.parent() + if gui is None: + return + m = gui.library_view.model() + count = 0 + + needs_rescan = False + duplicates = [] + + for fname, tdir in data.iteritems(): + paths = [os.path.join(self.worker.path, fname)] + sz = os.path.join(tdir, 'size.txt') + try: + with open(sz, 'rb') as f: + sz = int(f.read()) + if sz != os.stat(paths[0]).st_size: + raise Exception('Looks like the file was written to after' + ' we tried to read metadata') + except: + needs_rescan = True + try: + self.worker.staging.remove(fname) + except KeyError: + pass + + continue + + mi = os.path.join(tdir, 'metadata.opf') + if not os.access(mi, os.R_OK): + continue + mi = [OPF(open(mi, 'rb'), tdir, + populate_spine=False).to_book_metadata()] + dups, num = m.add_books(paths, + [os.path.splitext(fname)[1][1:].upper()], mi, + add_duplicates=not gprefs['auto_add_check_for_duplicates']) + if dups: + path = dups[0][0] + with open(os.path.join(tdir, 'dup_cache.'+dups[1][0].lower()), + 'wb') as dest, open(path, 'rb') as src: + shutil.copyfileobj(src, dest) + dups[0][0] = dest.name + duplicates.append(dups) + + try: + os.remove(paths[0]) + self.worker.staging.remove(fname) + except: + pass + count += num + + if duplicates: + paths, formats, metadata = [], [], [] + for p, f, mis in duplicates: + paths.extend(p) + formats.extend(f) + metadata.extend(mis) + files = [_('%s by %s')%(mi.title, mi.format_field('authors')[1]) + for mi in metadata] + if question_dialog(self.parent(), _('Duplicates found!'), + _('Books with the same title as the following already ' + 'exist in the database. Add them anyway?'), + '\n'.join(files)): + dups, num = m.add_books(paths, formats, metadata, + add_duplicates=True) + count += num + + for tdir in data.itervalues(): + try: + shutil.rmtree(tdir) + except: + pass + + if count > 0: + m.books_added(count) + gui.status_bar.show_message(_( + 'Added %d book(s) automatically from %s') % + (count, self.worker.path), 2000) + if hasattr(gui, 'db_images'): + gui.db_images.reset() + + if needs_rescan: + QTimer.singleShot(2000, self.dir_changed) + + diff --git a/src/calibre/gui2/bars.py b/src/calibre/gui2/bars.py index 64d5160045..50b3f3e7f5 100644 --- a/src/calibre/gui2/bars.py +++ b/src/calibre/gui2/bars.py @@ -199,8 +199,9 @@ class MenuBar(QMenuBar): # {{{ def update_lm_actions(self): for ac in self.added_actions: - if ac in self.location_manager.all_actions: - ac.setVisible(ac in self.location_manager.available_actions) + clone = getattr(ac, 'clone', None) + if clone is not None and clone in self.location_manager.all_actions: + ac.setVisible(clone in self.location_manager.available_actions) def init_bar(self, actions): for ac in self.added_actions: diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 6ad1aaf0c4..628f846aea 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -38,14 +38,24 @@ def render_html(mi, css, vertical, widget, all_fields=False): # {{{ ans = unicode(col.name()) return ans - f = QFontInfo(QApplication.font(widget)).pixelSize() + fi = QFontInfo(QApplication.font(widget)) + f = fi.pixelSize()+1 + fam = unicode(fi.family()).strip().replace('"', '') + if not fam: + fam = 'sans-serif' + c = color_to_string(QApplication.palette().color(QPalette.Normal, QPalette.WindowText)) templ = u'''\