diff --git a/format_docs/pdb/mobi.txt b/format_docs/pdb/mobi.txt index e378e1622b..bf08bd8b71 100644 --- a/format_docs/pdb/mobi.txt +++ b/format_docs/pdb/mobi.txt @@ -210,21 +210,23 @@ record type usual length name comments 114 versionnumber 115 sample 116 startreading -118 retail price (as text) -119 retail price currency (as text) -201 coveroffset -202 thumboffset +117 3 adult Mobipocket Creator adds this if Adult only is checked; contents: "yes" +118 retail price As text, e.g. "4.99" +119 retail price currency As text, e.g. "USD" +201 4 coveroffset Add to first image field in Mobi Header to find PDB record containing the cover image +202 4 thumboffset Add to first image field in Mobi Header to find PDB record containing the thumbnail cover image 203 hasfakecover -204 204 Unknown -205 205 Unknown -206 206 Unknown -207 207 Unknown -208 208 Unknown -300 300 Unknown -401 clippinglimit +204 4 Creator Software Records 204-207 are usually the same for all books from a certain source, e.g. 1-6-2-41 for Baen and 201-1-0-85 for project gutenberg, 200-1-0-85 for amazon when converted to a 32 bit integer. +205 4 Creator Major Version +206 4 Creator Minor Version +207 4 Creator Build Number +208 watermark +209 tamper proof keys Used by the Kindle (and Android app) for generating book-specific PIDs. +300 fontsignature +401 1 clippinglimit 402 publisherlimit -403 403 Unknown -404 404 ttsflag +403 403 Unknown 1 - Text to Speech disabled; 0 - Text to Speech enabled +404 1 404 ttsflag 501 4 cdetype PDOC - Personal Doc; EBOK - ebook; 502 lastupdatetime @@ -287,9 +289,9 @@ content at the beginning of the following record. The trailing entry ends with a byte containing a count of the overlapping bytes plus additional flags. offset bytes content comments -0 0-3 N terminal bytes +0 0-3 N terminal bytes of a multibyte - character + character N 1 Size & flags bits 1-2 encode N, use of bits 3-8 is unknown @@ -328,6 +330,102 @@ programs may ignore them entirely. They are stored at the end of the file itself so the full file needs to be scanned when loaded to find them. +Image Records +------------- + +If the file contains images, they follow the text blocks, with each image using a +single block. The 4096-byte record size in the PalmDoc header applies only to +text records; image records may be larger. + + +Magic Records +------------- + +In some cases, MobiPocket Creator adds a 2-zero-byte record after the text +records in a file. This record is not included in the "record count" of text +records in the PalmDoc header, and is also not used as the "first non-book +index" in the MOBI header. (If the 2-zero-byte record is present, the index of +the following block is used as the "first non-book index".) + +MobiPocket Creator also ends files with three records: 'FLIS', 'FCIS', and +'end-of-file', in that order. The 'FLIS' and 'FCIS' records do not seem to be +necessary for MobiPocket Reader or the Amazon Kindle 2 to read the file. The +'end-of-file' record might be necessary. + + +FLIS Record +----------- + +The FLIS record appears to have a fixed value. The meaning of the values is not known. + +offset bytes content comments +0 4 identifier the characters F L I S (0x46 0x4c 0x49 0x53) +4 4 ? fixed value: 8 +8 2 ? fixed value: 65 +10 2 ? fixed value: 0 +12 4 ? fixed value: 0 +16 4 ? fixed value: -1 +20 2 ? fixed value: 1 +22 2 ? fixed value: 3 +24 4 ? fixed value: 3 +28 4 ? fixed value: 1 +32 4 ? fixed value: -1 + + +FCIS Record +----------- + +The FCIS record appears to have mostly fixed values. + +offset bytes content comments +0 4 identifier the characters F C I S (0x46 0x43 0x49 0x53) +4 4 ? fixed value: 20 +8 4 ? fixed value: 16 +12 4 ? fixed value: 1 +16 4 ? fixed value: 0 +20 4 ? text length (the same value as "text length" in the PalmDoc header) +24 4 ? fixed value: 0 +28 4 ? fixed value: 32 +32 4 ? fixed value: 8 +36 2 ? fixed value: 1 +38 2 ? fixed value: 1 +40 4 ? fixed value: 0 + + +End-of-file Record +------------------ + +The end-of-file record is a fixed 4-byte record. While the last two bytes +appear to be a CRLF marker, the meaning of the first two bytes is unknown. + +offset bytes content comments +0 1 ? fixed value: 233 (0xe9) +1 1 ? fixed value: 142 (0x8e) +2 1 ? fixed value: 13 (0x0d) +3 1 ? fixed value: 10 (0x0a) + + +SRCS Record +----------- + +kindlegen creates a record whose content is a zip archive of all source files +(i.e., .opf, .ncx, .htm, .jpg, ...) given to the command and puts it in the +generated MOBI file. The record begins with the "SRCS" signature and is +located just before the #End-of-file Record. + +MOBI files created with Mobipocket creator, Amazon's Personal Document Service, +or Kindle Direct Publishing (former Amazon DTP) don't include SRCS record. +In a past, kindlegen had an undocumented option to suppress this record, but +the option was removed in 2010. + +offset bytes content comments +0 4 identifier "SRCS" (0x53 0x52 0x43 0x53) +4 4 ? fixed value(?): 0x00000010 +8 4 ? fixed value(?): 0x0000002f +12 4 ? fixed value(?): 0x00000001 +16 zip The zip archive continues to the end of this record + + MBP --- diff --git a/recipes/cracked_com.recipe b/recipes/cracked_com.recipe new file mode 100644 index 0000000000..49ed9d2279 --- /dev/null +++ b/recipes/cracked_com.recipe @@ -0,0 +1,83 @@ +from calibre.web.feeds.news import BasicNewsRecipe +import re + +class Cracked(BasicNewsRecipe): + title = u'Cracked.com' + __author__ = u'Nudgenudge' + language = 'en' + description = 'America''s Only Humor and Video Site, since 1958' + publisher = 'Cracked' + category = 'comedy, lists' + oldest_article = 2 + delay = 10 + max_articles_per_feed = 2 + no_stylesheets = True + encoding = 'cp1252' + remove_javascript = True + use_embedded_content = False + INDEX = u'http://www.cracked.com' + extra_css = """ + .pageheader_type{font-size: x-large; font-weight: bold; color: #828D74} + .pageheader_title{font-size: xx-large; color: #394128} + .pageheader_byline{font-size: small; font-weight: bold; color: #394128} + .score_bg {display: inline; width: 100%; margin-bottom: 2em} + .score_column_1{ padding-left: 10px; font-size: small; width: 50%} + .score_column_2{ padding-left: 10px; font-size: small; width: 50%} + .score_column_3{ padding-left: 10px; font-size: small; width: 50%} + .score_header{font-size: large; color: #50544A} + .bodytext{display: block} + body{font-family: Helvetica,Arial,sans-serif} + """ + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'linearize_tables' : True + } + + keep_only_tags = [ + dict(name='div', attrs={'class':['Column1']}) + ] + + feeds = [(u'Articles', u'http://feeds.feedburner.com/CrackedRSS')] + + def get_article_url(self, article): + return article.get('guid', None) + + def cleanup_page(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for alink in soup.findAll('a'): + if alink.string is not None: + tstr = alink.string + alink.replaceWith(tstr) + for div_to_remove in soup.findAll('div', attrs={'id':['googlead_1','fb-like-article','comments_section']}): + div_to_remove.extract() + for div_to_remove in soup.findAll('div', attrs={'class':['share_buttons_col_1','GenericModule1']}): + div_to_remove.extract() + for div_to_remove in soup.findAll('div', attrs={'class':re.compile("prev_next")}): + div_to_remove.extract() + for ul_to_remove in soup.findAll('ul', attrs={'class':['Nav6']}): + ul_to_remove.extract() + for image in soup.findAll('img', attrs={'alt': 'article image'}): + image.extract() + + def append_page(self, soup, appendtag, position): + pager = soup.find('a',attrs={'class':'next_arrow_active'}) + if pager: + nexturl = self.INDEX + pager['href'] + soup2 = self.index_to_soup(nexturl) + texttag = soup2.find('div', attrs={'class':re.compile("userStyled")}) + newpos = len(texttag.contents) + self.append_page(soup2,texttag,newpos) + texttag.extract() + self.cleanup_page(appendtag) + appendtag.insert(position,texttag) + else: + self.cleanup_page(appendtag) + + def preprocess_html(self, soup): + self.append_page(soup, soup.body, 3) + return self.adeify_images(soup) diff --git a/recipes/estadao.recipe b/recipes/estadao.recipe index 4e520c1135..86ab572398 100644 --- a/recipes/estadao.recipe +++ b/recipes/estadao.recipe @@ -1,63 +1,134 @@ #!/usr/bin/env python - -__license__ = 'GPL v3' -__copyright__ = '2010, elsuave' -''' -estadao.com.br -''' - from calibre.web.feeds.news import BasicNewsRecipe +from datetime import datetime, timedelta +from calibre.ebooks.BeautifulSoup import Tag,BeautifulSoup +from calibre.utils.magick import Image, PixelWand +from urllib2 import Request, urlopen, URLError class Estadao(BasicNewsRecipe): - title = 'O Estado de S. Paulo' - __author__ = 'elsuave (modified from Darko Miletic)' - description = 'News from Brasil in Portuguese' - publisher = 'O Estado de S. Paulo' - category = 'news, politics, Brasil' - oldest_article = 2 - max_articles_per_feed = 25 + THUMBALIZR_API = "0123456789abcdef01234567890" # ---->Get your at http://www.thumbalizr.com/ + LANGUAGE = 'pt_br' + language = 'pt' + LANGHTM = 'pt-br' + ENCODING = 'utf' + ENCHTM = 'utf-8' + directionhtm = 'ltr' + requires_version = (0,8,47) + news = True + publication_type = 'newsportal' + + title = u'Estadao' + __author__ = 'Euler Alves' + description = u'Brazilian news from Estad\xe3o' + publisher = u'Estad\xe3o' + category = 'news, rss' + + oldest_article = 4 + max_articles_per_feed = 100 + summary_length = 1000 + + remove_javascript = True no_stylesheets = True use_embedded_content = False - encoding = 'utf8' - cover_url = 'http://www.estadao.com.br/img/logo_estadao.png' - remove_javascript = True + remove_empty_feeds = True + timefmt = ' [%d %b %Y (%a)]' - html2lrf_options = [ - '--comment', description - , '--category', category - , '--publisher', publisher - ] + html2lrf_options = [ + '--comment', description + ,'--category', category + ,'--publisher', publisher + ] - html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' + html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' - keep_only_tags = [ - dict(name='div', attrs={'class':['bb-md-noticia','c5']}) - ] + hoje = datetime.now()-timedelta(days=2) + pubdate = hoje.strftime('%a, %d %b') + if hoje.hour<10: + hoje = hoje-timedelta(days=1) + CAPA = 'http://www.estadao.com.br/estadaodehoje/'+hoje.strftime('%Y%m%d')+'/img/capadodia.jpg' + SCREENSHOT = 'http://estadao.com.br/' + cover_margins = (0,0,'white') + masthead_url = 'http://www.estadao.com.br/estadao/novo/img/logo.png' + keep_only_tags = [dict(name='div', attrs={'class':['bb-md-noticia','corpo']})] remove_tags = [ - dict(name=['script','object','form','ul']) - ,dict(name='div', attrs={'class':['fnt2 Color_04 bold','right fnt2 innerTop15 dvTmFont','™_01 right outerLeft15','tituloBox','tags']}) - ,dict(name='div', attrs={'id':['bb-md-noticia-subcom']}) - ] + dict(name='div', + attrs={'id':[ + 'bb-md-noticia-tabs' + ]}) + ,dict(name='div', + attrs={'class':[ + 'tags' + ,'discussion' + ,'bb-gg adsense_container' + ]}) + + ,dict(name='a') + ,dict(name='iframe') + ,dict(name='link') + ,dict(name='script') + ] feeds = [ - (u'Manchetes Estadao', u'http://www.estadao.com.br/rss/manchetes.xml') - ,(u'Ultimas noticias', u'http://www.estadao.com.br/rss/ultimas.xml') - ,(u'Nacional', u'http://www.estadao.com.br/rss/nacional.xml') - ,(u'Internacional', u'http://www.estadao.com.br/rss/internacional.xml') - ,(u'Cidades', u'http://www.estadao.com.br/rss/cidades.xml') - ,(u'Esportes', u'http://www.estadao.com.br/rss/esportes.xml') - ,(u'Arte & Lazer', u'http://www.estadao.com.br/rss/arteelazer.xml') - ,(u'Economia', u'http://www.estadao.com.br/rss/economia.xml') - ,(u'Vida &', u'http://www.estadao.com.br/rss/vidae.xml') - ] + (u'\xDAltimas Not\xEDcias', u'http://www.estadao.com.br/rss/ultimas.xml') + ,(u'Manchetes', u'http://www.estadao.com.br/rss/manchetes.xml') + ,(u'Brasil', u'http://www.estadao.com.br/rss/brasil.xml') + ,(u'Internacional', u'http://www.estadao.com.br/rss/internacional.xml') + ,(u'Cinema', u'http://blogs.estadao.com.br/cinema/feed/') + ,(u'Planeta', u'http://www.estadao.com.br/rss/planeta.xml') + ,(u'Ci\xEAncia', u'http://www.estadao.com.br/rss/ciencia.xml') + ,(u'Sa\xFAde', u'http://www.estadao.com.br/rss/saude.xml') + ,(u'Pol\xEDtica', u'http://www.estadao.com.br/rss/politica.xml') + ] + conversion_options = { + 'title' : title + ,'comments' : description + ,'publisher' : publisher + ,'tags' : category + ,'language' : LANGUAGE + ,'linearize_tables': True + } + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + if not soup.find(attrs={'http-equiv':'Content-Language'}): + meta0 = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.LANGHTM)]) + soup.head.insert(0,meta0) + if not soup.find(attrs={'http-equiv':'Content-Type'}): + meta1 = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset="+self.ENCHTM)]) + soup.head.insert(0,meta1) + return soup - language = 'pt' - - def get_article_url(self, article): - url = BasicNewsRecipe.get_article_url(self, article) - if '/Multimidia/' not in url: - return url + def postprocess_html(self, soup, first): + #process all the images. assumes that the new html has the correct path + for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')): + iurl = tag['src'] + img = Image() + img.open(iurl) + width, height = img.size + print 'img is: ', iurl, 'width is: ', width, 'height is: ', height + pw = PixelWand() + if( width > height and width > 590) : + print 'Rotate image' + img.rotate(pw, -90) + img.save(iurl) + return soup + def get_cover_url(self): + cover_url = self.CAPA + pedido = Request(self.CAPA) + pedido.add_header('User-agent','Mozilla/5.0 (Windows; U; Windows NT 5.1; '+self.LANGHTM+'; userid='+self.THUMBALIZR_API+') Calibre/0.8.47 (like Gecko)') + pedido.add_header('Accept-Charset',self.ENCHTM) + pedido.add_header('Referer',self.SCREENSHOT) + try: + resposta = urlopen(pedido) + soup = BeautifulSoup(resposta) + cover_item = soup.find('body') + if cover_item: + cover_url='http://api.thumbalizr.com/?api_key='+self.THUMBALIZR_API+'&url='+self.SCREENSHOT+'&width=600&quality=90' + return cover_url + except URLError: + cover_url='http://api.thumbalizr.com/?api_key='+self.THUMBALIZR_API+'&url='+self.SCREENSHOT+'&width=600&quality=90' + return cover_url diff --git a/recipes/folhadesaopaulo.recipe b/recipes/folhadesaopaulo.recipe index 262a265020..40898672e6 100644 --- a/recipes/folhadesaopaulo.recipe +++ b/recipes/folhadesaopaulo.recipe @@ -1,74 +1,149 @@ -#!/usr/bin/env python - -__license__ = 'GPL v3' -__copyright__ = '2010, Saverio Palmieri Neto ' -''' -folha.uol.com.br -''' - from calibre.web.feeds.news import BasicNewsRecipe +from datetime import datetime, timedelta +from calibre.ebooks.BeautifulSoup import Tag,BeautifulSoup +from calibre.utils.magick import Image, PixelWand +from urllib2 import Request, urlopen, URLError class FolhaOnline(BasicNewsRecipe): - title = 'Folha de Sao Paulo' - __author__ = 'Saverio Palmieri Neto' - description = 'Brazilian news from Folha de Sao Paulo Online' - publisher = 'Folha de Sao Paulo' - category = 'Brasil, news' - oldest_article = 2 - max_articles_per_feed = 1000 - summary_length = 2048 + THUMBALIZR_API = "0123456789abcdef01234567890" # ---->Get your at http://www.thumbalizr.com/ + LANGUAGE = 'pt_br' + language = 'pt' + LANGHTM = 'pt-br' + ENCODING = 'cp1252' + ENCHTM = 'iso-8859-1' + directionhtm = 'ltr' + requires_version = (0,8,47) + news = True + publication_type = 'newsportal' + + title = u'Folha de S\xE3o Paulo' + __author__ = 'Euler Alves' + description = u'Brazilian news from Folha de S\xE3o Paulo' + publisher = u'Folha de S\xE3o Paulo' + category = 'news, rss' + + oldest_article = 4 + max_articles_per_feed = 100 + summary_length = 1000 + + remove_javascript = True no_stylesheets = True use_embedded_content = False + remove_empty_feeds = True timefmt = ' [%d %b %Y (%a)]' - encoding = 'cp1252' - cover_url = 'http://lh5.ggpht.com/_hEb7sFmuBvk/TFoiKLRS5dI/AAAAAAAAADM/kcVKggZwKnw/capa_folha.jpg' - cover_margins = (5,5,'white') - remove_javascript = True - keep_only_tags = [dict(name='div', attrs={'id':'articleNew'})] + html2lrf_options = [ + '--comment', description + ,'--category', category + ,'--publisher', publisher + ] - remove_tags = [ - dict(name='script') - ,dict(name='div', - attrs={'id':[ - 'articleButton' - ,'bookmarklets' - ,'ad-180x150-1' - ,'contextualAdsArticle' - ,'articleEnd' - ,'articleComments' - ]}) - ,dict(name='div', - attrs={'class':[ - 'openBox adslibraryArticle' - ]}) - ,dict(name='a') - ,dict(name='iframe') - ,dict(name='link') - ] + html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' + hoje = datetime.now() + pubdate = hoje.strftime('%a, %d %b') + if hoje.hour<6: + hoje = hoje-timedelta(days=1) + CAPA = 'http://www1.folha.uol.com.br/fsp/images/cp'+hoje.strftime('%d%m%Y')+'.jpg' + SCREENSHOT = 'http://www1.folha.uol.com.br/' + cover_margins = (0,0,'white') + masthead_url = 'http://f.i.uol.com.br/fsp/furniture/images/lgo-fsp-430x50-ffffff.gif' + + keep_only_tags = [dict(name='div', attrs={'id':'articleNew'})] + remove_tags = [ + dict(name='div', + attrs={'id':[ + 'articleButton' + ,'bookmarklets' + ,'ad-180x150-1' + ,'contextualAdsArticle' + ,'articleEnd' + ,'articleComments' + ]}) + ,dict(name='div', + attrs={'class':[ + 'openBox adslibraryArticle' + ]}) + + ,dict(name='a') + ,dict(name='iframe') + ,dict(name='link') + ,dict(name='script') + ] feeds = [ - (u'Em cima da hora', u'http://feeds.folha.uol.com.br/emcimadahora/rss091.xml') - ,(u'Ambiente', u'http://feeds.folha.uol.com.br/ambiente/rss091.xml') - ,(u'Bichos', u'http://feeds.folha.uol.com.br/bichos/rss091.xml') - ,(u'Poder', u'http://feeds.folha.uol.com.br/poder/rss091.xml') - ,(u'Ciencia', u'http://feeds.folha.uol.com.br/ciencia/rss091.xml') - ,(u'Cotidiano', u'http://feeds.folha.uol.com.br/cotidiado/rss091.xml') - ,(u'Saber', u'http://feeds.folha.uol.com.br/saber/rss091.xml') - ,(u'Equilíbrio e Saúde', u'http://feeds.folha.uol.com.br/equilibrioesaude/rss091.xml') - ,(u'Esporte', u'http://feeds.folha.uol.com.br/esporte/rss091.xml') - ,(u'Ilustrada', u'http://feeds.folha.uol.com.br/ilustrada/rss091.xml') - ,(u'Ilustríssima', u'http://feeds.folha.uol.com.br/ilustrissima/rss091.xml') - ,(u'Mercado', u'http://feeds.folha.uol.com.br/mercado/rss091.xml') - ,(u'Mundo', u'http://feeds.folha.uol.com.br/mundo/rss091.xml') - ,(u'Tec', u'http://feeds.folha.uol.com.br/tec/rss091.xml') - ,(u'Turismo', u'http://feeds.folha.uol.com.br/turismo/rss091.xml') - ] + (u'Em cima da hora', u'http://feeds.folha.uol.com.br/emcimadahora/rss091.xml') + ,(u'Ambiente', u'http://feeds.folha.uol.com.br/ambiente/rss091.xml') + ,(u'Bichos', u'http://feeds.folha.uol.com.br/bichos/rss091.xml') + ,(u'Ci\xEAncia', u'http://feeds.folha.uol.com.br/ciencia/rss091.xml') + ,(u'Poder', u'http://feeds.folha.uol.com.br/poder/rss091.xml') + ,(u'Equil\xEDbrio e Sa\xFAde', u'http://feeds.folha.uol.com.br/equilibrioesaude/rss091.xml') + ,(u'Turismo', u'http://feeds.folha.uol.com.br/folha/turismo/rss091.xml') + ,(u'Mundo', u'http://feeds.folha.uol.com.br/mundo/rss091.xml') + ,(u'Pelo Mundo', u'http://feeds.folha.uol.com.br/pelomundo.folha.rssblog.uol.com.br/') + ,(u'Circuito integrado', u'http://feeds.folha.uol.com.br/circuitointegrado.folha.rssblog.uol.com.br/') + ,(u'Blog do Fred', u'http://feeds.folha.uol.com.br/blogdofred.folha.rssblog.uol.com.br/') + ,(u'Maria In\xEAs Dolci', u'http://feeds.folha.uol.com.br/mariainesdolci.folha.blog.uol.com.br/') + ,(u'Eduardo Ohata', u'http://feeds.folha.uol.com.br/folha/pensata/eduardoohata/rss091.xml') + ,(u'Kennedy Alencar', u'http://feeds.folha.uol.com.br/folha/pensata/kennedyalencar/rss091.xml') + ,(u'Eliane Catanh\xEAde', u'http://feeds.folha.uol.com.br/folha/pensata/elianecantanhede/rss091.xml') + ,(u'Fernado Canzian', u'http://feeds.folha.uol.com.br/folha/pensata/fernandocanzian/rss091.xml') + ,(u'Gilberto Dimenstein', u'http://feeds.folha.uol.com.br/folha/pensata/gilbertodimenstein/rss091.xml') + ,(u'H\xE9lio Schwartsman', u'http://feeds.folha.uol.com.br/folha/pensata/helioschwartsman/rss091.xml') + ,(u'Jo\xE3o Pereira Coutinho', u'http://http://feeds.folha.uol.com.br/folha/pensata/joaopereiracoutinho/rss091.xml') + ,(u'Luiz Caversan', u'http://http://feeds.folha.uol.com.br/folha/pensata/luizcaversan/rss091.xml') + ,(u'S\xE9rgio Malbergier', u'http://http://feeds.folha.uol.com.br/folha/pensata/sergiomalbergier/rss091.xml') + ,(u'Valdo Cruz', u'http://http://feeds.folha.uol.com.br/folha/pensata/valdocruz/rss091.xml') + ] + + conversion_options = { + 'title' : title + ,'comments' : description + ,'publisher' : publisher + ,'tags' : category + ,'language' : LANGUAGE + ,'linearize_tables': True + } def preprocess_html(self, soup): for item in soup.findAll(style=True): del item['style'] + if not soup.find(attrs={'http-equiv':'Content-Language'}): + meta0 = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.LANGHTM)]) + soup.head.insert(0,meta0) + if not soup.find(attrs={'http-equiv':'Content-Type'}): + meta1 = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset="+self.ENCHTM)]) + soup.head.insert(0,meta1) return soup - language = 'pt' + def postprocess_html(self, soup, first): + #process all the images. assumes that the new html has the correct path + for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')): + iurl = tag['src'] + img = Image() + img.open(iurl) + width, height = img.size + print 'img is: ', iurl, 'width is: ', width, 'height is: ', height + pw = PixelWand() + if( width > height and width > 590) : + print 'Rotate image' + img.rotate(pw, -90) + img.save(iurl) + return soup + + def get_cover_url(self): + cover_url = self.CAPA + pedido = Request(self.CAPA) + pedido.add_header('User-agent','Mozilla/5.0 (Windows; U; Windows NT 5.1; '+self.LANGHTM+'; userid='+self.THUMBALIZR_API+') Calibre/0.8.47 (like Gecko)') + pedido.add_header('Accept-Charset',self.ENCHTM) + pedido.add_header('Referer',self.SCREENSHOT) + try: + resposta = urlopen(pedido) + soup = BeautifulSoup(resposta) + cover_item = soup.find('body') + if cover_item: + cover_url='http://api.thumbalizr.com/?api_key='+self.THUMBALIZR_API+'&url='+self.SCREENSHOT+'&width=600&quality=90' + return cover_url + except URLError: + cover_url='http://api.thumbalizr.com/?api_key='+self.THUMBALIZR_API+'&url='+self.SCREENSHOT+'&width=600&quality=90' + return cover_url diff --git a/recipes/wash_post.recipe b/recipes/wash_post.recipe index 3af89d502e..0ca6410b4e 100644 --- a/recipes/wash_post.recipe +++ b/recipes/wash_post.recipe @@ -39,14 +39,26 @@ class WashingtonPost(BasicNewsRecipe): {'class':lambda x: x and 'also-read' in x.split()}, {'class':lambda x: x and 'partners-content' in x.split()}, {'class':['module share', 'module ads', 'comment-vars', 'hidden', - 'share-icons-wrap', 'comments']}, - {'id':['right-rail']}, + 'share-icons-wrap', 'comments', 'flipper']}, + {'id':['right-rail', 'save-and-share']}, + {'width':'1', 'height':'1'}, + + ] - ] keep_only_tags = dict(id=['content', 'article']) + def get_article_url(self, *args): + ans = BasicNewsRecipe.get_article_url(self, *args) + ans = ans.rpartition('?')[0] + if ans.endswith('_video.html'): + return None + if 'ads.pheedo.com' in ans: + return None + #if not ans.endswith('_blog.html'): + # return None + return ans + def print_version(self, url): - url = url.rpartition('?')[0] return url.replace('_story.html', '_singlePage.html') diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 464c9d2cfd..1cf699efa3 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -363,3 +363,11 @@ maximum_cover_size = (1200, 1600) # the files will be sent to the location with the most free space. send_news_to_device_location = "main" +#: What interfaces should the content server listen on +# By default, the calibre content server listens on '0.0.0.0' which means that it +# accepts IPv4 connections on all interfaces. You can change this to, for +# example, '127.0.0.1' to only listen for connections from the local machine, or +# to '::' to listen to all incoming IPv6 and IPv4 connections (this may not +# work on all operating systems) +server_listen_on = '0.0.0.0' + diff --git a/setup/upload.py b/setup/upload.py index 3bad1dd8f3..6cd9ad3eca 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -1,14 +1,14 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, re, cStringIO, base64, httplib, subprocess, hashlib, shutil, time +import os, re, cStringIO, base64, httplib, subprocess, hashlib, shutil, time, glob from subprocess import check_call from tempfile import NamedTemporaryFile, mkdtemp +from zipfile import ZipFile from setup import Command, __version__, installer_name, __appname__ @@ -341,7 +341,25 @@ class UploadUserManual(Command): # {{{ description = 'Build and upload the User Manual' sub_commands = ['manual'] + def build_plugin_example(self, path): + from calibre import CurrentDir + with NamedTemporaryFile(suffix='.zip') as f: + with CurrentDir(self.d(path)): + with ZipFile(f, 'w') as zf: + for x in os.listdir('.'): + zf.write(x) + if os.path.isdir(x): + for y in os.listdir(x): + zf.write(os.path.join(x, y)) + bname = self.b(path) + '_plugin.zip' + subprocess.check_call(['scp', f.name, 'divok:%s/%s'%(DOWNLOADS, + bname)]) + def run(self, opts): + path = self.j(self.SRC, 'calibre', 'manual', 'plugin_examples') + for x in glob.glob(self.j(path, '*')): + self.build_plugin_example(x) + check_call(' '.join(['scp', '-r', 'src/calibre/manual/.build/html/*', 'divok:%s'%USER_MANUAL]), shell=True) # }}} diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 67cf822a17..816c275f72 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -4,9 +4,22 @@ __copyright__ = '2008, Kovid Goyal ' import os, sys, zipfile, importlib -from calibre.constants import numeric_version +from calibre.constants import numeric_version, iswindows, isosx from calibre.ptempfile import PersistentTemporaryFile +platform = 'linux' +if iswindows: + platform = 'windows' +elif isosx: + platform = 'osx' + + +class PluginNotFound(ValueError): + pass + +class InvalidPlugin(ValueError): + pass + class Plugin(object): # {{{ ''' @@ -512,13 +525,21 @@ class InterfaceActionBase(Plugin): # {{{ actual_plugin = None + def __init__(self, *args, **kwargs): + Plugin.__init__(self, *args, **kwargs) + self.actual_plugin_ = None + def load_actual_plugin(self, gui): ''' This method must return the actual interface action plugin object. ''' - mod, cls = self.actual_plugin.split(':') - return getattr(importlib.import_module(mod), cls)(gui, - self.site_customization) + ac = self.actual_plugin_ + if ac is None: + mod, cls = self.actual_plugin.split(':') + ac = getattr(importlib.import_module(mod), cls)(gui, + self.site_customization) + self.actual_plugin_ = ac + return ac # }}} diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 994d73d33b..7d21f8255e 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -853,7 +853,7 @@ class Columns(PreferencesPlugin): class Toolbar(PreferencesPlugin): name = 'Toolbar' icon = I('wizard.png') - gui_name = _('Customize the toolbar') + gui_name = _('Toolbar') category = 'Interface' gui_category = _('Interface') category_order = 1 @@ -865,7 +865,7 @@ class Toolbar(PreferencesPlugin): class Search(PreferencesPlugin): name = 'Search' icon = I('search.png') - gui_name = _('Customize searching') + gui_name = _('Searching') category = 'Interface' gui_category = _('Interface') category_order = 1 diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index 0f5508a89e..9c8f80544b 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -2,17 +2,16 @@ from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -import os, shutil, traceback, functools, sys, re -from contextlib import closing +import os, shutil, traceback, functools, sys -from calibre.customize import Plugin, CatalogPlugin, FileTypePlugin, \ - MetadataReaderPlugin, MetadataWriterPlugin, \ - InterfaceActionBase as InterfaceAction, \ - PreferencesPlugin +from calibre.customize import (CatalogPlugin, FileTypePlugin, PluginNotFound, + MetadataReaderPlugin, MetadataWriterPlugin, + InterfaceActionBase as InterfaceAction, + PreferencesPlugin, platform, InvalidPlugin) from calibre.customize.conversion import InputFormatPlugin, OutputFormatPlugin +from calibre.customize.zipplugin import loader from calibre.customize.profiles import InputProfile, OutputProfile from calibre.customize.builtins import plugins as builtin_plugins -from calibre.constants import numeric_version as version, iswindows, isosx from calibre.devices.interface import DevicePlugin from calibre.ebooks.metadata import MetaInformation from calibre.ebooks.metadata.covers import CoverDownload @@ -22,14 +21,6 @@ from calibre.utils.config import make_config_dir, Config, ConfigProxy, \ from calibre.ebooks.epub.fix import ePubFixer from calibre.ebooks.metadata.sources.base import Source -platform = 'linux' -if iswindows: - platform = 'windows' -elif isosx: - platform = 'osx' - -from zipfile import ZipFile - def _config(): c = Config('customize') c.add_opt('plugins', default={}, help=_('Installed plugins')) @@ -42,11 +33,6 @@ def _config(): config = _config() -class InvalidPlugin(ValueError): - pass - -class PluginNotFound(ValueError): - pass def find_plugin(name): for plugin in _initialized_plugins: @@ -60,38 +46,7 @@ def load_plugin(path_to_zip_file): # {{{ :return: A :class:`Plugin` instance. ''' - #print 'Loading plugin from', path_to_zip_file - if not os.access(path_to_zip_file, os.R_OK): - raise PluginNotFound - with closing(ZipFile(path_to_zip_file)) as zf: - for name in zf.namelist(): - if name.lower().endswith('plugin.py'): - locals = {} - raw = zf.read(name) - lines, encoding = raw.splitlines(), 'utf-8' - cr = re.compile(r'coding[:=]\s*([-\w.]+)') - raw = [] - for l in lines[:2]: - match = cr.search(l) - if match is not None: - encoding = match.group(1) - else: - raw.append(l) - raw += lines[2:] - raw = '\n'.join(raw) - raw = raw.decode(encoding) - raw = re.sub('\r\n', '\n', raw) - exec raw in locals - for x in locals.values(): - if isinstance(x, type) and issubclass(x, Plugin) and \ - x.name != 'Trivial Plugin': - if x.minimum_calibre_version > version or \ - platform not in x.supported_platforms: - continue - - return x - - raise InvalidPlugin(_('No valid plugin found in ')+path_to_zip_file) + return loader.load(path_to_zip_file) # }}} diff --git a/src/calibre/customize/zipplugin.py b/src/calibre/customize/zipplugin.py new file mode 100644 index 0000000000..ca07462b9c --- /dev/null +++ b/src/calibre/customize/zipplugin.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) +from future_builtins import map + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, zipfile, posixpath, importlib, threading, re, imp, sys +from collections import OrderedDict +from functools import partial + +from calibre import as_unicode +from calibre.customize import (Plugin, numeric_version, platform, + InvalidPlugin, PluginNotFound) + +# PEP 302 based plugin loading mechanism, works around the bug in zipimport in +# python 2.x that prevents importing from zip files in locations whose paths +# have non ASCII characters + +def get_resources(zfp, name_or_list_of_names): + ''' + Load resources from the plugin zip file + + :param name_or_list_of_names: List of paths to resources in the zip file using / as + separator, or a single path + + :return: A dictionary of the form ``{name : file_contents}``. Any names + that were not found in the zip file will not be present in the + dictionary. If a single path is passed in the return value will + be just the bytes of the resource or None if it wasn't found. + ''' + names = name_or_list_of_names + if isinstance(names, basestring): + names = [names] + ans = {} + with zipfile.ZipFile(zfp) as zf: + for name in names: + try: + ans[name] = zf.read(name) + except: + import traceback + traceback.print_exc() + if len(names) == 1: + ans = ans.pop(names[0], None) + + return ans + +def get_icons(zfp, name_or_list_of_names): + ''' + Load icons from the plugin zip file + + :param name_or_list_of_names: List of paths to resources in the zip file using / as + separator, or a single path + + :return: A dictionary of the form ``{name : QIcon}``. Any names + that were not found in the zip file will be null QIcons. + If a single path is passed in the return value will + be A QIcon. + ''' + from PyQt4.Qt import QIcon, QPixmap + names = name_or_list_of_names + ans = get_resources(zfp, names) + if isinstance(names, basestring): + names = [names] + if ans is None: + ans = {} + if isinstance(ans, basestring): + ans = dict([(names[0], ans)]) + + ians = {} + for name in names: + p = QPixmap() + raw = ans.get(name, None) + if raw: + p.loadFromData(raw) + ians[name] = QIcon(p) + if len(names) == 1: + ians = ians.pop(names[0]) + return ians + +class PluginLoader(object): + + def __init__(self): + self.loaded_plugins = {} + self._lock = threading.RLock() + self._identifier_pat = re.compile(r'[a-zA-Z][_0-9a-zA-Z]*') + + def _get_actual_fullname(self, fullname): + parts = fullname.split('.') + if parts[0] == 'calibre_plugins': + if len(parts) == 1: + return parts[0], None + plugin_name = parts[1] + with self._lock: + names = self.loaded_plugins.get(plugin_name, None) + if names is None: + raise ImportError('No plugin named %r loaded'%plugin_name) + names = names[1] + fullname = '.'.join(parts[2:]) + if not fullname: + fullname = '__init__' + if fullname in names: + return fullname, plugin_name + if fullname+'.__init__' in names: + return fullname+'.__init__', plugin_name + return None, None + + def find_module(self, fullname, path=None): + fullname, plugin_name = self._get_actual_fullname(fullname) + if fullname is None and plugin_name is None: + return None + return self + + def load_module(self, fullname): + import_name, plugin_name = self._get_actual_fullname(fullname) + if import_name is None and plugin_name is None: + raise ImportError('No plugin named %r is loaded'%fullname) + mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + mod.__file__ = "" + mod.__loader__ = self + + if import_name.endswith('.__init__') or import_name in ('__init__', + 'calibre_plugins'): + # We have a package + mod.__path__ = [] + + if plugin_name is not None: + # We have some actual code to load + with self._lock: + zfp, names = self.loaded_plugins.get(plugin_name, (None, None)) + if names is None: + raise ImportError('No plugin named %r loaded'%plugin_name) + zinfo = names.get(import_name, None) + if zinfo is None: + raise ImportError('Plugin %r has no module named %r' % + (plugin_name, import_name)) + with zipfile.ZipFile(zfp) as zf: + try: + code = zf.read(zinfo) + except: + # Maybe the zip file changed from under us + code = zf.read(zinfo.filename) + compiled = compile(code, 'calibre_plugins.%s.%s'%(plugin_name, + import_name), 'exec', dont_inherit=True) + mod.__dict__['get_resources'] = partial(get_resources, zfp) + mod.__dict__['get_icons'] = partial(get_icons, zfp) + exec compiled in mod.__dict__ + + return mod + + + def load(self, path_to_zip_file): + if not os.access(path_to_zip_file, os.R_OK): + raise PluginNotFound('Cannot access %r'%path_to_zip_file) + + with zipfile.ZipFile(path_to_zip_file) as zf: + plugin_name = self._locate_code(zf, path_to_zip_file) + + try: + ans = None + plugin_module = 'calibre_plugins.%s'%plugin_name + m = sys.modules.get(plugin_module, None) + if m is not None: + reload(m) + else: + m = importlib.import_module(plugin_module) + for obj in m.__dict__.itervalues(): + if isinstance(obj, type) and issubclass(obj, Plugin) and \ + obj.name != 'Trivial Plugin': + ans = obj + break + if ans is None: + raise InvalidPlugin('No plugin class found in %s:%s'%( + as_unicode(path_to_zip_file), plugin_name)) + + if ans.minimum_calibre_version > numeric_version: + raise InvalidPlugin( + 'The plugin at %s needs a version of calibre >= %s' % + (as_unicode(path_to_zip_file), '.'.join(map(unicode, + ans.minimum_calibre_version)))) + + if platform not in ans.supported_platforms: + raise InvalidPlugin( + 'The plugin at %s cannot be used on %s' % + (as_unicode(path_to_zip_file), platform)) + + return ans + except: + with self._lock: + del self.loaded_plugins[plugin_name] + raise + + + def _locate_code(self, zf, path_to_zip_file): + names = [x if isinstance(x, unicode) else x.decode('utf-8') for x in + zf.namelist()] + names = [x[1:] if x[0] == '/' else x for x in names] + + plugin_name = None + for name in names: + name, ext = posixpath.splitext(name) + if name.startswith('plugin-import-name-') and ext == '.txt': + plugin_name = name.rpartition('-')[-1] + + if plugin_name is None: + c = 0 + while True: + c += 1 + plugin_name = 'dummy%d'%c + if plugin_name not in self.loaded_plugins: + break + else: + if self._identifier_pat.match(plugin_name) is None: + raise InvalidPlugin(( + 'The plugin at %r uses an invalid import name: %r' % + (path_to_zip_file, plugin_name))) + + pynames = [x for x in names if x.endswith('.py')] + + candidates = [posixpath.dirname(x) for x in pynames if + x.endswith('/__init__.py')] + candidates.sort(key=lambda x: x.count('/')) + valid_packages = set() + + for candidate in candidates: + parts = candidate.split('/') + parent = '.'.join(parts[:-1]) + if parent and parent not in valid_packages: + continue + valid_packages.add('.'.join(parts)) + + names = OrderedDict() + + for candidate in pynames: + parts = posixpath.splitext(candidate)[0].split('/') + package = '.'.join(parts[:-1]) + if package and package not in valid_packages: + continue + name = '.'.join(parts) + names[name] = zf.getinfo(candidate) + + # Legacy plugins + if '__init__' not in names: + for name in list(names.iterkeys()): + if '.' not in name and name.endswith('plugin'): + names['__init__'] = names[name] + break + + if '__init__' not in names: + raise InvalidPlugin(('The plugin in %r is invalid. It does not ' + 'contain a top-level __init__.py file') + % path_to_zip_file) + + with self._lock: + self.loaded_plugins[plugin_name] = (path_to_zip_file, names) + + return plugin_name + + +loader = PluginLoader() +sys.meta_path.insert(0, loader) + + +if __name__ == '__main__': + from tempfile import NamedTemporaryFile + from calibre.customize.ui import add_plugin + from calibre import CurrentDir + path = sys.argv[-1] + with NamedTemporaryFile(suffix='.zip') as f: + with zipfile.ZipFile(f, 'w') as zf: + with CurrentDir(path): + for x in os.listdir('.'): + if x[0] != '.': + print ('Adding', x) + zf.write(x) + if os.path.isdir(x): + for y in os.listdir(x): + zf.write(os.path.join(x, y)) + add_plugin(f.name) + print ('Added plugin from', sys.argv[-1]) + diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 39f24ff6ee..54e4979524 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -94,14 +94,14 @@ class ANDROID(USBMS): VENDOR_NAME = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER', 'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS', - 'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE'] + 'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE', 'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H', 'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD', - '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM'] + '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7'] diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 0cfa8f1ab2..c12b533c4e 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -24,6 +24,29 @@ from calibre.utils.logging import Log from calibre.utils.zipfile import ZipFile + +class AppleOpenFeedback(OpenFeedback): + + def __init__(self): + OpenFeedback.__init__(self, u'') + + def custom_dialog(self, parent): + from PyQt4.Qt import (QDialog, QVBoxLayout, QLabel, QDialogButtonBox) + + class Dialog(QDialog): + + def __init__(self, p): + QDialog.__init__(self, p) + self.l = l = QVBoxLayout() + self.setLayout(l) + l.addWidget(QLabel('test')) + self.bb = QDialogButtonBox(QDialogButtonBox.OK) + l.addWidget(self.bb) + self.bb.accepted.connect(self.accept) + self.bb.rejected.connect(self.reject) + + return Dialog(parent) + from PIL import Image as PILImage from lxml import etree @@ -743,14 +766,8 @@ class ITUNES(DriverBase): self.log.info("ITUNES.open()") # Display a dialog recommending using 'Connect to iTunes' - if False and not self.settings().extra_customization[self.SKIP_CONNECT_TO_ITUNES_DIALOG]: - raise OpenFeedback('

' + ('Click the "Connect/Share" button and choose' - ' "Connect to iTunes" to send books from your calibre library' - ' to your Apple iDevice.

For more information, see ' - '' - 'Calibre + Apple iDevices FAQ.

' - 'After following the Quick Start steps outlined in the FAQ, ' - 'restart calibre.')) + if not self.settings().extra_customization[self.SKIP_CONNECT_TO_ITUNES_DIALOG]: + raise AppleOpenFeedback() if DEBUG: self.log.info(" advanced user mode, directly connecting to iDevice") diff --git a/src/calibre/devices/errors.py b/src/calibre/devices/errors.py index 3d88eb741f..7b11b6933f 100644 --- a/src/calibre/devices/errors.py +++ b/src/calibre/devices/errors.py @@ -41,6 +41,13 @@ class OpenFeedback(DeviceError): self.feedback_msg = msg DeviceError.__init__(self, msg) + def custom_dialog(self, parent): + ''' + If you need to show the user a custom dialog, instead if just + displaying the feedback_msg, create and return it here. + ''' + raise NotImplementedError + class DeviceBusy(ProtocolError): """ Raised when device is busy """ def __init__(self, uerr=""): diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index da1ef55786..2f962fc218 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -57,6 +57,7 @@ class PRS505(USBMS): MUST_READ_METADATA = True SUPPORTS_USE_AUTHOR_SORT = True EBOOK_DIR_MAIN = 'database/media/books' + SCAN_FROM_ROOT = False ALL_BY_TITLE = _('All by title') ALL_BY_AUTHOR = _('All by author') @@ -87,18 +88,27 @@ class PRS505(USBMS): _('Set this option if you want the cover thumbnails to have ' 'the same aspect ratio (width to height) as the cover. ' 'Unset it if you want the thumbnail to be the maximum size, ' - 'ignoring aspect ratio.') + 'ignoring aspect ratio.'), + _('Search for books in all folders') + + ':::' + + _('Setting this option tells calibre to look for books in all ' + 'folders on the device and its cards. This permits calibre to ' + 'find books put on the device by other software and by ' + 'wireless download.') ] EXTRA_CUSTOMIZATION_DEFAULT = [ ', '.join(['series', 'tags']), False, False, + True, True ] OPT_COLLECTIONS = 0 OPT_UPLOAD_COVERS = 1 OPT_REFRESH_COVERS = 2 + OPT_PRESERVE_ASPECT_RATIO = 3 + OPT_SCAN_FROM_ROOT = 4 plugboard = None plugboard_func = None @@ -147,11 +157,13 @@ class PRS505(USBMS): self.booklist_class.rebuild_collections = self.rebuild_collections # Set the thumbnail width to the theoretical max if the user has asked # that we do not preserve aspect ratio - if not self.settings().extra_customization[3]: + if not self.settings().extra_customization[self.OPT_PRESERVE_ASPECT_RATIO]: self.THUMBNAIL_WIDTH = 168 # Set WANTS_UPDATED_THUMBNAILS if the user has asked that thumbnails be # updated on every connect - self.WANTS_UPDATED_THUMBNAILS = self.settings().extra_customization[2] + self.WANTS_UPDATED_THUMBNAILS = \ + self.settings().extra_customization[self.OPT_REFRESH_COVERS] + self.SCAN_FROM_ROOT = self.settings().extra_customization[self.OPT_SCAN_FROM_ROOT] def filename_callback(self, fname, mi): if getattr(mi, 'application_id', None) is not None: diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 578c28b894..c8580f6741 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -55,6 +55,8 @@ class USBMS(CLI, Device): METADATA_CACHE = 'metadata.calibre' DRIVEINFO = 'driveinfo.calibre' + SCAN_FROM_ROOT = False + def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None): if not isinstance(dinfo, dict): dinfo = {} @@ -173,9 +175,13 @@ class USBMS(CLI, Device): ebook_dirs = [ebook_dirs] for ebook_dir in ebook_dirs: ebook_dir = self.path_to_unicode(ebook_dir) - ebook_dir = self.normalize_path( \ + if self.SCAN_FROM_ROOT: + ebook_dir = self.normalize_path(prefix) + else: + ebook_dir = self.normalize_path( \ os.path.join(prefix, *(ebook_dir.split('/'))) \ if ebook_dir else prefix) + debug_print('USBMS: scan from root', self.SCAN_FROM_ROOT, ebook_dir) if not os.path.exists(ebook_dir): continue # Get all books in the ebook_dir directory if self.SUPPORTS_SUB_DIRS: diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index d86a83e9b0..8d1164e026 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -111,6 +111,7 @@ class FB2MLizer(object): metadata['lang'] = u'en' metadata['id'] = None metadata['cover'] = self.get_cover() + metadata['genre'] = self.opts.fb2_genre metadata['author'] = u'' for auth in self.oeb_book.metadata.creator: @@ -159,7 +160,7 @@ class FB2MLizer(object): return u'' \ '' \ '' \ - 'antique' \ + '%(genre)s' \ '%(author)s' \ '%(title)s' \ '%(cover)s' \ diff --git a/src/calibre/ebooks/fb2/output.py b/src/calibre/ebooks/fb2/output.py index bce0a00029..bccc665d35 100644 --- a/src/calibre/ebooks/fb2/output.py +++ b/src/calibre/ebooks/fb2/output.py @@ -15,6 +15,133 @@ class FB2Output(OutputFormatPlugin): author = 'John Schember' file_type = 'fb2' + FB2_GENRES = [ + # Science Fiction & Fantasy + 'sf_history', # Alternative history + 'sf_action', # Action + 'sf_epic', # Epic + 'sf_heroic', # Heroic + 'sf_detective', # Detective + 'sf_cyberpunk', # Cyberpunk + 'sf_space', # Space + 'sf_social', # Social#philosophical + 'sf_horror', # Horror & mystic + 'sf_humor', # Humor + 'sf_fantasy', # Fantasy + 'sf', # Science Fiction + # Detectives & Thrillers + 'det_classic', # Classical detectives + 'det_police', # Police Stories + 'det_action', # Action + 'det_irony', # Ironical detectives + 'det_history', # Historical detectives + 'det_espionage', # Espionage detectives + 'det_crime', # Crime detectives + 'det_political', # Political detectives + 'det_maniac', # Maniacs + 'det_hard', # Hard#boiled + 'thriller', # Thrillers + 'detective', # Detectives + # Prose + 'prose_classic', # Classics prose + 'prose_history', # Historical prose + 'prose_contemporary', # Contemporary prose + 'prose_counter', # Counterculture + 'prose_rus_classic', # Russial classics prose + 'prose_su_classics', # Soviet classics prose + # Romance + 'love_contemporary', # Contemporary Romance + 'love_history', # Historical Romance + 'love_detective', # Detective Romance + 'love_short', # Short Romance + 'love_erotica', # Erotica + # Adventure + 'adv_western', # Western + 'adv_history', # History + 'adv_indian', # Indians + 'adv_maritime', # Maritime Fiction + 'adv_geo', # Travel & geography + 'adv_animal', # Nature & animals + 'adventure', # Other + # Children's + 'child_tale', # Fairy Tales + 'child_verse', # Verses + 'child_prose', # Prose + 'child_sf', # Science Fiction + 'child_det', # Detectives & Thrillers + 'child_adv', # Adventures + 'child_education', # Educational + 'children', # Other + # Poetry & Dramaturgy + 'poetry', # Poetry + 'dramaturgy', # Dramaturgy + # Antique literature + 'antique_ant', # Antique + 'antique_european', # European + 'antique_russian', # Old russian + 'antique_east', # Old east + 'antique_myths', # Myths. Legends. Epos + 'antique', # Other + # Scientific#educational + 'sci_history', # History + 'sci_psychology', # Psychology + 'sci_culture', # Cultural science + 'sci_religion', # Religious studies + 'sci_philosophy', # Philosophy + 'sci_politics', # Politics + 'sci_business', # Business literature + 'sci_juris', # Jurisprudence + 'sci_linguistic', # Linguistics + 'sci_medicine', # Medicine + 'sci_phys', # Physics + 'sci_math', # Mathematics + 'sci_chem', # Chemistry + 'sci_biology', # Biology + 'sci_tech', # Technical + 'science', # Other + # Computers & Internet + 'comp_www', # Internet + 'comp_programming', # Programming + 'comp_hard', # Hardware + 'comp_soft', # Software + 'comp_db', # Databases + 'comp_osnet', # OS & Networking + 'computers', # Other + # Reference + 'ref_encyc', # Encyclopedias + 'ref_dict', # Dictionaries + 'ref_ref', # Reference + 'ref_guide', # Guidebooks + 'reference', # Other + # Nonfiction + 'nonf_biography', # Biography & Memoirs + 'nonf_publicism', # Publicism + 'nonf_criticism', # Criticism + 'design', # Art & design + 'nonfiction', # Other + # Religion & Inspiration + 'religion_rel', # Religion + 'religion_esoterics', # Esoterics + 'religion_self', # Self#improvement + 'religion', # Other + # Humor + 'humor_anecdote', # Anecdote (funny stories) + 'humor_prose', # Prose + 'humor_verse', # Verses + 'humor', # Other + # Home & Family + 'home_cooking', # Cooking + 'home_pets', # Pets + 'home_crafts', # Hobbies & Crafts + 'home_entertain', # Entertaining + 'home_health', # Health + 'home_garden', # Garden + 'home_diy', # Do it yourself + 'home_sport', # Sports + 'home_sex', # Erotica & sex + 'home', # Other + ] + options = set([ OptionRecommendation(name='sectionize', recommended_value='files', level=OptionRecommendation.LOW, @@ -25,6 +152,11 @@ class FB2Output(OutputFormatPlugin): 'A value of "Table of Contents" turns the entries in the Table of Contents into titles and creates sections; ' 'if it fails, adjust the "Structure Detection" and/or "Table of Contents" settings ' '(turn on "Force use of auto-generated Table of Contents).')), + OptionRecommendation(name='fb2_genre', + recommended_value='antique', level=OptionRecommendation.LOW, + choices=FB2_GENRES, + help=_('Genre for the book. Choices: %s\n\n See: ' % FB2_GENRES) + 'http://www.fictionbook.org/index.php/Eng:FictionBook_2.1_genres ' \ + + _('for a complete list with descriptions.')), ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): diff --git a/src/calibre/ebooks/metadata/mobi.py b/src/calibre/ebooks/metadata/mobi.py index 963391dcf8..ffdef77d11 100644 --- a/src/calibre/ebooks/metadata/mobi.py +++ b/src/calibre/ebooks/metadata/mobi.py @@ -271,11 +271,11 @@ class MetadataUpdater(object): FILTER=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)]) N=0; result='' while src: - s,src = src[:length],src[length:] - hexa = ' '.join(["%02X"%ord(x) for x in s]) - s = s.translate(FILTER) - result += "%04X %-*s %s\n" % (N, length*3, hexa, s) - N+=length + s,src = src[:length],src[length:] + hexa = ' '.join(["%02X"%ord(x) for x in s]) + s = s.translate(FILTER) + result += "%04X %-*s %s\n" % (N, length*3, hexa, s) + N+=length print result def get_pdbrecords(self): @@ -323,6 +323,7 @@ class MetadataUpdater(object): "\tThis is a '%s' file of type '%s'" % (self.type[0:4], self.type[4:8])) recs = [] + added_501 = False try: from calibre.ebooks.conversion.config import load_defaults prefs = load_defaults('mobi_output') @@ -355,6 +356,7 @@ class MetadataUpdater(object): update_exth_record((105, normalize(subjects).encode(self.codec, 'replace'))) if kindle_pdoc and kindle_pdoc in mi.tags: + added_501 = True update_exth_record((501, str('PDOC'))) if mi.pubdate: @@ -370,6 +372,12 @@ class MetadataUpdater(object): update_exth_record((203, pack('>I', 0))) if self.thumbnail_record is not None: update_exth_record((202, pack('>I', self.thumbnail_rindex))) + # Add a 113 record if not present to allow Amazon syncing + if (113 not in self.original_exth_records and + self.original_exth_records.get(501, None) == 'EBOK' and + not added_501): + from uuid import uuid4 + update_exth_record((113, str(uuid4()))) if 503 in self.original_exth_records: update_exth_record((503, mi.title.encode(self.codec, 'replace'))) diff --git a/src/calibre/ebooks/mobi/writer.py b/src/calibre/ebooks/mobi/writer.py index 2be699e525..fccaad8811 100644 --- a/src/calibre/ebooks/mobi/writer.py +++ b/src/calibre/ebooks/mobi/writer.py @@ -1098,7 +1098,7 @@ class MobiWriter(object): nodeCountValue = 0x80 if nodeCountValue == 0 else nodeCountValue tbSequence += chr(nodeCountValue) else : - tbSequence += decint(0x00, DECINT_FORWARD) # arg1 = 0x80 + tbSequence += decint(0x00, DECINT_FORWARD) # arg1 = 0x80 tbSequence += decint(len(tbSequence) + 1, DECINT_FORWARD) # len @@ -1188,7 +1188,7 @@ class MobiWriter(object): toc = self._oeb.toc nodes = list(toc.iter())[1:] toc_conforms = True - for (i, child) in enumerate(nodes) : + for child in nodes: if child.klass == "periodical" and child.depth() != 3 or \ child.klass == "section" and child.depth() != 2 or \ child.klass == "article" and child.depth() != 1 : @@ -1644,14 +1644,14 @@ class MobiWriter(object): self._oeb.log('Generating INDX ...') self._primary_index_record = None - # Build the NCXEntries and INDX + # Build the NCXEntries and INDX indxt, indxt_count, indices, last_name = self._generate_indxt() if last_name is None: self._oeb.log.warn('Input document has no TOC. No index generated.') return - # Assemble the INDX0[0] and INDX1[0] output streams + # Assemble the INDX0[0] and INDX1[0] output streams indx1 = StringIO() indx1.write('INDX'+pack('>I', 0xc0)) # header length @@ -2310,10 +2310,8 @@ class MobiWriter(object): parentIndex = sectionParent.parentIndex self._write_section_node(indxt, indices, sectionParent.myCtocMapIndex, index, offset, length, c, firstArticle, lastArticle, parentIndex) - last_name = "%04X"%c - # articles - for (i, article) in enumerate(list(sectionParent.articles)) : + for article in list(sectionParent.articles): index = article.myCtocMapIndex offset = article.startAddress length = article.articleLength @@ -2413,7 +2411,6 @@ class MobiWriter(object): # Article(s) child.depth() = 1 # Section 2 - documentType = "unknown" sectionIndices = [] sectionParents = [] currentSection = 0 # Starting section number @@ -2421,7 +2418,6 @@ class MobiWriter(object): indxt, indices, c = StringIO(), StringIO(), 0 indices.write('IDXT') - c = 0 last_name = None # 'book', 'periodical' or None @@ -2449,8 +2445,8 @@ class MobiWriter(object): if self.opts.verbose > 3 : self._oeb.logger.info("unknown document type %12.12s \tdepth:%d" % (child.title, child.depth()) ) - # Original code starts here - # test first node for depth/class + # Original code starts here + # test first node for depth/class entries = list(toc.iter())[1:] for (i, child) in enumerate(entries): if not child.title or not child.title.strip(): diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index 4a09e0b1d4..ebc2f30d00 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -103,8 +103,8 @@ class OEBReader(object): data = self.oeb.container.read(None) data = self.oeb.decode(data) data = XMLDECL_RE.sub('', data) - data = data.replace('http://openebook.org/namespaces/oeb-package/1.0', - OPF1_NS) + data = re.sub(r'http://openebook.org/namespaces/oeb-package/1.0(/*)', + OPF1_NS, data) try: opf = etree.fromstring(data) except etree.XMLSyntaxError: diff --git a/src/calibre/ebooks/rtf2xml/make_lists.py b/src/calibre/ebooks/rtf2xml/make_lists.py index fc544ac499..b785bf85d2 100755 --- a/src/calibre/ebooks/rtf2xml/make_lists.py +++ b/src/calibre/ebooks/rtf2xml/make_lists.py @@ -291,9 +291,12 @@ class MakeLists: if self.__list_of_lists: # older RTF won't generate a list_of_lists index_of_list = self.__get_index_of_list(id) if index_of_list != None:# found a matching id - list_dict = self.__list_of_lists[index_of_list][0] + curlist = self.__list_of_lists[index_of_list] + list_dict = curlist[0] level = int(self.__level) + 1 - level_dict = self.__list_of_lists[index_of_list][level][0] + if level >= len(curlist): + level = len(curlist) - 1 + level_dict = curlist[level][0] list_type = level_dict.get('numbering-type') if list_type == 'bullet': list_type = 'unordered' diff --git a/src/calibre/ebooks/textile/functions.py b/src/calibre/ebooks/textile/functions.py index b01a7bcdb7..39f793face 100755 --- a/src/calibre/ebooks/textile/functions.py +++ b/src/calibre/ebooks/textile/functions.py @@ -64,6 +64,8 @@ import re import uuid from urlparse import urlparse +from calibre.utils.smartypants import smartyPants + def _normalize_newlines(string): out = re.sub(r'\r\n', '\n', string) out = re.sub(r'\n{3,}', '\n\n', out) @@ -262,10 +264,9 @@ class Textile(object): self.rel = ' rel="%s"' % rel text = self.getRefs(text) - text = self.block(text, int(head_offset)) - text = self.retrieve(text) + text = smartyPants(text, 'q') return text diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index cad55b8c3f..99f7035800 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -165,7 +165,6 @@ class TXTInput(InputFormatPlugin): elif options.formatting_type == 'textile': log.debug('Running text through textile conversion...') html = convert_textile(txt) - #setattr(options, 'smarten_punctuation', True) else: log.debug('Running text through basic conversion...') flow_size = getattr(options, 'flow_size', 0) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 9649a79cc1..4e550de6ad 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -29,6 +29,8 @@ gprefs.defaults['action-layout-toolbar'] = ( 'Connect Share', None, 'Remove Books', None, 'Help', 'Preferences', ) +gprefs.defaults['action-layout-toolbar-child'] = () + gprefs.defaults['action-layout-toolbar-device'] = ( 'Add Books', 'Edit Metadata', None, 'Convert Books', 'View', 'Send To Device', None, None, 'Location Manager', None, None, @@ -52,7 +54,6 @@ gprefs.defaults['show_splash_screen'] = True gprefs.defaults['toolbar_icon_size'] = 'medium' gprefs.defaults['automerge'] = 'ignore' gprefs.defaults['toolbar_text'] = 'auto' -gprefs.defaults['show_child_bar'] = False gprefs.defaults['font'] = None gprefs.defaults['tags_browser_partition_method'] = 'first letter' gprefs.defaults['tags_browser_collapse_at'] = 100 @@ -79,6 +80,8 @@ def _config(): help=_('Use Roman numerals for series number')) c.add_opt('sort_tags_by', default='name', help=_('Sort tags list by name, popularity, or rating')) + c.add_opt('match_tags_type', default='any', + help=_('Match tags by any or all.')) c.add_opt('cover_flow_queue_length', default=6, help=_('Number of covers to show in the cover browsing mode')) c.add_opt('LRF_conversion_defaults', default=[], diff --git a/src/calibre/gui2/actions/__init__.py b/src/calibre/gui2/actions/__init__.py index 8563956b28..c0dd40326d 100644 --- a/src/calibre/gui2/actions/__init__.py +++ b/src/calibre/gui2/actions/__init__.py @@ -75,7 +75,7 @@ class InterfaceAction(QObject): dont_remove_from = frozenset([]) all_locations = frozenset(['toolbar', 'toolbar-device', 'context-menu', - 'context-menu-device']) + 'context-menu-device', 'toolbar-child']) #: Type of action #: 'current' means acts on the current view diff --git a/src/calibre/gui2/actions/add_to_library.py b/src/calibre/gui2/actions/add_to_library.py index 05aea8f1dd..fd686e3833 100644 --- a/src/calibre/gui2/actions/add_to_library.py +++ b/src/calibre/gui2/actions/add_to_library.py @@ -12,7 +12,7 @@ class AddToLibraryAction(InterfaceAction): name = 'Add To Library' action_spec = (_('Add books to library'), 'add_book.png', _('Add books to your calibre library from the connected device'), None) - dont_add_to = frozenset(['toolbar', 'context-menu']) + dont_add_to = frozenset(['toolbar', 'context-menu', 'toolbar-child']) action_type = 'current' def genesis(self): diff --git a/src/calibre/gui2/actions/device.py b/src/calibre/gui2/actions/device.py index 0b0492228e..a4ca95a9bb 100644 --- a/src/calibre/gui2/actions/device.py +++ b/src/calibre/gui2/actions/device.py @@ -121,7 +121,7 @@ class SendToDeviceAction(InterfaceAction): name = 'Send To Device' action_spec = (_('Send to device'), 'sync.png', None, _('D')) dont_remove_from = frozenset(['toolbar-device']) - dont_add_to = frozenset(['toolbar', 'context-menu']) + dont_add_to = frozenset(['toolbar', 'context-menu', 'toolbar-child']) def genesis(self): self.qaction.triggered.connect(self.do_sync) diff --git a/src/calibre/gui2/actions/edit_collections.py b/src/calibre/gui2/actions/edit_collections.py index 7f5dd76538..c64a3249d4 100644 --- a/src/calibre/gui2/actions/edit_collections.py +++ b/src/calibre/gui2/actions/edit_collections.py @@ -12,7 +12,7 @@ class EditCollectionsAction(InterfaceAction): name = 'Edit Collections' action_spec = (_('Manage collections'), None, _('Manage the collections on this device'), None) - dont_add_to = frozenset(['toolbar', 'context-menu']) + dont_add_to = frozenset(['toolbar', 'context-menu', 'toolbar-child']) action_type = 'current' def genesis(self): diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 6c2cfb8126..3f053e5223 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -164,6 +164,8 @@ class EditMetadataAction(InterfaceAction): if d.row_delta == 0: break current_row += d.row_delta + self.gui.library_view.set_current_row(current_row) + self.gui.library_view.scroll_to_row(current_row) if changed: diff --git a/src/calibre/gui2/convert/fb2_output.py b/src/calibre/gui2/convert/fb2_output.py index 66296ee666..19d0995fc1 100644 --- a/src/calibre/gui2/convert/fb2_output.py +++ b/src/calibre/gui2/convert/fb2_output.py @@ -17,8 +17,10 @@ class PluginWidget(Widget, Ui_Form): ICON = I('mimetypes/fb2.png') def __init__(self, parent, get_option, get_help, db=None, book_id=None): - Widget.__init__(self, parent, ['sectionize']) + Widget.__init__(self, parent, ['sectionize', 'fb2_genre']) self.db, self.book_id = db, book_id for x in ('toc', 'files', 'nothing'): self.opt_sectionize.addItem(x) + for x in get_option('fb2_genre').option.choices: + self.opt_fb2_genre.addItem(x) self.initialize_options(get_option, get_help, db, book_id) diff --git a/src/calibre/gui2/convert/fb2_output.ui b/src/calibre/gui2/convert/fb2_output.ui index 891aa29857..c8d4230a99 100644 --- a/src/calibre/gui2/convert/fb2_output.ui +++ b/src/calibre/gui2/convert/fb2_output.ui @@ -14,7 +14,7 @@ Form - + Qt::Vertical @@ -29,21 +29,31 @@ - - Sectionize: - - - opt_sectionize - - - - - - - 20 + + Sectionize: + + + opt_sectionize + + + + 20 + + + + + + + Genre + + + + + + diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index ab2177cef1..6163c01d27 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -164,7 +164,7 @@ class DeviceManager(Thread): # {{{ dev.open(self.current_library_uuid) except OpenFeedback as e: if dev not in self.ejected_devices: - self.open_feedback_msg(dev.get_gui_name(), e.feedback_msg) + self.open_feedback_msg(dev.get_gui_name(), e) self.ejected_devices.add(dev) continue except: @@ -618,8 +618,11 @@ class DeviceMixin(object): # {{{ if tweaks['auto_connect_to_folder']: self.connect_to_folder_named(tweaks['auto_connect_to_folder']) - def show_open_feedback(self, devname, msg): - self.__of_dev_mem__ = d = info_dialog(self, devname, msg) + def show_open_feedback(self, devname, e): + try: + self.__of_dev_mem__ = d = e.custom_dialog(self) + except NotImplementedError: + self.__of_dev_mem__ = d = info_dialog(self, devname, e.feedback_msg) d.show() def auto_convert_question(self, msg, autos): diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py index e8a4e79384..41b415e25c 100644 --- a/src/calibre/gui2/layout.py +++ b/src/calibre/gui2/layout.py @@ -278,11 +278,14 @@ class ToolBar(QToolBar): # {{{ pass def build_bar(self): - self.child_bar.setVisible(gprefs['show_child_bar']) self.showing_donate = False showing_device = self.location_manager.has_device - actions = '-device' if showing_device else '' - actions = gprefs['action-layout-toolbar'+actions] + mactions = '-device' if showing_device else '' + mactions = gprefs['action-layout-toolbar'+mactions] + cactions = gprefs['action-layout-toolbar-child'] + + show_child = len(cactions) > 0 + self.child_bar.setVisible(show_child) for ac in self.added_actions: m = ac.menu() @@ -292,44 +295,30 @@ class ToolBar(QToolBar): # {{{ self.clear() self.child_bar.clear() self.added_actions = [] - self.spacers = [Spacer(self.child_bar), Spacer(self.child_bar), - Spacer(self), Spacer(self)] - self.child_bar.addWidget(self.spacers[0]) - if gprefs['show_child_bar']: - self.addWidget(self.spacers[2]) - for what in actions: - if what is None and not gprefs['show_child_bar']: - self.addSeparator() - elif what == 'Location Manager': - for ac in self.location_manager.available_actions: - self.addAction(ac) - self.added_actions.append(ac) - self.setup_tool_button(ac, QToolButton.MenuButtonPopup) - elif what == 'Donate': - self.d_widget = QWidget() - self.d_widget.setLayout(QVBoxLayout()) - self.d_widget.layout().addWidget(self.donate_button) - self.addWidget(self.d_widget) - self.showing_donate = True - elif what in self.gui.iactions: - action = self.gui.iactions[what] - bar = self - if action.action_type == 'current' and gprefs['show_child_bar']: - bar = self.child_bar - bar.addAction(action.qaction) - self.added_actions.append(action.qaction) - self.setup_tool_button(action.qaction, action.popup_type) + for bar, actions in ((self, mactions), (self.child_bar, cactions)): + for what in actions: + if what is None: + bar.addSeparator() + elif what == 'Location Manager': + for ac in self.location_manager.available_actions: + bar.addAction(ac) + bar.added_actions.append(ac) + bar.setup_tool_button(bar, ac, QToolButton.MenuButtonPopup) + elif what == 'Donate': + self.d_widget = QWidget() + self.d_widget.setLayout(QVBoxLayout()) + self.d_widget.layout().addWidget(self.donate_button) + bar.addWidget(self.d_widget) + self.showing_donate = True + elif what in self.gui.iactions: + action = self.gui.iactions[what] + bar.addAction(action.qaction) + self.added_actions.append(action.qaction) + self.setup_tool_button(bar, action.qaction, action.popup_type) - self.child_bar.addWidget(self.spacers[1]) - if gprefs['show_child_bar']: - self.addWidget(self.spacers[3]) - else: - for s in self.spacers[2:]: - s.setVisible(False) - - def setup_tool_button(self, ac, menu_mode=None): - ch = self.widgetForAction(ac) + def setup_tool_button(self, bar, ac, menu_mode=None): + ch = bar.widgetForAction(ac) if ch is None: ch = self.child_bar.widgetForAction(ac) ch.setCursor(Qt.PointingHandCursor) @@ -345,7 +334,7 @@ class ToolBar(QToolBar): # {{{ style = Qt.ToolButtonIconOnly if p == 'auto' and self.preferred_width > self.width()+35 and \ - not gprefs['show_child_bar']: + not gprefs['action-layout-toolbar-child']: style = Qt.ToolButtonIconOnly self.setToolButtonStyle(style) diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index a2d2236039..523a296a37 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -48,7 +48,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('disable_tray_notification', config) r('use_roman_numerals_for_series_number', config) r('separate_cover_flow', config, restart_required=True) - r('show_child_bar', gprefs) choices = [(_('Small'), 'small'), (_('Medium'), 'medium'), (_('Large'), 'large')] diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index bc965b89fa..996caeb653 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -6,8 +6,8 @@ 0 0 - 670 - 422 + 717 + 444 @@ -244,13 +244,6 @@ then the tags will be displayed each on their own line. - - - - &Split the toolbar into two toolbars - - - diff --git a/src/calibre/gui2/preferences/main.py b/src/calibre/gui2/preferences/main.py index 699087c4bb..d930fb3ebd 100644 --- a/src/calibre/gui2/preferences/main.py +++ b/src/calibre/gui2/preferences/main.py @@ -71,11 +71,11 @@ class Category(QWidget): # {{{ plugin_activated = pyqtSignal(object) - def __init__(self, name, plugins, parent=None): + def __init__(self, name, plugins, gui_name, parent=None): QWidget.__init__(self, parent) self._layout = QVBoxLayout() self.setLayout(self._layout) - self.label = QLabel(name) + self.label = QLabel(gui_name) self.sep = QFrame(self) self.bf = QFont() self.bf.setBold(True) @@ -118,12 +118,17 @@ class Browser(QScrollArea): # {{{ QScrollArea.__init__(self, parent) self.setWidgetResizable(True) - category_map = {} + category_map, category_names = {}, {} for plugin in preferences_plugins(): if plugin.category not in category_map: category_map[plugin.category] = plugin.category_order if category_map[plugin.category] < plugin.category_order: category_map[plugin.category] = plugin.category_order + if plugin.category not in category_names: + category_names[plugin.category] = (plugin.gui_category if + plugin.gui_category else plugin.category) + + self.category_names = category_names categories = list(category_map.keys()) categories.sort(cmp=lambda x, y: cmp(category_map[x], category_map[y])) @@ -145,7 +150,7 @@ class Browser(QScrollArea): # {{{ self.setWidget(self.container) for name, plugins in self.category_map.items(): - w = Category(name, plugins, self) + w = Category(name, plugins, self.category_names[name], parent=self) self.widgets.append(w) self._layout.addWidget(w) w.plugin_activated.connect(self.show_plugin.emit) diff --git a/src/calibre/gui2/preferences/toolbar.py b/src/calibre/gui2/preferences/toolbar.py index a0d48f3910..93079110a5 100644 --- a/src/calibre/gui2/preferences/toolbar.py +++ b/src/calibre/gui2/preferences/toolbar.py @@ -208,6 +208,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): LOCATIONS = [ ('toolbar', _('The main toolbar')), + ('toolbar-child', _('The optional second toolbar')), ('toolbar-device', _('The main toolbar when a device is connected')), ('context-menu', _('The context menu for the books in the ' 'calibre library')), diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 6b1ce2f851..f86e261443 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -158,11 +158,17 @@ class TagsView(QTreeView): # {{{ self.setContextMenuPolicy(Qt.CustomContextMenu) pop = config['sort_tags_by'] self.sort_by.setCurrentIndex(self.db.CATEGORY_SORTS.index(pop)) + try: + match_pop = self.db.MATCH_TYPE.index(config['match_tags_type']) + except ValueError: + match_pop = 0 + self.tag_match.setCurrentIndex(match_pop) if not self.made_connections: self.clicked.connect(self.toggle) self.customContextMenuRequested.connect(self.show_context_menu) self.refresh_required.connect(self.recount, type=Qt.QueuedConnection) self.sort_by.currentIndexChanged.connect(self.sort_changed) + self.tag_match.currentIndexChanged.connect(self.match_changed) self.made_connections = True self.refresh_signal_processed = True db.add_listener(self.database_changed) @@ -180,6 +186,12 @@ class TagsView(QTreeView): # {{{ config.set('sort_tags_by', self.db.CATEGORY_SORTS[pop]) self.recount() + def match_changed(self, pop): + try: + config.set('match_tags_type', self.db.MATCH_TYPE[pop]) + except: + pass + def set_search_restriction(self, s): if s: self.search_restriction = s @@ -2114,6 +2126,7 @@ class TagBrowserWidget(QWidget): # {{{ parent.sort_by.setCurrentIndex(0) self._layout.addWidget(parent.sort_by) + # Must be in the same order as db2.MATCH_TYPE parent.tag_match = QComboBox(parent) for x in (_('Match any'), _('Match all')): parent.tag_match.addItem(x) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 03aae37883..a0aa0138bc 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -382,6 +382,9 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ error_dialog(self, _('Failed to start content server'), unicode(self.content_server.exception)).exec_() + @property + def current_db(self): + return self.library_view.model().db def another_instance_wants_to_talk(self): try: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 0cae7be2f8..042899adbe 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1228,6 +1228,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ########## data structures for get_categories CATEGORY_SORTS = ('name', 'popularity', 'rating') + MATCH_TYPE = ('any', 'all') class TCat_Tag(object): diff --git a/src/calibre/library/server/__init__.py b/src/calibre/library/server/__init__.py index 244669f50a..950c881d8d 100644 --- a/src/calibre/library/server/__init__.py +++ b/src/calibre/library/server/__init__.py @@ -10,7 +10,7 @@ import os from calibre.utils.config import Config, StringConfig, config_dir, tweaks -listen_on = '0.0.0.0' +listen_on = tweaks['server_listen_on'] log_access_file = os.path.join(config_dir, 'server_access_log.txt') diff --git a/src/calibre/manual/creating_plugins.rst b/src/calibre/manual/creating_plugins.rst new file mode 100644 index 0000000000..dd62e4e416 --- /dev/null +++ b/src/calibre/manual/creating_plugins.rst @@ -0,0 +1,212 @@ + +.. include:: global.rst + +.. _pluginstutorial: + +Writing your own plugins to extend |app|'s functionality +==================================================================== + +|app| has a very modular design. Almost all functionality in |app| comes in the form of plugins. Plugins are used for conversion, for downloading news (though these are called recipes), for various components of the user interface, to connect to different devices, to process files when adding them to |app| and so on. You can get a complete list of all the builtin plugins in |app| by going to :guilabel:`Preferences->Plugins`. + +Here, we will teach you how to create your own plugins to add new features to |app|. + + +.. contents:: Contents + :depth: 2 + :local: + +.. note:: This only applies to calibre releases >= 0.7.53 + +Anatomy of a |app| plugin +--------------------------- + +A |app| plugin is very simple, it's just a zip file that contains some python code +and any other resources like image files needed by the plugin. Without further ado, +let's see a basic example. + +Suppose you have an installation of |app| that you are using to self publish various e-documents in EPUB and MOBI +formats. You would like all files generated by |app| to have their publisher set as "Hello world", here's how to do it. +Create a file named :file:`__init__.py` (this is a special name and must always be used for the main file of your plugin) +and enter the following Python code into it: + +.. literalinclude:: plugin_examples/helloworld/__init__.py + :lines: 10- + +That's all. To add this code to |app| as a plugin, simply create a zip file with:: + + zip plugin.zip __init__.py + +Add this plugin to |app| via :guilabel:`Preferences->Plugins`. + +You can download the Hello World plugin from +`helloworld_plugin.zip `_. + +Every time you use calibre to convert a book, the plugin's :meth:`run` method will be called and the +converted book will have its publisher set to "Hello World". This is a trivial plugin, lets move on to +a more complex example that actually adds a component to the user interface. + +A User Interface plugin +------------------------- + +This plugin will be spread over a few files (to keep the code clean). It will show you how to get resources +(images or data files) from the plugin zip file, allow users to configure your plugin, +how to create elements in the |app| user interface and how to access +and query the books database in |app|. + +You can download this plugin from `interface_demo_plugin.zip `_ + +The first thing to note is that this zip file has a lot more files in it, explained below, pay particular attention to +``plugin-import-name-interface_demo.txt``. + + **plugin-import-name-interface_demo.txt** + An empty text file used to enable the multi-file plugin magic. This file must be present in all plugins that use + more than one .py file. It should be empty and its filename must be of the form: plugin-import-name-**some_name**.txt + The presence of this file allows you to import code from the .py files present inside the zip file, using a statement like:: + + from calibre_plugins.some_name.some_module import some_object + + The prefix ``calibre_plugins`` must always be present. ``some_name`` comes from the filename of the empty text file. + ``some_module`` refers to :file:`some_module.py` file inside the zip file. Note that this importing is just as + powerful as regular python imports. You can create packages and subpackages of .py modules inside the zip file, + just like you would normally (by defining __init__.py in each sub directory), and everything should Just Work. + + The name you use for ``some_name`` enters a global namespace shared by all plugins, **so make it as unique as possible**. + But remember that it must be a valid python identifier (only alphabets, numbers and the underscore). + + **__init__.py** + As before, the file that defines the plugin class + + **main.py** + This file contains the actual code that does something useful + + **ui.py** + This file defines the interface part of the plugin + + **images/icon.png** + The icon for this plugin + + **about.txt** + A text file with information about the plugin + +Now let's look at the code. + +__init__.py +^^^^^^^^^^^^^ + +First, the obligatory ``__init__.py`` to define the plugin metadata: + +.. literalinclude:: plugin_examples/interface_demo/__init__.py + :lines: 10- + +The only noteworthy feature is the field :attr:`actual_plugin`. Since |app| has both command line and GUI interfaces, +GUI plugins like this one should not load any GUI libraries in __init__.py. The actual_plugin field does this for you, +by telling |app| that the actual plugin is to be found in another file inside your zip archive, which will only be loaded +in a GUI context. + +Remember that for this to work, you must have a plugin-import-name-some_name.txt file in your plugin zip file, +as discussed above. + +Also there are a couple of methods for enabling user configuration of the plugin. These are discussed below. + +ui.py +^^^^^^^^ + +Now let's look at ui.py which defines the actual GUI plugin. The source code is heavily commented and should be self explanatory: + +.. literalinclude:: plugin_examples/interface_demo/ui.py + :lines: 16- + +main.py +^^^^^^^^^ + +The actual logic to implement the Interface Plugin Demo dialog. + +.. literalinclude:: plugin_examples/interface_demo/main.py + :lines: 16- + +Getting resources from the plugin zip file +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +|app|'s plugin loading system defines a couple of builtin functions that allow you to conveniently get files from the plugin zip file. + + **get_resources(name_or_list_of_names)** + This function should be called with a list of paths to files inside the zip file. For example to access the file icon.png in + the directory images in the zip file, you would use: ``images/icon.png``. Always use a forward slash as the path separator, + even on windows. When you pass in a single name, the function will return the raw bytes of that file or None if the name + was not found in the zip file. If you pass in more than one name then it returns a dict mapping the names to bytes. + If a name is not found, it will not be present in the returned dict. + + **get_icons(name_or_list_of_names)** + A convenience wrapper for get_resources() that creates QIcon objects from the raw bytes returned by get_resources. + If a name is not found in the zip file the corresponding QIcon will be null. + +Enabling user configuration of your plugin +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To allow users to configure your plugin, you must define three methods in your base plugin class, '**is_customizable**, **config_widget** and **save_settings** as shown below: + +.. literalinclude:: plugin_examples/interface_demo/__init__.py + :pyobject: InterfacePluginDemo.is_customizable + +.. literalinclude:: plugin_examples/interface_demo/__init__.py + :pyobject: InterfacePluginDemo.config_widget + +.. literalinclude:: plugin_examples/interface_demo/__init__.py + :pyobject: InterfacePluginDemo.save_settings + +|app| has many different ways to store configuration data (a legacy of its long history). The recommended way is to use the **JSONConfig** class, which stores your configuration information in a .json file. + +The code to manage configuration data in the demo plugin is in config.py: + +.. literalinclude:: plugin_examples/interface_demo/config.py + :lines: 10- + +The ``prefs`` object is now available throughout the plugin code by a simple:: + + from calibre_plugins.interface_demo.config import prefs + + +You can see the ``prefs`` object being used in main.py: + +.. literalinclude:: plugin_examples/interface_demo/main.py + :pyobject: DemoDialog.config + + +The different types of plugins +-------------------------------- + +As you may have noticed above, a plugin in |app| is a class. There are different classes for the different types of plugins in |app|. +Details on each class, including the base class of all plugins can be found in :ref:`plugins`. + +Debugging plugins +------------------- + +The first, most important step is to run |app| in debug mode. You can do this from the command line with:: + + calibre-debug -g + +Or from within calibre by clicking the arrow next to the preferences button or using the `Ctrl+Shift+R` keyboard shortcut. + +When running from the command line, debug output will be printed to the console, when running from within |app| the output will go to a txt file. + +You can insert print statements anywhere in your plugin code, they will be output in debug mode. Remember, this is python, you really shouldn't need anything more than print statements to debug ;) I developed all of |app| using just this debugging technique. + +It can get tiresome to keep re-adding a plugin to calibre to test small changes. The plugin zip files are stored in the calibre config directory in plugins/ (goto Preferences->Misc and click open config directory to see the config directory). + +Once you've located the zip file of your plugin you can then directly update it with your changes instead of re-adding it each time. To do so from the command line, in the directory that contains your plugin source code, use:: + + zip -R /path/to/plugin/zip/file.zip * + +This will automatically update all changed files. It relies on the freely available zip command line tool. + +More plugin examples +---------------------- + +You can find a list of many, sophisticated |app| plugins `here `_. + +Sharing your plugins with others +---------------------------------- + +If you would like to share the plugins you have created with other users of |app|, post your plugin in a new thread in the +`calibre plugins forum `_. + diff --git a/src/calibre/manual/customize.rst b/src/calibre/manual/customize.rst index 6218bf8112..fe33100576 100644 --- a/src/calibre/manual/customize.rst +++ b/src/calibre/manual/customize.rst @@ -17,6 +17,11 @@ use *plugins* to add functionality to |app|. :depth: 2 :local: +.. toctree:: + :hidden: + + plugins + Environment variables ----------------------- @@ -53,148 +58,10 @@ You should not change the files in this resources folder, as your changes will g For example, if you wanted to change the icon for the :guilabel:`Remove books` action, you would first look in the builtin resources folder and see that the relevant file is :file:`resources/images/trash.svg`. Assuming you have an alternate icon in svg format called :file:`mytrash.svg` you would save it in the configuration directory as :file:`resources/images/trash.svg`. All the icons used by the calibre user interface are in :file:`resources/images` and its sub-folders. -A Hello World plugin ------------------------- +Customizing |app| with plugins +-------------------------------- -Suppose you have an installation of |app| that you are using to self publish various e-documents in EPUB and LRF -format. You would like all file generated by |app| to have their publisher set as "Hello world", here's how to do it. -Create a file name :file:`my_plugin.py` (the file name must end with plugin.py) and enter the following Python code into it: +|app| has a very modular design. Almost all functionality in |app| comes in the form of plugins. Plugins are used for conversion, for downloading news (though these are called recipes), for various components of the user interface, to connect to different devices, to process files when adding them to |app| and so on. You can get a complete list of all the builtin plugins in |app| by going to :guilabel:`Preferences->Plugins`. -.. code-block:: python +You can write your own plugins to customize and extend the behavior of |app|. The plugin architecture in |app| is very simple, see the tutorial :ref:`pluginstutorial`. - import os - from calibre.customize import FileTypePlugin - - class HelloWorld(FileTypePlugin): - - name = 'Hello World Plugin' # Name of the plugin - description = 'Set the publisher to Hello World for all new conversions' - supported_platforms = ['windows', 'osx', 'linux'] # Platforms this plugin will run on - author = 'Acme Inc.' # The author of this plugin - version = (1, 0, 0) # The version number of this plugin - file_types = set(['epub', 'lrf']) # The file types that this plugin will be applied to - on_postprocess = True # Run this plugin after conversion is complete - - def run(self, path_to_ebook): - from calibre.ebooks.metadata.meta import get_metadata, set_metadata - file = open(path_to_ebook, 'r+b') - ext = os.path.splitext(path_to_ebook)[-1][1:].lower() - mi = get_metadata(file, ext) - mi.publisher = 'Hello World' - set_metadata(file, mi, ext) - return path_to_ebook - -That's all. To add this code to |app| as a plugin, simply create a zip file with:: - - zip plugin.zip my_plugin.py - -You can download the Hello World plugin from -`helloworld_plugin.zip `_. -Now either use the configuration dialog in |app| GUI to add this zip file as a plugin, or -use the command:: - - calibre-customize -a plugin.zip - -Every time you use calibre to convert a book, the plugin's :meth:`run` method will be called and the -converted book will have its publisher set to "Hello World". For more information about -|app|'s plugin system, read on... - - -A Hello World GUI plugin ---------------------------- - -Here's a simple Hello World plugin for the |app| GUI. It will cause a box to popup with the message "Hellooo World!" when you press Ctrl+Shift+H - -.. note:: Only available in calibre versions ``>= 0.7.32``. - -.. code-block:: python - - from calibre.customize import InterfaceActionBase - - class HelloWorldBase(InterfaceActionBase): - - name = 'Hello World GUI' - author = 'The little green man' - - def load_actual_plugin(self, gui): - from calibre.gui2.actions import InterfaceAction - - class HelloWorld(InterfaceAction): - name = 'Hello World GUI' - action_spec = ('Hello World!', 'add_book.png', None, - _('Ctrl+Shift+H')) - - def genesis(self): - self.qaction.triggered.connect(self.hello_world) - - def hello_world(self, *args): - from calibre.gui2 import info_dialog - info_dialog(self.gui, 'Hello World!', 'Hellooo World!', - show=True) - - return HelloWorld(gui, self.site_customization) - -You can also have it show up in the toolbars/context menu by going to Preferences->Toolbars and adding this plugin to the locations you want it to be in. - -While this plugin is utterly useless, note that all calibre GUI actions like adding/saving/removing/viewing/etc. are implemented as plugins, so there is no limit to what you can achieve. The key thing to remember is that the plugin has access to the full |app| GUI via ``self.gui``. - - -The Plugin base class ------------------------- - -As you may have noticed above, all |app| plugins are classes. The Plugin classes are organized in a hierarchy at the top of which -is :class:`calibre.customize.Plugin`. The has excellent in source documentation for its various features, here I will discuss a -few of the important ones. - -First, all plugins must supply a list of platforms they have been tested on by setting the ``supported_platforms`` member as in the -example above. - -If the plugin needs to do any initialization, it should implement the :meth:`initialize` method. The path to the plugin zip file -is available as ``self.plugin_path``. The initialization method could be used to load any needed resources from the zip file. - -If the plugin needs to be customized (i.e. it needs some information from the user), it should implement the :meth:`customization_help` -method, to indicate to |app| that it needs user input. This can be useful, for example, to ask the user to input the path to a needed system -binary or the URL of a website, etc. When |app| asks the user for the customization information, the string retuned by the :meth:`customization_help` -method is used as help text to le thte user know what information is needed. - -Another useful method is :meth:`temporary_file`, which returns a file handle to an opened temporary file. If your plugin needs to make use -of temporary files, it should use this method. Temporary file cleanup is then taken care of automatically. - -In addition, whenever plugins are run, their zip files are automatically added to the start of ``sys.path``, so you can directly import -any python files you bundle in the zip files. Note that this is not available when the plugin is being initialized, only when it is being run. - -Finally, plugins can have a priority (a positive integer). Higher priority plugins are run in preference tolower priority ones in a given context. -By default all plugins have priority 1. You can change that by setting the member :attr:'priority` in your subclass. - -See :ref:`pluginsPlugin` for details. - -File type plugins -------------------- - -File type plugins are intended to be associated with specific file types (as identified by extension). They can be run on several different occassions. - - * When books/formats are added ot the |app| database (if :attr:`on_import` is set to True). - * Just before an any2whatever converter is run on an input file (if :attr:`on_preprocess` is set to True). - * After an any2whatever converter has run, on the output file (if :attr:`on_postprocess` is set to True). - -File type plugins specify which file types they are associated with by specifying the :attr:`file_types` member as in the above example. -the actual work should be done in the :meth:`run` method, which must return the path to the modified ebook (it can be the same as the original -if the modifcations are done in place). - -See :ref:`pluginsFTPlugin` for details. - -Metadata plugins -------------------- - -Metadata plugins add the ability to read/write metadata from ebook files to |app|. See :ref:`pluginsMetadataPlugin` for details. - -.. toctree:: - :hidden: - - plugins - -Metadata download plugins ----------------------------- - -Metadata download plugins add various sources that |app| uses to download metadata based on title/author/isbn etc. See :ref:`pluginsMetadataSource` -for details. diff --git a/src/calibre/manual/plugin_examples/helloworld/__init__.py b/src/calibre/manual/plugin_examples/helloworld/__init__.py new file mode 100644 index 0000000000..3a49123e07 --- /dev/null +++ b/src/calibre/manual/plugin_examples/helloworld/__init__.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os +from calibre.customize import FileTypePlugin + +class HelloWorld(FileTypePlugin): + + name = 'Hello World Plugin' # Name of the plugin + description = 'Set the publisher to Hello World for all new conversions' + supported_platforms = ['windows', 'osx', 'linux'] # Platforms this plugin will run on + author = 'Acme Inc.' # The author of this plugin + version = (1, 0, 0) # The version number of this plugin + file_types = set(['epub', 'mobi']) # The file types that this plugin will be applied to + on_postprocess = True # Run this plugin after conversion is complete + minimum_calibre_version = (0, 7, 53) + + def run(self, path_to_ebook): + from calibre.ebooks.metadata.meta import get_metadata, set_metadata + file = open(path_to_ebook, 'r+b') + ext = os.path.splitext(path_to_ebook)[-1][1:].lower() + mi = get_metadata(file, ext) + mi.publisher = 'Hello World' + set_metadata(file, mi, ext) + return path_to_ebook + + diff --git a/src/calibre/manual/plugin_examples/interface_demo/__init__.py b/src/calibre/manual/plugin_examples/interface_demo/__init__.py new file mode 100644 index 0000000000..ac7d9c6ec1 --- /dev/null +++ b/src/calibre/manual/plugin_examples/interface_demo/__init__.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +# The class that all Interface Action plugin wrappers must inherit from +from calibre.customize import InterfaceActionBase + +class InterfacePluginDemo(InterfaceActionBase): + ''' + This class is a simple wrapper that provides information about the actual + plugin class. The actual interface plugin class is called InterfacePlugin + and is defined in the ui.py file, as specified in the actual_plugin field + below. + + The reason for having two classes is that it allows the command line + calibre utilities to run without needing to load the GUI libraries. + ''' + name = 'Interface Plugin Demo' + description = 'An advanced plugin demo' + supported_platforms = ['windows', 'osx', 'linux'] + author = 'Kovid Goyal' + version = (1, 0, 0) + minimum_calibre_version = (0, 7, 53) + + #: This field defines the GUI plugin class that contains all the code + #: that actually does something. Its format is module_path:class_name + #: The specified class must be defined in the specified module. + actual_plugin = 'calibre_plugins.interface_demo.ui:InterfacePlugin' + + def is_customizable(self): + ''' + This method must return True to enable customization via + Preferences->Plugins + ''' + return True + + def config_widget(self): + ''' + Implement this method and :meth:`save_settings` in your plugin to + use a custom configuration dialog. + + This method, if implemented, must return a QWidget. The widget can have + an optional method validate() that takes no arguments and is called + immediately after the user clicks OK. Changes are applied if and only + if the method returns True. + + If for some reason you cannot perform the configuration at this time, + return a tuple of two strings (message, details), these will be + displayed as a warning dialog to the user and the process will be + aborted. + + The base class implementation of this method raises NotImplementedError + so by default no user configuration is possible. + ''' + # It is important to put this import statement here rather than at the + # top of the module as importing the config class will also cause the + # GUI libraries to be loaded, which we do not want when using calibre + # from the command line + from calibre_plugins.interface_demo.config import ConfigWidget + return ConfigWidget() + + def save_settings(self, config_widget): + ''' + Save the settings specified by the user with config_widget. + + :param config_widget: The widget returned by :meth:`config_widget`. + ''' + config_widget.save_settings() + + # Apply the changes + ac = self.actual_plugin_ + if ac is not None: + ac.apply_settings() + + diff --git a/src/calibre/manual/plugin_examples/interface_demo/about.txt b/src/calibre/manual/plugin_examples/interface_demo/about.txt new file mode 100644 index 0000000000..f35b1e7196 --- /dev/null +++ b/src/calibre/manual/plugin_examples/interface_demo/about.txt @@ -0,0 +1,7 @@ +The Interface Plugin Demo +=========================== + +Created by Kovid Goyal + +Requires calibre >= 0.7.53 + diff --git a/src/calibre/manual/plugin_examples/interface_demo/config.py b/src/calibre/manual/plugin_examples/interface_demo/config.py new file mode 100644 index 0000000000..fd391ce944 --- /dev/null +++ b/src/calibre/manual/plugin_examples/interface_demo/config.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from PyQt4.Qt import QWidget, QHBoxLayout, QLabel, QLineEdit + +from calibre.utils.config import JSONConfig + +# This is where all preferences for this plugin will be stored +# Remember that this name (i.e. plugins/interface_demo) is also +# in a global namespace, so make it as unique as possible. +# You should always prefix your config file name with plugins/, +# so as to ensure you dont accidentally clobber a calibre config file +prefs = JSONConfig('plugins/interface_demo') + +# Set defaults +prefs.defaults['hello_world_msg'] = 'Hello, World!' + +class ConfigWidget(QWidget): + + def __init__(self): + QWidget.__init__(self) + self.l = QHBoxLayout() + self.setLayout(self.l) + + self.label = QLabel('Hello world &message:') + self.l.addWidget(self.label) + + self.msg = QLineEdit(self) + self.msg.setText(prefs['hello_world_msg']) + self.l.addWidget(self.msg) + self.label.setBuddy(self.msg) + + def save_settings(self): + prefs['hello_world_msg'] = unicode(self.msg.text()) + diff --git a/src/calibre/manual/plugin_examples/interface_demo/images/icon.png b/src/calibre/manual/plugin_examples/interface_demo/images/icon.png new file mode 100644 index 0000000000..ad823e2ff4 Binary files /dev/null and b/src/calibre/manual/plugin_examples/interface_demo/images/icon.png differ diff --git a/src/calibre/manual/plugin_examples/interface_demo/main.py b/src/calibre/manual/plugin_examples/interface_demo/main.py new file mode 100644 index 0000000000..f23664b1de --- /dev/null +++ b/src/calibre/manual/plugin_examples/interface_demo/main.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +if False: + # This is here to keep my python error checker from complaining about + # the builtin functions that will be defined by the plugin loading system + # You do not need this code in your plugins + get_icons = get_resources = None + +from PyQt4.Qt import QDialog, QVBoxLayout, QPushButton, QMessageBox, QLabel + +from calibre_plugins.interface_demo.config import prefs + +class DemoDialog(QDialog): + + def __init__(self, gui, icon, do_user_config): + QDialog.__init__(self, gui) + self.gui = gui + self.do_user_config = do_user_config + + # The current database shown in the GUI + # db is an instance of the class LibraryDatabase2 from database.py + # This class has many, many methods that allow you to do a lot of + # things. + self.db = gui.current_db + + self.l = QVBoxLayout() + self.setLayout(self.l) + + self.label = QLabel(prefs['hello_world_msg']) + self.l.addWidget(self.label) + + self.setWindowTitle('Interface Plugin Demo') + self.setWindowIcon(icon) + + self.about_button = QPushButton('About', self) + self.about_button.clicked.connect(self.about) + self.l.addWidget(self.about_button) + + self.marked_button = QPushButton( + 'Show books with only one format in the calibre GUI', self) + self.marked_button.clicked.connect(self.marked) + self.l.addWidget(self.marked_button) + + self.view_button = QPushButton( + 'View the most recently added book', self) + self.view_button.clicked.connect(self.view) + self.l.addWidget(self.view_button) + + self.conf_button = QPushButton( + 'Configure this plugin', self) + self.conf_button.clicked.connect(self.config) + self.l.addWidget(self.conf_button) + + self.resize(self.sizeHint()) + + def about(self): + # Get the about text from a file inside the plugin zip file + # The get_resources function is a builtin function defined for all your + # plugin code. It loads files from the plugin zip file. It returns + # the bytes from the specified file. + # + # Note that if you are loading more than one file, for performance, you + # should pass a list of names to get_resources. In this case, + # get_resources will return a dictionary mapping names to bytes. Names that + # are not found in the zip file will not be in the returned dictionary. + text = get_resources('about.txt') + QMessageBox.about(self, 'About the Interface Plugin Demo', + text.decode('utf-8')) + + def marked(self): + fmt_idx = self.db.FIELD_MAP['formats'] + matched_ids = set() + for record in self.db.data.iterall(): + # Iterate over all records + fmts = record[fmt_idx] + # fmts is either None or a comma separated list of formats + if fmts and ',' not in fmts: + matched_ids.add(record[0]) + # Mark the records with the matching ids + self.db.set_marked_ids(matched_ids) + + # Tell the GUI to search for all marked records + self.gui.search.setEditText('marked:true') + self.gui.search.do_search() + + def view(self): + most_recent = most_recent_id = None + timestamp_idx = self.db.FIELD_MAP['timestamp'] + + for record in self.db.data: + # Iterate over all currently showing records + timestamp = record[timestamp_idx] + if most_recent is None or timestamp > most_recent: + most_recent = timestamp + most_recent_id = record[0] + + if most_recent_id is not None: + # Get the row number of the id as shown in the GUI + row_number = self.db.row(most_recent_id) + # Get a reference to the View plugin + view_plugin = self.gui.iactions['View'] + # Ask the view plugin to launch the viewer for row_number + view_plugin._view_books([row_number]) + + def config(self): + self.do_user_config(parent=self) + # Apply the changes + self.label.setText(prefs['hello_world_msg']) + diff --git a/src/calibre/manual/plugin_examples/interface_demo/plugin-import-name-interface_demo.txt b/src/calibre/manual/plugin_examples/interface_demo/plugin-import-name-interface_demo.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/calibre/manual/plugin_examples/interface_demo/ui.py b/src/calibre/manual/plugin_examples/interface_demo/ui.py new file mode 100644 index 0000000000..5026269b60 --- /dev/null +++ b/src/calibre/manual/plugin_examples/interface_demo/ui.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +if False: + # This is here to keep my python error checker from complaining about + # the builtin functions that will be defined by the plugin loading system + # You do not need this code in your plugins + get_icons = get_resources = None + +# The class that all interface action plugins must inherit from +from calibre.gui2.actions import InterfaceAction +from calibre_plugins.interface_demo.main import DemoDialog + +class InterfacePlugin(InterfaceAction): + + name = 'Interface Plugin Demo' + + # Declare the main action associated with this plugin + # The keyboard shortcut can be None if you dont want to use a keyboard + # shortcut. Remember that currently calibre has no central management for + # keyboard shortcuts, so try to use an unusual/unused shortcut. + action_spec = ('Interface Plugin Demo', None, + 'Run the Interface Plugin Demo', 'Ctrl+Shift+F1') + + def genesis(self): + # This method is called once per plugin, do initial setup here + + # Set the icon for this interface action + # The get_icons function is a builtin function defined for all your + # plugin code. It loads icons from the plugin zip file. It returns + # QIcon objects, if you want the actual data, use the analogous + # get_resources builtin function. + # + # Note that if you are loading more than one icon, for performance, you + # should pass a list of names to get_icons. In this case, get_icons + # will return a dictionary mapping names to QIcons. Names that + # are not found in the zip file will result in null QIcons. + icon = get_icons('images/icon.png') + + # The qaction is automatically created from the action_spec defined + # above + self.qaction.setIcon(icon) + self.qaction.triggered.connect(self.show_dialog) + + def show_dialog(self): + # The base plugin object defined in __init__.py + base_plugin_object = self.interface_action_base_plugin + # Show the config dialog + # The config dialog can also be shown from within + # Preferences->Plugins, which is why the do_user_config + # method is defined on the base plugin class + do_user_config = base_plugin_object.do_user_config + + # self.gui is the main calibre GUI. It acts as the gateway to access + # all the elements of the calibre user interface, it should also be the + # parent of the dialog + d = DemoDialog(self.gui, self.qaction.icon(), do_user_config) + d.show() + + def apply_settings(self): + from calibre_plugins.interface_demo.config import prefs + # In an actual non trivial plugin, you would probably need to + # do something based on the settings in prefs + prefs + diff --git a/src/calibre/manual/tutorials.rst b/src/calibre/manual/tutorials.rst index c156167cb1..84f0281eaa 100644 --- a/src/calibre/manual/tutorials.rst +++ b/src/calibre/manual/tutorials.rst @@ -18,4 +18,5 @@ Here you will find tutorials to get you started using |app|'s more advanced feat regexp portable server + creating_plugins diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py index 9594f64ae4..a891d09f3d 100644 --- a/src/calibre/utils/ipc/worker.py +++ b/src/calibre/utils/ipc/worker.py @@ -12,6 +12,8 @@ from threading import Thread from Queue import Queue from contextlib import closing from binascii import unhexlify +from zipimport import ZipImportError + from calibre import prints from calibre.constants import iswindows, isosx @@ -75,7 +77,14 @@ class Progress(Thread): def get_func(name): module, func, notification = PARALLEL_FUNCS[name] - module = importlib.import_module(module) + try: + module = importlib.import_module(module) + except ZipImportError: + # Something windows weird happened, try clearing the zip import cache + # incase the zipfile was changed from under us + from zipimport import _zip_directory_cache as zdc + zdc.clear() + module = importlib.import_module(module) func = getattr(module, func) return func, notification diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index f2aeb4e4bd..325fcf5209 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -71,7 +71,8 @@ class BasicNewsRecipe(Recipe): #: Number of levels of links to follow on article webpages recursions = 0 - #: Delay between consecutive downloads in seconds + #: Delay between consecutive downloads in seconds. The argument may be a + #: floating point number to indicate a more precise time. delay = 0 #: Publication type diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index 64a2c32fb3..5a76c4fa2e 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -486,7 +486,7 @@ def option_parser(usage=_('%prog URL\n\nWhere URL is for example http://google.c type='int', dest='max_recursions') parser.add_option('-n', '--max-files', default=sys.maxint, type='int', dest='max_files', help=_('The maximum number of files to download. This only applies to files from tags. Default is %default')) - parser.add_option('--delay', default=0, dest='delay', type='int', + parser.add_option('--delay', default=0, dest='delay', type='float', help=_('Minimum interval in seconds between consecutive fetches. Default is %default s')) parser.add_option('--encoding', default=None, help=_('The character encoding for the websites you are trying to download. The default is to try and guess the encoding.'))