diff --git a/recipes/der_spiegel.recipe b/recipes/der_spiegel.recipe new file mode 100644 index 0000000000..1e94785233 --- /dev/null +++ b/recipes/der_spiegel.recipe @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2011, Nikolas Mangold ' +''' +spiegel.de +''' +from calibre.web.feeds.news import BasicNewsRecipe +from calibre import strftime +from calibre import re + +class DerSpiegel(BasicNewsRecipe): + title = 'Der Spiegel' + __author__ = 'Nikolas Mangold' + description = 'Der Spiegel, Printed Edition. Access to paid content.' + publisher = 'SPIEGEL-VERLAG RUDOLF AUGSTEIN GMBH & CO. KG' + category = 'news, politics, Germany' + no_stylesheets = True + encoding = 'cp1252' + needs_subscription = True + remove_empty_feeds = True + delay = 1 + PREFIX = 'http://m.spiegel.de' + INDEX = PREFIX + '/spiegel/print/epaper/index-heftaktuell.html' + use_embedded_content = False + masthead_url = 'http://upload.wikimedia.org/wikipedia/en/thumb/1/17/Der_Spiegel_logo.svg/200px-Der_Spiegel_logo.svg.png' + language = 'de' + publication_type = 'magazine' + extra_css = ' body{font-family: Arial,Helvetica,sans-serif} ' + timefmt = '[%W/%Y]' + empty_articles = ['Titelbild'] + preprocess_regexps = [ + (re.compile(r'

◆

', re.DOTALL|re.IGNORECASE), lambda match: '
'), + ] + + def get_browser(self): + def has_login_name(form): + try: + form.find_control(name="f.loginName") + except: + return False + else: + return True + + br = BasicNewsRecipe.get_browser() + if self.username is not None and self.password is not None: + br.open(self.PREFIX + '/meinspiegel/login.html') + br.select_form(predicate=has_login_name) + br['f.loginName' ] = self.username + br['f.password'] = self.password + br.submit() + return br + + remove_tags_before = dict(attrs={'class':'spArticleContent'}) + remove_tags_after = dict(attrs={'class':'spArticleCredit'}) + + def parse_index(self): + soup = self.index_to_soup(self.INDEX) + + cover = soup.find('img', width=248) + if cover is not None: + self.cover_url = cover['src'] + + index = soup.find('dl') + + feeds = [] + for section in index.findAll('dt'): + section_title = self.tag_to_string(section).strip() + self.log('Found section ', section_title) + + articles = [] + for article in section.findNextSiblings(['dd','dt']): + if article.name == 'dt': + break + link = article.find('a') + title = self.tag_to_string(link).strip() + if title in self.empty_articles: + continue + self.log('Found article ', title) + url = self.PREFIX + link['href'] + articles.append({'title' : title, 'date' : strftime(self.timefmt), 'url' : url}) + feeds.append((section_title,articles)) + return feeds; diff --git a/recipes/handelsblatt.recipe b/recipes/handelsblatt.recipe index 945dac0560..056fcfb26b 100644 --- a/recipes/handelsblatt.recipe +++ b/recipes/handelsblatt.recipe @@ -1,4 +1,3 @@ - from calibre.web.feeds.news import BasicNewsRecipe class Handelsblatt(BasicNewsRecipe): @@ -7,14 +6,11 @@ class Handelsblatt(BasicNewsRecipe): oldest_article = 7 max_articles_per_feed = 100 no_stylesheets = True - cover_url = 'http://www.handelsblatt.com/images/logo/logo_handelsblatt.com.png' +# cover_url = 'http://www.handelsblatt.com/images/logo/logo_handelsblatt.com.png' language = 'de' - # keep_only_tags = [] - keep_only_tags = (dict(name = 'div', attrs = {'class': ['hcf-detail-abstract hcf-teaser ajaxify','hcf-detail','hcf-author-wrapper']})) - # keep_only_tags.append(dict(name = 'div', attrs = {'id': 'fullText'})) - remove_tags = [dict(name='img', attrs = {'src': 'http://www.handelsblatt.com/images/icon/loading.gif'}) - ,dict(name='ul' , attrs={'class':['hcf-detail-tools']}) - ] + + remove_tags_before = dict(attrs={'class':'hcf-overline'}) + remove_tags_after = dict(attrs={'class':'hcf-footer'}) feeds = [ (u'Handelsblatt Exklusiv',u'http://www.handelsblatt.com/rss/exklusiv'), @@ -28,17 +24,16 @@ class Handelsblatt(BasicNewsRecipe): (u'Handelsblatt Magazin',u'http://www.handelsblatt.com/rss/magazin/'), (u'Handelsblatt Weblogs',u'http://www.handelsblatt.com/rss/blogs') ] + extra_css = ''' - .hcf-headline {font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:x-large;} - .hcf-overline {font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:x-large;} - .hcf-exclusive {font-family:Arial,Helvetica,sans-serif; font-style:italic;font-weight:bold; margin-right:5pt;} - p{font-family:Arial,Helvetica,sans-serif;} - .hcf-location-mark{font-weight:bold; margin-right:5pt;} - .MsoNormal{font-family:Helvetica,Arial,sans-serif;} - .hcf-author-wrapper{font-style:italic;} - .hcf-article-date{font-size:x-small;} - .hcf-caption {font-style:italic;font-size:small;} - img {align:left;} + h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;} + h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} + p{font-family:Arial,Helvetica,sans-serif;font-size:small;} + body{font-family:Helvetica,Arial,sans-serif;font-size:small;} ''' - + def print_version(self, url): + url = url.split('/') + url[-1] = 'v_detail_tab_print,'+url[-1] + url = '/'.join(url) + return url diff --git a/recipes/ilsole24ore.recipe b/recipes/ilsole24ore.recipe index 920c703222..0cf1ddc6bf 100644 --- a/recipes/ilsole24ore.recipe +++ b/recipes/ilsole24ore.recipe @@ -1,71 +1,65 @@ -#!/usr/bin/env python -__license__ = 'GPL v3' -__author__ = 'Lorenzo Vigentini & Edwin van Maastrigt' -__copyright__ = '2009, Lorenzo Vigentini and Edwin van Maastrigt ' -__description__ = 'Financial news daily paper - v1.02 (30, January 2010)' +__author__ = 'Marco Saraceno' +__copyright__ = '2010, Marco Saraceno ' +description = 'Italian daily newspaper - v 1.1 (Mar14,2011)' ''' -http://www.ilsole24ore.com/ +http://www.ilsole24ore.com ''' from calibre.web.feeds.news import BasicNewsRecipe +class IlSole24Ore(BasicNewsRecipe): + __author__ = 'Marco Saraceno' + description = 'Italian financial daily newspaper' -class ilsole24Ore(BasicNewsRecipe): - author = 'Lorenzo Vigentini & Edwin van Maastrigt' - description = 'Financial news daily paper' - - cover_url = 'http://www.ilsole24ore.com/img2007/print_header.gif' - - title = u'il Sole 24 Ore New' - publisher = 'italiaNews' - category = 'News, finance, economy, politics' + cover_url = 'http://www.shopping24.ilsole24ore.com/ProductRelated/rds/img/logo_sole.gif' + title = u'Il Sole 24 Ore' + publisher = 'Gruppo editoriale GRUPPO 24ORE' + category = 'News, politics, culture, economy, financial, Italian' language = 'it' timefmt = '[%a, %d %b, %Y]' oldest_article = 2 - max_articles_per_feed = 50 + max_articles_per_feed = 100 use_embedded_content = False + extra_css = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt }' + + + remove_tags = [ + dict(name='div', attrs={'class':['header','titolo']}), + dict(name='table', attrs={'class':['footer1024','footerdown']}), + ] - remove_javascript = True - no_stylesheets = True def get_article_url(self, article): - return article.get('id', article.get('guid', None)) + link = article.get('link', None) + if link is None: + return article + if link.split('/')[-1]=="story01.htm": + link=link.split('/')[-2] + a=['0B','0C','0D','0E','0F','0G','0N' ,'0L0S','0A'] + b=['.' ,'/' ,'?' ,'-' ,'=' ,'&' ,'.com','www.','0'] + for i in range(0,len(a)): + link=link.replace(a[i],b[i]) + link="http://"+link + return link + + feeds = [ + (u'Notizie Italia', u'http://www.ilsole24ore.com/rss/notizie/italia.xml'), + (u'Notizie Europa', u'http://www.ilsole24ore.com/rss/notizie/europa.xml'), + (u'Notizie USA', u'http://www.ilsole24ore.com/rss/notizie/usa.xml'), + (u'Notizie Americhe', u'http://www.ilsole24ore.com/rss/notizie/americhe.xml'), + (u'Notizie Medio Oriente e Africa', u'http://www.ilsole24ore.com/rss/notizie/medio-oriente-e-africa.xml'), + (u'Notizie Asia e Oceania', u'http://www.ilsole24ore.com/rss/notizie/asia-e-oceania.xml'), + (u'Commenti', u'http://www.ilsole24ore.com/rss/commenti-e-idee.xml'), + (u'Norme e tributi', u'http://www.ilsole24ore.com/rss/norme-e-tributi.xml'), + (u'Finanza', u'http://www.ilsole24ore.com/rss/finanza-e-mercati.xml'), + (u'Economia', u'http://www.ilsole24ore.com/rss/economia.xml'), + (u'Tecnologia', u'http://www.ilsole24ore.com/rss/tecnologie.xml'), + (u'Cultura', u'http://www.ilsole24ore.com/rss/cultura.xml'), + ] def print_version(self, url): - link, sep, params = url.rpartition('?') - if link is None: - return link.replace('_1.php', '_php') - return link.replace('.shtml', '_PRN.shtml') - - keep_only_tags = [ - dict(name='div', attrs={'class':'txt'}) - ] -# remove_tags = [dict(name='br')] - - feeds = [ - (u'Prima pagina', u'http://www.ilsole24ore.com/rss/primapagina.xml'), - (u'Norme e tributi', u'http://www.ilsole24ore.com/rss/norme-tributi.xml'), - (u'Finanza e mercati', u'http://www.ilsole24ore.com/rss/finanza-mercati.xml'), - (u'Economia e lavoro', u'http://www.ilsole24ore.com/rss/economia-lavoro.xml'), - (u'Italia', u'http://www.ilsole24ore.com/rss/italia.xml'), - (u'Mondo', u'http://www.ilsole24ore.com/rss/mondo.xml'), - (u'Tecnologia e business', u'http://www.ilsole24ore.com/rss/tecnologia-business.xml'), - (u'Cultura e tempo libero', u'http://www.ilsole24ore.com/rss/tempolibero-cultura.xml'), - (u'Sport', u'http://www.ilsole24ore.com/rss/sport.xml'), - (u'Professionisti 24', u'http://www.ilsole24ore.com/rss/prof_home.xml'), - (u'Ambiente e Sicurezza',u'http://www.ilsole24ore.com/rss/prof_as.xml') - ] - - extra_css = ''' - html, body, table, tr, td, h1, h2, h3, h4, h5, h6, p, a, span, br, img {margin:0;padding:0;border:0;font-size:12px;font-family:"Georgia","Times New Roman";} - .linkHighlight {color:#0292c6;} - .txt {border-bottom:1px solid #7c7c7c;padding-bottom:20px};text-align:justify;font-family:"serif"} - .txt p {line-height:18px;} - .txt span {line-height:22px;} - .title h3 {color:#7b7b7b;} - .title h4 {color:#08526e;font-size:26px;font-family:"Times New Roman";font-weight:normal;} - ''' + return url.replace('.shtml', '_PRN.shtml') diff --git a/recipes/newsweek.recipe b/recipes/newsweek.recipe index 73837c1872..97abd69aac 100644 --- a/recipes/newsweek.recipe +++ b/recipes/newsweek.recipe @@ -1,4 +1,3 @@ -import string from calibre.web.feeds.news import BasicNewsRecipe class Newsweek(BasicNewsRecipe): @@ -11,7 +10,6 @@ class Newsweek(BasicNewsRecipe): no_stylesheets = True BASE_URL = 'http://www.newsweek.com' - INDEX = BASE_URL+'/topics.html' keep_only_tags = dict(name='article', attrs={'class':'article-text'}) remove_tags = [dict(attrs={'data-dartad':True})] @@ -23,11 +21,14 @@ class Newsweek(BasicNewsRecipe): return soup def newsweek_sections(self): - soup = self.index_to_soup(self.INDEX) - for a in soup.findAll('a', title='Primary tag', href=True): - yield (string.capitalize(self.tag_to_string(a)), - self.BASE_URL+a['href']) - + return [ + ('Nation', 'http://www.newsweek.com/tag/nation.html'), + ('Society', 'http://www.newsweek.com/tag/society.html'), + ('Culture', 'http://www.newsweek.com/tag/culture.html'), + ('World', 'http://www.newsweek.com/tag/world.html'), + ('Politics', 'http://www.newsweek.com/tag/politics.html'), + ('Business', 'http://www.newsweek.com/tag/business.html'), + ] def newsweek_parse_section_page(self, soup): for article in soup.findAll('article', about=True, diff --git a/recipes/the_journal.recipe b/recipes/the_journal.recipe new file mode 100644 index 0000000000..e65d7e272e --- /dev/null +++ b/recipes/the_journal.recipe @@ -0,0 +1,26 @@ +__license__ = 'GPL v3' +__copyright__ = '2011 Phil Burns' +''' +TheJournal.ie +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class TheJournal(BasicNewsRecipe): + + __author_ = 'Phil Burns' + title = u'TheJournal.ie' + oldest_article = 1 + max_articles_per_feed = 100 + encoding = 'utf8' + language = 'en_IE' + timefmt = ' (%A, %B %d, %Y)' + + no_stylesheets = True + remove_tags = [dict(name='div', attrs={'class':'footer'}), + dict(name=['script', 'noscript'])] + + extra_css = 'p, div { margin: 0pt; border: 0pt; text-indent: 0.5em }' + + feeds = [ + (u'Latest News', u'http://www.thejournal.ie/feed/')] diff --git a/resources/template-functions.json b/resources/template-functions.json index c19627c6c7..cf858c7691 100644 --- a/resources/template-functions.json +++ b/resources/template-functions.json @@ -5,6 +5,7 @@ "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n i = 0\n res = ''\n for i in range(0, len(args)):\n res += args[i]\n return res\n", "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n if val:\n return val\n else:\n return value_if_empty\n", + "booksize": "def evaluate(self, formatter, kwargs, mi, locals):\n if mi.book_size is not None:\n try:\n return str(mi.book_size)\n except:\n pass\n return ''\n", "select": "def evaluate(self, formatter, kwargs, mi, locals, val, key):\n if not val:\n return ''\n vals = [v.strip() for v in val.split(',')]\n for v in vals:\n if v.startswith(key+':'):\n return v[len(key)+1:]\n return ''\n", "field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n return formatter.get_value(name, [], kwargs)\n", "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n x = float(x if x else 0)\n y = float(y if y else 0)\n return unicode(x - y)\n", @@ -25,9 +26,9 @@ "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n return capitalize(val)\n", "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n return unicode(len(val.split(sep)))\n", "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n return val.lower()\n", - "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n locals[target] = value\n return value\n", - "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n if (len(args) % 2) != 1:\n raise ValueError(_('switch requires an odd number of arguments'))\n i = 0\n while i < len(args):\n if i + 1 >= len(args):\n return args[i]\n if re.search(args[i], val):\n return args[i+1]\n i += 2\n", "strcmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n v = strcmp(x, y)\n if v < 0:\n return lt\n if v == 0:\n return eq\n return gt\n", + "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n if (len(args) % 2) != 1:\n raise ValueError(_('switch requires an odd number of arguments'))\n i = 0\n while i < len(args):\n if i + 1 >= len(args):\n return args[i]\n if re.search(args[i], val):\n return args[i+1]\n i += 2\n", + "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n locals[target] = value\n return value\n", "raw_field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n return unicode(getattr(mi, name, None))\n", "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n x = float(x if x else 0)\n y = float(y if y else 0)\n if x < y:\n return lt\n if x == y:\n return eq\n return gt\n" } \ No newline at end of file diff --git a/setup/__init__.py b/setup/__init__.py index 9e62fb377d..61bafd2282 100644 --- a/setup/__init__.py +++ b/setup/__init__.py @@ -24,8 +24,10 @@ def initialize_constants(): global __version__, __appname__, modules, functions, basenames, scripts src = open('src/calibre/constants.py', 'rb').read() - __version__ = re.search(r'__version__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1) - __appname__ = re.search(r'__appname__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1) + nv = re.search(r'numeric_version\s+=\s+\((\d+), (\d+), (\d+)\)', src) + __version__ = '%s.%s.%s'%(nv.group(1), nv.group(2), nv.group(3)) + __appname__ = re.search(r'__appname__\s+=\s+(u{0,1})[\'"]([^\'"]+)[\'"]', + src).group(2) epsrc = re.compile(r'entry_points = (\{.*?\})', re.DOTALL).\ search(open('src/calibre/linux.py', 'rb').read()).group(1) entry_points = eval(epsrc, {'__appname__': __appname__}) diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index cf4dcd5f9d..f666427598 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -13,7 +13,8 @@ from setup import Command, modules, functions, basenames, __version__, \ from setup.build_environment import msvc, MT, RC from setup.installer.windows.wix import WixMixIn -QT_DIR = 'Q:\\Qt\\4.7.1' +OPENSSL_DIR = r'Q:\openssl' +QT_DIR = 'Q:\\Qt\\4.7.2' QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns'] LIBUSB_DIR = 'C:\\libusb' LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll' @@ -108,6 +109,8 @@ class Win32Freeze(Command, WixMixIn): self.dll_dir = self.j(self.base, 'DLLs') shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir, ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*')) + for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')): + shutil.copy2(x, self.dll_dir) for x in QT_DLLS: x += '4.dll' if not x.startswith('phonon'): x = 'Qt'+x diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 5dfd956ce2..ce6ca650a4 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -53,12 +53,25 @@ SQLite Put sqlite3*.h from the sqlite windows amlgamation in ~/sw/include +OpenSSL +-------- + +First install ActiveState Perl if you dont already have perl in windows +Download and untar the openssl tarball, follow the instructions in INSTALL.W32 (use no-asm) +to install use prefix q:\openssl + +perl Configure VC-WIN32 no-asm enable-static-engine --prefix=Q:/openssl +ms\do_ms.bat +nmake -f ms\ntdll.mak +nmake -f ms\ntdll.mak test +nmake -f ms\ntdll.mak install + Qt -------- Extract Qt sourcecode to C:\Qt\4.x.x. Run configure and make:: - configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs && nmake + configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake SIP ----- diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 0fddb9de9d..29c69a6799 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -3,11 +3,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import uuid, sys, os, re, logging, time, random, \ - __builtin__, warnings, multiprocessing -from contextlib import closing -from urllib import getproxies -from urllib2 import unquote as urllib2_unquote +import sys, os, re, time, random, __builtin__, warnings __builtin__.__dict__['dynamic_property'] = lambda(func): func(None) from htmlentitydefs import name2codepoint from math import floor @@ -16,25 +12,51 @@ from functools import partial warnings.simplefilter('ignore', DeprecationWarning) -from calibre.constants import iswindows, isosx, islinux, isfreebsd, isfrozen, \ - terminal_controller, preferred_encoding, \ - __appname__, __version__, __author__, \ - win32event, win32api, winerror, fcntl, \ - filesystem_encoding, plugins, config_dir -from calibre.startup import winutil, winutilerror, guess_type +from calibre.constants import (iswindows, isosx, islinux, isfreebsd, isfrozen, + preferred_encoding, __appname__, __version__, __author__, + win32event, win32api, winerror, fcntl, + filesystem_encoding, plugins, config_dir) +from calibre.startup import winutil, winutilerror -if islinux and not getattr(sys, 'frozen', False): - # Imported before PyQt4 to workaround PyQt4 util-linux conflict on gentoo +if False and islinux and not getattr(sys, 'frozen', False): + # Imported before PyQt4 to workaround PyQt4 util-linux conflict discovered on gentoo + # See http://bugs.gentoo.org/show_bug.cgi?id=317557 + # Importing uuid is slow so get rid of this at some point, maybe in a few + # years when even Debian has caught up + # Also remember to remove it from site.py in the binary builds + import uuid uuid.uuid4() if False: # Prevent pyflakes from complaining winutil, winutilerror, __appname__, islinux, __version__ - fcntl, win32event, isfrozen, __author__, terminal_controller - winerror, win32api, isfreebsd, guess_type + fcntl, win32event, isfrozen, __author__ + winerror, win32api, isfreebsd -import cssutils -cssutils.log.setLevel(logging.WARN) +_mt_inited = False +def _init_mimetypes(): + global _mt_inited + import mimetypes + mimetypes.init([P('mime.types')]) + _mt_inited = True + +def guess_type(*args, **kwargs): + import mimetypes + if not _mt_inited: + _init_mimetypes() + return mimetypes.guess_type(*args, **kwargs) + +def guess_all_extensions(*args, **kwargs): + import mimetypes + if not _mt_inited: + _init_mimetypes() + return mimetypes.guess_all_extensions(*args, **kwargs) + +def get_types_map(): + import mimetypes + if not _mt_inited: + _init_mimetypes() + return mimetypes.types_map def to_unicode(raw, encoding='utf-8', errors='strict'): if isinstance(raw, unicode): @@ -182,6 +204,7 @@ class CommandLineError(Exception): pass def setup_cli_handlers(logger, level): + import logging if os.environ.get('CALIBRE_WORKER', None) is not None and logger.handlers: return logger.setLevel(level) @@ -243,6 +266,7 @@ def extract(path, dir): extractor(path, dir) def get_proxies(debug=True): + from urllib import getproxies proxies = getproxies() for key, proxy in list(proxies.items()): if not proxy or '..' in proxy: @@ -386,6 +410,7 @@ class StreamReadWrapper(object): def detect_ncpus(): """Detects the number of effective CPUs in the system""" + import multiprocessing ans = -1 try: ans = multiprocessing.cpu_count() @@ -550,6 +575,9 @@ def get_download_filename(url, cookie_file=None): ''' Get a local filename for a URL using the content disposition header ''' + from contextlib import closing + from urllib2 import unquote as urllib2_unquote + filename = '' br = browser() @@ -679,4 +707,3 @@ main() ipshell() sys.argv = old_argv - diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 6f26a63940..28dbcc4299 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -1,28 +1,32 @@ +from future_builtins import map + __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' -__appname__ = 'calibre' -__version__ = '0.7.56' -__author__ = "Kovid Goyal " - -import re, importlib -_ver = __version__.split('.') -_ver = [int(re.search(r'(\d+)', x).group(1)) for x in _ver] -numeric_version = tuple(_ver) +__appname__ = u'calibre' +numeric_version = (0, 7, 56) +__version__ = u'.'.join(map(unicode, numeric_version)) +__author__ = u"Kovid Goyal " ''' Various run time constants. ''' -import sys, locale, codecs, os -from calibre.utils.terminfo import TerminalController +import sys, locale, codecs, os, importlib, collections -terminal_controller = TerminalController(sys.stdout) +_tc = None +def terminal_controller(): + global _tc + if _tc is None: + from calibre.utils.terminfo import TerminalController + _tc = TerminalController(sys.stdout) + return _tc -iswindows = 'win32' in sys.platform.lower() or 'win64' in sys.platform.lower() -isosx = 'darwin' in sys.platform.lower() -isnewosx = isosx and getattr(sys, 'new_app_bundle', False) -isfreebsd = 'freebsd' in sys.platform.lower() +_plat = sys.platform.lower() +iswindows = 'win32' in _plat or 'win64' in _plat +isosx = 'darwin' in _plat +isnewosx = isosx and getattr(sys, 'new_app_bundle', False) +isfreebsd = 'freebsd' in _plat islinux = not(iswindows or isosx or isfreebsd) isfrozen = hasattr(sys, 'frozen') isunix = isosx or islinux @@ -48,15 +52,12 @@ def debug(): DEBUG = True # plugins {{{ -plugins = None -if plugins is None: - # Load plugins - def load_plugins(): - plugins = {} - plugin_path = sys.extensions_location - sys.path.insert(0, plugin_path) - for plugin in [ +class Plugins(collections.Mapping): + + def __init__(self): + self._plugins = {} + plugins = [ 'pictureflow', 'lzx', 'msdes', @@ -70,19 +71,44 @@ if plugins is None: 'chm_extra', 'icu', 'speedup', - ] + \ - (['winutil'] if iswindows else []) + \ - (['usbobserver'] if isosx else []): - try: - p, err = importlib.import_module(plugin), '' - except Exception as err: - p = None - err = str(err) - plugins[plugin] = (p, err) - sys.path.remove(plugin_path) - return plugins + ] + if iswindows: + plugins.append('winutil') + if isosx: + plugins.append('usbobserver') + self.plugins = frozenset(plugins) - plugins = load_plugins() + def load_plugin(self, name): + if name in self._plugins: + return + sys.path.insert(0, sys.extensions_location) + try: + p, err = importlib.import_module(name), '' + except Exception as err: + p = None + err = str(err) + self._plugins[name] = (p, err) + sys.path.remove(sys.extensions_location) + + def __iter__(self): + return iter(self.plugins) + + def __len__(self): + return len(self.plugins) + + def __contains__(self, name): + return name in self.plugins + + def __getitem__(self, name): + if name not in self.plugins: + raise KeyError('No plugin named %r'%name) + self.load_plugin(name) + return self._plugins[name] + + +plugins = None +if plugins is None: + plugins = Plugins() # }}} # config_dir {{{ diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 75c02c7e00..00af4e5117 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -9,7 +9,6 @@ from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \ from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf -from calibre.ebooks.oeb.base import OEB_IMAGES from calibre.utils.config import test_eight_code # To archive plugins {{{ @@ -98,6 +97,8 @@ class TXT2TXTZ(FileTypePlugin): on_import = True def _get_image_references(self, txt, base_dir): + from calibre.ebooks.oeb.base import OEB_IMAGES + images = [] # Textile diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 86a0477811..8d65c37bbf 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -106,7 +106,7 @@ def migrate(old, new): from calibre.library.database import LibraryDatabase from calibre.library.database2 import LibraryDatabase2 from calibre.utils.terminfo import ProgressBar - from calibre import terminal_controller + from calibre.constants import terminal_controller class Dummy(ProgressBar): def setLabelText(self, x): pass def setAutoReset(self, y): pass @@ -119,7 +119,7 @@ def migrate(old, new): db = LibraryDatabase(old) db2 = LibraryDatabase2(new) - db2.migrate_old(db, Dummy(terminal_controller, 'Migrating database...')) + db2.migrate_old(db, Dummy(terminal_controller(), 'Migrating database...')) prefs['library_path'] = os.path.abspath(new) print 'Database migrated to', os.path.abspath(new) diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index c63eada0c8..9b729a3561 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -8,7 +8,7 @@ manner. import sys, os, re from threading import RLock -from calibre import iswindows, isosx, plugins, islinux +from calibre.constants import iswindows, isosx, plugins, islinux osx_scanner = win_scanner = linux_scanner = None diff --git a/src/calibre/ebooks/chm/reader.py b/src/calibre/ebooks/chm/reader.py index 7c9a6bf48a..24814a34f9 100644 --- a/src/calibre/ebooks/chm/reader.py +++ b/src/calibre/ebooks/chm/reader.py @@ -5,8 +5,8 @@ __copyright__ = '2008, Kovid Goyal ,' \ ' and Alex Bramley .' import os, re -from mimetypes import guess_type as guess_mimetype +from calibre import guess_type as guess_mimetype from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString from calibre.constants import iswindows, filesystem_encoding from calibre.utils.chm.chm import CHMFile diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index b26befe075..96ea3e5884 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -14,7 +14,8 @@ from calibre.ebooks.conversion.preprocess import HTMLPreProcessor from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.date import parse_date from calibre.utils.zipfile import ZipFile -from calibre import extract, walk, isbytestring, filesystem_encoding +from calibre import (extract, walk, isbytestring, filesystem_encoding, + get_types_map) from calibre.constants import __version__ DEBUG_README=u''' @@ -875,6 +876,9 @@ OptionRecommendation(name='sr3_replace', if self.opts.verbose: self.log.filter_level = self.log.DEBUG self.flush() + import cssutils, logging + cssutils.log.setLevel(logging.WARN) + get_types_map() # Ensure the mimetypes module is intialized if self.opts.debug_pipeline is not None: self.opts.verbose = max(self.opts.verbose, 4) diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 8d1164e026..b45f8f9f9e 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -10,7 +10,6 @@ Transform OEB content into FB2 markup from base64 import b64encode from datetime import datetime -from mimetypes import types_map import re import uuid @@ -18,9 +17,6 @@ from lxml import etree from calibre import prepare_string_for_xml from calibre.constants import __appname__, __version__ -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace -from calibre.ebooks.oeb.stylizer import Stylizer -from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES, OPF from calibre.utils.magick import Image class FB2MLizer(object): @@ -71,7 +67,7 @@ class FB2MLizer(object): return u'' + output def clean_text(self, text): - # Condense empty paragraphs into a line break. + # Condense empty paragraphs into a line break. text = re.sub(r'(?miu)(

\s*

\s*){3,}', '', text) # Remove empty paragraphs. text = re.sub(r'(?miu)

\s*

', '', text) @@ -100,6 +96,7 @@ class FB2MLizer(object): return text def fb2_header(self): + from calibre.ebooks.oeb.base import OPF metadata = {} metadata['title'] = self.oeb_book.metadata.title[0].value metadata['appname'] = __appname__ @@ -180,6 +177,8 @@ class FB2MLizer(object): return u'' def get_cover(self): + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES + cover_href = None # Get the raster cover if it's available. @@ -213,6 +212,8 @@ class FB2MLizer(object): return u'' def get_text(self): + from calibre.ebooks.oeb.base import XHTML + from calibre.ebooks.oeb.stylizer import Stylizer text = [''] # Create main section if there are no others to create @@ -248,6 +249,8 @@ class FB2MLizer(object): ''' This function uses the self.image_hrefs dictionary mapping. It is populated by the dump_text function. ''' + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES + images = [] for item in self.oeb_book.manifest: # Don't write the image if it's not referenced in the document's text. @@ -255,7 +258,7 @@ class FB2MLizer(object): continue if item.media_type in OEB_RASTER_IMAGES: try: - if not item.media_type == types_map['.jpeg'] or not item.media_type == types_map['.jpg']: + if item.media_type != 'image/jpeg': im = Image() im.load(item.data) im.set_compression_quality(70) @@ -344,6 +347,8 @@ class FB2MLizer(object): @return: List of string representing the XHTML converted to FB2 markup. ''' + from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace + # Ensure what we are converting is not a string and that the fist tag is part of the XHTML namespace. if not isinstance(elem_tree.tag, basestring) or namespace(elem_tree.tag) != XHTML_NS: return [] diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index dd0a247a67..079e990de3 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -315,7 +315,8 @@ class HTMLInput(InputFormatPlugin): from calibre import guess_type from calibre.ebooks.oeb.transforms.metadata import \ meta_info_to_oeb_metadata - import cssutils + import cssutils, logging + cssutils.log.setLevel(logging.WARN) self.OEB_STYLES = OEB_STYLES oeb = create_oebbook(log, None, opts, self, encoding=opts.input_encoding, populate=False) diff --git a/src/calibre/ebooks/html/meta.py b/src/calibre/ebooks/html/meta.py index 9a088efb16..07cf9236fc 100644 --- a/src/calibre/ebooks/html/meta.py +++ b/src/calibre/ebooks/html/meta.py @@ -4,7 +4,6 @@ __copyright__ = '2010, Fabian Grassl ' __docformat__ = 'restructuredtext en' -from calibre.ebooks.oeb.base import namespace, barename, DC11_NS class EasyMeta(object): @@ -12,6 +11,7 @@ class EasyMeta(object): self.meta = meta def __iter__(self): + from calibre.ebooks.oeb.base import namespace, barename, DC11_NS meta = self.meta for item_name in meta.items: for item in meta[item_name]: diff --git a/src/calibre/ebooks/html/output.py b/src/calibre/ebooks/html/output.py index 5c984162ac..fe7b4cf274 100644 --- a/src/calibre/ebooks/html/output.py +++ b/src/calibre/ebooks/html/output.py @@ -12,7 +12,6 @@ from os.path import dirname, abspath, relpath, exists, basename from lxml import etree from templite import Templite -from calibre.ebooks.oeb.base import element from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation from calibre import CurrentDir from calibre.ptempfile import PersistentTemporaryDirectory @@ -51,6 +50,7 @@ class HTMLOutput(OutputFormatPlugin): ''' Generate table of contents ''' + from calibre.ebooks.oeb.base import element with CurrentDir(output_dir): def build_node(current_node, parent=None): if parent is None: diff --git a/src/calibre/ebooks/htmlz/output.py b/src/calibre/ebooks/htmlz/output.py index 03fe12c89e..6d2ad54a12 100644 --- a/src/calibre/ebooks/htmlz/output.py +++ b/src/calibre/ebooks/htmlz/output.py @@ -12,7 +12,6 @@ from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME from calibre.ptempfile import TemporaryDirectory from calibre.utils.zipfile import ZipFile @@ -42,6 +41,8 @@ class HTMLZOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME + # HTML if opts.htmlz_css_type == 'inline': from calibre.ebooks.htmlz.oeb2html import OEB2HTMLInlineCSSizer @@ -72,7 +73,7 @@ class HTMLZOutput(OutputFormatPlugin): for item in oeb_book.manifest: if item.media_type in OEB_IMAGES and item.href in images: if item.media_type == SVG_MIME: - data = unicode(etree.tostring(item.data, encoding=unicode)) + data = unicode(etree.tostring(item.data, encoding=unicode)) else: data = item.data fname = os.path.join(tdir, 'images', images[item.href]) diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 6078a0aa94..2ae5f3ade5 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -6,11 +6,11 @@ __docformat__ = 'restructuredtext en' """ Provides abstraction for metadata reading.writing from a variety of ebook formats. """ -import os, mimetypes, sys, re +import os, sys, re from urllib import unquote, quote from urlparse import urlparse -from calibre import relpath +from calibre import relpath, guess_type from calibre.utils.config import tweaks @@ -118,7 +118,7 @@ class Resource(object): self.path = None self.fragment = '' try: - self.mime_type = mimetypes.guess_type(href_or_path)[0] + self.mime_type = guess_type(href_or_path)[0] except: self.mime_type = None if self.mime_type is None: diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 167ae52fa3..faac8e98b1 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -592,7 +592,7 @@ class Metadata(object): elif datatype == 'bool': res = _('Yes') if res else _('No') elif datatype == 'rating': - res = res/2 + res = res/2.0 return (name, unicode(res), orig_res, cmeta) # convert top-level ids into their value @@ -625,6 +625,8 @@ class Metadata(object): res = res + ' [%s]'%self.format_series_index() elif datatype == 'datetime': res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy')) + elif datatype == 'rating': + res = res/2.0 return (name, unicode(res), orig_res, fmeta) return (None, None, None, None) diff --git a/src/calibre/ebooks/metadata/fb2.py b/src/calibre/ebooks/metadata/fb2.py index 2d6192f949..21f15b05ae 100644 --- a/src/calibre/ebooks/metadata/fb2.py +++ b/src/calibre/ebooks/metadata/fb2.py @@ -5,11 +5,12 @@ __copyright__ = '2008, Anatoly Shipitsin ' '''Read meta information from fb2 files''' -import mimetypes, os +import os from base64 import b64decode from lxml import etree from calibre.ebooks.metadata import MetaInformation from calibre.ebooks.chardet import xml_to_unicode +from calibre import guess_all_extensions XLINK_NS = 'http://www.w3.org/1999/xlink' def XLINK(name): @@ -71,7 +72,7 @@ def get_metadata(stream): binary = XPath('//fb2:binary[@id="%s"]'%id)(root) if binary: mt = binary[0].get('content-type', 'image/jpeg') - exts = mimetypes.guess_all_extensions(mt) + exts = guess_all_extensions(mt) if not exts: exts = ['.jpg'] cdata = (exts[0][1:], b64decode(tostring(binary[0]))) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index d360451e2e..58c887bfdb 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' lxml based OPF parser. ''' -import re, sys, unittest, functools, os, mimetypes, uuid, glob, cStringIO, json +import re, sys, unittest, functools, os, uuid, glob, cStringIO, json from urllib import unquote from urlparse import urlparse @@ -20,7 +20,7 @@ from calibre.ebooks.metadata import string_to_authors, MetaInformation, check_is from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.date import parse_date, isoformat from calibre.utils.localization import get_lang -from calibre import prints +from calibre import prints, guess_type from calibre.utils.cleantext import clean_ascii_chars class Resource(object): # {{{ @@ -42,7 +42,7 @@ class Resource(object): # {{{ self.path = None self.fragment = '' try: - self.mime_type = mimetypes.guess_type(href_or_path)[0] + self.mime_type = guess_type(href_or_path)[0] except: self.mime_type = None if self.mime_type is None: @@ -1000,7 +1000,7 @@ class OPF(object): # {{{ for t in ('cover', 'other.ms-coverimage-standard', 'other.ms-coverimage'): for item in self.guide: if item.type.lower() == t: - self.create_manifest_item(item.href(), mimetypes.guess_type(path)[0]) + self.create_manifest_item(item.href(), guess_type(path)[0]) return property(fget=fget, fset=fset) diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index 6295efa0c0..4d21a0c210 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -382,7 +382,11 @@ def identify(log, abort, # {{{ log(plog) log('\n'+'*'*80) + dummy = Metadata(_('Unknown')) for i, result in enumerate(presults): + for f in plugin.prefs['ignore_fields']: + if ':' not in f: + setattr(result, f, getattr(dummy, f)) result.relevance_in_source = i result.has_cached_cover_url = (plugin.cached_cover_url_is_reliable and plugin.get_cached_cover_url(result.identifiers) is not diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index c8a2dc6d8b..dc28a6ced5 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' ''' Fetch metadata using Overdrive Content Reserve ''' -import re, random, mechanize, copy +import re, random, mechanize, copy, json from threading import RLock from Queue import Queue, Empty @@ -40,10 +40,6 @@ class OverDrive(Source): supports_gzip_transfer_encoding = False cached_cover_url_is_reliable = True - def __init__(self, *args, **kwargs): - Source.__init__(self, *args, **kwargs) - self.prefs.defaults['ignore_fields'] =['tags', 'pubdate', 'comments', 'identifier:isbn', 'language'] - def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ identifiers={}, timeout=30): ovrdrv_id = identifiers.get('overdrive', None) @@ -231,7 +227,7 @@ class OverDrive(Source): def sort_ovrdrv_results(self, raw, title=None, title_tokens=None, author=None, author_tokens=None, ovrdrv_id=None): close_matches = [] raw = re.sub('.*?\[\[(?P.*?)\]\].*', '[[\g]]', raw) - results = eval(raw) + results = json.loads(raw) #print results # The search results are either from a keyword search or a multi-format list from a single ID, # sort through the results for closest match/format @@ -393,9 +389,14 @@ class OverDrive(Source): if pub_date: from calibre.utils.date import parse_date - mi.pubdate = parse_date(pub_date[0].strip()) + try: + mi.pubdate = parse_date(pub_date[0].strip()) + except: + pass if lang: - mi.language = lang[0].strip() + lang = lang[0].strip().lower() + mi.language = {'english':'en', 'french':'fr', 'german':'de', + 'spanish':'es'}.get(lang, None) if ebook_isbn: #print "ebook isbn is "+str(ebook_isbn[0]) diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index 58083f807f..f2c9696976 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -8,23 +8,18 @@ __copyright__ = '2008, Marshall T. Vandegrift ' __docformat__ = 'restructuredtext en' import os, re, uuid, logging -from mimetypes import types_map from collections import defaultdict from itertools import count from urlparse import urldefrag, urlparse, urlunparse, urljoin from urllib import unquote as urlunquote from lxml import etree, html -from cssutils import CSSParser, parseString, parseStyle, replaceUrls -from cssutils.css import CSSRule - -import calibre -from calibre.constants import filesystem_encoding +from calibre.constants import filesystem_encoding, __version__ from calibre.translations.dynamic import translate from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.oeb.entitydefs import ENTITYDEFS from calibre.ebooks.conversion.preprocess import CSSPreProcessor -from calibre import isbytestring, as_unicode +from calibre import isbytestring, as_unicode, get_types_map RECOVER_PARSER = etree.XMLParser(recover=True, no_network=True) @@ -179,6 +174,9 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False): If the ``link_repl_func`` returns None, the attribute or tag text will be removed completely. ''' + from cssutils import parseString, parseStyle, replaceUrls, log + log.setLevel(logging.WARN) + if resolve_base_href: resolve_base_href(root) for el, attrib, link, pos in iterlinks(root, find_links_in_css=False): @@ -248,7 +246,7 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False): el.attrib['style'] = repl - +types_map = get_types_map() EPUB_MIME = types_map['.epub'] XHTML_MIME = types_map['.xhtml'] CSS_MIME = types_map['.css'] @@ -1075,7 +1073,9 @@ class Manifest(object): def _parse_css(self, data): - + from cssutils.css import CSSRule + from cssutils import CSSParser, log + log.setLevel(logging.WARN) def get_style_rules_from_import(import_rule): ans = [] if not import_rule.styleSheet: @@ -2011,7 +2011,7 @@ class OEBBook(object): name='dtb:uid', content=unicode(self.uid)) etree.SubElement(head, NCX('meta'), name='dtb:depth', content=str(self.toc.depth())) - generator = ''.join(['calibre (', calibre.__version__, ')']) + generator = ''.join(['calibre (', __version__, ')']) etree.SubElement(head, NCX('meta'), name='dtb:generator', content=generator) etree.SubElement(head, NCX('meta'), diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index ebc2f30d00..6c10436038 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -10,11 +10,9 @@ import sys, os, uuid, copy, re, cStringIO from itertools import izip from urlparse import urldefrag, urlparse from urllib import unquote as urlunquote -from mimetypes import guess_type from collections import defaultdict from lxml import etree -import cssutils from calibre.ebooks.oeb.base import OPF1_NS, OPF2_NS, OPF2_NSMAP, DC11_NS, \ DC_NSES, OPF, xml2text @@ -30,6 +28,7 @@ from calibre.ebooks.oeb.entitydefs import ENTITYDEFS from calibre.utils.localization import get_lang from calibre.ptempfile import TemporaryDirectory from calibre.constants import __appname__, __version__ +from calibre import guess_type __all__ = ['OEBReader'] @@ -172,6 +171,7 @@ class OEBReader(object): return bad def _manifest_add_missing(self, invalid): + import cssutils manifest = self.oeb.manifest known = set(manifest.hrefs) unchecked = set(manifest.values()) diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index 634f7f5fce..39ab41eede 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -12,17 +12,18 @@ import os, itertools, re, logging, copy, unicodedata from weakref import WeakKeyDictionary from xml.dom import SyntaxErr as CSSSyntaxError import cssutils -from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \ - CSSValueList, CSSFontFaceRule, cssproperties +from cssutils.css import (CSSStyleRule, CSSPageRule, CSSStyleDeclaration, + CSSValueList, CSSFontFaceRule, cssproperties) from cssutils import profile as cssprofiles from lxml import etree from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError - from calibre import force_unicode from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES from calibre.ebooks.oeb.base import XPNSMAP, xpath, urlnormalize from calibre.ebooks.oeb.profile import PROFILES +cssutils.log.setLevel(logging.WARN) + _html_css_stylesheet = None def html_css_stylesheet(): diff --git a/src/calibre/ebooks/oeb/transforms/filenames.py b/src/calibre/ebooks/oeb/transforms/filenames.py index bad75b9a6f..c3c7f091c3 100644 --- a/src/calibre/ebooks/oeb/transforms/filenames.py +++ b/src/calibre/ebooks/oeb/transforms/filenames.py @@ -9,7 +9,6 @@ import posixpath from urlparse import urldefrag, urlparse from lxml import etree -import cssutils from calibre.ebooks.oeb.base import rewrite_links, urlnormalize @@ -25,6 +24,7 @@ class RenameFiles(object): # {{{ self.renamed_items_map = renamed_items_map def __call__(self, oeb, opts): + import cssutils self.log = oeb.logger self.opts = opts self.oeb = oeb diff --git a/src/calibre/ebooks/oeb/transforms/trimmanifest.py b/src/calibre/ebooks/oeb/transforms/trimmanifest.py index 0baacfd1f9..95501dbb9b 100644 --- a/src/calibre/ebooks/oeb/transforms/trimmanifest.py +++ b/src/calibre/ebooks/oeb/transforms/trimmanifest.py @@ -8,8 +8,6 @@ __copyright__ = '2008, Marshall T. Vandegrift ' from urlparse import urldefrag -import cssutils - from calibre.ebooks.oeb.base import CSS_MIME, OEB_DOCS from calibre.ebooks.oeb.base import urlnormalize, iterlinks @@ -23,6 +21,7 @@ class ManifestTrimmer(object): return cls() def __call__(self, oeb, context): + import cssutils oeb.logger.info('Trimming unused files from manifest...') self.opts = context used = set() diff --git a/src/calibre/ebooks/pdb/ereader/writer.py b/src/calibre/ebooks/pdb/ereader/writer.py index 4fbd343a6b..eb023c594b 100644 --- a/src/calibre/ebooks/pdb/ereader/writer.py +++ b/src/calibre/ebooks/pdb/ereader/writer.py @@ -21,7 +21,6 @@ except ImportError: import cStringIO from calibre.ebooks.pdb.formatwriter import FormatWriter -from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES from calibre.ebooks.pdb.header import PdbHeaderBuilder from calibre.ebooks.pml.pmlml import PMLMLizer @@ -135,6 +134,7 @@ class Writer(FormatWriter): 62-...: Raw image data in 8 bit PNG format. ''' images = [] + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES for item in manifest: if item.media_type in OEB_RASTER_IMAGES and item.href in image_hrefs.keys(): diff --git a/src/calibre/ebooks/pml/output.py b/src/calibre/ebooks/pml/output.py index 9d2ddc6ca6..63d8a8b220 100644 --- a/src/calibre/ebooks/pml/output.py +++ b/src/calibre/ebooks/pml/output.py @@ -18,7 +18,6 @@ from calibre.customize.conversion import OutputFormatPlugin from calibre.customize.conversion import OptionRecommendation from calibre.ptempfile import TemporaryDirectory from calibre.utils.zipfile import ZipFile -from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES from calibre.ebooks.pml.pmlml import PMLMLizer class PMLOutput(OutputFormatPlugin): @@ -60,6 +59,7 @@ class PMLOutput(OutputFormatPlugin): pmlz.add_dir(tdir) def write_images(self, manifest, image_hrefs, out_dir, opts): + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES for item in manifest: if item.media_type in OEB_RASTER_IMAGES and item.href in image_hrefs.keys(): if opts.full_image_depth: diff --git a/src/calibre/ebooks/pml/pmlml.py b/src/calibre/ebooks/pml/pmlml.py index 779e75d713..b04aaacaec 100644 --- a/src/calibre/ebooks/pml/pmlml.py +++ b/src/calibre/ebooks/pml/pmlml.py @@ -12,8 +12,6 @@ import re from lxml import etree -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace -from calibre.ebooks.oeb.stylizer import Stylizer from calibre.ebooks.pdb.ereader import image_name from calibre.ebooks.pml import unipmlcode @@ -110,6 +108,9 @@ class PMLMLizer(object): return output def get_cover_page(self): + from calibre.ebooks.oeb.stylizer import Stylizer + from calibre.ebooks.oeb.base import XHTML + output = u'' if 'cover' in self.oeb_book.guide: output += '\\m="cover.png"\n' @@ -125,6 +126,9 @@ class PMLMLizer(object): return output def get_text(self): + from calibre.ebooks.oeb.stylizer import Stylizer + from calibre.ebooks.oeb.base import XHTML + text = [u''] for item in self.oeb_book.spine: self.log.debug('Converting %s to PML markup...' % item.href) @@ -180,7 +184,7 @@ class PMLMLizer(object): links = set(re.findall(r'(?<=\\q="#).+?(?=")', text)) for unused in anchors.difference(links): text = text.replace('\\Q="%s"' % unused, '') - + # Remove \Cn tags that are within \x and \Xn tags text = re.sub(ur'(?msu)(?P\\(x|X[0-4]))(?P.*?)(?P\\C[0-4]\s*=\s*"[^"]*")(?P.*?)(?P=t)', '\g\g\g\g', text) @@ -214,6 +218,8 @@ class PMLMLizer(object): return text def dump_text(self, elem, stylizer, page, tag_stack=[]): + from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace + if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: return [] diff --git a/src/calibre/ebooks/rb/rbml.py b/src/calibre/ebooks/rb/rbml.py index 50153d7d4d..8cf63e334c 100644 --- a/src/calibre/ebooks/rb/rbml.py +++ b/src/calibre/ebooks/rb/rbml.py @@ -11,8 +11,6 @@ Transform OEB content into RB compatible markup. import re from calibre import prepare_string_for_xml -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace -from calibre.ebooks.oeb.stylizer import Stylizer from calibre.ebooks.rb import unique_name TAGS = [ @@ -81,6 +79,8 @@ class RBMLizer(object): return output def get_cover_page(self): + from calibre.ebooks.oeb.stylizer import Stylizer + from calibre.ebooks.oeb.base import XHTML output = u'' if 'cover' in self.oeb_book.guide: if self.name_map.get(self.oeb_book.guide['cover'].href, None): @@ -109,6 +109,9 @@ class RBMLizer(object): return ''.join(toc) def get_text(self): + from calibre.ebooks.oeb.stylizer import Stylizer + from calibre.ebooks.oeb.base import XHTML + output = [u''] for item in self.oeb_book.spine: self.log.debug('Converting %s to RocketBook HTML...' % item.href) @@ -137,6 +140,8 @@ class RBMLizer(object): return text def dump_text(self, elem, stylizer, page, tag_stack=[]): + from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace + if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: return [u''] diff --git a/src/calibre/ebooks/rb/writer.py b/src/calibre/ebooks/rb/writer.py index c8908ee95f..f71b103fbd 100644 --- a/src/calibre/ebooks/rb/writer.py +++ b/src/calibre/ebooks/rb/writer.py @@ -18,7 +18,6 @@ import cStringIO from calibre.ebooks.rb.rbml import RBMLizer from calibre.ebooks.rb import HEADER from calibre.ebooks.rb import unique_name -from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES from calibre.constants import __appname__, __version__ TEXT_RECORD_SIZE = 4096 @@ -111,6 +110,7 @@ class RBWriter(object): return (size, pages) def _images(self, manifest): + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES images = [] used_names = [] diff --git a/src/calibre/ebooks/rtf/rtfml.py b/src/calibre/ebooks/rtf/rtfml.py index f739207018..97fa175d1a 100644 --- a/src/calibre/ebooks/rtf/rtfml.py +++ b/src/calibre/ebooks/rtf/rtfml.py @@ -14,9 +14,6 @@ import cStringIO from lxml import etree -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace, \ - OEB_RASTER_IMAGES -from calibre.ebooks.oeb.stylizer import Stylizer from calibre.ebooks.metadata import authors_to_string from calibre.utils.filenames import ascii_text from calibre.utils.magick.draw import save_cover_data_to, identify_data @@ -100,6 +97,8 @@ class RTFMLizer(object): return self.mlize_spine() def mlize_spine(self): + from calibre.ebooks.oeb.base import XHTML + from calibre.ebooks.oeb.stylizer import Stylizer output = self.header() if 'titlepage' in self.oeb_book.guide: href = self.oeb_book.guide['titlepage'].href @@ -154,6 +153,8 @@ class RTFMLizer(object): return ' }' def insert_images(self, text): + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES + for item in self.oeb_book.manifest: if item.media_type in OEB_RASTER_IMAGES: src = os.path.basename(item.href) @@ -201,6 +202,8 @@ class RTFMLizer(object): return text def dump_text(self, elem, stylizer, tag_stack=[]): + from calibre.ebooks.oeb.base import XHTML_NS, namespace, barename + if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: p = elem.getparent() diff --git a/src/calibre/ebooks/snb/input.py b/src/calibre/ebooks/snb/input.py index 100ac1447f..13b1ca45f9 100755 --- a/src/calibre/ebooks/snb/input.py +++ b/src/calibre/ebooks/snb/input.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' import os, uuid from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.oeb.base import DirContainer from calibre.ebooks.snb.snbfile import SNBFile from calibre.ptempfile import TemporaryDirectory from calibre.utils.filenames import ascii_filename @@ -30,6 +29,7 @@ class SNBInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.oeb.base import DirContainer log.debug("Parsing SNB file...") snbFile = SNBFile() try: diff --git a/src/calibre/ebooks/snb/snbfile.py b/src/calibre/ebooks/snb/snbfile.py index 1a0986baf4..be4e537825 100644 --- a/src/calibre/ebooks/snb/snbfile.py +++ b/src/calibre/ebooks/snb/snbfile.py @@ -5,7 +5,8 @@ __copyright__ = '2010, Li Fanxi ' __docformat__ = 'restructuredtext en' import sys, struct, zlib, bz2, os -from mimetypes import types_map + +from calibre import guess_type class FileStream: def IsBinary(self): @@ -180,7 +181,7 @@ class SNBFile: file = open(os.path.join(path, fname), 'wb') file.write(f.fileBody) file.close() - fileNames.append((fname, types_map[ext])) + fileNames.append((fname, guess_type('a'+ext)[0])) return fileNames def Output(self, outputFile): diff --git a/src/calibre/ebooks/snb/snbml.py b/src/calibre/ebooks/snb/snbml.py index 078e7ebe76..a501de1ff0 100644 --- a/src/calibre/ebooks/snb/snbml.py +++ b/src/calibre/ebooks/snb/snbml.py @@ -13,8 +13,6 @@ import re from lxml import etree -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace -from calibre.ebooks.oeb.stylizer import Stylizer def ProcessFileName(fileName): # Flat the path @@ -81,6 +79,8 @@ class SNBMLizer(object): body.append(entity) def mlize(self): + from calibre.ebooks.oeb.base import XHTML + from calibre.ebooks.oeb.stylizer import Stylizer output = [ u'' ] stylizer = Stylizer(self.item.data, self.item.href, self.oeb_book, self.opts, self.opts.output_profile) content = unicode(etree.tostring(self.item.data.find(XHTML('body')), encoding=unicode)) @@ -208,6 +208,7 @@ class SNBMLizer(object): return text def dump_text(self, subitems, elem, stylizer, end='', pre=False, li = ''): + from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: diff --git a/src/calibre/ebooks/txt/output.py b/src/calibre/ebooks/txt/output.py index 4e54a97b45..ac63690996 100644 --- a/src/calibre/ebooks/txt/output.py +++ b/src/calibre/ebooks/txt/output.py @@ -11,7 +11,6 @@ from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.oeb.base import OEB_IMAGES from calibre.ebooks.txt.txtml import TXTMLizer from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines from calibre.ptempfile import TemporaryDirectory, TemporaryFile @@ -103,12 +102,13 @@ class TXTOutput(OutputFormatPlugin): class TXTZOutput(TXTOutput): - + name = 'TXTZ Output' author = 'John Schember' file_type = 'txtz' def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.oeb.base import OEB_IMAGES with TemporaryDirectory('_txtz_output') as tdir: # TXT with TemporaryFile('index.txt') as tf: @@ -123,10 +123,10 @@ class TXTZOutput(TXTOutput): os.makedirs(path) with open(os.path.join(tdir, item.href), 'wb') as imgf: imgf.write(item.data) - + # Metadata - with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: + with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: mdataf.write(etree.tostring(oeb_book.metadata.to_opf1())) - + txtz = ZipFile(output_path, 'w') txtz.add_dir(tdir) diff --git a/src/calibre/ebooks/txt/txtml.py b/src/calibre/ebooks/txt/txtml.py index fa7bfbb380..2320fbbbc7 100644 --- a/src/calibre/ebooks/txt/txtml.py +++ b/src/calibre/ebooks/txt/txtml.py @@ -12,8 +12,6 @@ import re from lxml import etree -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace -from calibre.ebooks.oeb.stylizer import Stylizer BLOCK_TAGS = [ 'div', @@ -58,12 +56,14 @@ class TXTMLizer(object): self.toc_titles = [] self.toc_ids = [] self.last_was_heading = False - + self.create_flat_toc(self.oeb_book.toc) return self.mlize_spine() def mlize_spine(self): + from calibre.ebooks.oeb.base import XHTML + from calibre.ebooks.oeb.stylizer import Stylizer output = [u''] output.append(self.get_toc()) for item in self.oeb_book.spine: @@ -139,7 +139,7 @@ class TXTMLizer(object): # when remove paragraph spacing is enabled. text = re.sub('(?imu)^[ ]+', '', text) text = re.sub('(?imu)[ ]+$', '', text) - + # Remove empty space and newlines at the beginning of the document. text = re.sub(r'(?u)^[ \n]+', '', text) @@ -185,6 +185,7 @@ class TXTMLizer(object): @stylizer: The style information attached to the element. @page: OEB page used to determine absolute urls. ''' + from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 773aea3002..de066359ed 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -4,19 +4,17 @@ __copyright__ = '2008, Kovid Goyal ' import os, sys, Queue, threading from threading import RLock from urllib import unquote - -from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \ - QByteArray, QTranslator, QCoreApplication, QThread, \ - QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, \ - QFileDialog, QFileIconProvider, \ - QIcon, QApplication, QDialog, QUrl, QFont +from PyQt4.Qt import (QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, + QByteArray, QTranslator, QCoreApplication, QThread, + QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, + QFileDialog, QFileIconProvider, + QIcon, QApplication, QDialog, QUrl, QFont) ORG_NAME = 'KovidsBrain' APP_UID = 'libprs500' from calibre.constants import islinux, iswindows, isfreebsd, isfrozen, isosx from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig from calibre.utils.localization import set_qt_translator -from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats from calibre.ebooks.metadata import MetaInformation from calibre.utils.date import UNDEFINED_DATE @@ -156,7 +154,9 @@ def _config(): c.add_opt('plugin_search_history', default=[], help='Search history for the recipe scheduler') c.add_opt('worker_limit', default=6, - help=_('Maximum number of waiting worker processes')) + help=_( + 'Maximum number of simultaneous conversion/news download jobs. ' + 'This number is twice the actual value for historical reasons.')) c.add_opt('get_social_metadata', default=True, help=_('Download social metadata (tags/rating/etc.)')) c.add_opt('overwrite_author_title_metadata', default=True, @@ -330,6 +330,7 @@ class GetMetadata(QObject): id, args, kwargs) def _from_formats(self, id, args, kwargs): + from calibre.ebooks.metadata.meta import metadata_from_formats try: mi = metadata_from_formats(*args, **kwargs) except: @@ -337,6 +338,7 @@ class GetMetadata(QObject): self.emit(SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'), id, mi) def _get_metadata(self, id, args, kwargs): + from calibre.ebooks.metadata.meta import get_metadata try: mi = get_metadata(*args, **kwargs) except: @@ -738,3 +740,4 @@ def build_forms(srcdir, info=None): _df = os.environ.get('CALIBRE_DEVELOP_FROM', None) if _df and os.path.exists(_df): build_forms(_df) + diff --git a/src/calibre/gui2/actions/annotate.py b/src/calibre/gui2/actions/annotate.py index 48397936fb..f934a4a53c 100644 --- a/src/calibre/gui2/actions/annotate.py +++ b/src/calibre/gui2/actions/annotate.py @@ -22,7 +22,7 @@ class FetchAnnotationsAction(InterfaceAction): action_type = 'current' def genesis(self): - pass + self.qaction.triggered.connect(self.fetch_annotations) def fetch_annotations(self, *args): # Generate a path_map from selected ids @@ -52,6 +52,10 @@ class FetchAnnotationsAction(InterfaceAction): return path_map device = self.gui.device_manager.device + if not getattr(device, 'SUPPORTS_ANNOTATIONS', False): + return error_dialog(self.gui, _('Not supported'), + _('Fetching annotations is not supported for this device'), + show=True) if self.gui.current_view() is not self.gui.library_view: return error_dialog(self.gui, _('Use library only'), diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 99cb5848ba..49542abdc1 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -7,7 +7,7 @@ import os, traceback, Queue, time, cStringIO, re, sys from threading import Thread from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, \ - Qt, pyqtSignal, QDialog + Qt, pyqtSignal, QDialog, QObject from calibre.customize.ui import available_input_formats, available_output_formats, \ device_plugins @@ -25,7 +25,6 @@ from calibre.devices.errors import FreeSpaceError from calibre.devices.apple.driver import ITUNES_ASYNC from calibre.devices.folder_device.driver import FOLDER_DEVICE from calibre.devices.bambook.driver import BAMBOOK, BAMBOOKWifi -from calibre.ebooks.metadata.meta import set_metadata from calibre.constants import DEBUG from calibre.utils.config import prefs, tweaks from calibre.utils.magick.draw import thumbnail @@ -334,6 +333,7 @@ class DeviceManager(Thread): # {{{ def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None): '''Upload books to device: ''' + from calibre.ebooks.metadata.meta import set_metadata if hasattr(self.connected_device, 'set_plugboards') and \ callable(self.connected_device.set_plugboards): self.connected_device.set_plugboards(plugboards, find_plugboard) @@ -587,18 +587,26 @@ class DeviceMenu(QMenu): # {{{ # }}} -class DeviceMixin(object): # {{{ - +class DeviceSignals(QObject): #: This signal is emitted once, after metadata is downloaded from the #: connected device. #: The sequence: gui.device_manager.is_device_connected will become True, + #: and the device_connection_changed signal will be emitted, #: then sometime later gui.device_metadata_available will be signaled. #: This does not mean that there are no more jobs running. Automatic metadata #: management might have kicked off a sync_booklists to write new metadata onto #: the device, and that job might still be running when the signal is emitted. device_metadata_available = pyqtSignal() + + #: This signal is emitted once when the device is detected and once when + #: it is disconnected. If the parameter is True, then it is a connection, + #: otherwise a disconnection. device_connection_changed = pyqtSignal(object) +device_signals = DeviceSignals() + +class DeviceMixin(object): # {{{ + def __init__(self): self.device_error_dialog = error_dialog(self, _('Error'), _('Error communicating with device'), ' ') @@ -745,7 +753,7 @@ class DeviceMixin(object): # {{{ self.location_manager.update_devices() self.library_view.set_device_connected(self.device_connected) self.refresh_ondevice() - self.device_connection_changed.emit(connected) + device_signals.device_connection_changed.emit(connected) def info_read(self, job): ''' @@ -784,7 +792,7 @@ class DeviceMixin(object): # {{{ self.sync_news() self.sync_catalogs() self.refresh_ondevice() - self.device_metadata_available.emit() + device_signals.device_metadata_available.emit() def refresh_ondevice(self, reset_only = False): ''' diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 8a97183ffe..66cf55a9b2 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -13,7 +13,6 @@ from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog from calibre.gui2.dialogs.tag_editor import TagEditor from calibre.ebooks.metadata import string_to_authors, authors_to_string, title_sort from calibre.ebooks.metadata.book.base import composite_formatter -from calibre.ebooks.metadata.meta import get_metadata from calibre.gui2.custom_column_widgets import populate_metadata_page from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE, \ gprefs, question_dialog @@ -26,6 +25,7 @@ from calibre.utils.magick.draw import identify_data from calibre.utils.date import qt_to_dt def get_cover_data(path): # {{{ + from calibre.ebooks.metadata.meta import get_metadata old = prefs['read_file_metadata'] if not old: prefs['read_file_metadata'] = True diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index f6b7b94453..4776562c29 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -25,7 +25,6 @@ from calibre.ebooks import BOOK_EXTENSIONS from calibre.ebooks.metadata import string_to_authors, \ authors_to_string, check_isbn, title_sort from calibre.ebooks.metadata.covers import download_cover -from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.metadata import MetaInformation from calibre.utils.config import prefs, tweaks from calibre.utils.date import qt_to_dt, local_tz, utcfromtimestamp @@ -353,6 +352,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.formats_changed = True def get_selected_format_metadata(self): + from calibre.ebooks.metadata.meta import get_metadata old = prefs['read_file_metadata'] if not old: prefs['read_file_metadata'] = True diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py index c72b074463..7250103615 100644 --- a/src/calibre/gui2/layout.py +++ b/src/calibre/gui2/layout.py @@ -156,8 +156,6 @@ class SearchBar(QWidget): # {{{ x = ComboBoxWithHelp(self) x.setMaximumSize(QSize(150, 16777215)) x.setObjectName("search_restriction") - x.setToolTip(_('Books display will be restricted to those matching the ' - 'selected saved search')) l.addWidget(x) parent.search_restriction = x diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 8d89ec76ed..0bd3f2133a 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -18,7 +18,6 @@ from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import tweaks, prefs from calibre.utils.date import dt_factory, qt_to_dt, isoformat from calibre.utils.icu import sort_key -from calibre.ebooks.metadata.meta import set_metadata as _set_metadata from calibre.utils.search_query_parser import SearchQueryParser from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ REGEXP_MATCH, MetadataBackup, force_to_bool @@ -478,6 +477,7 @@ class BooksModel(QAbstractTableModel): # {{{ def get_preferred_formats_from_ids(self, ids, formats, set_metadata=False, specific_format=None, exclude_auto=False, mode='r+b'): + from calibre.ebooks.metadata.meta import set_metadata as _set_metadata ans = [] need_auto = [] if specific_format is not None: @@ -526,6 +526,7 @@ class BooksModel(QAbstractTableModel): # {{{ def get_preferred_formats(self, rows, formats, paths=False, set_metadata=False, specific_format=None, exclude_auto=False): + from calibre.ebooks.metadata.meta import set_metadata as _set_metadata ans = [] need_auto = [] if specific_format is not None: diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index c67ec8c2b4..ee18d8e9ca 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -19,6 +19,9 @@ from calibre.utils.config import prefs, dynamic from calibre.library.database2 import LibraryDatabase2 from calibre.library.sqlite import sqlite, DatabaseException +if iswindows: + winutil = plugins['winutil'][0] + def option_parser(): parser = _option_parser('''\ %prog [opts] [path_to_ebook] @@ -80,8 +83,7 @@ def get_library_path(parent=None): if library_path is None: # Need to migrate to new database layout base = os.path.expanduser('~') if iswindows: - base = plugins['winutil'][0].special_folder_path( - plugins['winutil'][0].CSIDL_PERSONAL) + base = winutil.special_folder_path(winutil.CSIDL_PERSONAL) if not base or not os.path.exists(base): from PyQt4.Qt import QDir base = unicode(QDir.homePath()).replace('/', os.sep) diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index 4aa4561078..82804b9c96 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -242,7 +242,7 @@ def apply_metadata(job, gui, q, result): q.finished.disconnect() if result != q.Accepted: return - id_map, failed_ids, failed_covers, title_map = job.result + id_map, failed_ids, failed_covers, title_map, all_failed = job.result id_map = dict([(k, v) for k, v in id_map.iteritems() if k not in failed_ids]) if not id_map: @@ -279,38 +279,47 @@ def apply_metadata(job, gui, q, result): def proceed(gui, job): gui.status_bar.show_message(_('Metadata download completed'), 3000) - id_map, failed_ids, failed_covers, title_map = job.result - fmsg = det_msg = '' - if failed_ids or failed_covers: - fmsg = '

'+_('Could not download metadata and/or covers for %d of the books. Click' - ' "Show details" to see which books.')%len(failed_ids) - det_msg = [] - for i in failed_ids | failed_covers: - title = title_map[i] - if i in failed_ids: - title += (' ' + _('(Failed metadata)')) - if i in failed_covers: - title += (' ' + _('(Failed cover)')) - det_msg.append(title) - msg = '

' + _('Finished downloading metadata for %d book(s). ' - 'Proceed with updating the metadata in your library?')%len(id_map) - q = MessageBox(MessageBox.QUESTION, _('Download complete'), - msg + fmsg, det_msg='\n'.join(det_msg), show_copy_button=bool(failed_ids), - parent=gui) + id_map, failed_ids, failed_covers, title_map, all_failed = job.result + det_msg = [] + for i in failed_ids | failed_covers: + title = title_map[i] + if i in failed_ids: + title += (' ' + _('(Failed metadata)')) + if i in failed_covers: + title += (' ' + _('(Failed cover)')) + det_msg.append(title) + det_msg = '\n'.join(det_msg) + + if all_failed: + q = error_dialog(gui, _('Download failed'), + _('Failed to download metadata or covers for any of the %d' + ' book(s).') % len(id_map), det_msg=det_msg) + else: + fmsg = det_msg = '' + if failed_ids or failed_covers: + fmsg = '

'+_('Could not download metadata and/or covers for %d of the books. Click' + ' "Show details" to see which books.')%len(failed_ids) + msg = '

' + _('Finished downloading metadata for %d book(s). ' + 'Proceed with updating the metadata in your library?')%len(id_map) + q = MessageBox(MessageBox.QUESTION, _('Download complete'), + msg + fmsg, det_msg=det_msg, show_copy_button=bool(failed_ids), + parent=gui) + q.finished.connect(partial(apply_metadata, job, gui, q)) + q.vlb = q.bb.addButton(_('View log'), q.bb.ActionRole) q.vlb.setIcon(QIcon(I('debug.png'))) q.vlb.clicked.connect(partial(view_log, job, q)) q.det_msg_toggle.setVisible(bool(failed_ids | failed_covers)) q.setModal(False) q.show() - q.finished.connect(partial(apply_metadata, job, gui, q)) # }}} def merge_result(oldmi, newmi): dummy = Metadata(_('Unknown')) for f in msprefs['ignore_fields']: - setattr(newmi, f, getattr(dummy, f)) + if ':' not in f: + setattr(newmi, f, getattr(dummy, f)) fields = set() for plugin in metadata_plugins(['identify']): fields |= plugin.touched_fields @@ -335,6 +344,7 @@ def download(ids, db, do_identify, covers, title_map = {} ans = {} count = 0 + all_failed = True for i, mi in izip(ids, metadata): if abort.is_set(): log.error('Aborting...') @@ -349,6 +359,7 @@ def download(ids, db, do_identify, covers, except: pass if results: + all_failed = False mi = merge_result(mi, results[0]) identifiers = mi.identifiers if not mi.is_null('rating'): @@ -366,6 +377,7 @@ def download(ids, db, do_identify, covers, with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: f.write(cdata[-1]) mi.cover = f.name + all_failed = False else: failed_covers.add(i) ans[i] = mi @@ -373,7 +385,7 @@ def download(ids, db, do_identify, covers, notifications.put((count/len(ids), _('Downloaded %d of %d')%(count, len(ids)))) log('Download complete, with %d failures'%len(failed_ids)) - return (ans, failed_ids, failed_covers, title_map) + return (ans, failed_ids, failed_covers, title_map, all_failed) diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index d527dda022..63d4499966 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -326,7 +326,8 @@ class MetadataSingleDialogBase(ResizableDialog): mi = d.book dummy = Metadata(_('Unknown')) for f in msprefs['ignore_fields']: - setattr(mi, f, getattr(dummy, f)) + if ':' not in f: + setattr(mi, f, getattr(dummy, f)) if mi is not None: self.update_from_mi(mi) if d.cover_pixmap is not None: diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py index 330332a716..ead5da4ce4 100644 --- a/src/calibre/gui2/preferences/misc.py +++ b/src/calibre/gui2/preferences/misc.py @@ -6,19 +6,27 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from calibre.gui2.preferences import ConfigWidgetBase, test_widget +from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting from calibre.gui2.preferences.misc_ui import Ui_Form from calibre.gui2 import error_dialog, config, open_local_file, info_dialog from calibre.constants import isosx -# Check Integrity {{{ +class WorkersSetting(Setting): + + def set_gui_val(self, val): + val = val//2 + Setting.set_gui_val(self, val) + + def get_gui_val(self): + val = Setting.get_gui_val(self) + return val * 2 class ConfigWidget(ConfigWidgetBase, Ui_Form): def genesis(self, gui): self.gui = gui r = self.register - r('worker_limit', config, restart_required=True) + r('worker_limit', config, restart_required=True, setting=WorkersSetting) r('enforce_cpu_limit', config, restart_required=True) self.device_detection_button.clicked.connect(self.debug_device_detection) self.button_open_config_dir.clicked.connect(self.open_config_dir) diff --git a/src/calibre/gui2/preferences/misc.ui b/src/calibre/gui2/preferences/misc.ui index c036cb971b..8b0189b0a1 100644 --- a/src/calibre/gui2/preferences/misc.ui +++ b/src/calibre/gui2/preferences/misc.ui @@ -17,7 +17,7 @@ - &Maximum number of waiting worker processes (needs restart): + Max. simultaneous conversion/news download jobs: opt_worker_limit @@ -27,13 +27,7 @@ - 2 - - - 10000 - - - 2 + 1 diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index 8ef02b34b0..ffebc9e131 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -17,6 +17,10 @@ class SearchRestrictionMixin(object): self.search_restriction.setMinimumContentsLength(10) self.search_restriction.setStatusTip(self.search_restriction.toolTip()) self.search_count.setText(_("(all books)")) + self.search_restriction_tooltip = \ + _('Books display will be restricted to those matching a ' + 'selected saved search') + self.search_restriction.setToolTip(self.search_restriction_tooltip) def apply_named_search_restriction(self, name): if not name: @@ -30,29 +34,38 @@ class SearchRestrictionMixin(object): self.apply_search_restriction(r) def apply_text_search_restriction(self, search): + search = unicode(search) if not search: - self.search_restriction.setItemText(1, _('*Current search')) self.search_restriction.setCurrentIndex(0) else: - self.search_restriction.setCurrentIndex(1) - self.search_restriction.setItemText(1, search) + s = '*' + search + if self.search_restriction.count() > 1: + txt = unicode(self.search_restriction.itemText(2)) + if txt.startswith('*'): + self.search_restriction.setItemText(2, s) + else: + self.search_restriction.insertItem(2, s) + else: + self.search_restriction.insertItem(2, s) + self.search_restriction.setCurrentIndex(2) + self.search_restriction.setToolTip('

' + + self.search_restriction_tooltip + + _(' or the search ') + "'" + search + "'

") self._apply_search_restriction(search) def apply_search_restriction(self, i): - self.search_restriction.setItemText(1, _('*Current search')) if i == 1: - restriction = unicode(self.search.currentText()) - if not restriction: - self.search_restriction.setCurrentIndex(0) - else: - self.search_restriction.setItemText(1, restriction) + self.apply_text_search_restriction(unicode(self.search.currentText())) + elif i == 2 and unicode(self.search_restriction.currentText()).startswith('*'): + self.apply_text_search_restriction( + unicode(self.search_restriction.currentText())[1:]) else: r = unicode(self.search_restriction.currentText()) if r is not None and r != '': restriction = 'search:"%s"'%(r) else: restriction = '' - self._apply_search_restriction(restriction) + self._apply_search_restriction(restriction) def _apply_search_restriction(self, restriction): self.saved_search.clear() diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index 49c265d7fe..b65748ac57 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -28,15 +28,15 @@ from calibre.utils.config import DynamicConfig from calibre.utils.icu import sort_key class MobileReadStore(BasicStoreConfig, StorePlugin): - + def genesis(self): self.config = DynamicConfig('store_' + self.name) self.rlock = RLock() - + def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() url = 'http://www.mobileread.com/' - + if external or settings.get(self.name + '_open_external', False): open_url(QUrl(detail_item if detail_item else url)) else: @@ -71,7 +71,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): ratio += s.ratio() if ratio > 0: matches.append((ratio, x)) - + # Move the best scorers to head of list. matches = heapq.nlargest(max_results, matches) for score, book in matches: @@ -81,21 +81,21 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): def update_book_list(self, timeout=10): with self.rlock: url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html' - + last_download = self.config.get(self.name + '_last_download', None) # Don't update the book list if our cache is less than one week old. if last_download and (time.time() - last_download) < 604800: return - + # Download the book list HTML file from MobileRead. br = browser() raw_data = None with closing(br.open(url, timeout=timeout)) as f: raw_data = f.read() - + if not raw_data: return - + # Turn books listed in the HTML file into BookRef's. books = [] try: @@ -105,7 +105,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): book.detail_item = ''.join(book_data.xpath('.//a/@href')) book.format = ''.join(book_data.xpath('.//i/text()')) book.format = book.format.strip() - + text = ''.join(book_data.xpath('.//a/text()')) if ':' in text: book.author, q, text = text.partition(':') @@ -114,7 +114,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): books.append(book) except: pass - + # Save the book list and it's create time. if books: self.config[self.name + '_last_download'] = time.time() @@ -126,21 +126,21 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): class BookRef(SearchResult): - + def __init__(self): SearchResult.__init__(self) - + self.format = '' class MobeReadStoreDialog(QDialog, Ui_Dialog): - + def __init__(self, plugin, *args): QDialog.__init__(self, *args) self.setupUi(self) self.plugin = plugin - + self.model = BooksModel() self.results_view.setModel(self.model) self.results_view.model().set_books(self.plugin.get_book_list()) @@ -150,14 +150,14 @@ class MobeReadStoreDialog(QDialog, Ui_Dialog): self.search_query.textChanged.connect(self.model.set_filter) self.results_view.model().total_changed.connect(self.total.setText) self.finished.connect(self.dialog_closed) - + self.restore_state() - + def open_store(self, index): result = self.results_view.model().get_book(index) if result: self.plugin.open(self, result.detail_item) - + def restore_state(self): geometry = self.plugin.config['store_mobileread_dialog_geometry'] if geometry: @@ -172,7 +172,7 @@ class MobeReadStoreDialog(QDialog, Ui_Dialog): else: for i in xrange(self.results_view.model().columnCount()): self.results_view.resizeColumnToContents(i) - + self.results_view.model().sort_col = self.plugin.config.get('store_mobileread_dialog_sort_col', 0) self.results_view.model().sort_order = self.plugin.config.get('store_mobileread_dialog_sort_order', Qt.AscendingOrder) self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order) @@ -189,7 +189,7 @@ class MobeReadStoreDialog(QDialog, Ui_Dialog): class BooksModel(QAbstractItemModel): - + total_changed = pyqtSignal(unicode) HEADERS = [_('Title'), _('Author(s)'), _('Format')] @@ -205,7 +205,7 @@ class BooksModel(QAbstractItemModel): def set_books(self, books): self.books = books self.all_books = books - + self.sort(self.sort_col, self.sort_order) def get_book(self, index): @@ -214,11 +214,11 @@ class BooksModel(QAbstractItemModel): return self.books[row] else: return None - + def set_filter(self, filter): #self.layoutAboutToBeChanged.emit() self.beginResetModel() - + self.filter = unicode(filter) self.books = [] if self.filter: @@ -241,7 +241,7 @@ class BooksModel(QAbstractItemModel): self.endResetModel() #self.layoutChanged.emit() - + def index(self, row, column, parent=QModelIndex()): return self.createIndex(row, column) @@ -255,7 +255,7 @@ class BooksModel(QAbstractItemModel): def columnCount(self, *args): return len(self.HEADERS) - + def headerData(self, section, orientation, role): if role != Qt.DisplayRole: return NONE @@ -295,7 +295,7 @@ class BooksModel(QAbstractItemModel): if not self.books: return - descending = order == Qt.DescendingOrder + descending = order == Qt.DescendingOrder self.books.sort(None, lambda x: sort_key(unicode(self.data_as_text(x, col))), descending) diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index ea0d2570e5..a7ecdf7b88 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -19,7 +19,6 @@ from calibre.gui2 import NONE, error_dialog, pixmap_to_data, gprefs from calibre.gui2.filename_pattern_ui import Ui_Form from calibre import fit_image from calibre.ebooks import BOOK_EXTENSIONS -from calibre.ebooks.metadata.meta import metadata_from_filename from calibre.utils.config import prefs, XMLConfig, tweaks from calibre.gui2.progress_indicator import ProgressIndicator as _ProgressIndicator from calibre.gui2.dnd import dnd_has_image, dnd_get_image, dnd_get_files, \ @@ -95,6 +94,7 @@ class FilenamePattern(QWidget, Ui_Form): self.re.setCurrentIndex(0) def do_test(self): + from calibre.ebooks.metadata.meta import metadata_from_filename try: pat = self.pattern() except Exception as err: diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 9523795f28..92c5ca9b3c 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -707,7 +707,10 @@ class ResultCache(SearchQueryParser): # {{{ for loc in location: # location is now an array of field indices if loc == db_col['authors']: ### DB stores authors with commas changed to bars, so change query - q = query.replace(',', '|'); + if matchkind == REGEXP_MATCH: + q = query.replace(',', r'\|'); + else: + q = query.replace(',', '|'); else: q = query diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index ffa08eaed2..717e8e2c6b 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -15,7 +15,6 @@ from calibre.customize import CatalogPlugin from calibre.customize.conversion import OptionRecommendation, DummyReporter from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString from calibre.ebooks.chardet import substitute_entites -from calibre.ebooks.oeb.base import XHTML_NS from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.config import config_dir from calibre.utils.date import format_date, isoformat, is_date_undefined, now as nowf @@ -4322,6 +4321,8 @@ Author '{0}': ''' Generate description header from template ''' + from calibre.ebooks.oeb.base import XHTML_NS + def generate_html(): args = dict( author=author, diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index b1a8236151..61e7ec334d 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -10,8 +10,7 @@ Command line interface to the calibre database. import sys, os, cStringIO, re from textwrap import TextWrapper -from calibre import terminal_controller, preferred_encoding, prints, \ - isbytestring +from calibre import preferred_encoding, prints, isbytestring from calibre.utils.config import OptionParser, prefs, tweaks from calibre.ebooks.metadata.meta import get_metadata from calibre.library.database2 import LibraryDatabase2 @@ -53,6 +52,8 @@ def get_db(dbpath, options): def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, separator, prefix, subtitle='Books in the calibre database'): + from calibre.constants import terminal_controller as tc + terminal_controller = tc() if sort_by: db.sort(sort_by, ascending) if search_text: @@ -1087,6 +1088,9 @@ def command_list_categories(args, dbpath): fields = ['category', 'tag_name', 'count', 'rating'] def do_list(): + from calibre.constants import terminal_controller as tc + terminal_controller = tc() + separator = ' ' widths = list(map(lambda x : 0, fields)) for i in data: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index bdcefd13a2..d7f6c22925 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -15,7 +15,8 @@ from math import ceil from PyQt4.QtGui import QImage from calibre import prints -from calibre.ebooks.metadata import title_sort, author_to_author_sort +from calibre.ebooks.metadata import (title_sort, author_to_author_sort, + string_to_authors, authors_to_string) from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.library.database import LibraryDatabase from calibre.library.field_metadata import FieldMetadata, TagsIcons @@ -24,9 +25,7 @@ from calibre.library.caches import ResultCache from calibre.library.custom_columns import CustomColumns from calibre.library.sqlite import connect, IntegrityError from calibre.library.prefs import DBPrefs -from calibre.ebooks.metadata import string_to_authors, authors_to_string from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding from calibre.ptempfile import PersistentTemporaryFile from calibre.customize.ui import run_plugins_on_import @@ -853,6 +852,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.pubdate = row[fm['pubdate']] mi.uuid = row[fm['uuid']] mi.title_sort = row[fm['sort']] + mi.book_size = row[fm['size']] mi.last_modified = row[fm['last_modified']] formats = row[fm['formats']] if not formats: @@ -1378,13 +1378,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for (cat, dex, mult, is_comp) in md: if not book[dex]: continue + tid_cat = tids[cat] + tcats_cat = tcategories[cat] if not mult: val = book[dex] if is_comp: - item = tcategories[cat].get(val, None) + item = tcats_cat.get(val, None) if not item: item = tag_class(val, val) - tcategories[cat][val] = item + tcats_cat[val] = item item.c += 1 item.id = val if rating > 0: @@ -1392,11 +1394,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): item.rc += 1 continue try: - (item_id, sort_val) = tids[cat][val] # let exceptions fly - item = tcategories[cat].get(val, None) + (item_id, sort_val) = tid_cat[val] # let exceptions fly + item = tcats_cat.get(val, None) if not item: item = tag_class(val, sort_val) - tcategories[cat][val] = item + tcats_cat[val] = item item.c += 1 item.id_set.add(book[0]) item.id = item_id @@ -1410,21 +1412,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if is_comp: vals = [v.strip() for v in vals if v.strip()] for val in vals: - if val not in tids: - tids[cat][val] = (val, val) - item = tcategories[cat].get(val, None) - if not item: - item = tag_class(val, val) - tcategories[cat][val] = item - item.c += 1 - item.id = val + if val not in tid_cat: + tid_cat[val] = (val, val) for val in vals: try: - (item_id, sort_val) = tids[cat][val] # let exceptions fly - item = tcategories[cat].get(val, None) + (item_id, sort_val) = tid_cat[val] # let exceptions fly + item = tcats_cat.get(val, None) if not item: item = tag_class(val, sort_val) - tcategories[cat][val] = item + tcats_cat[val] = item item.c += 1 item.id_set.add(book[0]) item.id = item_id @@ -2732,6 +2728,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.set_identifier(id_, 'isbn', isbn, notify=notify, commit=commit) def add_catalog(self, path, title): + from calibre.ebooks.metadata.meta import get_metadata + format = os.path.splitext(path)[1][1:].lower() with lopen(path, 'rb') as stream: matches = self.data.get_matches('title', '='+title) @@ -2767,6 +2765,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def add_news(self, path, arg): + from calibre.ebooks.metadata.meta import get_metadata + format = os.path.splitext(path)[1][1:].lower() stream = path if hasattr(path, 'read') else lopen(path, 'rb') stream.seek(0) @@ -3160,6 +3160,8 @@ books_series_link feeds yield formats def import_book_directory_multiple(self, dirpath, callback=None): + from calibre.ebooks.metadata.meta import metadata_from_formats + duplicates = [] for formats in self.find_books_in_directory(dirpath, False): mi = metadata_from_formats(formats) @@ -3175,6 +3177,7 @@ books_series_link feeds return duplicates def import_book_directory(self, dirpath, callback=None): + from calibre.ebooks.metadata.meta import metadata_from_formats dirpath = os.path.abspath(dirpath) formats = self.find_books_in_directory(dirpath, True) formats = list(formats)[0] diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 3c57af40a8..f7f5559412 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -14,7 +14,6 @@ from calibre.utils.formatter import TemplateFormatter from calibre.utils.filenames import shorten_components_to, supports_long_names, \ ascii_filename from calibre.ebooks.metadata.opf2 import metadata_to_opf -from calibre.ebooks.metadata.meta import set_metadata from calibre.constants import preferred_encoding from calibre.ebooks.metadata import fmt_sidx from calibre.ebooks.metadata import title_sort @@ -198,7 +197,6 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, for key in custom_metadata: if key in format_args: cm = custom_metadata[key] - ## TODO: NEWMETA: should ratings be divided by 2? The standard rating isn't... if cm['datatype'] == 'series': format_args[key] = title_sort(format_args[key], order=tsorder) if key+'_index' in format_args: @@ -252,6 +250,7 @@ def save_book_to_disk(id_, db, root, opts, length): def do_save_book_to_disk(id_, mi, cover, plugboards, format_map, root, opts, length): + from calibre.ebooks.metadata.meta import set_metadata available_formats = [x.lower().strip() for x in format_map.keys()] if opts.formats == 'all': asked_formats = available_formats diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index ef4da23826..3dce13f144 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -549,17 +549,6 @@ How do I run calibre from my USB stick? A portable version of calibre is available at: `portableapps.com `_. However, this is usually out of date. You can also setup your own portable calibre install by following :ref:`these instructions `. -Why are there so many calibre-parallel processes on my system? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -|app| maintains two separate worker process pools. One is used for adding books/saving to disk and the other for conversions. You can control the number of worker processes via :guilabel:`Preferences->Advanced->Miscellaneous`. So if you set it to 6 that means a maximum of 3 conversions will run simultaneously. And that is why you will see the number of worker processes changes by two when you use the up and down arrows. On windows, you can set the priority that these processes run with. This can be useful on older, single CPU machines, if you find them slowing down to a crawl when conversions are running. - -In addition to this some conversion plugins run tasks in their own pool of processes, so for example if you bulk convert comics, each comic conversion will use three separate processes to render the images. The job manager knows this so it will run only a single comic conversion simultaneously. - -And since I'm sure someone will ask: The reason adding/saving books are in separate processes is because of PDF. PDF processing libraries can crash on reading PDFs and I dont want the crash to take down all of calibre. Also when adding EPUB books, in order to extract the cover you have to sometimes render the HTML of the first page, which means that it either has to run in the GUI thread of the main process or in a separate process. - -Finally, the reason calibre keep workers alive and idle instead of launching on demand is to workaround the slow startup time of python processes. - How do I run parts of |app| like news download and the content server on my own linux server? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst index cdb8df2e2b..a77f0d1697 100644 --- a/src/calibre/manual/template_lang.rst +++ b/src/calibre/manual/template_lang.rst @@ -230,6 +230,7 @@ The following functions are available in addition to those described in single-f * ``add(x, y)`` -- returns x + y. Throws an exception if either x or y are not numbers. * ``assign(id, val)`` -- assigns val to id, then returns val. id must be an identifier, not an expression + * ``booksize()`` -- returns the value of the |app| 'size' field. Returns '' if there are no formats. * ``cmp(x, y, lt, eq, gt)`` -- compares x and y after converting both to numbers. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``. * ``divide(x, y)`` -- returns x / y. Throws an exception if either x or y are not numbers. * ``field(name)`` -- returns the metadata field named by ``name``. diff --git a/src/calibre/startup.py b/src/calibre/startup.py index c883c43e8a..fd9ef01141 100644 --- a/src/calibre/startup.py +++ b/src/calibre/startup.py @@ -163,10 +163,6 @@ if not _run_once: __builtin__.__dict__['icu_upper'] = icu_upper __builtin__.__dict__['icu_title'] = title_case - import mimetypes - mimetypes.init([P('mime.types')]) - guess_type = mimetypes.guess_type - def test_lopen(): from calibre.ptempfile import TemporaryDirectory from calibre import CurrentDir diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py index 66316d051b..8b23cf3071 100644 --- a/src/calibre/utils/config.py +++ b/src/calibre/utils/config.py @@ -6,22 +6,25 @@ __docformat__ = 'restructuredtext en' ''' Manage application-wide preferences. ''' -import os, re, cPickle, textwrap, traceback, plistlib, json, base64, datetime +import os, re, cPickle, base64, datetime, json, plistlib from copy import deepcopy -from functools import partial from optparse import OptionParser as _OptionParser from optparse import IndentedHelpFormatter -from collections import defaultdict -from calibre.constants import terminal_controller, config_dir, CONFIG_DIR_MODE, \ - __appname__, __version__, __author__ -from calibre.utils.lock import LockError, ExclusiveFile +from calibre.constants import (config_dir, CONFIG_DIR_MODE, __appname__, + __version__, __author__, terminal_controller) +from calibre.utils.lock import ExclusiveFile +from calibre.utils.config_base import (make_config_dir, Option, OptionValues, + OptionSet, ConfigInterface, Config, prefs, StringConfig, ConfigProxy, + read_raw_tweaks, read_tweaks, write_tweaks, tweaks, plugin_dir) -plugin_dir = os.path.join(config_dir, 'plugins') +if False: + # Make pyflakes happy + Config, ConfigProxy, Option, OptionValues, StringConfig + OptionSet, ConfigInterface, read_tweaks, write_tweaks + read_raw_tweaks, tweaks, plugin_dir -def make_config_dir(): - if not os.path.exists(plugin_dir): - os.makedirs(plugin_dir, mode=CONFIG_DIR_MODE) +test_eight_code = tweaks.get('test_eight_code', False) def check_config_write_access(): return os.access(config_dir, os.W_OK) and os.access(config_dir, os.X_OK) @@ -29,23 +32,28 @@ def check_config_write_access(): class CustomHelpFormatter(IndentedHelpFormatter): def format_usage(self, usage): - return _("%sUsage%s: %s\n") % (terminal_controller.BLUE, terminal_controller.NORMAL, usage) + tc = terminal_controller() + return _("%sUsage%s: %s\n") % (tc.BLUE, tc.NORMAL, usage) def format_heading(self, heading): - return "%*s%s%s%s:\n" % (self.current_indent, terminal_controller.BLUE, - "", heading, terminal_controller.NORMAL) + tc = terminal_controller() + return "%*s%s%s%s:\n" % (self.current_indent, tc.BLUE, + "", heading, tc.NORMAL) def format_option(self, option): + import textwrap + tc = terminal_controller() + result = [] opts = self.option_strings[option] opt_width = self.help_position - self.current_indent - 2 if len(opts) > opt_width: opts = "%*s%s\n" % (self.current_indent, "", - terminal_controller.GREEN+opts+terminal_controller.NORMAL) + tc.GREEN+opts+tc.NORMAL) indent_first = self.help_position else: # start help on same line as opts - opts = "%*s%-*s " % (self.current_indent, "", opt_width + len(terminal_controller.GREEN + terminal_controller.NORMAL), - terminal_controller.GREEN + opts + terminal_controller.NORMAL) + opts = "%*s%-*s " % (self.current_indent, "", opt_width + + len(tc.GREEN + tc.NORMAL), tc.GREEN + opts + tc.NORMAL) indent_first = 0 result.append(opts) if option.help: @@ -71,9 +79,12 @@ class OptionParser(_OptionParser): gui_mode=False, conflict_handler='resolve', **kwds): + import textwrap + tc = terminal_controller() + usage = textwrap.dedent(usage) if epilog is None: - epilog = _('Created by ')+terminal_controller.RED+__author__+terminal_controller.NORMAL + epilog = _('Created by ')+tc.RED+__author__+tc.NORMAL usage += '\n\n'+_('''Whenever you pass arguments to %prog that have spaces in them, ''' '''enclose the arguments in quotation marks.''') _OptionParser.__init__(self, usage=usage, version=version, epilog=epilog, @@ -146,353 +157,6 @@ class OptionParser(_OptionParser): upper.__dict__[dest] = lower.__dict__[dest] - -class Option(object): - - def __init__(self, name, switches=[], help='', type=None, choices=None, - check=None, group=None, default=None, action=None, metavar=None): - if choices: - type = 'choice' - - self.name = name - self.switches = switches - self.help = help.replace('%default', repr(default)) if help else None - self.type = type - if self.type is None and action is None and choices is None: - if isinstance(default, float): - self.type = 'float' - elif isinstance(default, int) and not isinstance(default, bool): - self.type = 'int' - - self.choices = choices - self.check = check - self.group = group - self.default = default - self.action = action - self.metavar = metavar - - def __eq__(self, other): - return self.name == getattr(other, 'name', other) - - def __repr__(self): - return 'Option: '+self.name - - def __str__(self): - return repr(self) - -class OptionValues(object): - - def copy(self): - return deepcopy(self) - -class OptionSet(object): - - OVERRIDE_PAT = re.compile(r'#{3,100} Override Options #{15}(.*?)#{3,100} End Override #{3,100}', - re.DOTALL|re.IGNORECASE) - - def __init__(self, description=''): - self.description = description - self.defaults = {} - self.preferences = [] - self.group_list = [] - self.groups = {} - self.set_buffer = {} - - def has_option(self, name_or_option_object): - if name_or_option_object in self.preferences: - return True - for p in self.preferences: - if p.name == name_or_option_object: - return True - return False - - def get_option(self, name_or_option_object): - idx = self.preferences.index(name_or_option_object) - if idx > -1: - return self.preferences[idx] - for p in self.preferences: - if p.name == name_or_option_object: - return p - - def add_group(self, name, description=''): - if name in self.group_list: - raise ValueError('A group by the name %s already exists in this set'%name) - self.groups[name] = description - self.group_list.append(name) - return partial(self.add_opt, group=name) - - def update(self, other): - for name in other.groups.keys(): - self.groups[name] = other.groups[name] - if name not in self.group_list: - self.group_list.append(name) - for pref in other.preferences: - if pref in self.preferences: - self.preferences.remove(pref) - self.preferences.append(pref) - - def smart_update(self, opts1, opts2): - ''' - Updates the preference values in opts1 using only the non-default preference values in opts2. - ''' - for pref in self.preferences: - new = getattr(opts2, pref.name, pref.default) - if new != pref.default: - setattr(opts1, pref.name, new) - - def remove_opt(self, name): - if name in self.preferences: - self.preferences.remove(name) - - - def add_opt(self, name, switches=[], help=None, type=None, choices=None, - group=None, default=None, action=None, metavar=None): - ''' - Add an option to this section. - - :param name: The name of this option. Must be a valid Python identifier. - Must also be unique in this OptionSet and all its subsets. - :param switches: List of command line switches for this option - (as supplied to :module:`optparse`). If empty, this - option will not be added to the command line parser. - :param help: Help text. - :param type: Type checking of option values. Supported types are: - `None, 'choice', 'complex', 'float', 'int', 'string'`. - :param choices: List of strings or `None`. - :param group: Group this option belongs to. You must previously - have created this group with a call to :method:`add_group`. - :param default: The default value for this option. - :param action: The action to pass to optparse. Supported values are: - `None, 'count'`. For choices and boolean options, - action is automatically set correctly. - ''' - pref = Option(name, switches=switches, help=help, type=type, choices=choices, - group=group, default=default, action=action, metavar=None) - if group is not None and group not in self.groups.keys(): - raise ValueError('Group %s has not been added to this section'%group) - if pref in self.preferences: - raise ValueError('An option with the name %s already exists in this set.'%name) - self.preferences.append(pref) - self.defaults[name] = default - - def option_parser(self, user_defaults=None, usage='', gui_mode=False): - parser = OptionParser(usage, gui_mode=gui_mode) - groups = defaultdict(lambda : parser) - for group, desc in self.groups.items(): - groups[group] = parser.add_option_group(group.upper(), desc) - - for pref in self.preferences: - if not pref.switches: - continue - g = groups[pref.group] - action = pref.action - if action is None: - action = 'store' - if pref.default is True or pref.default is False: - action = 'store_' + ('false' if pref.default else 'true') - args = dict( - dest=pref.name, - help=pref.help, - metavar=pref.metavar, - type=pref.type, - choices=pref.choices, - default=getattr(user_defaults, pref.name, pref.default), - action=action, - ) - g.add_option(*pref.switches, **args) - - - return parser - - def get_override_section(self, src): - match = self.OVERRIDE_PAT.search(src) - if match: - return match.group() - return '' - - def parse_string(self, src): - options = {'cPickle':cPickle} - if src is not None: - try: - if not isinstance(src, unicode): - src = src.decode('utf-8') - exec src in options - except: - print 'Failed to parse options string:' - print repr(src) - traceback.print_exc() - opts = OptionValues() - for pref in self.preferences: - val = options.get(pref.name, pref.default) - formatter = __builtins__.get(pref.type, None) - if callable(formatter): - val = formatter(val) - setattr(opts, pref.name, val) - - return opts - - def render_group(self, name, desc, opts): - prefs = [pref for pref in self.preferences if pref.group == name] - lines = ['### Begin group: %s'%(name if name else 'DEFAULT')] - if desc: - lines += map(lambda x: '# '+x, desc.split('\n')) - lines.append(' ') - for pref in prefs: - lines.append('# '+pref.name.replace('_', ' ')) - if pref.help: - lines += map(lambda x: '# ' + x, pref.help.split('\n')) - lines.append('%s = %s'%(pref.name, - self.serialize_opt(getattr(opts, pref.name, pref.default)))) - lines.append(' ') - return '\n'.join(lines) - - def serialize_opt(self, val): - if val is val is True or val is False or val is None or \ - isinstance(val, (int, float, long, basestring)): - return repr(val) - from PyQt4.QtCore import QString - if isinstance(val, QString): - return repr(unicode(val)) - pickle = cPickle.dumps(val, -1) - return 'cPickle.loads(%s)'%repr(pickle) - - def serialize(self, opts): - src = '# %s\n\n'%(self.description.replace('\n', '\n# ')) - groups = [self.render_group(name, self.groups.get(name, ''), opts) \ - for name in [None] + self.group_list] - return src + '\n\n'.join(groups) - -class ConfigInterface(object): - - def __init__(self, description): - self.option_set = OptionSet(description=description) - self.add_opt = self.option_set.add_opt - self.add_group = self.option_set.add_group - self.remove_opt = self.remove = self.option_set.remove_opt - self.parse_string = self.option_set.parse_string - self.get_option = self.option_set.get_option - self.preferences = self.option_set.preferences - - def update(self, other): - self.option_set.update(other.option_set) - - def option_parser(self, usage='', gui_mode=False): - return self.option_set.option_parser(user_defaults=self.parse(), - usage=usage, gui_mode=gui_mode) - - def smart_update(self, opts1, opts2): - self.option_set.smart_update(opts1, opts2) - - -class Config(ConfigInterface): - ''' - A file based configuration. - ''' - - def __init__(self, basename, description=''): - ConfigInterface.__init__(self, description) - self.config_file_path = os.path.join(config_dir, basename+'.py') - - - def parse(self): - src = '' - if os.path.exists(self.config_file_path): - try: - with ExclusiveFile(self.config_file_path) as f: - try: - src = f.read().decode('utf-8') - except ValueError: - print "Failed to parse", self.config_file_path - traceback.print_exc() - except LockError: - raise IOError('Could not lock config file: %s'%self.config_file_path) - return self.option_set.parse_string(src) - - def as_string(self): - if not os.path.exists(self.config_file_path): - return '' - try: - with ExclusiveFile(self.config_file_path) as f: - return f.read().decode('utf-8') - except LockError: - raise IOError('Could not lock config file: %s'%self.config_file_path) - - def set(self, name, val): - if not self.option_set.has_option(name): - raise ValueError('The option %s is not defined.'%name) - try: - if not os.path.exists(config_dir): - make_config_dir() - with ExclusiveFile(self.config_file_path) as f: - src = f.read() - opts = self.option_set.parse_string(src) - setattr(opts, name, val) - footer = self.option_set.get_override_section(src) - src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n' - f.seek(0) - f.truncate() - if isinstance(src, unicode): - src = src.encode('utf-8') - f.write(src) - except LockError: - raise IOError('Could not lock config file: %s'%self.config_file_path) - -class StringConfig(ConfigInterface): - ''' - A string based configuration - ''' - - def __init__(self, src, description=''): - ConfigInterface.__init__(self, description) - self.src = src - - def parse(self): - return self.option_set.parse_string(self.src) - - def set(self, name, val): - if not self.option_set.has_option(name): - raise ValueError('The option %s is not defined.'%name) - opts = self.option_set.parse_string(self.src) - setattr(opts, name, val) - footer = self.option_set.get_override_section(self.src) - self.src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n' - -class ConfigProxy(object): - ''' - A Proxy to minimize file reads for widely used config settings - ''' - - def __init__(self, config): - self.__config = config - self.__opts = None - - @property - def defaults(self): - return self.__config.option_set.defaults - - def refresh(self): - self.__opts = self.__config.parse() - - def __getitem__(self, key): - return self.get(key) - - def __setitem__(self, key, val): - return self.set(key, val) - - def get(self, key): - if self.__opts is None: - self.refresh() - return getattr(self.__opts, key) - - def set(self, key, val): - if self.__opts is None: - self.refresh() - setattr(self.__opts, key, val) - return self.__config.set(key, val) - - def help(self, key): - return self.__config.get_option(key).help - class DynamicConfig(dict): ''' A replacement for QSettings that supports dynamic config keys. @@ -690,101 +354,6 @@ class JSONConfig(XMLConfig): -def _prefs(): - c = Config('global', 'calibre wide preferences') - c.add_opt('database_path', - default=os.path.expanduser('~/library1.db'), - help=_('Path to the database in which books are stored')) - c.add_opt('filename_pattern', default=ur'(?P.+) - (?P<author>[^_]+)', - help=_('Pattern to guess metadata from filenames')) - c.add_opt('isbndb_com_key', default='', - help=_('Access key for isbndb.com')) - c.add_opt('network_timeout', default=5, - help=_('Default timeout for network operations (seconds)')) - c.add_opt('library_path', default=None, - help=_('Path to directory in which your library of books is stored')) - c.add_opt('language', default=None, - help=_('The language in which to display the user interface')) - c.add_opt('output_format', default='EPUB', - help=_('The default output format for ebook conversions.')) - c.add_opt('input_format_order', default=['EPUB', 'MOBI', 'LIT', 'PRC', - 'FB2', 'HTML', 'HTM', 'XHTM', 'SHTML', 'XHTML', 'ZIP', 'ODT', 'RTF', 'PDF', - 'TXT'], - help=_('Ordered list of formats to prefer for input.')) - c.add_opt('read_file_metadata', default=True, - help=_('Read metadata from files')) - c.add_opt('worker_process_priority', default='normal', - help=_('The priority of worker processes. A higher priority ' - 'means they run faster and consume more resources. ' - 'Most tasks like conversion/news download/adding books/etc. ' - 'are affected by this setting.')) - c.add_opt('swap_author_names', default=False, - help=_('Swap author first and last names when reading metadata')) - c.add_opt('add_formats_to_existing', default=False, - help=_('Add new formats to existing book records')) - c.add_opt('installation_uuid', default=None, help='Installation UUID') - c.add_opt('new_book_tags', default=[], help=_('Tags to apply to books added to the library')) - - # these are here instead of the gui preferences because calibredb and - # calibre server can execute searches - c.add_opt('saved_searches', default={}, help=_('List of named saved searches')) - c.add_opt('user_categories', default={}, help=_('User-created tag browser categories')) - c.add_opt('manage_device_metadata', default='manual', - help=_('How and when calibre updates metadata on the device.')) - c.add_opt('limit_search_columns', default=False, - help=_('When searching for text without using lookup ' - 'prefixes, as for example, Red instead of title:Red, ' - 'limit the columns searched to those named below.')) - c.add_opt('limit_search_columns_to', - default=['title', 'authors', 'tags', 'series', 'publisher'], - help=_('Choose columns to be searched when not using prefixes, ' - 'as for example, when searching for Redd instead of ' - 'title:Red. Enter a list of search/lookup names ' - 'separated by commas. Only takes effect if you set the option ' - 'to limit search columns above.')) - - c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.') - return c - -prefs = ConfigProxy(_prefs()) -if prefs['installation_uuid'] is None: - import uuid - prefs['installation_uuid'] = str(uuid.uuid4()) - -# Read tweaks -def read_raw_tweaks(): - make_config_dir() - default_tweaks = P('default_tweaks.py', data=True, - allow_user_override=False) - tweaks_file = os.path.join(config_dir, 'tweaks.py') - if not os.path.exists(tweaks_file): - with open(tweaks_file, 'wb') as f: - f.write(default_tweaks) - with open(tweaks_file, 'rb') as f: - return default_tweaks, f.read() - -def read_tweaks(): - default_tweaks, tweaks = read_raw_tweaks() - l, g = {}, {} - try: - exec tweaks in g, l - except: - print 'Failed to load custom tweaks file' - traceback.print_exc() - dl, dg = {}, {} - exec default_tweaks in dg, dl - dl.update(l) - return dl - -def write_tweaks(raw): - make_config_dir() - tweaks_file = os.path.join(config_dir, 'tweaks.py') - with open(tweaks_file, 'wb') as f: - f.write(raw) - - -tweaks = read_tweaks() -test_eight_code = tweaks.get('test_eight_code', False) def migrate(): if hasattr(os, 'geteuid') and os.geteuid() == 0: diff --git a/src/calibre/utils/config_base.py b/src/calibre/utils/config_base.py new file mode 100644 index 0000000000..7660370353 --- /dev/null +++ b/src/calibre/utils/config_base.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' +__docformat__ = 'restructuredtext en' + +import os, re, cPickle, traceback +from functools import partial +from collections import defaultdict +from copy import deepcopy + +from calibre.utils.lock import LockError, ExclusiveFile +from calibre.constants import config_dir, CONFIG_DIR_MODE + +plugin_dir = os.path.join(config_dir, 'plugins') + +def make_config_dir(): + if not os.path.exists(plugin_dir): + os.makedirs(plugin_dir, mode=CONFIG_DIR_MODE) + +class Option(object): + + def __init__(self, name, switches=[], help='', type=None, choices=None, + check=None, group=None, default=None, action=None, metavar=None): + if choices: + type = 'choice' + + self.name = name + self.switches = switches + self.help = help.replace('%default', repr(default)) if help else None + self.type = type + if self.type is None and action is None and choices is None: + if isinstance(default, float): + self.type = 'float' + elif isinstance(default, int) and not isinstance(default, bool): + self.type = 'int' + + self.choices = choices + self.check = check + self.group = group + self.default = default + self.action = action + self.metavar = metavar + + def __eq__(self, other): + return self.name == getattr(other, 'name', other) + + def __repr__(self): + return 'Option: '+self.name + + def __str__(self): + return repr(self) + +class OptionValues(object): + + def copy(self): + return deepcopy(self) + +class OptionSet(object): + + OVERRIDE_PAT = re.compile(r'#{3,100} Override Options #{15}(.*?)#{3,100} End Override #{3,100}', + re.DOTALL|re.IGNORECASE) + + def __init__(self, description=''): + self.description = description + self.defaults = {} + self.preferences = [] + self.group_list = [] + self.groups = {} + self.set_buffer = {} + + def has_option(self, name_or_option_object): + if name_or_option_object in self.preferences: + return True + for p in self.preferences: + if p.name == name_or_option_object: + return True + return False + + def get_option(self, name_or_option_object): + idx = self.preferences.index(name_or_option_object) + if idx > -1: + return self.preferences[idx] + for p in self.preferences: + if p.name == name_or_option_object: + return p + + def add_group(self, name, description=''): + if name in self.group_list: + raise ValueError('A group by the name %s already exists in this set'%name) + self.groups[name] = description + self.group_list.append(name) + return partial(self.add_opt, group=name) + + def update(self, other): + for name in other.groups.keys(): + self.groups[name] = other.groups[name] + if name not in self.group_list: + self.group_list.append(name) + for pref in other.preferences: + if pref in self.preferences: + self.preferences.remove(pref) + self.preferences.append(pref) + + def smart_update(self, opts1, opts2): + ''' + Updates the preference values in opts1 using only the non-default preference values in opts2. + ''' + for pref in self.preferences: + new = getattr(opts2, pref.name, pref.default) + if new != pref.default: + setattr(opts1, pref.name, new) + + def remove_opt(self, name): + if name in self.preferences: + self.preferences.remove(name) + + + def add_opt(self, name, switches=[], help=None, type=None, choices=None, + group=None, default=None, action=None, metavar=None): + ''' + Add an option to this section. + + :param name: The name of this option. Must be a valid Python identifier. + Must also be unique in this OptionSet and all its subsets. + :param switches: List of command line switches for this option + (as supplied to :module:`optparse`). If empty, this + option will not be added to the command line parser. + :param help: Help text. + :param type: Type checking of option values. Supported types are: + `None, 'choice', 'complex', 'float', 'int', 'string'`. + :param choices: List of strings or `None`. + :param group: Group this option belongs to. You must previously + have created this group with a call to :method:`add_group`. + :param default: The default value for this option. + :param action: The action to pass to optparse. Supported values are: + `None, 'count'`. For choices and boolean options, + action is automatically set correctly. + ''' + pref = Option(name, switches=switches, help=help, type=type, choices=choices, + group=group, default=default, action=action, metavar=None) + if group is not None and group not in self.groups.keys(): + raise ValueError('Group %s has not been added to this section'%group) + if pref in self.preferences: + raise ValueError('An option with the name %s already exists in this set.'%name) + self.preferences.append(pref) + self.defaults[name] = default + + def option_parser(self, user_defaults=None, usage='', gui_mode=False): + from calibre.utils.config import OptionParser + parser = OptionParser(usage, gui_mode=gui_mode) + groups = defaultdict(lambda : parser) + for group, desc in self.groups.items(): + groups[group] = parser.add_option_group(group.upper(), desc) + + for pref in self.preferences: + if not pref.switches: + continue + g = groups[pref.group] + action = pref.action + if action is None: + action = 'store' + if pref.default is True or pref.default is False: + action = 'store_' + ('false' if pref.default else 'true') + args = dict( + dest=pref.name, + help=pref.help, + metavar=pref.metavar, + type=pref.type, + choices=pref.choices, + default=getattr(user_defaults, pref.name, pref.default), + action=action, + ) + g.add_option(*pref.switches, **args) + + + return parser + + def get_override_section(self, src): + match = self.OVERRIDE_PAT.search(src) + if match: + return match.group() + return '' + + def parse_string(self, src): + options = {'cPickle':cPickle} + if src is not None: + try: + if not isinstance(src, unicode): + src = src.decode('utf-8') + exec src in options + except: + print 'Failed to parse options string:' + print repr(src) + traceback.print_exc() + opts = OptionValues() + for pref in self.preferences: + val = options.get(pref.name, pref.default) + formatter = __builtins__.get(pref.type, None) + if callable(formatter): + val = formatter(val) + setattr(opts, pref.name, val) + + return opts + + def render_group(self, name, desc, opts): + prefs = [pref for pref in self.preferences if pref.group == name] + lines = ['### Begin group: %s'%(name if name else 'DEFAULT')] + if desc: + lines += map(lambda x: '# '+x, desc.split('\n')) + lines.append(' ') + for pref in prefs: + lines.append('# '+pref.name.replace('_', ' ')) + if pref.help: + lines += map(lambda x: '# ' + x, pref.help.split('\n')) + lines.append('%s = %s'%(pref.name, + self.serialize_opt(getattr(opts, pref.name, pref.default)))) + lines.append(' ') + return '\n'.join(lines) + + def serialize_opt(self, val): + if val is val is True or val is False or val is None or \ + isinstance(val, (int, float, long, basestring)): + return repr(val) + from PyQt4.QtCore import QString + if isinstance(val, QString): + return repr(unicode(val)) + pickle = cPickle.dumps(val, -1) + return 'cPickle.loads(%s)'%repr(pickle) + + def serialize(self, opts): + src = '# %s\n\n'%(self.description.replace('\n', '\n# ')) + groups = [self.render_group(name, self.groups.get(name, ''), opts) \ + for name in [None] + self.group_list] + return src + '\n\n'.join(groups) + +class ConfigInterface(object): + + def __init__(self, description): + self.option_set = OptionSet(description=description) + self.add_opt = self.option_set.add_opt + self.add_group = self.option_set.add_group + self.remove_opt = self.remove = self.option_set.remove_opt + self.parse_string = self.option_set.parse_string + self.get_option = self.option_set.get_option + self.preferences = self.option_set.preferences + + def update(self, other): + self.option_set.update(other.option_set) + + def option_parser(self, usage='', gui_mode=False): + return self.option_set.option_parser(user_defaults=self.parse(), + usage=usage, gui_mode=gui_mode) + + def smart_update(self, opts1, opts2): + self.option_set.smart_update(opts1, opts2) + + +class Config(ConfigInterface): + ''' + A file based configuration. + ''' + + def __init__(self, basename, description=''): + ConfigInterface.__init__(self, description) + self.config_file_path = os.path.join(config_dir, basename+'.py') + + + def parse(self): + src = '' + if os.path.exists(self.config_file_path): + try: + with ExclusiveFile(self.config_file_path) as f: + try: + src = f.read().decode('utf-8') + except ValueError: + print "Failed to parse", self.config_file_path + traceback.print_exc() + except LockError: + raise IOError('Could not lock config file: %s'%self.config_file_path) + return self.option_set.parse_string(src) + + def as_string(self): + if not os.path.exists(self.config_file_path): + return '' + try: + with ExclusiveFile(self.config_file_path) as f: + return f.read().decode('utf-8') + except LockError: + raise IOError('Could not lock config file: %s'%self.config_file_path) + + def set(self, name, val): + if not self.option_set.has_option(name): + raise ValueError('The option %s is not defined.'%name) + try: + if not os.path.exists(config_dir): + make_config_dir() + with ExclusiveFile(self.config_file_path) as f: + src = f.read() + opts = self.option_set.parse_string(src) + setattr(opts, name, val) + footer = self.option_set.get_override_section(src) + src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n' + f.seek(0) + f.truncate() + if isinstance(src, unicode): + src = src.encode('utf-8') + f.write(src) + except LockError: + raise IOError('Could not lock config file: %s'%self.config_file_path) + +class StringConfig(ConfigInterface): + ''' + A string based configuration + ''' + + def __init__(self, src, description=''): + ConfigInterface.__init__(self, description) + self.src = src + + def parse(self): + return self.option_set.parse_string(self.src) + + def set(self, name, val): + if not self.option_set.has_option(name): + raise ValueError('The option %s is not defined.'%name) + opts = self.option_set.parse_string(self.src) + setattr(opts, name, val) + footer = self.option_set.get_override_section(self.src) + self.src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n' + +class ConfigProxy(object): + ''' + A Proxy to minimize file reads for widely used config settings + ''' + + def __init__(self, config): + self.__config = config + self.__opts = None + + @property + def defaults(self): + return self.__config.option_set.defaults + + def refresh(self): + self.__opts = self.__config.parse() + + def __getitem__(self, key): + return self.get(key) + + def __setitem__(self, key, val): + return self.set(key, val) + + def get(self, key): + if self.__opts is None: + self.refresh() + return getattr(self.__opts, key) + + def set(self, key, val): + if self.__opts is None: + self.refresh() + setattr(self.__opts, key, val) + return self.__config.set(key, val) + + def help(self, key): + return self.__config.get_option(key).help + + + +def _prefs(): + c = Config('global', 'calibre wide preferences') + c.add_opt('database_path', + default=os.path.expanduser('~/library1.db'), + help=_('Path to the database in which books are stored')) + c.add_opt('filename_pattern', default=ur'(?P<title>.+) - (?P<author>[^_]+)', + help=_('Pattern to guess metadata from filenames')) + c.add_opt('isbndb_com_key', default='', + help=_('Access key for isbndb.com')) + c.add_opt('network_timeout', default=5, + help=_('Default timeout for network operations (seconds)')) + c.add_opt('library_path', default=None, + help=_('Path to directory in which your library of books is stored')) + c.add_opt('language', default=None, + help=_('The language in which to display the user interface')) + c.add_opt('output_format', default='EPUB', + help=_('The default output format for ebook conversions.')) + c.add_opt('input_format_order', default=['EPUB', 'MOBI', 'LIT', 'PRC', + 'FB2', 'HTML', 'HTM', 'XHTM', 'SHTML', 'XHTML', 'ZIP', 'ODT', 'RTF', 'PDF', + 'TXT'], + help=_('Ordered list of formats to prefer for input.')) + c.add_opt('read_file_metadata', default=True, + help=_('Read metadata from files')) + c.add_opt('worker_process_priority', default='normal', + help=_('The priority of worker processes. A higher priority ' + 'means they run faster and consume more resources. ' + 'Most tasks like conversion/news download/adding books/etc. ' + 'are affected by this setting.')) + c.add_opt('swap_author_names', default=False, + help=_('Swap author first and last names when reading metadata')) + c.add_opt('add_formats_to_existing', default=False, + help=_('Add new formats to existing book records')) + c.add_opt('installation_uuid', default=None, help='Installation UUID') + c.add_opt('new_book_tags', default=[], help=_('Tags to apply to books added to the library')) + + # these are here instead of the gui preferences because calibredb and + # calibre server can execute searches + c.add_opt('saved_searches', default={}, help=_('List of named saved searches')) + c.add_opt('user_categories', default={}, help=_('User-created tag browser categories')) + c.add_opt('manage_device_metadata', default='manual', + help=_('How and when calibre updates metadata on the device.')) + c.add_opt('limit_search_columns', default=False, + help=_('When searching for text without using lookup ' + 'prefixes, as for example, Red instead of title:Red, ' + 'limit the columns searched to those named below.')) + c.add_opt('limit_search_columns_to', + default=['title', 'authors', 'tags', 'series', 'publisher'], + help=_('Choose columns to be searched when not using prefixes, ' + 'as for example, when searching for Redd instead of ' + 'title:Red. Enter a list of search/lookup names ' + 'separated by commas. Only takes effect if you set the option ' + 'to limit search columns above.')) + + c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.') + return c + +prefs = ConfigProxy(_prefs()) +if prefs['installation_uuid'] is None: + import uuid + prefs['installation_uuid'] = str(uuid.uuid4()) + +# Read tweaks +def read_raw_tweaks(): + make_config_dir() + default_tweaks = P('default_tweaks.py', data=True, + allow_user_override=False) + tweaks_file = os.path.join(config_dir, 'tweaks.py') + if not os.path.exists(tweaks_file): + with open(tweaks_file, 'wb') as f: + f.write(default_tweaks) + with open(tweaks_file, 'rb') as f: + return default_tweaks, f.read() + +def read_tweaks(): + default_tweaks, tweaks = read_raw_tweaks() + l, g = {}, {} + try: + exec tweaks in g, l + except: + import traceback + print 'Failed to load custom tweaks file' + traceback.print_exc() + dl, dg = {}, {} + exec default_tweaks in dg, dl + dl.update(l) + return dl + +def write_tweaks(raw): + make_config_dir() + tweaks_file = os.path.join(config_dir, 'tweaks.py') + with open(tweaks_file, 'wb') as f: + f.write(raw) + + +tweaks = read_tweaks() + + diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 7957bd0749..aa8e4fb3a3 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -549,8 +549,22 @@ class BuiltinCapitalize(BuiltinFormatterFunction): def evaluate(self, formatter, kwargs, mi, locals, val): return capitalize(val) +class BuiltinBooksize(BuiltinFormatterFunction): + name = 'booksize' + arg_count = 0 + doc = _('booksize() -- return value of the field capitalized') + + def evaluate(self, formatter, kwargs, mi, locals): + if mi.book_size is not None: + try: + return str(mi.book_size) + except: + pass + return '' + builtin_add = BuiltinAdd() builtin_assign = BuiltinAssign() +builtin_booksize = BuiltinBooksize() builtin_capitalize = BuiltinCapitalize() builtin_cmp = BuiltinCmp() builtin_contains = BuiltinContains() diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index f17ff1b17f..d5bef449c4 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -10,7 +10,7 @@ import sys from functools import partial from calibre.constants import plugins -from calibre.utils.config import tweaks +from calibre.utils.config_base import tweaks _icu = _collator = None _locale = None diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index e3b7bfd449..ea6ce88ad6 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -17,7 +17,7 @@ from binascii import hexlify from calibre.utils.ipc.launch import Worker from calibre.utils.ipc.worker import PARALLEL_FUNCS from calibre import detect_ncpus as cpu_count -from calibre.constants import iswindows +from calibre.constants import iswindows, DEBUG from calibre.ptempfile import base_dir _counter = 0 @@ -106,13 +106,14 @@ class Server(Thread): self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() self.kill_queue = Queue() self.waiting_jobs = [] - self.pool, self.workers = deque(), deque() + self.workers = deque() self.launched_worker_count = 0 self._worker_launch_lock = RLock() self.start() def launch_worker(self, gui=False, redirect_output=None): + start = time.time() with self._worker_launch_lock: self.launched_worker_count += 1 id = self.launched_worker_count @@ -136,6 +137,8 @@ class Server(Thread): break if isinstance(cw, basestring): raise CriticalError('Failed to launch worker process:\n'+cw) + if DEBUG: + print 'Worker Launch took:', time.time() - start return cw def do_launch(self, env, gui, redirect_output, rfile): @@ -204,13 +207,6 @@ class Server(Thread): job.duration = time.time() - job.start_time self.changed_jobs_queue.put(job) - # Start new workers - if len(self.pool) + len(self.workers) < self.pool_size: - try: - self.pool.append(self.launch_worker()) - except Exception: - pass - # Start waiting jobs sj = self.suitable_waiting_job() if sj is not None: @@ -222,7 +218,7 @@ class Server(Thread): job.killed = job.failed = True job.result = None else: - worker = self.pool.pop() + worker = self.launch_worker() worker.start_job(job) self.workers.append(worker) job.log_path = worker.log_path @@ -236,7 +232,7 @@ class Server(Thread): break def suitable_waiting_job(self): - available_workers = len(self.pool) + available_workers = self.pool_size - len(self.workers) for worker in self.workers: job = worker.job if job.core_usage == -1: @@ -302,11 +298,6 @@ class Server(Thread): worker.kill() except: pass - for worker in list(self.pool): - try: - worker.kill() - except: - pass def __enter__(self): return self diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py index f676b99e43..533fd03457 100644 --- a/src/calibre/utils/localization.py +++ b/src/calibre/utils/localization.py @@ -24,7 +24,7 @@ def available_translations(): def get_lang(): 'Try to figure out what language to display the interface in' - from calibre.utils.config import prefs + from calibre.utils.config_base import prefs lang = prefs['language'] lang = os.environ.get('CALIBRE_OVERRIDE_LANG', lang) if lang is not None: