Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/kovidgoyal/calibre

Browse Source
This commit is contained in:
GRiker 2013-06-28 03:46:13 -07:00
commit ee556b867c
45 changed files with 1118 additions and 473 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
Changelog.yaml
View File

@ -20,6 +20,50 @@
# new recipes:
# - title:
- version: 0.9.37
date: 2013-06-28
new features:
- title: "Conversion: Add option to embed all referenced fonts"
type: major
description: "Add an option to embed all fonts that are referenced in the input document but are not already embedded. This will search your system for the referenced font, and if found, the font will be embedded. Only works if the output format supports font embedding (for example: EPUB or AZW3). The option is under the Look & Feel section of the conversion dialog."
- title: "ToC Editor: When generating a ToC from files, if the file has no text, do not skip it. Instead create an entry using the filename of the file."
- title: "AZW3 Input: Add support for the page-progression-direction that is used to indicate page turns should happen from right to left. The attribute is passed into EPUB when converting."
tickets: [1194766]
- title: "ebook-convert: Add a --from-opf option to read metadata from OPF files directly, instead of having to run ebook-meta --from-opf after conversion"
bug fixes:
- title: "PDF Output: Fix Table of Contents being added to the end of the PDF even without the Add Table of Contents option being enabled."
tickets: [1194836]
- title: "When auto-merging books on add, also merge identifiers."
- title: "Fix an error when using the Template Editor to create a template that uses custom columns."
tickets: [1193763]
- title: "LRF Output: Fix " entities in attribute values causing problems"
- title: "News download: Apply the default page margin conversion settings. Also, when converting to PDF, apply the pdf conversion defaults."
tickets: [1193912]
- title: "Fix a regression that broke scanning for books on all devices that used the Aluratek Color driver."
tickets: [1192940]
- title: "fetch-ebbok-metadata: Fix --opf argument erroneously requiring a value"
- title: "When waiting before sending email, log the wait."
tickets: [1195173]
improved recipes:
- taz.de (RSS)
- Miradas al sur
- Frontline
- La Nacion (Costa Rica)
- version: 0.9.36
date: 2013-06-21

BIN
manual/resources/simple_donate_button.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.1 KiB

2
manual/templates/layout.html
View File

@ -62,7 +62,7 @@
<form action="https://www.paypal.com/cgi-bin/webscr" method="post" title="Contribute to support calibre development">
<input type="hidden" name="cmd" value="_s-xclick" />
<input type="hidden" name="hosted_button_id" value="AF4H3B8QVDG6N" />
<input type="image" src="http://manual.calibre-ebook.com/simple_donate_button.gif" border="0" name="submit" alt="Contribute to support calibre development" style="border:0pt" />
<input type="image" src="_static/simple_donate_button.gif" border="0" name="submit" alt="Contribute to support calibre development" style="border:0pt" />
<img alt="" border="0" src="https://www.paypalobjects.com/en_GB/i/scr/pixel.gif" width="1" height="1" />
</form>
<hr/>

51
recipes/frontlineonnet.recipe
View File

@ -46,35 +46,34 @@ class Frontlineonnet(BasicNewsRecipe):
keep_only_tags= [
dict(name='div', attrs={'id':'content'})
#,dict(attrs={'class':'byline'})
]
#remove_attributes=['size','noshade','border']
#def preprocess_html(self, soup):
#for item in soup.findAll(style=True):
#del item['style']
#for item in soup.findAll('img'):
#if not item.has_key('alt'):
#item['alt'] = 'image'
#return soup
remove_attributes=['size','noshade','border']
def parse_index(self):
articles = []
current_section = None
feeds = []
soup = self.index_to_soup(self.INDEX)
for feed_link in soup.findAll('div', id='headseccol'):
a = feed_link.find('a', href=True)
title = self.tag_to_string(a)
url = a['href']
articles.append({
'title' :title
,'date' :''
,'url' :url
,'description':''
})
return [('Frontline', articles)]
for h3 in soup.findAll('h3'):
if h3.get('class', None) == 'artListSec':
if articles:
feeds.append((current_section, articles))
articles = []
current_section = self.tag_to_string(h3).strip()
self.log(current_section)
elif h3.get('id', None) in {'headseccol', 'headsec'}:
a = h3.find('a', href=True)
if a is not None:
title = self.tag_to_string(a)
url = a['href']
articles.append({
'title' :title
,'date' :''
,'url' :url
,'description':''
})
self.log('\t', title, url)
if articles:
feeds.append((current_section, articles))
return feeds
#def print_version(self, url):
#return "http://www.hinduonnet.com/thehindu/thscrip/print.pl?prd=fline&file=" + url.rpartition('/')[2]
#def image_url_processor(self, baseurl, url):
#return url.replace('../images/', self.INDEX + 'images/').strip()

BIN
recipes/icons/miradasalsur.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.1 KiB

2
recipes/la_nacion_cr.recipe
View File

@ -20,7 +20,7 @@ class crnews(BasicNewsRecipe):
no_stylesheets = True
feeds = [(u'Portada', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=portada'), (u'Ultima Hora', u'http://www.nacion.com/Generales/RSS/UltimaHoraRss.aspx'), (u'Nacionales', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=elpais'), (u'Entretenimiento', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=entretenimiento'), (u'Sucesos', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=sucesos'), (u'Deportes', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=deportes'), (u'Internacionales', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=mundo'), (u'Economia', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=economia'), (u'Aldea Global', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=aldeaglobal'), (u'Tecnologia', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=tecnologia'), (u'Opinion', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=opinion')]
feeds = [(u'Portada', u'http://www.nacion.com/rss/'), (u'Ultima Hora', u'http://www.nacion.com/rss/latest/'), (u'Nacionales', u'http://www.nacion.com/rss/nacional/'), (u'Entretenimiento', u'http://www.nacion.com/rss/ocio/'), (u'Sucesos', u'http://www.nacion.com/rss/sucesos/'), (u'Deportes', u'http://www.nacion.com/rss/deportes/'), (u'Internacionales', u'http://www.nacion.com/rss/mundo/'), (u'Economia', u'http://www.nacion.com/rss/economia/'), (u'Vivir', u'http://www.nacion.com/rss/vivir/'), (u'Tecnologia', u'http://www.nacion.com/rss/tecnologia/'), (u'Opinion', u'http://www.nacion.com/rss/opinion/')]
def get_cover_url(self):
index = 'http://kiosko.net/cr/np/cr_nacion.html'

16
recipes/living_digital.recipe
View File

@ -1,16 +0,0 @@
from calibre.web.feeds.news import CalibrePeriodical
class LivingDigital(CalibrePeriodical):
title = 'Living Digital'
calibre_periodicals_slug = 'living-digital'
description = '''
Catch the latest buzz in the digital world with Living Digital. Enjoy
reviews, news, features and recommendations on a wide range of consumer
technology products - from smartphones to flat panel TVs, netbooks to
cameras, and many more consumer lifestyle gadgets. To subscribe, visit
<a href="http://news.calibre-ebook.com/periodical/living-digital">calibre
Periodicals</a>.
'''
language = 'en_IN'

95
recipes/miradasalsur.recipe
View File

@ -1,18 +1,15 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2009-2013, Darko Miletic <darko.miletic at gmail.com>'
'''
elargentino.com
sur.infonews.com
'''
import datetime
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import Tag
class MiradasAlSur(BasicNewsRecipe):
title = 'Miradas al Sur'
__author__ = 'Darko Miletic'
description = 'Revista Argentina'
description = 'Semanario Argentino'
publisher = 'ElArgentino.com'
category = 'news, politics, Argentina'
oldest_article = 7
@ -20,53 +17,51 @@ class MiradasAlSur(BasicNewsRecipe):
no_stylesheets = True
use_embedded_content = False
encoding = 'utf-8'
language = 'es_AR'
language = 'es_AR'
remove_empty_feeds = True
masthead_url = 'http://sur.infonews.com/sites/default/files/www_miradas_al_sur_com_logo.gif'
extra_css = """
body{font-family: Arial,Helvetica,sans-serif}
h1{font-family: Georgia,Times,serif}
.field-field-story-author{color: gray; font-size: small}
"""
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
, 'series' : title
}
lang = 'es-AR'
direction = 'ltr'
INDEX = 'http://www.elargentino.com/medios/123/Miradas-al-Sur.html'
extra_css = ' .titulo{font-size: x-large; font-weight: bold} .volantaImp{font-size: small; font-weight: bold} '
keep_only_tags = [dict(name='div', attrs={'id':['content-header', 'content-area']})]
remove_tags = [
dict(name=['link','meta','iframe','embed','object']),
dict(name='form', attrs={'class':'fivestar-widget'}),
dict(attrs={'class':lambda x: x and 'terms-inline' in x.split()})
]
html2lrf_options = [
'--comment' , description
, '--category' , category
, '--publisher', publisher
]
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0cm; margin-top: 0em; margin-bottom: 0.5em} "'
keep_only_tags = [dict(name='div', attrs={'class':'ContainerPop'})]
remove_tags = [dict(name='link')]
feeds = [(u'Articulos', u'http://www.elargentino.com/Highlights.aspx?ParentType=Section&ParentId=123&Content-Type=text/xml&ChannelDesc=Miradas%20al%20Sur')]
def print_version(self, url):
main, sep, article_part = url.partition('/nota-')
article_id, rsep, rrest = article_part.partition('-')
return u'http://www.elargentino.com/Impresion.aspx?Id=' + article_id
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
soup.html['lang'] = self.lang
soup.html['dir' ] = self.direction
mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)])
mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=utf-8")])
soup.head.insert(0,mlang)
soup.head.insert(1,mcharset)
return soup
feeds = [
(u'Politica' , u'http://sur.infonews.com/taxonomy/term/1/0/feed'),
(u'Internacional' , u'http://sur.infonews.com/taxonomy/term/2/0/feed'),
(u'Informe Especial' , u'http://sur.infonews.com/taxonomy/term/14/0/feed'),
(u'Delitos y pesquisas', u'http://sur.infonews.com/taxonomy/term/6/0/feed'),
(u'Lesa Humanidad' , u'http://sur.infonews.com/taxonomy/term/7/0/feed'),
(u'Cultura' , u'http://sur.infonews.com/taxonomy/term/8/0/feed'),
(u'Deportes' , u'http://sur.infonews.com/taxonomy/term/9/0/feed'),
(u'Contratapa' , u'http://sur.infonews.com/taxonomy/term/10/0/feed'),
]
def get_cover_url(self):
# determine the series number, unfortunately not gonna happen now
#self.conversion_options.update({'series_index':seriesnr})
cover_url = None
soup = self.index_to_soup(self.INDEX)
cover_item = soup.find('div',attrs={'class':'colder'})
cdate = datetime.date.today()
todayweekday = cdate.isoweekday()
if (todayweekday != 7):
cdate -= datetime.timedelta(days=todayweekday)
cover_page_url = cdate.strftime('http://sur.infonews.com/ediciones/%Y-%m-%d/tapa')
soup = self.index_to_soup(cover_page_url)
cover_item = soup.find('img', attrs={'class':lambda x: x and 'imagecache-tapa_edicion_full' in x.split()})
if cover_item:
clean_url = self.image_url_processor(None,cover_item.div.img['src'])
cover_url = 'http://www.elargentino.com' + clean_url + '&height=600'
cover_url = cover_item['src']
return cover_url
def image_url_processor(self, baseurl, url):
base, sep, rest = url.rpartition('?Id=')
img, sep2, rrest = rest.partition('&')
return base + sep + img

16
recipes/pc_quest_india.recipe
View File

@ -1,16 +0,0 @@
from calibre.web.feeds.news import CalibrePeriodical
class PCQ(CalibrePeriodical):
title = 'PCQuest'
calibre_periodicals_slug = 'pc-quest-india'
description = '''
Buying a tech product? Seeking a tech solution? Consult PCQuest, India's
market-leading selection and implementation guide for the latest
technologies: servers, business apps, security, open source, gadgets and
more. To subscribe visit, <a
href="http://news.calibre-ebook.com/periodical/pc-quest-india">calibre
Periodicals</a>.
'''
language = 'en_IN'

1
recipes/taz_rss.recipe
View File

@ -18,7 +18,6 @@ class TazRSSRecipe(BasicNewsRecipe):
feeds = [(u'TAZ main feed', u'http://www.taz.de/rss.xml')]
keep_only_tags = [dict(name='div', attrs={'class': 'sect sect_article'})]
remove_tags_after = dict(name='div', attrs={'class': 'rack'})
remove_tags = [
dict(name=['div'], attrs={'class': 'artikelwerbung'}),
dict(name=['ul'], attrs={'class': 'toolbar'}),]

47
setup/file_hosting_servers.rst Normal file
View File

@ -0,0 +1,47 @@
Provisioning a file hosting server
====================================
Create the ssh authorized keys file.
Edit /etc/ssh/sshd_config and change PermitRootLogin to without-password.
Restart sshd.
hostname whatever
Edit /etc/hosts and put in FQDN in the appropriate places, for example::
27.0.1.1 download.calibre-ebook.com download
46.28.49.116 download.calibre-ebook.com download
apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools mosh
chsh -s /bin/zsh
mkdir -p /root/staging /root/work/vim /srv/download /srv/manual
scp .zshrc .vimrc server:
scp -r ~/work/vim/zsh-syntax-highlighting server:work/vim
If the server has a backup hard-disk, mount it at /mnt/backup and edit /etc/fstab so that it is auto-mounted.
Then, add the following to crontab::
@daily /usr/bin/rsync -ha /srv /mnt/backup
@daily /usr/bin/rsync -ha /etc /mnt/backup
Nginx
------
Copy over /etc/nginx/sites-available/default from another file server. When
copying, remember to use cat instead of cp to preserve hardlinks (the file is a
hardlink to /etc/nginx/sites-enabled/default)
rsync /srv from another file server
service nginx start
Services
---------
SSH into sourceforge and downloadbestsoftware so that their host keys are
stored.
ssh -oStrictHostKeyChecking=no kovid@www.downloadbestsoft-mirror1.com
ssh -oStrictHostKeyChecking=no kovidgoyal,calibre@frs.sourceforge.net
ssh -oStrictHostKeyChecking=no files.calibre-ebook.com (and whatever other mirrors are present)

183
setup/hosting.py
View File

@ -7,16 +7,14 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib
import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil
from argparse import ArgumentParser, FileType
from subprocess import check_call
from tempfile import NamedTemporaryFile#, mkdtemp
from tempfile import NamedTemporaryFile
from collections import OrderedDict
import mechanize
from lxml import html
def login_to_google(username, password):
def login_to_google(username, password): # {{{
import mechanize
br = mechanize.Browser()
br.addheaders = [('User-agent',
'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')]
@ -30,15 +28,16 @@ def login_to_google(username, password):
x = re.search(br'(?is)<title>.*?</title>', raw)
if x is not None:
print ('Title of post login page: %s'%x.group())
#open('/tmp/goog.html', 'wb').write(raw)
# open('/tmp/goog.html', 'wb').write(raw)
raise ValueError(('Failed to login to google with credentials: %s %s'
'\nGoogle sometimes requires verification when logging in from a '
'new IP address. Use lynx to login and supply the verification, '
'at: lynx -accept_all_cookies https://accounts.google.com/ServiceLogin?service=code')
%(username, password))
return br
# }}}
class ReadFileWithProgressReporting(file): # {{{
class ReadFileWithProgressReporting(file): # {{{
def __init__(self, path, mode='rb'):
file.__init__(self, path, mode)
@ -101,7 +100,7 @@ class Base(object): # {{{
#}}}
class GoogleCode(Base):# {{{
class GoogleCode(Base): # {{{
def __init__(self,
# A mapping of filenames to file descriptions. The descriptions are
@ -141,7 +140,7 @@ class GoogleCode(Base):# {{{
# The pattern to match filenames for the files being uploaded and
# extract version information from them. Must have a named group
# named version
filename_pattern=r'{appname}-(?:portable-installer-)?(?P<version>.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)'
filename_pattern=r'{appname}-(?:portable-installer-)?(?P<version>.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' # noqa
):
self.username, self.password, = username, password
@ -227,7 +226,8 @@ class GoogleCode(Base):# {{{
paths = eval(raw) if raw else {}
paths.update(self.paths)
rem = [x for x in paths if self.version not in x]
for x in rem: paths.pop(x)
for x in rem:
paths.pop(x)
raw = ['%r : %r,'%(k, v) for k, v in paths.items()]
raw = '{\n\n%s\n\n}\n'%('\n'.join(raw))
with NamedTemporaryFile() as t:
@ -244,6 +244,7 @@ class GoogleCode(Base):# {{{
return login_to_google(self.username, self.gmail_password)
def get_files_hosted_by_google_code(self):
from lxml import html
self.info('Getting existing files in google code:', self.gc_project)
raw = urllib2.urlopen(self.files_list).read()
root = html.fromstring(raw)
@ -347,7 +348,7 @@ class GoogleCode(Base):# {{{
# }}}
class SourceForge(Base): # {{{
class SourceForge(Base): # {{{
# Note that you should manually ssh once to username,project@frs.sourceforge.net
# on the staging server so that the host key is setup
@ -378,6 +379,156 @@ class SourceForge(Base): # {{{
# }}}
def generate_index(): # {{{
os.chdir('/srv/download')
releases = set()
for x in os.listdir('.'):
if os.path.isdir(x) and '.' in x:
releases.add(tuple((int(y) for y in x.split('.'))))
rmap = OrderedDict()
for rnum in sorted(releases, reverse=True):
series = rnum[:2] if rnum[0] == 0 else rnum[:1]
if series not in rmap:
rmap[series] = []
rmap[series].append(rnum)
template = '''<!DOCTYPE html>\n<html lang="en"> <head> <meta charset="utf-8"> <title>{title}</title> <style type="text/css"> {style} </style> </head> <body> <h1>{title}</h1> <p>{msg}</p> {body} </body> </html> ''' # noqa
style = '''
body { font-family: sans-serif; background-color: #eee; }
a { text-decoration: none; }
a:visited { color: blue }
a:hover { color: red }
ul { list-style-type: none }
li { padding-bottom: 1ex }
dd li { text-indent: 0; margin: 0 }
dd ul { padding: 0; margin: 0 }
dt { font-weight: bold }
dd { margin-bottom: 2ex }
'''
body = []
for series in rmap:
body.append('<li><a href="{0}.html" title="Releases in the {0}.x series">{0}.x</a>\xa0\xa0\xa0<span style="font-size:smaller">[{1} releases]</span></li>'.format( # noqa
'.'.join(map(type(''), series)), len(rmap[series])))
body = '<ul>{0}</ul>'.format(' '.join(body))
index = template.format(title='Previous calibre releases', style=style, msg='Choose a series of calibre releases', body=body)
with open('index.html', 'wb') as f:
f.write(index.encode('utf-8'))
for series, releases in rmap.iteritems():
sname = '.'.join(map(type(''), series))
body = [
'<li><a href="{0}/" title="Release {0}">{0}</a></li>'.format('.'.join(map(type(''), r)))
for r in releases]
body = '<ul class="release-list">{0}</ul>'.format(' '.join(body))
index = template.format(title='Previous calibre releases (%s.x)' % sname, style=style,
msg='Choose a calibre release', body=body)
with open('%s.html' % sname, 'wb') as f:
f.write(index.encode('utf-8'))
for r in releases:
rname = '.'.join(map(type(''), r))
os.chdir(rname)
try:
body = []
files = os.listdir('.')
windows = [x for x in files if x.endswith('.msi')]
if windows:
windows = ['<li><a href="{0}" title="{1}">{1}</a></li>'.format(
x, 'Windows 64-bit Installer' if '64bit' in x else 'Windows 32-bit Installer')
for x in windows]
body.append('<dt>Windows</dt><dd><ul>{0}</ul></dd>'.format(' '.join(windows)))
portable = [x for x in files if '-portable-' in x]
if portable:
body.append('<dt>Calibre Portable</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(
portable[0], 'Calibre Portable Installer'))
osx = [x for x in files if x.endswith('.dmg')]
if osx:
body.append('<dt>Apple Mac</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(
osx[0], 'OS X Disk Image (.dmg)'))
linux = [x for x in files if x.endswith('.bz2')]
if linux:
linux = ['<li><a href="{0}" title="{1}">{1}</a></li>'.format(
x, 'Linux 64-bit binary' if 'x86_64' in x else 'Linux 32-bit binary')
for x in linux]
body.append('<dt>Linux</dt><dd><ul>{0}</ul></dd>'.format(' '.join(linux)))
source = [x for x in files if x.endswith('.xz') or x.endswith('.gz')]
if source:
body.append('<dt>Source Code</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(
source[0], 'Source code (all platforms)'))
body = '<dl>{0}</dl>'.format(''.join(body))
index = template.format(title='calibre release (%s)' % rname, style=style,
msg='', body=body)
with open('index.html', 'wb') as f:
f.write(index.encode('utf-8'))
finally:
os.chdir('..')
# }}}
def upload_to_servers(files, version): # {{{
base = '/srv/download/'
dest = os.path.join(base, version)
if not os.path.exists(dest):
os.mkdir(dest)
for src in files:
shutil.copyfile(src, os.path.join(dest, os.path.basename(src)))
cwd = os.getcwd()
try:
generate_index()
finally:
os.chdir(cwd)
for server, rdir in {'files':'/srv/download/'}.iteritems():
print('Uploading to server:', server)
server = '%s.calibre-ebook.com' % server
# Copy the generated index files
print ('Copying generated index')
check_call(['rsync', '-hza', '-e', 'ssh -x', '--include', '*.html',
'--filter', '-! */', base, 'root@%s:%s' % (server, rdir)])
# Copy the release files
rdir = '%s%s/' % (rdir, version)
for x in files:
start = time.time()
print ('Uploading', x)
for i in range(5):
try:
check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
'root@%s:%s'%(server, rdir)])
except KeyboardInterrupt:
raise SystemExit(1)
except:
print ('\nUpload failed, trying again in 30 seconds')
time.sleep(30)
else:
break
print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
# }}}
def upload_to_dbs(files, version): # {{{
print('Uploading to downloadbestsoftware.com')
server = 'www.downloadbestsoft-mirror1.com'
rdir = 'release/'
check_call(['ssh', 'kovid@%s' % server, 'rm -f release/*'])
for x in files:
start = time.time()
print ('Uploading', x)
for i in range(5):
try:
check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
'kovid@%s:%s'%(server, rdir)])
except KeyboardInterrupt:
raise SystemExit(1)
except:
print ('\nUpload failed, trying again in 30 seconds')
time.sleep(30)
else:
break
print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
check_call(['ssh', 'kovid@%s' % server, '/home/kovid/uploadFiles'])
# }}}
# CLI {{{
def cli_parser():
epilog='Copyright Kovid Goyal 2012'
@ -409,6 +560,8 @@ def cli_parser():
sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge',
epilog=epilog)
cron = subparsers.add_parser('cron', help='Call script from cron')
subparsers.add_parser('calibre', help='Upload to calibre file servers')
subparsers.add_parser('dbs', help='Upload to downloadbestsoftware.com')
a = gc.add_argument
@ -471,8 +624,14 @@ def main(args=None):
sf()
elif args.service == 'cron':
login_to_google(args.username, args.password)
elif args.service == 'calibre':
upload_to_servers(ofiles, args.version)
elif args.service == 'dbs':
upload_to_dbs(ofiles, args.version)
if __name__ == '__main__':
main()
# }}}

48
setup/upload.py
View File

@ -19,10 +19,9 @@ from setup import Command, __version__, installer_name, __appname__
PREFIX = "/var/www/calibre-ebook.com"
DOWNLOADS = PREFIX+"/htdocs/downloads"
BETAS = DOWNLOADS +'/betas'
USER_MANUAL = '/var/www/localhost/htdocs/'
HTML2LRF = "calibre/ebooks/lrf/html/demo"
TXT2LRF = "src/calibre/ebooks/lrf/txt/demo"
STAGING_HOST = '67.207.135.179'
STAGING_HOST = 'download.calibre-ebook.com'
STAGING_USER = 'root'
STAGING_DIR = '/root/staging'
@ -81,7 +80,7 @@ class ReUpload(Command): # {{{
# Data {{{
def get_google_data():
with open(os.path.expanduser('~/work/kde/conf/googlecodecalibre'), 'rb') as f:
with open(os.path.expanduser('~/work/env/private/googlecodecalibre'), 'rb') as f:
gc_password, ga_un, pw = f.read().strip().split('|')
return {
@ -111,6 +110,12 @@ def sf_cmdline(ver, sdata):
return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'],
sdata['username']]
def calibre_cmdline(ver):
return [__appname__, ver, 'fmap', 'calibre']
def dbs_cmdline(ver):
return [__appname__, ver, 'fmap', 'dbs']
def run_remote_upload(args):
print 'Running remotely:', ' '.join(args)
subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST),
@ -129,22 +134,37 @@ class UploadInstallers(Command): # {{{
available = set(glob.glob('dist/*'))
files = {x:installer_description(x) for x in
all_possible.intersection(available)}
sizes = {os.path.basename(x):os.path.getsize(x) for x in files}
self.record_sizes(sizes)
tdir = mkdtemp()
backup = os.path.join('/mnt/external/calibre/%s' % __version__)
if not os.path.exists(backup):
os.mkdir(backup)
try:
self.upload_to_staging(tdir, files)
self.upload_to_staging(tdir, backup, files)
self.upload_to_calibre()
self.upload_to_sourceforge()
self.upload_to_google(opts.replace)
self.upload_to_dbs()
# self.upload_to_google(opts.replace)
finally:
shutil.rmtree(tdir, ignore_errors=True)
def upload_to_staging(self, tdir, files):
def record_sizes(self, sizes):
print ('\nRecording dist sizes')
args = ['%s:%s:%s' % (__version__, fname, size) for fname, size in sizes.iteritems()]
check_call(['ssh', 'divok', 'dist_sizes'] + args)
def upload_to_staging(self, tdir, backup, files):
os.mkdir(tdir+'/dist')
hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'hosting.py')
shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py'))
for f in files:
shutil.copyfile(f, os.path.join(tdir, f))
for x in (tdir+'/dist', backup):
dest = os.path.join(x, os.path.basename(f))
shutil.copy2(f, x)
os.chmod(dest, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH)
with open(os.path.join(tdir, 'fmap'), 'wb') as fo:
for f, desc in files.iteritems():
@ -170,6 +190,12 @@ class UploadInstallers(Command): # {{{
sdata = get_sourceforge_data()
args = sf_cmdline(__version__, sdata)
run_remote_upload(args)
def upload_to_calibre(self):
run_remote_upload(calibre_cmdline(__version__))
def upload_to_dbs(self):
run_remote_upload(dbs_cmdline(__version__))
# }}}
class UploadUserManual(Command): # {{{
@ -199,9 +225,9 @@ class UploadUserManual(Command): # {{{
for x in glob.glob(self.j(path, '*')):
self.build_plugin_example(x)
check_call(' '.join(['rsync', '-z', '-r', '--progress',
'manual/.build/html/',
'bugs:%s'%USER_MANUAL]), shell=True)
for host in ('download', 'files'):
check_call(' '.join(['rsync', '-z', '-r', '--progress',
'manual/.build/html/', '%s:/srv/manual/' % host]), shell=True)
# }}}
class UploadDemo(Command): # {{{
@ -229,8 +255,6 @@ class UploadToServer(Command): # {{{
description = 'Upload miscellaneous data to calibre server'
def run(self, opts):
check_call('ssh divok rm -f %s/calibre-\*.tar.xz'%DOWNLOADS, shell=True)
# check_call('scp dist/calibre-*.tar.xz divok:%s/'%DOWNLOADS, shell=True)
check_call('gpg --armor --detach-sign dist/calibre-*.tar.xz',
shell=True)
check_call('scp dist/calibre-*.tar.xz.asc divok:%s/signatures/'%DOWNLOADS,

14
src/calibre/__init__.py
View File

@ -310,9 +310,9 @@ def get_parsed_proxy(typ='http', debug=True):
proxy = proxies.get(typ, None)
if proxy:
pattern = re.compile((
'(?:ptype://)?' \
'(?:(?P<user>\w+):(?P<pass>.*)@)?' \
'(?P<host>[\w\-\.]+)' \
'(?:ptype://)?'
'(?:(?P<user>\w+):(?P<pass>.*)@)?'
'(?P<host>[\w\-\.]+)'
'(?::(?P<port>\d+))?').replace('ptype', typ)
)
@ -535,7 +535,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252',
ent = match.group(1)
if ent in exceptions:
return '&'+ent+';'
if ent in {'apos', 'squot'}: # squot is generated by some broken CMS software
if ent in {'apos', 'squot'}: # squot is generated by some broken CMS software
return check("'")
if ent == 'hellips':
ent = 'hellip'
@ -565,7 +565,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252',
return '&'+ent+';'
_ent_pat = re.compile(r'&(\S+?);')
xml_entity_to_unicode = partial(entity_to_unicode, result_exceptions = {
xml_entity_to_unicode = partial(entity_to_unicode, result_exceptions={
'"' : '&quot;',
"'" : '&apos;',
'<' : '&lt;',
@ -670,8 +670,8 @@ def human_readable(size, sep=' '):
""" Convert a size in bytes into a human readable form """
divisor, suffix = 1, "B"
for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
if size < 1024**(i+1):
divisor, suffix = 1024**(i), candidate
if size < (1 << ((i + 1) * 10)):
divisor, suffix = (1 << (i * 10)), candidate
break
size = str(float(size)/divisor)
if size.find(".") > -1:

2
src/calibre/constants.py
View File

@ -4,7 +4,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
__appname__ = u'calibre'
numeric_version = (0, 9, 36)
numeric_version = (0, 9, 37)
__version__ = u'.'.join(map(unicode, numeric_version))
__author__ = u"Kovid Goyal <kovid@kovidgoyal.net>"

36
src/calibre/db/cache.py
View File

@ -843,9 +843,25 @@ class Cache(object):
@write_api
def set_metadata(self, book_id, mi, ignore_errors=False, force_changes=False,
set_title=True, set_authors=True):
if callable(getattr(mi, 'to_book_metadata', None)):
'''
Set metadata for the book `id` from the `Metadata` object `mi`
Setting force_changes=True will force set_metadata to update fields even
if mi contains empty values. In this case, 'None' is distinguished from
'empty'. If mi.XXX is None, the XXX is not replaced, otherwise it is.
The tags, identifiers, and cover attributes are special cases. Tags and
identifiers cannot be set to None so then will always be replaced if
force_changes is true. You must ensure that mi contains the values you
want the book to have. Covers are always changed if a new cover is
provided, but are never deleted. Also note that force_changes has no
effect on setting title or authors.
'''
try:
# Handle code passing in an OPF object instead of a Metadata object
mi = mi.to_book_metadata()
except (AttributeError, TypeError):
pass
def set_field(name, val, **kwargs):
self._set_field(name, {book_id:val}, **kwargs)
@ -864,7 +880,7 @@ class Cache(object):
set_field('authors', authors, do_path_update=False)
if path_changed:
self._update_path((book_id,))
self._update_path({book_id})
def protected_set_field(name, val, **kwargs):
try:
@ -890,12 +906,16 @@ class Cache(object):
if cdata is not None:
self._set_cover({book_id: cdata})
for field in ('title_sort', 'author_sort', 'publisher', 'series',
'tags', 'comments', 'languages', 'pubdate'):
for field in ('author_sort', 'publisher', 'series', 'tags', 'comments',
'languages', 'pubdate'):
val = mi.get(field, None)
if (force_changes and val is not None) or not mi.is_null(field):
protected_set_field(field, val)
val = mi.get('title_sort', None)
if (force_changes and val is not None) or not mi.is_null('title_sort'):
protected_set_field('sort', val)
# identifiers will always be replaced if force_changes is True
mi_idents = mi.get_identifiers()
if force_changes:
@ -917,9 +937,11 @@ class Cache(object):
val = mi.get(key, None)
if force_changes or val is not None:
protected_set_field(key, val)
extra = mi.get_extra(key)
if extra is not None:
protected_set_field(key+'_index', extra)
idx = key + '_index'
if idx in self.fields:
extra = mi.get_extra(key)
if extra is not None or force_changes:
protected_set_field(idx, extra)
# }}}

6
src/calibre/db/tests/base.py
View File

@ -78,7 +78,7 @@ class BaseTest(unittest.TestCase):
def cloned_library(self):
return self.clone_library(self.library_path)
def compare_metadata(self, mi1, mi2):
def compare_metadata(self, mi1, mi2, exclude=()):
allfk1 = mi1.all_field_keys()
allfk2 = mi2.all_field_keys()
self.assertEqual(allfk1, allfk2)
@ -88,7 +88,7 @@ class BaseTest(unittest.TestCase):
'ondevice_col', 'last_modified', 'has_cover',
'cover_data'}.union(allfk1)
for attr in all_keys:
if attr == 'user_metadata':
if attr == 'user_metadata' or attr in exclude:
continue
attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr)
if attr == 'formats':
@ -97,7 +97,7 @@ class BaseTest(unittest.TestCase):
attr1, attr2 = set(attr1), set(attr2)
self.assertEqual(attr1, attr2,
'%s not the same: %r != %r'%(attr, attr1, attr2))
if attr.startswith('#'):
if attr.startswith('#') and attr + '_index' not in exclude:
attr1, attr2 = mi1.get_extra(attr), mi2.get_extra(attr)
self.assertEqual(attr1, attr2,
'%s {#extra} not the same: %r != %r'%(attr, attr1, attr2))

3
src/calibre/db/tests/legacy.py
View File

@ -120,7 +120,8 @@ class LegacyTest(BaseTest):
for attr in dir(db):
if attr in SKIP_ATTRS:
continue
self.assertTrue(hasattr(ndb, attr), 'The attribute %s is missing' % attr)
if not hasattr(ndb, attr):
raise AssertionError('The attribute %s is missing' % attr)
obj, nobj = getattr(db, attr), getattr(ndb, attr)
if attr not in SKIP_ARGSPEC:
try:

40
src/calibre/db/tests/writing.py
View File

@ -376,7 +376,43 @@ class WritingTest(BaseTest):
self.assertTrue(old.has_cover(book_id))
# }}}
def test_set_metadata(self):
def test_set_metadata(self): # {{{
' Test setting of metadata '
self.assertTrue(False, 'TODO: test set_metadata()')
ae = self.assertEqual
cache = self.init_cache(self.cloned_library)
# Check that changing title/author updates the path
mi = cache.get_metadata(1)
old_path = cache.field_for('path', 1)
old_title, old_author = mi.title, mi.authors[0]
ae(old_path, '%s/%s (1)' % (old_author, old_title))
mi.title, mi.authors = 'New Title', ['New Author']
cache.set_metadata(1, mi)
ae(cache.field_for('path', 1), '%s/%s (1)' % (mi.authors[0], mi.title))
p = cache.format_abspath(1, 'FMT1')
self.assertTrue(mi.authors[0] in p and mi.title in p)
# Compare old and new set_metadata()
db = self.init_old(self.cloned_library)
mi = db.get_metadata(1, index_is_id=True, get_cover=True, cover_as_data=True)
mi2 = db.get_metadata(3, index_is_id=True, get_cover=True, cover_as_data=True)
db.set_metadata(2, mi)
db.set_metadata(1, mi2, force_changes=True)
oldmi = db.get_metadata(2, index_is_id=True, get_cover=True, cover_as_data=True)
oldmi2 = db.get_metadata(1, index_is_id=True, get_cover=True, cover_as_data=True)
db.close()
del db
cache = self.init_cache(self.cloned_library)
cache.set_metadata(2, mi)
nmi = cache.get_metadata(2, get_cover=True, cover_as_data=True)
ae(oldmi.cover_data, nmi.cover_data)
self.compare_metadata(nmi, oldmi, exclude={'last_modified', 'format_metadata'})
cache.set_metadata(1, mi2, force_changes=True)
nmi2 = cache.get_metadata(1, get_cover=True, cover_as_data=True)
# The new code does not allow setting of #series_index to None, instead
# it is reset to 1.0
ae(nmi2.get_extra('#series'), 1.0)
self.compare_metadata(nmi2, oldmi2, exclude={'last_modified', 'format_metadata', '#series_index'})
# }}}

2
src/calibre/db/write.py
View File

@ -212,7 +212,7 @@ def custom_series_index(book_id_val_map, db, field, *args):
ids = series_field.ids_for_book(book_id)
if ids:
sequence.append((sidx, book_id, ids[0]))
field.table.book_col_map[book_id] = sidx
field.table.book_col_map[book_id] = sidx
if sequence:
db.conn.executemany('UPDATE %s SET %s=? WHERE book=? AND value=?'%(
field.metadata['table'], field.metadata['column']), sequence)

1
src/calibre/devices/misc.py
View File

@ -211,6 +211,7 @@ class ALURATEK_COLOR(USBMS):
VENDOR_NAME = ['USB_2.0', 'EZREADER', 'C4+']
WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['USB_FLASH_DRIVER', '.', 'TOUCH']
SCAN_FROM_ROOT = True
SUPPORTS_SUB_DIRS_FOR_SCAN = True
class TREKSTOR(USBMS):

9
src/calibre/ebooks/conversion/cli.py
View File

@ -94,6 +94,8 @@ def option_recommendation_to_cli_option(add_option, rec):
if opt.long_switch == 'verbose':
attrs['action'] = 'count'
attrs.pop('type', '')
if opt.name == 'read_metadata_from_opf':
switches.append('--from-opf')
if opt.name in DEFAULT_TRUE_OPTIONS and rec.recommended_value is True:
switches = ['--disable-'+opt.long_switch]
add_option(Option(*switches, **attrs))
@ -136,7 +138,7 @@ def add_pipeline_options(parser, plumber):
[
'base_font_size', 'disable_font_rescaling',
'font_size_mapping', 'embed_font_family',
'subset_embedded_fonts',
'subset_embedded_fonts', 'embed_all_fonts',
'line_height', 'minimum_line_height',
'linearize_tables',
'extra_css', 'filter_css',
@ -190,7 +192,7 @@ def add_pipeline_options(parser, plumber):
),
'METADATA' : (_('Options to set metadata in the output'),
plumber.metadata_option_names,
plumber.metadata_option_names + ['read_metadata_from_opf'],
),
'DEBUG': (_('Options to help with debugging the conversion'),
[
@ -320,7 +322,7 @@ def main(args=sys.argv):
opts.search_replace = read_sr_patterns(opts.search_replace, log)
recommendations = [(n.dest, getattr(opts, n.dest),
OptionRecommendation.HIGH) \
OptionRecommendation.HIGH)
for n in parser.options_iter()
if n.dest]
plumber.merge_ui_recommendations(recommendations)
@ -342,3 +344,4 @@ def main(args=sys.argv):
if __name__ == '__main__':
sys.exit(main())

17
src/calibre/ebooks/conversion/plumber.py
View File

@ -205,6 +205,16 @@ OptionRecommendation(name='embed_font_family',
'with some output formats, principally EPUB and AZW3.')
),
OptionRecommendation(name='embed_all_fonts',
recommended_value=False, level=OptionRecommendation.LOW,
help=_(
'Embed every font that is referenced in the input document '
'but not already embedded. This will search your system for the '
'fonts, and if found, they will be embedded. Embedding will only work '
'if the format you are converting to supports embedded fonts, such as '
'EPUB, AZW3 or PDF.'
)),
OptionRecommendation(name='subset_embedded_fonts',
recommended_value=False, level=OptionRecommendation.LOW,
help=_(
@ -965,6 +975,9 @@ OptionRecommendation(name='search_replace',
if self.for_regex_wizard and hasattr(self.opts, 'no_process'):
self.opts.no_process = True
self.flush()
if self.opts.embed_all_fonts or self.opts.embed_font_family:
# Start the threaded font scanner now, for performance
from calibre.utils.fonts.scanner import font_scanner # noqa
import cssutils, logging
cssutils.log.setLevel(logging.WARN)
get_types_map() # Ensure the mimetypes module is intialized
@ -1129,6 +1142,10 @@ OptionRecommendation(name='search_replace',
RemoveFakeMargins()(self.oeb, self.log, self.opts)
RemoveAdobeMargins()(self.oeb, self.log, self.opts)
if self.opts.embed_all_fonts:
from calibre.ebooks.oeb.transforms.embed_fonts import EmbedFonts
EmbedFonts()(self.oeb, self.log, self.opts)
if self.opts.subset_embedded_fonts and self.output_plugin.file_type != 'pdf':
from calibre.ebooks.oeb.transforms.subset import SubsetFonts
SubsetFonts()(self.oeb, self.log, self.opts)

2
src/calibre/ebooks/lrf/html/convert_from.py
View File

@ -104,7 +104,7 @@ class HTMLConverter(object):
# Replace entities
(re.compile(ur'&(\S+?);'), partial(entity_to_unicode,
exceptions=['lt', 'gt', 'amp'])),
exceptions=['lt', 'gt', 'amp', 'quot'])),
# Remove comments from within style tags as they can mess up BeatifulSoup
(re.compile(r'(<style.*?</style>)', re.IGNORECASE|re.DOTALL),
strip_style_comments),

11
src/calibre/ebooks/metadata/opf2.py
View File

@ -1047,6 +1047,14 @@ class OPF(object): # {{{
if raw:
return raw.rpartition(':')[-1]
@property
def page_progression_direction(self):
spine = self.XPath('descendant::*[re:match(name(), "spine", "i")][1]')(self.root)
if spine:
for k, v in spine[0].attrib.iteritems():
if k == 'page-progression-direction' or k.endswith('}page-progression-direction'):
return v
def guess_cover(self):
'''
Try to guess a cover. Needed for some old/badly formed OPF files.
@ -1185,6 +1193,7 @@ class OPFCreator(Metadata):
'''
Metadata.__init__(self, title='', other=other)
self.base_path = os.path.abspath(base_path)
self.page_progression_direction = None
if self.application_id is None:
self.application_id = str(uuid.uuid4())
if not isinstance(self.toc, TOC):
@ -1356,6 +1365,8 @@ class OPFCreator(Metadata):
spine = E.spine()
if self.toc is not None:
spine.set('toc', 'ncx')
if self.page_progression_direction is not None:
spine.set('page-progression-direction', self.page_progression_direction)
if self.spine is not None:
for ref in self.spine:
if ref.id is not None:

9
src/calibre/ebooks/metadata/sources/cli.py
View File

@ -34,9 +34,9 @@ def option_parser():
parser.add_option('-i', '--isbn', help='Book ISBN')
parser.add_option('-v', '--verbose', default=False, action='store_true',
help='Print the log to the console (stderr)')
parser.add_option('-o', '--opf', help='Output the metadata in OPF format')
parser.add_option('-o', '--opf', help='Output the metadata in OPF format instead of human readable text.', action='store_true', default=False)
parser.add_option('-c', '--cover',
help='Specify a filename. The cover, if available, will be saved to it')
help='Specify a filename. The cover, if available, will be saved to it. Without this option, no cover will be downloaded.')
parser.add_option('-d', '--timeout', default='30',
help='Timeout in seconds. Default is 30')
@ -71,16 +71,14 @@ def main(args=sys.argv):
if opts.cover and results:
cover = download_cover(log, title=opts.title, authors=authors,
identifiers=result.identifiers, timeout=int(opts.timeout))
if cover is None:
if cover is None and not opts.opf:
prints('No cover found', file=sys.stderr)
else:
save_cover_data_to(cover[-1], opts.cover)
result.cover = cf = opts.cover
log = buf.getvalue()
result = (metadata_to_opf(result) if opts.opf else
unicode(result).encode('utf-8'))
@ -95,3 +93,4 @@ def main(args=sys.argv):
if __name__ == '__main__':
sys.exit(main())

12
src/calibre/ebooks/mobi/reader/mobi8.py
View File

@ -20,7 +20,7 @@ from calibre.ebooks.mobi.reader.ncx import read_ncx, build_toc
from calibre.ebooks.mobi.reader.markup import expand_mobi8_markup
from calibre.ebooks.metadata.opf2 import Guide, OPFCreator
from calibre.ebooks.metadata.toc import TOC
from calibre.ebooks.mobi.utils import read_font_record
from calibre.ebooks.mobi.utils import read_font_record, read_resc_record
from calibre.ebooks.oeb.parse_utils import parse_html
from calibre.ebooks.oeb.base import XPath, XHTML, xml2text
from calibre.utils.imghdr import what
@ -65,6 +65,7 @@ class Mobi8Reader(object):
self.mobi6_reader, self.log = mobi6_reader, log
self.header = mobi6_reader.book_header
self.encrypted_fonts = []
self.resc_data = {}
def __call__(self):
self.mobi6_reader.check_for_drm()
@ -389,9 +390,11 @@ class Mobi8Reader(object):
data = sec[0]
typ = data[:4]
href = None
if typ in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n',
b'RESC', b'BOUN', b'FDST', b'DATP', b'AUDI', b'VIDE'}:
if typ in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n', b'BOUN',
b'FDST', b'DATP', b'AUDI', b'VIDE'}:
pass # Ignore these records
elif typ == b'RESC':
self.resc_data = read_resc_record(data)
elif typ == b'FONT':
font = read_font_record(data)
href = "fonts/%05d.%s" % (fname_idx, font['ext'])
@ -452,6 +455,9 @@ class Mobi8Reader(object):
opf.create_manifest_from_files_in([os.getcwdu()], exclude=exclude)
opf.create_spine(spine)
opf.set_toc(toc)
ppd = self.resc_data.get('page-progression-direction', None)
if ppd:
opf.page_progression_direction = ppd
with open('metadata.opf', 'wb') as of, open('toc.ncx', 'wb') as ncx:
opf.render(of, ncx, 'toc.ncx')

11
src/calibre/ebooks/mobi/utils.py
View File

@ -7,7 +7,7 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import struct, string, zlib, os
import struct, string, zlib, os, re
from collections import OrderedDict
from io import BytesIO
@ -393,6 +393,15 @@ def mobify_image(data):
data = im.export('gif')
return data
def read_resc_record(data):
ans = {}
match = re.search(br'''<spine [^>]*page-progression-direction=['"](.+?)['"]''', data)
if match is not None:
ppd = match.group(1).lower()
if ppd in {b'ltr', b'rtl'}:
ans['page-progression-direction'] = ppd.decode('ascii')
return ans
# Font records {{{
def read_font_record(data, extent=1040):
'''

3
src/calibre/ebooks/oeb/base.py
View File

@ -1210,6 +1210,7 @@ class Spine(object):
def __init__(self, oeb):
self.oeb = oeb
self.items = []
self.page_progression_direction = None
def _linear(self, linear):
if isinstance(linear, basestring):
@ -1896,4 +1897,6 @@ class OEBBook(object):
attrib={'media-type': PAGE_MAP_MIME})
spine.attrib['page-map'] = id
results[PAGE_MAP_MIME] = (href, self.pages.to_page_map())
if self.spine.page_progression_direction in {'ltr', 'rtl'}:
spine.attrib['page-progression-direction'] = self.spine.page_progression_direction
return results

9
src/calibre/ebooks/oeb/polish/toc.py
View File

@ -281,15 +281,18 @@ def find_text(node):
def from_files(container):
toc = TOC()
for spinepath in container.spine_items:
for i, spinepath in enumerate(container.spine_items):
name = container.abspath_to_name(spinepath)
root = container.parsed(name)
body = XPath('//h:body')(root)
if not body:
continue
text = find_text(body[0])
if text:
toc.add(text, name)
if not text:
text = name.rpartition('/')[-1]
if i == 0 and text.rpartition('.')[0].lower() in {'titlepage', 'cover'}:
text = _('Cover')
toc.add(text, name)
return toc
def add_id(container, name, loc):

3
src/calibre/ebooks/oeb/reader.py
View File

@ -330,6 +330,9 @@ class OEBReader(object):
if len(spine) == 0:
raise OEBError("Spine is empty")
self._spine_add_extra()
for val in xpath(opf, '/o2:package/o2:spine/@page-progression-direction'):
if val in {'ltr', 'rtl'}:
spine.page_progression_direction = val
def _guide_from_opf(self, opf):
guide = self.oeb.guide

233
src/calibre/ebooks/oeb/transforms/embed_fonts.py Normal file
View File

@ -0,0 +1,233 @@
#!/usr/bin/env python
# vim:fileencoding=utf-8
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import logging
from collections import defaultdict
import cssutils
from lxml import etree
from calibre import guess_type
from calibre.ebooks.oeb.base import XPath, CSS_MIME, XHTML
from calibre.ebooks.oeb.transforms.subset import get_font_properties, find_font_face_rules, elem_style
from calibre.utils.filenames import ascii_filename
from calibre.utils.fonts.scanner import font_scanner, NoFonts
def used_font(style, embedded_fonts):
ff = [unicode(f) for f in style.get('font-family', []) if unicode(f).lower() not in {
'serif', 'sansserif', 'sans-serif', 'fantasy', 'cursive', 'monospace'}]
if not ff:
return False, None
lnames = {unicode(x).lower() for x in ff}
matching_set = []
# Filter on font-family
for ef in embedded_fonts:
flnames = {x.lower() for x in ef.get('font-family', [])}
if not lnames.intersection(flnames):
continue
matching_set.append(ef)
if not matching_set:
return True, None
# Filter on font-stretch
widths = {x:i for i, x in enumerate(('ultra-condensed',
'extra-condensed', 'condensed', 'semi-condensed', 'normal',
'semi-expanded', 'expanded', 'extra-expanded', 'ultra-expanded'
))}
width = widths[style.get('font-stretch', 'normal')]
for f in matching_set:
f['width'] = widths[style.get('font-stretch', 'normal')]
min_dist = min(abs(width-f['width']) for f in matching_set)
if min_dist > 0:
return True, None
nearest = [f for f in matching_set if abs(width-f['width']) ==
min_dist]
if width <= 4:
lmatches = [f for f in nearest if f['width'] <= width]
else:
lmatches = [f for f in nearest if f['width'] >= width]
matching_set = (lmatches or nearest)
# Filter on font-style
fs = style.get('font-style', 'normal')
matching_set = [f for f in matching_set if f.get('font-style', 'normal') == fs]
# Filter on font weight
fw = int(style.get('font-weight', '400'))
matching_set = [f for f in matching_set if f.get('weight', 400) == fw]
if not matching_set:
return True, None
return True, matching_set[0]
class EmbedFonts(object):
'''
Embed all referenced fonts, if found on system. Must be called after CSS flattening.
'''
def __call__(self, oeb, log, opts):
self.oeb, self.log, self.opts = oeb, log, opts
self.sheet_cache = {}
self.find_style_rules()
self.find_embedded_fonts()
self.parser = cssutils.CSSParser(loglevel=logging.CRITICAL, log=logging.getLogger('calibre.css'))
self.warned = set()
self.warned2 = set()
for item in oeb.spine:
if not hasattr(item.data, 'xpath'):
continue
sheets = []
for href in XPath('//h:link[@href and @type="text/css"]/@href')(item.data):
sheet = self.oeb.manifest.hrefs.get(item.abshref(href), None)
if sheet is not None:
sheets.append(sheet)
if sheets:
self.process_item(item, sheets)
def find_embedded_fonts(self):
'''
Find all @font-face rules and extract the relevant info from them.
'''
self.embedded_fonts = []
for item in self.oeb.manifest:
if not hasattr(item.data, 'cssRules'):
continue
self.embedded_fonts.extend(find_font_face_rules(item, self.oeb))
def find_style_rules(self):
'''
Extract all font related style information from all stylesheets into a
dict mapping classes to font properties specified by that class. All
the heavy lifting has already been done by the CSS flattening code.
'''
rules = defaultdict(dict)
for item in self.oeb.manifest:
if not hasattr(item.data, 'cssRules'):
continue
for i, rule in enumerate(item.data.cssRules):
if rule.type != rule.STYLE_RULE:
continue
props = {k:v for k,v in
get_font_properties(rule).iteritems() if v}
if not props:
continue
for sel in rule.selectorList:
sel = sel.selectorText
if sel and sel.startswith('.'):
# We dont care about pseudo-selectors as the worst that
# can happen is some extra characters will remain in
# the font
sel = sel.partition(':')[0]
rules[sel[1:]].update(props)
self.style_rules = dict(rules)
def get_page_sheet(self):
if self.page_sheet is None:
manifest = self.oeb.manifest
id_, href = manifest.generate('page_css', 'page_styles.css')
self.page_sheet = manifest.add(id_, href, CSS_MIME, data=self.parser.parseString('', validate=False))
head = self.current_item.xpath('//*[local-name()="head"][1]')
if head:
href = self.current_item.relhref(href)
l = etree.SubElement(head[0], XHTML('link'),
rel='stylesheet', type=CSS_MIME, href=href)
l.tail = '\n'
else:
self.log.warn('No <head> cannot embed font rules')
return self.page_sheet
def process_item(self, item, sheets):
ff_rules = []
self.current_item = item
self.page_sheet = None
for sheet in sheets:
if 'page_css' in sheet.id:
ff_rules.extend(find_font_face_rules(sheet, self.oeb))
self.page_sheet = sheet
base = {'font-family':['serif'], 'font-weight': '400',
'font-style':'normal', 'font-stretch':'normal'}
for body in item.data.xpath('//*[local-name()="body"]'):
self.find_usage_in(body, base, ff_rules)
def find_usage_in(self, elem, inherited_style, ff_rules):
style = elem_style(self.style_rules, elem.get('class', '') or '', inherited_style)
for child in elem:
self.find_usage_in(child, style, ff_rules)
has_font, existing = used_font(style, ff_rules)
if not has_font:
return
if existing is None:
in_book = used_font(style, self.embedded_fonts)[1]
if in_book is None:
# Try to find the font in the system
added = self.embed_font(style)
if added is not None:
ff_rules.append(added)
self.embedded_fonts.append(added)
else:
# TODO: Create a page rule from the book rule (cannot use it
# directly as paths might be different)
item = in_book['item']
sheet = self.parser.parseString(in_book['rule'].cssText, validate=False)
rule = sheet.cssRules[0]
page_sheet = self.get_page_sheet()
href = page_sheet.abshref(item.href)
rule.style.setProperty('src', 'url(%s)' % href)
ff_rules.append(find_font_face_rules(sheet, self.oeb)[0])
page_sheet.data.insertRule(rule, len(page_sheet.data.cssRules))
def embed_font(self, style):
ff = [unicode(f) for f in style.get('font-family', []) if unicode(f).lower() not in {
'serif', 'sansserif', 'sans-serif', 'fantasy', 'cursive', 'monospace'}]
if not ff:
return
ff = ff[0]
if ff in self.warned:
return
try:
fonts = font_scanner.fonts_for_family(ff)
except NoFonts:
self.log.warn('Failed to find fonts for family:', ff, 'not embedding')
self.warned.add(ff)
return
try:
weight = int(style.get('font-weight', '400'))
except (ValueError, TypeError, AttributeError):
w = style['font-weight']
if w not in self.warned2:
self.log.warn('Invalid weight in font style: %r' % w)
self.warned2.add(w)
return
for f in fonts:
if f['weight'] == weight and f['font-style'] == style.get('font-style', 'normal') and f['font-stretch'] == style.get('font-stretch', 'normal'):
self.log('Embedding font %s from %s' % (f['full_name'], f['path']))
data = font_scanner.get_font_data(f)
name = f['full_name']
ext = 'otf' if f['is_otf'] else 'ttf'
name = ascii_filename(name).replace(' ', '-').replace('(', '').replace(')', '')
fid, href = self.oeb.manifest.generate(id=u'font', href=u'fonts/%s.%s'%(name, ext))
item = self.oeb.manifest.add(fid, href, guess_type('dummy.'+ext)[0], data=data)
item.unload_data_from_memory()
page_sheet = self.get_page_sheet()
href = page_sheet.relhref(item.href)
css = '''@font-face { font-family: "%s"; font-weight: %s; font-style: %s; font-stretch: %s; src: url(%s) }''' % (
f['font-family'], f['font-weight'], f['font-style'], f['font-stretch'], href)
sheet = self.parser.parseString(css, validate=False)
page_sheet.data.insertRule(sheet.cssRules[0], len(page_sheet.data.cssRules))
return find_font_face_rules(sheet, self.oeb)[0]

2
src/calibre/ebooks/oeb/transforms/flatcss.py
View File

@ -194,7 +194,7 @@ class CSSFlattener(object):
for i, font in enumerate(faces):
ext = 'otf' if font['is_otf'] else 'ttf'
fid, href = self.oeb.manifest.generate(id=u'font',
href=u'%s.%s'%(ascii_filename(font['full_name']).replace(u' ', u'-'), ext))
href=u'fonts/%s.%s'%(ascii_filename(font['full_name']).replace(u' ', u'-'), ext))
item = self.oeb.manifest.add(fid, href,
guess_type('dummy.'+ext)[0],
data=font_scanner.get_font_data(font))

208
src/calibre/ebooks/oeb/transforms/subset.py
View File

@ -12,6 +12,111 @@ from collections import defaultdict
from calibre.ebooks.oeb.base import urlnormalize
from calibre.utils.fonts.sfnt.subset import subset, NoGlyphs, UnsupportedFont
def get_font_properties(rule, default=None):
'''
Given a CSS rule, extract normalized font properties from
it. Note that shorthand font property should already have been expanded
by the CSS flattening code.
'''
props = {}
s = rule.style
for q in ('font-family', 'src', 'font-weight', 'font-stretch',
'font-style'):
g = 'uri' if q == 'src' else 'value'
try:
val = s.getProperty(q).propertyValue[0]
val = getattr(val, g)
if q == 'font-family':
val = [x.value for x in s.getProperty(q).propertyValue]
if val and val[0] == 'inherit':
val = None
except (IndexError, KeyError, AttributeError, TypeError, ValueError):
val = None if q in {'src', 'font-family'} else default
if q in {'font-weight', 'font-stretch', 'font-style'}:
val = unicode(val).lower() if (val or val == 0) else val
if val == 'inherit':
val = default
if q == 'font-weight':
val = {'normal':'400', 'bold':'700'}.get(val, val)
if val not in {'100', '200', '300', '400', '500', '600', '700',
'800', '900', 'bolder', 'lighter'}:
val = default
if val == 'normal':
val = '400'
elif q == 'font-style':
if val not in {'normal', 'italic', 'oblique'}:
val = default
elif q == 'font-stretch':
if val not in {'normal', 'ultra-condensed', 'extra-condensed',
'condensed', 'semi-condensed', 'semi-expanded',
'expanded', 'extra-expanded', 'ultra-expanded'}:
val = default
props[q] = val
return props
def find_font_face_rules(sheet, oeb):
'''
Find all @font-face rules in the given sheet and extract the relevant info from them.
sheet can be either a ManifestItem or a CSSStyleSheet.
'''
ans = []
try:
rules = sheet.data.cssRules
except AttributeError:
rules = sheet.cssRules
for i, rule in enumerate(rules):
if rule.type != rule.FONT_FACE_RULE:
continue
props = get_font_properties(rule, default='normal')
if not props['font-family'] or not props['src']:
continue
try:
path = sheet.abshref(props['src'])
except AttributeError:
path = props['src']
ff = oeb.manifest.hrefs.get(urlnormalize(path), None)
if not ff:
continue
props['item'] = ff
if props['font-weight'] in {'bolder', 'lighter'}:
props['font-weight'] = '400'
props['weight'] = int(props['font-weight'])
props['rule'] = rule
props['chars'] = set()
ans.append(props)
return ans
def elem_style(style_rules, cls, inherited_style):
'''
Find the effective style for the given element.
'''
classes = cls.split()
style = inherited_style.copy()
for cls in classes:
style.update(style_rules.get(cls, {}))
wt = style.get('font-weight', None)
pwt = inherited_style.get('font-weight', '400')
if wt == 'bolder':
style['font-weight'] = {
'100':'400',
'200':'400',
'300':'400',
'400':'700',
'500':'700',
}.get(pwt, '900')
elif wt == 'lighter':
style['font-weight'] = {
'600':'400', '700':'400',
'800':'700', '900':'700'}.get(pwt, '100')
return style
class SubsetFonts(object):
'''
@ -76,72 +181,15 @@ class SubsetFonts(object):
self.log('Reduced total font size to %.1f%% of original'%
(totals[0]/totals[1] * 100))
def get_font_properties(self, rule, default=None):
'''
Given a CSS rule, extract normalized font properties from
it. Note that shorthand font property should already have been expanded
by the CSS flattening code.
'''
props = {}
s = rule.style
for q in ('font-family', 'src', 'font-weight', 'font-stretch',
'font-style'):
g = 'uri' if q == 'src' else 'value'
try:
val = s.getProperty(q).propertyValue[0]
val = getattr(val, g)
if q == 'font-family':
val = [x.value for x in s.getProperty(q).propertyValue]
if val and val[0] == 'inherit':
val = None
except (IndexError, KeyError, AttributeError, TypeError, ValueError):
val = None if q in {'src', 'font-family'} else default
if q in {'font-weight', 'font-stretch', 'font-style'}:
val = unicode(val).lower() if (val or val == 0) else val
if val == 'inherit':
val = default
if q == 'font-weight':
val = {'normal':'400', 'bold':'700'}.get(val, val)
if val not in {'100', '200', '300', '400', '500', '600', '700',
'800', '900', 'bolder', 'lighter'}:
val = default
if val == 'normal': val = '400'
elif q == 'font-style':
if val not in {'normal', 'italic', 'oblique'}:
val = default
elif q == 'font-stretch':
if val not in { 'normal', 'ultra-condensed', 'extra-condensed',
'condensed', 'semi-condensed', 'semi-expanded',
'expanded', 'extra-expanded', 'ultra-expanded'}:
val = default
props[q] = val
return props
def find_embedded_fonts(self):
'''
Find all @font-face rules and extract the relevant info from them.
'''
self.embedded_fonts = []
for item in self.oeb.manifest:
if not hasattr(item.data, 'cssRules'): continue
for i, rule in enumerate(item.data.cssRules):
if rule.type != rule.FONT_FACE_RULE:
continue
props = self.get_font_properties(rule, default='normal')
if not props['font-family'] or not props['src']:
continue
path = item.abshref(props['src'])
ff = self.oeb.manifest.hrefs.get(urlnormalize(path), None)
if not ff:
continue
props['item'] = ff
if props['font-weight'] in {'bolder', 'lighter'}:
props['font-weight'] = '400'
props['weight'] = int(props['font-weight'])
props['chars'] = set()
props['rule'] = rule
self.embedded_fonts.append(props)
if not hasattr(item.data, 'cssRules'):
continue
self.embedded_fonts.extend(find_font_face_rules(item, self.oeb))
def find_style_rules(self):
'''
@ -151,12 +199,13 @@ class SubsetFonts(object):
'''
rules = defaultdict(dict)
for item in self.oeb.manifest:
if not hasattr(item.data, 'cssRules'): continue
if not hasattr(item.data, 'cssRules'):
continue
for i, rule in enumerate(item.data.cssRules):
if rule.type != rule.STYLE_RULE:
continue
props = {k:v for k,v in
self.get_font_properties(rule).iteritems() if v}
get_font_properties(rule).iteritems() if v}
if not props:
continue
for sel in rule.selectorList:
@ -172,41 +221,17 @@ class SubsetFonts(object):
def find_font_usage(self):
for item in self.oeb.manifest:
if not hasattr(item.data, 'xpath'): continue
if not hasattr(item.data, 'xpath'):
continue
for body in item.data.xpath('//*[local-name()="body"]'):
base = {'font-family':['serif'], 'font-weight': '400',
'font-style':'normal', 'font-stretch':'normal'}
self.find_usage_in(body, base)
def elem_style(self, cls, inherited_style):
'''
Find the effective style for the given element.
'''
classes = cls.split()
style = inherited_style.copy()
for cls in classes:
style.update(self.style_rules.get(cls, {}))
wt = style.get('font-weight', None)
pwt = inherited_style.get('font-weight', '400')
if wt == 'bolder':
style['font-weight'] = {
'100':'400',
'200':'400',
'300':'400',
'400':'700',
'500':'700',
}.get(pwt, '900')
elif wt == 'lighter':
style['font-weight'] = {
'600':'400', '700':'400',
'800':'700', '900':'700'}.get(pwt, '100')
return style
def used_font(self, style):
'''
Given a style find the embedded font that matches it. Returns None if
no match is found ( can happen if not family matches).
no match is found (can happen if no family matches).
'''
ff = style.get('font-family', [])
lnames = {unicode(x).lower() for x in ff}
@ -222,7 +247,7 @@ class SubsetFonts(object):
return None
# Filter on font-stretch
widths = {x:i for i, x in enumerate(( 'ultra-condensed',
widths = {x:i for i, x in enumerate(('ultra-condensed',
'extra-condensed', 'condensed', 'semi-condensed', 'normal',
'semi-expanded', 'expanded', 'extra-expanded', 'ultra-expanded'
))}
@ -280,7 +305,7 @@ class SubsetFonts(object):
return ans
def find_usage_in(self, elem, inherited_style):
style = self.elem_style(elem.get('class', '') or '', inherited_style)
style = elem_style(self.style_rules, elem.get('class', '') or '', inherited_style)
for child in elem:
self.find_usage_in(child, style)
font = self.used_font(style)
@ -290,3 +315,4 @@ class SubsetFonts(object):
font['chars'] |= chars

2
src/calibre/ebooks/pdf/render/from_html.py
View File

@ -253,7 +253,7 @@ class PDFWriter(QObject):
return self.loop.exit(1)
try:
if not self.render_queue:
if self.toc is not None and len(self.toc) > 0 and not hasattr(self, 'rendered_inline_toc'):
if self.opts.pdf_add_toc and self.toc is not None and len(self.toc) > 0 and not hasattr(self, 'rendered_inline_toc'):
return self.render_inline_toc()
self.loop.exit()
else:

18
src/calibre/gui2/actions/edit_metadata.py
View File

@ -399,8 +399,7 @@ class EditMetadataAction(InterfaceAction):
if safe_merge:
if not confirm('<p>'+_(
'Book formats and metadata from the selected books '
'will be added to the <b>first selected book</b> (%s). '
'ISBN will <i>not</i> be merged.<br><br> '
'will be added to the <b>first selected book</b> (%s).<br> '
'The second and subsequently selected books will not '
'be deleted or changed.<br><br>'
'Please confirm you want to proceed.')%title
@ -413,7 +412,7 @@ class EditMetadataAction(InterfaceAction):
'Book formats from the selected books will be merged '
'into the <b>first selected book</b> (%s). '
'Metadata in the first selected book will not be changed. '
'Author, Title, ISBN and all other metadata will <i>not</i> be merged.<br><br>'
'Author, Title and all other metadata will <i>not</i> be merged.<br><br>'
'After merger the second and subsequently '
'selected books, with any metadata they have will be <b>deleted</b>. <br><br>'
'All book formats of the first selected book will be kept '
@ -427,8 +426,7 @@ class EditMetadataAction(InterfaceAction):
else:
if not confirm('<p>'+_(
'Book formats and metadata from the selected books will be merged '
'into the <b>first selected book</b> (%s). '
'ISBN will <i>not</i> be merged.<br><br>'
'into the <b>first selected book</b> (%s).<br><br>'
'After merger the second and '
'subsequently selected books will be <b>deleted</b>. <br><br>'
'All book formats of the first selected book will be kept '
@ -490,11 +488,13 @@ class EditMetadataAction(InterfaceAction):
def merge_metadata(self, dest_id, src_ids):
db = self.gui.library_view.model().db
dest_mi = db.get_metadata(dest_id, index_is_id=True)
merged_identifiers = db.get_identifiers(dest_id, index_is_id=True)
orig_dest_comments = dest_mi.comments
dest_cover = db.cover(dest_id, index_is_id=True)
had_orig_cover = bool(dest_cover)
for src_id in src_ids:
src_mi = db.get_metadata(src_id, index_is_id=True)
if src_mi.comments and orig_dest_comments != src_mi.comments:
if not dest_mi.comments:
dest_mi.comments = src_mi.comments
@ -523,7 +523,15 @@ class EditMetadataAction(InterfaceAction):
if not dest_mi.series:
dest_mi.series = src_mi.series
dest_mi.series_index = src_mi.series_index
src_identifiers = db.get_identifiers(src_id, index_is_id=True)
src_identifiers.update(merged_identifiers)
merged_identifiers = src_identifiers.copy()
if merged_identifiers:
dest_mi.set_identifiers(merged_identifiers)
db.set_metadata(dest_id, dest_mi, ignore_errors=False)
if not had_orig_cover and dest_cover:
db.set_cover(dest_id, dest_cover)

2
src/calibre/gui2/convert/look_and_feel.py
View File

@ -32,7 +32,7 @@ class LookAndFeelWidget(Widget, Ui_Form):
Widget.__init__(self, parent,
['change_justification', 'extra_css', 'base_font_size',
'font_size_mapping', 'line_height', 'minimum_line_height',
'embed_font_family', 'subset_embedded_fonts',
'embed_font_family', 'embed_all_fonts', 'subset_embedded_fonts',
'smarten_punctuation', 'unsmarten_punctuation',
'disable_font_rescaling', 'insert_blank_line',
'remove_paragraph_spacing',

371
src/calibre/gui2/convert/look_and_feel.ui
View File

@ -14,6 +14,70 @@
<string>Form</string>
</property>
<layout class="QGridLayout" name="gridLayout">
<item row="12" column="1" colspan="2">
<widget class="QCheckBox" name="opt_keep_ligatures">
<property name="text">
<string>Keep &amp;ligatures</string>
</property>
</widget>
</item>
<item row="12" column="3">
<widget class="QCheckBox" name="opt_linearize_tables">
<property name="text">
<string>&amp;Linearize tables</string>
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QLabel" name="label_18">
<property name="text">
<string>Base &amp;font size:</string>
</property>
<property name="buddy">
<cstring>opt_base_font_size</cstring>
</property>
</widget>
</item>
<item row="9" column="3">
<widget class="QLabel" name="label_7">
<property name="text">
<string>&amp;Line size:</string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>opt_insert_blank_line_size</cstring>
</property>
</widget>
</item>
<item row="5" column="1" colspan="2">
<widget class="EncodingComboBox" name="opt_input_encoding">
<property name="editable">
<bool>true</bool>
</property>
</widget>
</item>
<item row="8" column="0" colspan="2">
<widget class="QCheckBox" name="opt_remove_paragraph_spacing">
<property name="text">
<string>Remove &amp;spacing between paragraphs</string>
</property>
</widget>
</item>
<item row="8" column="3">
<widget class="QLabel" name="label_4">
<property name="text">
<string>&amp;Indent size:</string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>opt_remove_paragraph_spacing_indent_size</cstring>
</property>
</widget>
</item>
<item row="3" column="4">
<widget class="QDoubleSpinBox" name="opt_line_height">
<property name="suffix">
@ -24,6 +88,57 @@
</property>
</widget>
</item>
<item row="9" column="0" colspan="2">
<widget class="QCheckBox" name="opt_insert_blank_line">
<property name="text">
<string>Insert &amp;blank line between paragraphs</string>
</property>
</widget>
</item>
<item row="9" column="4">
<widget class="QDoubleSpinBox" name="opt_insert_blank_line_size">
<property name="suffix">
<string> em</string>
</property>
<property name="decimals">
<number>1</number>
</property>
</widget>
</item>
<item row="10" column="0">
<widget class="QLabel" name="label_5">
<property name="text">
<string>Text &amp;justification:</string>
</property>
<property name="buddy">
<cstring>opt_change_justification</cstring>
</property>
</widget>
</item>
<item row="10" column="2" colspan="3">
<widget class="QComboBox" name="opt_change_justification"/>
</item>
<item row="11" column="0">
<widget class="QCheckBox" name="opt_smarten_punctuation">
<property name="text">
<string>Smarten &amp;punctuation</string>
</property>
</widget>
</item>
<item row="11" column="1" colspan="4">
<widget class="QCheckBox" name="opt_asciiize">
<property name="text">
<string>&amp;Transliterate unicode characters to ASCII</string>
</property>
</widget>
</item>
<item row="12" column="0">
<widget class="QCheckBox" name="opt_unsmarten_punctuation">
<property name="text">
<string>&amp;UnSmarten punctuation</string>
</property>
</widget>
</item>
<item row="3" column="3">
<widget class="QLabel" name="label">
<property name="text">
@ -44,51 +159,6 @@
</property>
</widget>
</item>
<item row="3" column="1">
<widget class="QDoubleSpinBox" name="opt_minimum_line_height">
<property name="suffix">
<string> %</string>
</property>
<property name="decimals">
<number>1</number>
</property>
<property name="maximum">
<double>900.000000000000000</double>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="QDoubleSpinBox" name="opt_base_font_size">
<property name="suffix">
<string> pt</string>
</property>
<property name="decimals">
<number>1</number>
</property>
<property name="minimum">
<double>0.000000000000000</double>
</property>
<property name="maximum">
<double>50.000000000000000</double>
</property>
<property name="singleStep">
<double>1.000000000000000</double>
</property>
<property name="value">
<double>15.000000000000000</double>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="label_2">
<property name="text">
<string>Font size &amp;key:</string>
</property>
<property name="buddy">
<cstring>opt_font_size_mapping</cstring>
</property>
</widget>
</item>
<item row="2" column="1" colspan="3">
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
@ -133,56 +203,72 @@
</property>
</widget>
</item>
<item row="5" column="1" colspan="2">
<widget class="EncodingComboBox" name="opt_input_encoding">
<property name="editable">
<bool>true</bool>
</property>
</widget>
</item>
<item row="7" column="0" colspan="2">
<widget class="QCheckBox" name="opt_remove_paragraph_spacing">
<property name="text">
<string>Remove &amp;spacing between paragraphs</string>
</property>
</widget>
</item>
<item row="7" column="3">
<widget class="QLabel" name="label_4">
<property name="text">
<string>&amp;Indent size:</string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>opt_remove_paragraph_spacing_indent_size</cstring>
</property>
</widget>
</item>
<item row="7" column="4">
<widget class="QDoubleSpinBox" name="opt_remove_paragraph_spacing_indent_size">
<property name="toolTip">
<string>&lt;p&gt;When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent.</string>
</property>
<property name="specialValueText">
<string>No change</string>
</property>
<item row="3" column="1">
<widget class="QDoubleSpinBox" name="opt_minimum_line_height">
<property name="suffix">
<string> em</string>
<string> %</string>
</property>
<property name="decimals">
<number>1</number>
</property>
<property name="maximum">
<double>900.000000000000000</double>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="QDoubleSpinBox" name="opt_base_font_size">
<property name="suffix">
<string> pt</string>
</property>
<property name="decimals">
<number>1</number>
</property>
<property name="minimum">
<double>-0.100000000000000</double>
<double>0.000000000000000</double>
</property>
<property name="maximum">
<double>50.000000000000000</double>
</property>
<property name="singleStep">
<double>0.100000000000000</double>
<double>1.000000000000000</double>
</property>
<property name="value">
<double>15.000000000000000</double>
</property>
</widget>
</item>
<item row="12" column="0" colspan="5">
<item row="0" column="0" colspan="5">
<widget class="QCheckBox" name="opt_disable_font_rescaling">
<property name="text">
<string>&amp;Disable font size rescaling</string>
</property>
</widget>
</item>
<item row="6" column="1" colspan="2">
<widget class="FontFamilyChooser" name="opt_embed_font_family" native="true"/>
</item>
<item row="2" column="0">
<widget class="QLabel" name="label_2">
<property name="text">
<string>Font size &amp;key:</string>
</property>
<property name="buddy">
<cstring>opt_font_size_mapping</cstring>
</property>
</widget>
</item>
<item row="6" column="0">
<widget class="QLabel" name="label_10">
<property name="text">
<string>&amp;Embed font family:</string>
</property>
<property name="buddy">
<cstring>opt_embed_font_family</cstring>
</property>
</widget>
</item>
<item row="13" column="0" colspan="5">
<widget class="QTabWidget" name="tabWidget">
<property name="currentIndex">
<number>0</number>
@ -300,121 +386,42 @@
</widget>
</widget>
</item>
<item row="8" column="0" colspan="2">
<widget class="QCheckBox" name="opt_insert_blank_line">
<property name="text">
<string>Insert &amp;blank line between paragraphs</string>
</property>
</widget>
</item>
<item row="8" column="4">
<widget class="QDoubleSpinBox" name="opt_insert_blank_line_size">
<widget class="QDoubleSpinBox" name="opt_remove_paragraph_spacing_indent_size">
<property name="toolTip">
<string>&lt;p&gt;When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent.</string>
</property>
<property name="specialValueText">
<string>No change</string>
</property>
<property name="suffix">
<string> em</string>
</property>
<property name="decimals">
<number>1</number>
</property>
</widget>
</item>
<item row="9" column="0">
<widget class="QLabel" name="label_5">
<property name="text">
<string>Text &amp;justification:</string>
<property name="minimum">
<double>-0.100000000000000</double>
</property>
<property name="buddy">
<cstring>opt_change_justification</cstring>
<property name="singleStep">
<double>0.100000000000000</double>
</property>
</widget>
</item>
<item row="9" column="2" colspan="3">
<widget class="QComboBox" name="opt_change_justification"/>
</item>
<item row="10" column="0">
<widget class="QCheckBox" name="opt_smarten_punctuation">
<property name="text">
<string>Smarten &amp;punctuation</string>
</property>
</widget>
</item>
<item row="10" column="1" colspan="4">
<widget class="QCheckBox" name="opt_asciiize">
<property name="text">
<string>&amp;Transliterate unicode characters to ASCII</string>
</property>
</widget>
</item>
<item row="11" column="0">
<widget class="QCheckBox" name="opt_unsmarten_punctuation">
<property name="text">
<string>&amp;UnSmarten punctuation</string>
</property>
</widget>
</item>
<item row="11" column="1" colspan="2">
<widget class="QCheckBox" name="opt_keep_ligatures">
<property name="text">
<string>Keep &amp;ligatures</string>
</property>
</widget>
</item>
<item row="11" column="3">
<widget class="QCheckBox" name="opt_linearize_tables">
<property name="text">
<string>&amp;Linearize tables</string>
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QLabel" name="label_18">
<property name="text">
<string>Base &amp;font size:</string>
</property>
<property name="buddy">
<cstring>opt_base_font_size</cstring>
</property>
</widget>
</item>
<item row="8" column="3">
<widget class="QLabel" name="label_7">
<property name="text">
<string>&amp;Line size:</string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>opt_insert_blank_line_size</cstring>
</property>
</widget>
</item>
<item row="6" column="0">
<widget class="QLabel" name="label_10">
<property name="text">
<string>&amp;Embed font family:</string>
</property>
<property name="buddy">
<cstring>opt_embed_font_family</cstring>
</property>
</widget>
</item>
<item row="0" column="0" colspan="5">
<widget class="QCheckBox" name="opt_disable_font_rescaling">
<property name="text">
<string>&amp;Disable font size rescaling</string>
</property>
</widget>
</item>
<item row="6" column="1" colspan="2">
<widget class="FontFamilyChooser" name="opt_embed_font_family" native="true"/>
</item>
<item row="6" column="3" colspan="2">
<item row="7" column="3">
<widget class="QCheckBox" name="opt_subset_embedded_fonts">
<property name="text">
<string>&amp;Subset all embedded fonts</string>
</property>
</widget>
</item>
<item row="7" column="0" colspan="3">
<widget class="QCheckBox" name="opt_embed_all_fonts">
<property name="text">
<string>&amp;Embed referenced fonts</string>
</property>
</widget>
</item>
</layout>
</widget>
<customwidgets>

2
src/calibre/gui2/dialogs/template_dialog.py
View File

@ -262,6 +262,8 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
self.mi.rating = 4.0
self.mi.tags = [_('Tag 1'), _('Tag 2')]
self.mi.languages = ['eng']
if fm is not None:
self.mi.set_all_user_metadata(fm.custom_field_metadata())
# Remove help icon on title bar
icon = self.windowIcon()

21
src/calibre/gui2/email.py
View File

@ -32,7 +32,7 @@ class Worker(Thread):
self.func, self.args = func, args
def run(self):
#time.sleep(1000)
# time.sleep(1000)
try:
self.func(*self.args)
except Exception as e:
@ -46,7 +46,7 @@ class Worker(Thread):
class Sendmail(object):
MAX_RETRIES = 1
TIMEOUT = 15 * 60 # seconds
TIMEOUT = 15 * 60 # seconds
def __init__(self):
self.calculate_rate_limit()
@ -92,7 +92,11 @@ class Sendmail(object):
raise worker.exception
def sendmail(self, attachment, aname, to, subject, text, log):
logged = False
while time.time() - self.last_send_time <= self.rate_limit:
if not logged and self.rate_limit > 0:
log('Waiting %s seconds before sending, to avoid being marked as spam.\nYou can control this delay via Preferences->Tweaks' % self.rate_limit)
logged = True
time.sleep(1)
try:
opts = email_config().parse()
@ -162,7 +166,7 @@ def email_news(mi, remove, get_fmts, done, job_manager):
plugboard_email_value = 'email'
plugboard_email_formats = ['epub', 'mobi', 'azw3']
class EmailMixin(object): # {{{
class EmailMixin(object): # {{{
def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None,
do_auto_convert=True, specific_format=None):
@ -204,10 +208,10 @@ class EmailMixin(object): # {{{
if not components:
components = [mi.title]
subjects.append(os.path.join(*components))
a = authors_to_string(mi.authors if mi.authors else \
a = authors_to_string(mi.authors if mi.authors else
[_('Unknown')])
texts.append(_('Attached, you will find the e-book') + \
'\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' + \
texts.append(_('Attached, you will find the e-book') +
'\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' +
_('in the %s format.') %
os.path.splitext(f)[1][1:].upper())
prefix = ascii_filename(t+' - '+a)
@ -227,7 +231,7 @@ class EmailMixin(object): # {{{
auto = []
if _auto_ids != []:
for id in _auto_ids:
if specific_format == None:
if specific_format is None:
dbfmts = self.library_view.model().db.formats(id, index_is_id=True)
formats = [f.lower() for f in (dbfmts.split(',') if dbfmts else
[])]
@ -298,8 +302,9 @@ class EmailMixin(object): # {{{
sent_mails = email_news(mi, remove,
get_fmts, self.email_sent, self.job_manager)
if sent_mails:
self.status_bar.show_message(_('Sent news to')+' '+\
self.status_bar.show_message(_('Sent news to')+' '+
', '.join(sent_mails), 3000)
# }}}

5
src/calibre/gui2/preferences/save_template.py
View File

@ -24,7 +24,7 @@ class SaveTemplate(QWidget, Ui_Form):
Ui_Form.__init__(self)
self.setupUi(self)
def initialize(self, name, default, help):
def initialize(self, name, default, help, field_metadata):
variables = sorted(FORMAT_ARG_DESCS.keys())
rows = []
for var in variables:
@ -36,6 +36,7 @@ class SaveTemplate(QWidget, Ui_Form):
table = u'<table>%s</table>'%(u'\n'.join(rows))
self.template_variables.setText(table)
self.field_metadata = field_metadata
self.opt_template.initialize(name+'_template_history',
default, help)
self.opt_template.editTextChanged.connect(self.changed)
@ -44,7 +45,7 @@ class SaveTemplate(QWidget, Ui_Form):
self.open_editor.clicked.connect(self.do_open_editor)
def do_open_editor(self):
t = TemplateDialog(self, self.opt_template.text())
t = TemplateDialog(self, self.opt_template.text(), fm=self.field_metadata)
t.setWindowTitle(_('Edit template'))
if t.exec_():
self.opt_template.set_value(t.rule[1])

3
src/calibre/gui2/preferences/saving.py
View File

@ -34,7 +34,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
ConfigWidgetBase.initialize(self)
self.save_template.blockSignals(True)
self.save_template.initialize('save_to_disk', self.proxy['template'],
self.proxy.help('template'))
self.proxy.help('template'),
self.gui.library_view.model().db.field_metadata)
self.save_template.blockSignals(False)
def restore_defaults(self):

3
src/calibre/gui2/preferences/sending.py
View File

@ -44,7 +44,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
ConfigWidgetBase.initialize(self)
self.send_template.blockSignals(True)
self.send_template.initialize('send_to_device', self.proxy['send_template'],
self.proxy.help('send_template'))
self.proxy.help('send_template'),
self.gui.library_view.model().db.field_metadata)
self.send_template.blockSignals(False)
def restore_defaults(self):

24
src/calibre/gui2/tools.py
View File

@ -24,7 +24,7 @@ from calibre.ebooks.conversion.config import GuiRecommendations, \
load_defaults, load_specifics, save_specifics
from calibre.gui2.convert import bulk_defaults_for_input_format
def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
out_format=None, show_no_format_warning=True):
changed = False
jobs = []
@ -47,7 +47,7 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
result = d.exec_()
if result == QDialog.Accepted:
#if not convert_existing(parent, db, [book_id], d.output_format):
# if not convert_existing(parent, db, [book_id], d.output_format):
# continue
mi = db.get_metadata(book_id, True)
@ -116,7 +116,6 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
msg = _('This book has no actual ebook files')
res.append('%s - %s'%(title, msg))
msg = '%s' % '\n'.join(res)
warning_dialog(parent, _('Could not convert some books'),
_('Could not convert %(num)d of %(tot)d books, because no supported source'
@ -254,7 +253,7 @@ class QueueBulk(QProgressDialog):
# }}}
def fetch_scheduled_recipe(arg): # {{{
def fetch_scheduled_recipe(arg): # {{{
fmt = prefs['output_format'].lower()
# Never use AZW3 for periodicals...
if fmt == 'azw3':
@ -266,6 +265,10 @@ def fetch_scheduled_recipe(arg): # {{{
if 'output_profile' in ps:
recs.append(('output_profile', ps['output_profile'],
OptionRecommendation.HIGH))
for edge in ('left', 'top', 'bottom', 'right'):
edge = 'margin_' + edge
if edge in ps:
recs.append((edge, ps[edge], OptionRecommendation.HIGH))
lf = load_defaults('look_and_feel')
if lf.get('base_font_size', 0.0) != 0.0:
@ -283,18 +286,24 @@ def fetch_scheduled_recipe(arg): # {{{
if epub.get('epub_flatten', False):
recs.append(('epub_flatten', True, OptionRecommendation.HIGH))
if fmt == 'pdf':
pdf = load_defaults('pdf_output')
from calibre.customize.ui import plugin_for_output_format
p = plugin_for_output_format('pdf')
for opt in p.options:
recs.append(opt.name, pdf.get(opt.name, opt.recommended_value), OptionRecommendation.HIGH)
args = [arg['recipe'], pt.name, recs]
if arg['username'] is not None:
recs.append(('username', arg['username'], OptionRecommendation.HIGH))
if arg['password'] is not None:
recs.append(('password', arg['password'], OptionRecommendation.HIGH))
return 'gui_convert', args, _('Fetch news from ')+arg['title'], fmt.upper(), [pt]
# }}}
def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{
def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{
from calibre.gui2.dialogs.catalog import Catalog
# Build the Catalog dialog in gui2.dialogs.catalog
@ -354,7 +363,7 @@ def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{
d.catalog_title
# }}}
def convert_existing(parent, db, book_ids, output_format): # {{{
def convert_existing(parent, db, book_ids, output_format): # {{{
already_converted_ids = []
already_converted_titles = []
for book_id in book_ids:
@ -372,3 +381,4 @@ def convert_existing(parent, db, book_ids, output_format): # {{{
return book_ids
# }}}

2
src/calibre/web/fetch/javascript.py
View File

@ -128,6 +128,8 @@ def download_resources(browser, resource_cache, output_dir):
else:
img_counter += 1
ext = what(None, raw) or 'jpg'
if ext == 'jpeg':
ext = 'jpg' # Apparently Moon+ cannot handle .jpeg
href = 'img_%d.%s' % (img_counter, ext)
dest = os.path.join(output_dir, href)
resource_cache[h] = dest