From 27678a85c6342aeb48e56b3464e0fdc84ed0f374 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Jun 2013 16:33:43 +0530 Subject: [PATCH 01/37] Add tests for set_metadata() --- src/calibre/db/cache.py | 36 +++++++++++++++++++++++------ src/calibre/db/tests/base.py | 6 ++--- src/calibre/db/tests/writing.py | 40 +++++++++++++++++++++++++++++++-- src/calibre/db/write.py | 2 +- 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index b79ff2a31b..88f06b43ba 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -843,9 +843,25 @@ class Cache(object): @write_api def set_metadata(self, book_id, mi, ignore_errors=False, force_changes=False, set_title=True, set_authors=True): - if callable(getattr(mi, 'to_book_metadata', None)): + ''' + Set metadata for the book `id` from the `Metadata` object `mi` + + Setting force_changes=True will force set_metadata to update fields even + if mi contains empty values. In this case, 'None' is distinguished from + 'empty'. If mi.XXX is None, the XXX is not replaced, otherwise it is. + The tags, identifiers, and cover attributes are special cases. Tags and + identifiers cannot be set to None so then will always be replaced if + force_changes is true. You must ensure that mi contains the values you + want the book to have. Covers are always changed if a new cover is + provided, but are never deleted. Also note that force_changes has no + effect on setting title or authors. + ''' + + try: # Handle code passing in an OPF object instead of a Metadata object mi = mi.to_book_metadata() + except (AttributeError, TypeError): + pass def set_field(name, val, **kwargs): self._set_field(name, {book_id:val}, **kwargs) @@ -864,7 +880,7 @@ class Cache(object): set_field('authors', authors, do_path_update=False) if path_changed: - self._update_path((book_id,)) + self._update_path({book_id}) def protected_set_field(name, val, **kwargs): try: @@ -890,12 +906,16 @@ class Cache(object): if cdata is not None: self._set_cover({book_id: cdata}) - for field in ('title_sort', 'author_sort', 'publisher', 'series', - 'tags', 'comments', 'languages', 'pubdate'): + for field in ('author_sort', 'publisher', 'series', 'tags', 'comments', + 'languages', 'pubdate'): val = mi.get(field, None) if (force_changes and val is not None) or not mi.is_null(field): protected_set_field(field, val) + val = mi.get('title_sort', None) + if (force_changes and val is not None) or not mi.is_null('title_sort'): + protected_set_field('sort', val) + # identifiers will always be replaced if force_changes is True mi_idents = mi.get_identifiers() if force_changes: @@ -917,9 +937,11 @@ class Cache(object): val = mi.get(key, None) if force_changes or val is not None: protected_set_field(key, val) - extra = mi.get_extra(key) - if extra is not None: - protected_set_field(key+'_index', extra) + idx = key + '_index' + if idx in self.fields: + extra = mi.get_extra(key) + if extra is not None or force_changes: + protected_set_field(idx, extra) # }}} diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index cc8da89b05..b57b017ba3 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -78,7 +78,7 @@ class BaseTest(unittest.TestCase): def cloned_library(self): return self.clone_library(self.library_path) - def compare_metadata(self, mi1, mi2): + def compare_metadata(self, mi1, mi2, exclude=()): allfk1 = mi1.all_field_keys() allfk2 = mi2.all_field_keys() self.assertEqual(allfk1, allfk2) @@ -88,7 +88,7 @@ class BaseTest(unittest.TestCase): 'ondevice_col', 'last_modified', 'has_cover', 'cover_data'}.union(allfk1) for attr in all_keys: - if attr == 'user_metadata': + if attr == 'user_metadata' or attr in exclude: continue attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr) if attr == 'formats': @@ -97,7 +97,7 @@ class BaseTest(unittest.TestCase): attr1, attr2 = set(attr1), set(attr2) self.assertEqual(attr1, attr2, '%s not the same: %r != %r'%(attr, attr1, attr2)) - if attr.startswith('#'): + if attr.startswith('#') and attr + '_index' not in exclude: attr1, attr2 = mi1.get_extra(attr), mi2.get_extra(attr) self.assertEqual(attr1, attr2, '%s {#extra} not the same: %r != %r'%(attr, attr1, attr2)) diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 6d3169b905..5d04c11def 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -376,7 +376,43 @@ class WritingTest(BaseTest): self.assertTrue(old.has_cover(book_id)) # }}} - def test_set_metadata(self): + def test_set_metadata(self): # {{{ ' Test setting of metadata ' - self.assertTrue(False, 'TODO: test set_metadata()') + ae = self.assertEqual + cache = self.init_cache(self.cloned_library) + + # Check that changing title/author updates the path + mi = cache.get_metadata(1) + old_path = cache.field_for('path', 1) + old_title, old_author = mi.title, mi.authors[0] + ae(old_path, '%s/%s (1)' % (old_author, old_title)) + mi.title, mi.authors = 'New Title', ['New Author'] + cache.set_metadata(1, mi) + ae(cache.field_for('path', 1), '%s/%s (1)' % (mi.authors[0], mi.title)) + p = cache.format_abspath(1, 'FMT1') + self.assertTrue(mi.authors[0] in p and mi.title in p) + + # Compare old and new set_metadata() + db = self.init_old(self.cloned_library) + mi = db.get_metadata(1, index_is_id=True, get_cover=True, cover_as_data=True) + mi2 = db.get_metadata(3, index_is_id=True, get_cover=True, cover_as_data=True) + db.set_metadata(2, mi) + db.set_metadata(1, mi2, force_changes=True) + oldmi = db.get_metadata(2, index_is_id=True, get_cover=True, cover_as_data=True) + oldmi2 = db.get_metadata(1, index_is_id=True, get_cover=True, cover_as_data=True) + db.close() + del db + cache = self.init_cache(self.cloned_library) + cache.set_metadata(2, mi) + nmi = cache.get_metadata(2, get_cover=True, cover_as_data=True) + ae(oldmi.cover_data, nmi.cover_data) + self.compare_metadata(nmi, oldmi, exclude={'last_modified', 'format_metadata'}) + cache.set_metadata(1, mi2, force_changes=True) + nmi2 = cache.get_metadata(1, get_cover=True, cover_as_data=True) + # The new code does not allow setting of #series_index to None, instead + # it is reset to 1.0 + ae(nmi2.get_extra('#series'), 1.0) + self.compare_metadata(nmi2, oldmi2, exclude={'last_modified', 'format_metadata', '#series_index'}) + + # }}} diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py index 7fdb2070c0..9bae3e6abb 100644 --- a/src/calibre/db/write.py +++ b/src/calibre/db/write.py @@ -212,7 +212,7 @@ def custom_series_index(book_id_val_map, db, field, *args): ids = series_field.ids_for_book(book_id) if ids: sequence.append((sidx, book_id, ids[0])) - field.table.book_col_map[book_id] = sidx + field.table.book_col_map[book_id] = sidx if sequence: db.conn.executemany('UPDATE %s SET %s=? WHERE book=? AND value=?'%( field.metadata['table'], field.metadata['column']), sequence) From 4a105ef459fe5204d6218c4b2bdf8084ba5dc78c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Jun 2013 16:40:47 +0530 Subject: [PATCH 02/37] ... --- src/calibre/db/tests/legacy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 48408d6105..af6b977aef 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -120,7 +120,8 @@ class LegacyTest(BaseTest): for attr in dir(db): if attr in SKIP_ATTRS: continue - self.assertTrue(hasattr(ndb, attr), 'The attribute %s is missing' % attr) + if not hasattr(ndb, attr): + raise AssertionError('The attribute %s is missing' % attr) obj, nobj = getattr(db, attr), getattr(ndb, attr) if attr not in SKIP_ARGSPEC: try: From a2aad03af904e174463464a9139247e665ca685e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Jun 2013 22:25:37 +0530 Subject: [PATCH 03/37] Stop using googlecode for file hosting --- setup/hosting.py | 44 ++++++++++++++++++++++++++++++++++++-------- setup/upload.py | 12 ++++++++++-- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/setup/hosting.py b/setup/hosting.py index 8707388181..9853f181a4 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -10,13 +10,13 @@ __docformat__ = 'restructuredtext en' import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib from argparse import ArgumentParser, FileType from subprocess import check_call -from tempfile import NamedTemporaryFile#, mkdtemp +from tempfile import NamedTemporaryFile from collections import OrderedDict import mechanize from lxml import html -def login_to_google(username, password): +def login_to_google(username, password): # {{{ br = mechanize.Browser() br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')] @@ -30,15 +30,16 @@ def login_to_google(username, password): x = re.search(br'(?is).*?', raw) if x is not None: print ('Title of post login page: %s'%x.group()) - #open('/tmp/goog.html', 'wb').write(raw) + # open('/tmp/goog.html', 'wb').write(raw) raise ValueError(('Failed to login to google with credentials: %s %s' '\nGoogle sometimes requires verification when logging in from a ' 'new IP address. Use lynx to login and supply the verification, ' 'at: lynx -accept_all_cookies https://accounts.google.com/ServiceLogin?service=code') %(username, password)) return br +# }}} -class ReadFileWithProgressReporting(file): # {{{ +class ReadFileWithProgressReporting(file): # {{{ def __init__(self, path, mode='rb'): file.__init__(self, path, mode) @@ -101,7 +102,7 @@ class Base(object): # {{{ #}}} -class GoogleCode(Base):# {{{ +class GoogleCode(Base): # {{{ def __init__(self, # A mapping of filenames to file descriptions. The descriptions are @@ -141,7 +142,7 @@ class GoogleCode(Base):# {{{ # The pattern to match filenames for the files being uploaded and # extract version information from them. Must have a named group # named version - filename_pattern=r'{appname}-(?:portable-installer-)?(?P.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' + filename_pattern=r'{appname}-(?:portable-installer-)?(?P.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' # noqa ): self.username, self.password, = username, password @@ -227,7 +228,8 @@ class GoogleCode(Base):# {{{ paths = eval(raw) if raw else {} paths.update(self.paths) rem = [x for x in paths if self.version not in x] - for x in rem: paths.pop(x) + for x in rem: + paths.pop(x) raw = ['%r : %r,'%(k, v) for k, v in paths.items()] raw = '{\n\n%s\n\n}\n'%('\n'.join(raw)) with NamedTemporaryFile() as t: @@ -347,7 +349,7 @@ class GoogleCode(Base):# {{{ # }}} -class SourceForge(Base): # {{{ +class SourceForge(Base): # {{{ # Note that you should manually ssh once to username,project@frs.sourceforge.net # on the staging server so that the host key is setup @@ -378,6 +380,28 @@ class SourceForge(Base): # {{{ # }}} +def upload_to_servers(files, version): # {{{ + for server, rdir in {'files':'/usr/share/nginx/html'}.iteritems(): + print('Uploading to server:', server) + server = '%s.calibre-ebook.com' % server + rdir = '%s/%s/' % (rdir, version) + for x in files: + start = time.time() + print ('Uploading', x) + for i in range(5): + try: + check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x, + 'root@%s:%s'%(server, rdir)]) + except KeyboardInterrupt: + raise SystemExit(1) + except: + print ('\nUpload failed, trying again in 30 seconds') + time.sleep(30) + else: + break + print ('Uploaded in', int(time.time() - start), 'seconds\n\n') +# }}} + # CLI {{{ def cli_parser(): epilog='Copyright Kovid Goyal 2012' @@ -409,6 +433,7 @@ def cli_parser(): sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge', epilog=epilog) cron = subparsers.add_parser('cron', help='Call script from cron') + subparsers.add_parser('calibre', help='Upload to calibre file servers') a = gc.add_argument @@ -471,8 +496,11 @@ def main(args=None): sf() elif args.service == 'cron': login_to_google(args.username, args.password) + elif args.service == 'calibre': + upload_to_servers(ofiles, args.version) if __name__ == '__main__': main() # }}} + diff --git a/setup/upload.py b/setup/upload.py index 673f9f4679..1c7348bfe9 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -81,7 +81,7 @@ class ReUpload(Command): # {{{ # Data {{{ def get_google_data(): - with open(os.path.expanduser('~/work/kde/conf/googlecodecalibre'), 'rb') as f: + with open(os.path.expanduser('~/work/env/private/googlecodecalibre'), 'rb') as f: gc_password, ga_un, pw = f.read().strip().split('|') return { @@ -111,6 +111,9 @@ def sf_cmdline(ver, sdata): return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'], sdata['username']] +def calibre_cmdline(ver): + return [__appname__, ver, 'fmap', 'calibre'] + def run_remote_upload(args): print 'Running remotely:', ' '.join(args) subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST), @@ -133,7 +136,8 @@ class UploadInstallers(Command): # {{{ try: self.upload_to_staging(tdir, files) self.upload_to_sourceforge() - self.upload_to_google(opts.replace) + self.upload_to_calibre() + # self.upload_to_google(opts.replace) finally: shutil.rmtree(tdir, ignore_errors=True) @@ -170,6 +174,10 @@ class UploadInstallers(Command): # {{{ sdata = get_sourceforge_data() args = sf_cmdline(__version__, sdata) run_remote_upload(args) + + def upload_to_calibre(self): + run_remote_upload(calibre_cmdline(__version__)) + # }}} class UploadUserManual(Command): # {{{ From b06d080987897347494bbc202872e70e6fbb5bae Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Jun 2013 07:05:24 +0530 Subject: [PATCH 04/37] Update La Nacion (Costa Rica) --- recipes/la_nacion_cr.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/la_nacion_cr.recipe b/recipes/la_nacion_cr.recipe index ae320064d6..99b927edbb 100644 --- a/recipes/la_nacion_cr.recipe +++ b/recipes/la_nacion_cr.recipe @@ -20,7 +20,7 @@ class crnews(BasicNewsRecipe): no_stylesheets = True - feeds = [(u'Portada', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=portada'), (u'Ultima Hora', u'http://www.nacion.com/Generales/RSS/UltimaHoraRss.aspx'), (u'Nacionales', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=elpais'), (u'Entretenimiento', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=entretenimiento'), (u'Sucesos', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=sucesos'), (u'Deportes', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=deportes'), (u'Internacionales', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=mundo'), (u'Economia', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=economia'), (u'Aldea Global', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=aldeaglobal'), (u'Tecnologia', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=tecnologia'), (u'Opinion', u'http://www.nacion.com/Generales/RSS/EdicionRss.aspx?section=opinion')] + feeds = [(u'Portada', u'http://www.nacion.com/rss/'), (u'Ultima Hora', u'http://www.nacion.com/rss/latest/'), (u'Nacionales', u'http://www.nacion.com/rss/nacional/'), (u'Entretenimiento', u'http://www.nacion.com/rss/ocio/'), (u'Sucesos', u'http://www.nacion.com/rss/sucesos/'), (u'Deportes', u'http://www.nacion.com/rss/deportes/'), (u'Internacionales', u'http://www.nacion.com/rss/mundo/'), (u'Economia', u'http://www.nacion.com/rss/economia/'), (u'Vivir', u'http://www.nacion.com/rss/vivir/'), (u'Tecnologia', u'http://www.nacion.com/rss/tecnologia/'), (u'Opinion', u'http://www.nacion.com/rss/opinion/')] def get_cover_url(self): index = 'http://kiosko.net/cr/np/cr_nacion.html' From ec2bd359abbaa941dd7cc505b45d21753d914ade Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Jun 2013 07:30:27 +0530 Subject: [PATCH 05/37] Copy calibre releases to my backup hdd --- setup/upload.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup/upload.py b/setup/upload.py index 1c7348bfe9..a113eef703 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -133,15 +133,18 @@ class UploadInstallers(Command): # {{{ files = {x:installer_description(x) for x in all_possible.intersection(available)} tdir = mkdtemp() + backup = os.path.join('/mnt/external/calibre/%s' % __version__) + if not os.path.exists(backup): + os.mkdir(backup) try: - self.upload_to_staging(tdir, files) + self.upload_to_staging(tdir, backup, files) self.upload_to_sourceforge() self.upload_to_calibre() # self.upload_to_google(opts.replace) finally: shutil.rmtree(tdir, ignore_errors=True) - def upload_to_staging(self, tdir, files): + def upload_to_staging(self, tdir, backup, files): os.mkdir(tdir+'/dist') hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'hosting.py') @@ -149,6 +152,7 @@ class UploadInstallers(Command): # {{{ for f in files: shutil.copyfile(f, os.path.join(tdir, f)) + shutil.copyfile(f, os.path.join(backup, f)) with open(os.path.join(tdir, 'fmap'), 'wb') as fo: for f, desc in files.iteritems(): From bf1055abd97934eacc3c015769a9a82cd54b362a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Jun 2013 07:42:30 +0530 Subject: [PATCH 06/37] fetch-ebbok-metadata: Fix --opf argument requiring a value --- src/calibre/ebooks/metadata/sources/cli.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/cli.py b/src/calibre/ebooks/metadata/sources/cli.py index f8b9c6b7a9..b20c6a5bfa 100644 --- a/src/calibre/ebooks/metadata/sources/cli.py +++ b/src/calibre/ebooks/metadata/sources/cli.py @@ -34,9 +34,9 @@ def option_parser(): parser.add_option('-i', '--isbn', help='Book ISBN') parser.add_option('-v', '--verbose', default=False, action='store_true', help='Print the log to the console (stderr)') - parser.add_option('-o', '--opf', help='Output the metadata in OPF format') + parser.add_option('-o', '--opf', help='Output the metadata in OPF format instead of human readable text.', action='store_true', default=False) parser.add_option('-c', '--cover', - help='Specify a filename. The cover, if available, will be saved to it') + help='Specify a filename. The cover, if available, will be saved to it. Without this option, no cover will be downloaded.') parser.add_option('-d', '--timeout', default='30', help='Timeout in seconds. Default is 30') @@ -71,16 +71,14 @@ def main(args=sys.argv): if opts.cover and results: cover = download_cover(log, title=opts.title, authors=authors, identifiers=result.identifiers, timeout=int(opts.timeout)) - if cover is None: + if cover is None and not opts.opf: prints('No cover found', file=sys.stderr) else: save_cover_data_to(cover[-1], opts.cover) result.cover = cf = opts.cover - log = buf.getvalue() - result = (metadata_to_opf(result) if opts.opf else unicode(result).encode('utf-8')) @@ -95,3 +93,4 @@ def main(args=sys.argv): if __name__ == '__main__': sys.exit(main()) + From 6a134427c84491d7d5df488a3eb1aa55c68a7109 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Jun 2013 07:59:40 +0530 Subject: [PATCH 07/37] Update Frontline --- recipes/frontlineonnet.recipe | 51 +++++++++++++++++------------------ 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/recipes/frontlineonnet.recipe b/recipes/frontlineonnet.recipe index dc1d16cfd4..73d866c3b3 100644 --- a/recipes/frontlineonnet.recipe +++ b/recipes/frontlineonnet.recipe @@ -46,35 +46,34 @@ class Frontlineonnet(BasicNewsRecipe): keep_only_tags= [ dict(name='div', attrs={'id':'content'}) - #,dict(attrs={'class':'byline'}) ] - #remove_attributes=['size','noshade','border'] - - #def preprocess_html(self, soup): - #for item in soup.findAll(style=True): - #del item['style'] - #for item in soup.findAll('img'): - #if not item.has_key('alt'): - #item['alt'] = 'image' - #return soup + remove_attributes=['size','noshade','border'] def parse_index(self): articles = [] + current_section = None + feeds = [] soup = self.index_to_soup(self.INDEX) - for feed_link in soup.findAll('div', id='headseccol'): - a = feed_link.find('a', href=True) - title = self.tag_to_string(a) - url = a['href'] - articles.append({ - 'title' :title - ,'date' :'' - ,'url' :url - ,'description':'' - }) - return [('Frontline', articles)] + for h3 in soup.findAll('h3'): + if h3.get('class', None) == 'artListSec': + if articles: + feeds.append((current_section, articles)) + articles = [] + current_section = self.tag_to_string(h3).strip() + self.log(current_section) + elif h3.get('id', None) in {'headseccol', 'headsec'}: + a = h3.find('a', href=True) + if a is not None: + title = self.tag_to_string(a) + url = a['href'] + articles.append({ + 'title' :title + ,'date' :'' + ,'url' :url + ,'description':'' + }) + self.log('\t', title, url) + if articles: + feeds.append((current_section, articles)) + return feeds - #def print_version(self, url): - #return "http://www.hinduonnet.com/thehindu/thscrip/print.pl?prd=fline&file=" + url.rpartition('/')[2] - - #def image_url_processor(self, baseurl, url): - #return url.replace('../images/', self.INDEX + 'images/').strip() From 2ee5ad2e301c7acfb7a3b137a998b71d8ac87d52 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Jun 2013 08:30:59 +0530 Subject: [PATCH 08/37] Ensure dist files have correct permissions --- setup/upload.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/upload.py b/setup/upload.py index a113eef703..784c0cf9f8 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -151,8 +151,10 @@ class UploadInstallers(Command): # {{{ shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py')) for f in files: - shutil.copyfile(f, os.path.join(tdir, f)) - shutil.copyfile(f, os.path.join(backup, f)) + for x in (tdir, backup): + dest = os.path.join(x, f) + shutil.copyfile(f, dest) + os.chmod(dest, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH) with open(os.path.join(tdir, 'fmap'), 'wb') as fo: for f, desc in files.iteritems(): From 4fe86065f9ecef37c1c3ef9a85170bc36e5bfacf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 23 Jun 2013 09:09:45 +0530 Subject: [PATCH 09/37] Fix scanning for books on Aluratek Color Fix a regression that broke scanning for books on all devices that used the Aluratek Color driver. Fixes #1192940 [No longer seeing books on Odys Leon (Aluratek colour)](https://bugs.launchpad.net/calibre/+bug/1192940) --- src/calibre/devices/misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 4a2e6aa864..b20ec3ca6e 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -211,6 +211,7 @@ class ALURATEK_COLOR(USBMS): VENDOR_NAME = ['USB_2.0', 'EZREADER', 'C4+'] WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['USB_FLASH_DRIVER', '.', 'TOUCH'] SCAN_FROM_ROOT = True + SUPPORTS_SUB_DIRS_FOR_SCAN = True class TREKSTOR(USBMS): From c8889644bd77beee40423f608301071be1743f77 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 23 Jun 2013 10:47:29 +0530 Subject: [PATCH 10/37] Remove obsolete recipes --- recipes/living_digital.recipe | 16 ---------------- recipes/pc_quest_india.recipe | 16 ---------------- 2 files changed, 32 deletions(-) delete mode 100644 recipes/living_digital.recipe delete mode 100644 recipes/pc_quest_india.recipe diff --git a/recipes/living_digital.recipe b/recipes/living_digital.recipe deleted file mode 100644 index 2fcf50dfab..0000000000 --- a/recipes/living_digital.recipe +++ /dev/null @@ -1,16 +0,0 @@ -from calibre.web.feeds.news import CalibrePeriodical - -class LivingDigital(CalibrePeriodical): - - title = 'Living Digital' - calibre_periodicals_slug = 'living-digital' - - description = ''' - Catch the latest buzz in the digital world with Living Digital. Enjoy - reviews, news, features and recommendations on a wide range of consumer - technology products - from smartphones to flat panel TVs, netbooks to - cameras, and many more consumer lifestyle gadgets. To subscribe, visit - calibre - Periodicals. - ''' - language = 'en_IN' diff --git a/recipes/pc_quest_india.recipe b/recipes/pc_quest_india.recipe deleted file mode 100644 index e45272a2df..0000000000 --- a/recipes/pc_quest_india.recipe +++ /dev/null @@ -1,16 +0,0 @@ -from calibre.web.feeds.news import CalibrePeriodical - -class PCQ(CalibrePeriodical): - - title = 'PCQuest' - calibre_periodicals_slug = 'pc-quest-india' - - description = ''' - Buying a tech product? Seeking a tech solution? Consult PCQuest, India's - market-leading selection and implementation guide for the latest - technologies: servers, business apps, security, open source, gadgets and - more. To subscribe visit, calibre - Periodicals. - ''' - language = 'en_IN' From e8e4dbc35b0a3070c446b5562e4508b18ad346c8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 24 Jun 2013 09:56:07 +0530 Subject: [PATCH 11/37] Update Miradasalsur. Fixes #1193922 [Updated recipe for Miradas al Sur](https://bugs.launchpad.net/calibre/+bug/1193922) --- recipes/icons/miradasalsur.png | Bin 0 -> 1120 bytes recipes/miradasalsur.recipe | 97 ++++++++++++++++----------------- 2 files changed, 47 insertions(+), 50 deletions(-) create mode 100644 recipes/icons/miradasalsur.png diff --git a/recipes/icons/miradasalsur.png b/recipes/icons/miradasalsur.png new file mode 100644 index 0000000000000000000000000000000000000000..9cb7d033addb5d783f10eda8764433907d27df0c GIT binary patch literal 1120 zcmdr~?OT&o7{8xAVo zdH?_w9Tl+}d->RMC$ZH8%Mt)ch+mNyIf8hsh{cLr*l;w9c#g->D8h~xkbpZp5)k4L zgCQ;)UIgKydMh8$qmqt1K>$B6f%o zForc(*VvM>?df^WN53N%H(XF|&N(_#S=*m?jA{J!Rmthzk}|4@9a#{UMWlc9UlaZMF)CeHN3}Ez;U*F54p4{zOf{u zE2-(tE;XvcOyh%*xt~Lh;f8w;<=Peq=}gUX_H?|u)BKpwyo^g3)R%OFrfqMjQ8}+& zG1p#y)8wstu`=BpzwJ4wn3Gkugsokm?c1@xiS&J$oc25ipUSUq&%wu2bnTOLbY-;> zeEylAG*of1S*&iAg+2vUrzWcZyx}Q1tutzabyq>FTTov{ZujJ{C!lVTg}1o{b;WG4 zR9~_t>Mia zKm`z&Yl$m{wn@e&*?K6JR`Na@jI$4`87^3>@v z*!|_6oV{P|%l$epzo2mcfg<>1d)lYz8J~T=W9P2SEd3YRu Date: Mon, 24 Jun 2013 10:18:24 +0530 Subject: [PATCH 12/37] News download: Respect more default conversion settings News download: Apply the default page margin conversion settings. Also, when converting to PDF, apply the pdf conversion defaults. Fixes #1193763 [Private bug](https://bugs.launchpad.net/calibre/+bug/1193763) --- src/calibre/gui2/tools.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index eda60a4fec..32e3174c8d 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -24,7 +24,7 @@ from calibre.ebooks.conversion.config import GuiRecommendations, \ load_defaults, load_specifics, save_specifics from calibre.gui2.convert import bulk_defaults_for_input_format -def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ +def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ out_format=None, show_no_format_warning=True): changed = False jobs = [] @@ -47,7 +47,7 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ result = d.exec_() if result == QDialog.Accepted: - #if not convert_existing(parent, db, [book_id], d.output_format): + # if not convert_existing(parent, db, [book_id], d.output_format): # continue mi = db.get_metadata(book_id, True) @@ -116,7 +116,6 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ msg = _('This book has no actual ebook files') res.append('%s - %s'%(title, msg)) - msg = '%s' % '\n'.join(res) warning_dialog(parent, _('Could not convert some books'), _('Could not convert %(num)d of %(tot)d books, because no supported source' @@ -254,7 +253,7 @@ class QueueBulk(QProgressDialog): # }}} -def fetch_scheduled_recipe(arg): # {{{ +def fetch_scheduled_recipe(arg): # {{{ fmt = prefs['output_format'].lower() # Never use AZW3 for periodicals... if fmt == 'azw3': @@ -266,6 +265,10 @@ def fetch_scheduled_recipe(arg): # {{{ if 'output_profile' in ps: recs.append(('output_profile', ps['output_profile'], OptionRecommendation.HIGH)) + for edge in ('left', 'top', 'bottom', 'right'): + edge = 'margin_' + edge + if edge in ps: + recs.append((edge, ps[edge], OptionRecommendation.HIGH)) lf = load_defaults('look_and_feel') if lf.get('base_font_size', 0.0) != 0.0: @@ -283,18 +286,24 @@ def fetch_scheduled_recipe(arg): # {{{ if epub.get('epub_flatten', False): recs.append(('epub_flatten', True, OptionRecommendation.HIGH)) + if fmt == 'pdf': + pdf = load_defaults('pdf_output') + from calibre.customize.ui import plugin_for_output_format + p = plugin_for_output_format('pdf') + for opt in p.options: + recs.append(opt.name, pdf.get(opt.name, opt.recommended_value), OptionRecommendation.HIGH) + args = [arg['recipe'], pt.name, recs] if arg['username'] is not None: recs.append(('username', arg['username'], OptionRecommendation.HIGH)) if arg['password'] is not None: recs.append(('password', arg['password'], OptionRecommendation.HIGH)) - return 'gui_convert', args, _('Fetch news from ')+arg['title'], fmt.upper(), [pt] # }}} -def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{ +def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{ from calibre.gui2.dialogs.catalog import Catalog # Build the Catalog dialog in gui2.dialogs.catalog @@ -354,7 +363,7 @@ def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{ d.catalog_title # }}} -def convert_existing(parent, db, book_ids, output_format): # {{{ +def convert_existing(parent, db, book_ids, output_format): # {{{ already_converted_ids = [] already_converted_titles = [] for book_id in book_ids: @@ -372,3 +381,4 @@ def convert_existing(parent, db, book_ids, output_format): # {{{ return book_ids # }}} + From f58b8aee4fd223ad6f2434389de276956765eb88 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 24 Jun 2013 10:21:42 +0530 Subject: [PATCH 13/37] Update taz.de (RSS) Fixes #18 (RSS Recipe broken: Taz RSS) --- recipes/taz_rss.recipe | 1 - 1 file changed, 1 deletion(-) diff --git a/recipes/taz_rss.recipe b/recipes/taz_rss.recipe index 90cf27a303..3ccbe2a4f1 100644 --- a/recipes/taz_rss.recipe +++ b/recipes/taz_rss.recipe @@ -18,7 +18,6 @@ class TazRSSRecipe(BasicNewsRecipe): feeds = [(u'TAZ main feed', u'http://www.taz.de/rss.xml')] keep_only_tags = [dict(name='div', attrs={'class': 'sect sect_article'})] - remove_tags_after = dict(name='div', attrs={'class': 'rack'}) remove_tags = [ dict(name=['div'], attrs={'class': 'artikelwerbung'}), dict(name=['ul'], attrs={'class': 'toolbar'}),] From d2292a759d5718ea6cb82360d7e3026652020f18 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 24 Jun 2013 18:04:07 +0530 Subject: [PATCH 14/37] Upload installers to downloadbestsoftware.com as well --- setup/hosting.py | 26 ++++++++++++++++++++++++++ setup/upload.py | 6 ++++++ 2 files changed, 32 insertions(+) diff --git a/setup/hosting.py b/setup/hosting.py index 9853f181a4..76ab3992a0 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -402,6 +402,29 @@ def upload_to_servers(files, version): # {{{ print ('Uploaded in', int(time.time() - start), 'seconds\n\n') # }}} +def upload_to_dbs(files, version): # {{{ + print('Uploading to downloadbestsoftware.com') + server = 'www.downloadbestsoft-mirror1.com' + rdir = 'release/' + check_call(['ssh', 'kovid@%s' % server, 'rm -f release/*']) + for x in files: + start = time.time() + print ('Uploading', x) + for i in range(5): + try: + check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x, + 'kovid@%s:%s'%(server, rdir)]) + except KeyboardInterrupt: + raise SystemExit(1) + except: + print ('\nUpload failed, trying again in 30 seconds') + time.sleep(30) + else: + break + print ('Uploaded in', int(time.time() - start), 'seconds\n\n') + check_call(['ssh', 'kovid@%s' % server, '/home/kovid/uploadFiles']) +# }}} + # CLI {{{ def cli_parser(): epilog='Copyright Kovid Goyal 2012' @@ -434,6 +457,7 @@ def cli_parser(): epilog=epilog) cron = subparsers.add_parser('cron', help='Call script from cron') subparsers.add_parser('calibre', help='Upload to calibre file servers') + subparsers.add_parser('dbs', help='Upload to downloadbestsoftware.com') a = gc.add_argument @@ -498,6 +522,8 @@ def main(args=None): login_to_google(args.username, args.password) elif args.service == 'calibre': upload_to_servers(ofiles, args.version) + elif args.service == 'dbs': + upload_to_dbs(ofiles, args.version) if __name__ == '__main__': main() diff --git a/setup/upload.py b/setup/upload.py index 784c0cf9f8..8a4e467dd0 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -114,6 +114,9 @@ def sf_cmdline(ver, sdata): def calibre_cmdline(ver): return [__appname__, ver, 'fmap', 'calibre'] +def dbs_cmdline(ver): + return [__appname__, ver, 'fmap', 'dbs'] + def run_remote_upload(args): print 'Running remotely:', ' '.join(args) subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST), @@ -140,6 +143,7 @@ class UploadInstallers(Command): # {{{ self.upload_to_staging(tdir, backup, files) self.upload_to_sourceforge() self.upload_to_calibre() + self.upload_to_dbs() # self.upload_to_google(opts.replace) finally: shutil.rmtree(tdir, ignore_errors=True) @@ -184,6 +188,8 @@ class UploadInstallers(Command): # {{{ def upload_to_calibre(self): run_remote_upload(calibre_cmdline(__version__)) + def upload_to_dbs(self): + run_remote_upload(dbs_cmdline(__version__)) # }}} class UploadUserManual(Command): # {{{ From 7089c66a986ec8741216449d76d6fd79e4afdc9d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 24 Jun 2013 18:45:05 +0530 Subject: [PATCH 15/37] LRF Output: Fix " entities in attribute values causing problems --- src/calibre/ebooks/lrf/html/convert_from.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/lrf/html/convert_from.py b/src/calibre/ebooks/lrf/html/convert_from.py index c755cabd92..2b9a3617dc 100644 --- a/src/calibre/ebooks/lrf/html/convert_from.py +++ b/src/calibre/ebooks/lrf/html/convert_from.py @@ -104,7 +104,7 @@ class HTMLConverter(object): # Replace entities (re.compile(ur'&(\S+?);'), partial(entity_to_unicode, - exceptions=['lt', 'gt', 'amp'])), + exceptions=['lt', 'gt', 'amp', 'quot'])), # Remove comments from within style tags as they can mess up BeatifulSoup (re.compile(r'()', re.IGNORECASE|re.DOTALL), strip_style_comments), From 5b07091d5939c0f25e3122d66817ed86f22968e1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Jun 2013 09:12:04 +0530 Subject: [PATCH 16/37] Make the manual self contained --- manual/resources/simple_donate_button.gif | Bin 0 -> 2132 bytes manual/templates/layout.html | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 manual/resources/simple_donate_button.gif diff --git a/manual/resources/simple_donate_button.gif b/manual/resources/simple_donate_button.gif new file mode 100644 index 0000000000000000000000000000000000000000..42dd2c3c88f13d0a8eebeecfff7088effdcf8566 GIT binary patch literal 2132 zcmb`G`#;kQ1AxD~glRIYNRqj2<~D{D)!c7WG$hAm8*`@_38`j7t~K}N3}Zx5jU^@J z)?#LbbO@cdP;YhW{m_+9y}s``|HJ!yo}ZuJ9xoqH#{)D5pbPu}fOCvP4utbgc4vK$ zq_`hs`5Z|nIc9~EqmoYer-qSP5gyD)zgS98LP$_@*l`BsWMXJkCe1TD+T%i^+i99# zR*YX(bTGv;Cd4N;%rA!GnGot5Pj+D)_qh;3$)z5nMpLM?kaLW%vx$+}xv8g4(a)sP zax)mG&m^*PlQPm7x!DQXrA+I(Y!ksb)5=Wq+8n&#f;l_RiJRnHa_Z2f47*a+!OGKi zwP%T(WY^+UuTqv*L1s#6PEy_}7B?fk^laKd4OGHY8>rrK?IOB8e@|DLp12IvU9Q_( zg_cz5J*?l;lxHqr6R#E+wHISM%M3*Y=GThx;&KB?rNKZA=HV60a2+5O0P-g2kN_oZ z*bC-C!7>_BIAq(t!BU>o^He6W1p72$K7n>4VL*K~d7( z7S`j=W24uDWj6yyL{X#HPs(pZD!M6?lCYWHpg-<~EZz@!c01z5ov_(^;Y;_!RtCd{ zMf9<2(ekdC@f*o=14(1ISfIicoc4P=68u3L{7DwFJR0^%9{FwZ_{P)Y>yx3so>9I{ zo&5Pc9Gs7MIZS&kjsEup(Ao<`!W@4QjW)#SeaiYT5)Zy z%N=aWog7LIIhMO0F7!B3|MdQO0#MlkrV)DHVn1zb6)k-D27@+IxWgzX4_DFh=%TiICT6@_G-Q?Wy-3fe zr&U&kqg)S{^%ysl>mMk5?mn)7o0r^VreZMqI0>n!MZPH9xxf1u!_9;EE9v!8Sz1U8*lGxF@r!iSDh&A`=! zA1TpyowwHS_>&#GUk|3jNa8%7NXRWAOUd})v z%!i#}<*NLF9OpYHoRiAQ>C_CjpvYO`GWl z^MV|;V?v;F0MlbUgv#^STc}4Lw?8>6@X|l?Zf$70wm0*!vej(z_%2;4h7AR9Boc~< zWU~Ra@H{CD2@qgvNvmnzQdVxDjr3wDPf0h&CbM8r-O< z5u_$u!yT{`tbg4ZBMkJ?Pv391U+NUI=8jg$O}ZC*_YhlY4h02_ z2}*X37~o_U2~Hh?o84|k6?O2nrvhy}aE2NMMI0E83b*YVMjnVKsXY!T#0JpRGD&d5 zWj4%iei&stz|}M+L2$g+$gMzcDDo3PbQ)+z)XNQ2>Iiw;bRcShLf5qqgl#&r1`u=y z2|dZ=+D=g++xae+K6M^YK--f`G$S2iu9xWA{e&%MnuVbeEc=5t-zs<(CwLLR#73x7O_VN>Hn zX6O>H$lXotqRjjgZHe_yKfW=XwdGkEx75EvwmXjbJYm&qT`7KL>b1&zlG*>IVJ-iS zUcM{J`W>cm!(lSc`J3=)PRr=_bQPRF6d#bcOcN?wzSC3cpDM7mYgSLexgrv$@(Pm;G$y93r-%lC)Bga!y^>S_ literal 0 HcmV?d00001 diff --git a/manual/templates/layout.html b/manual/templates/layout.html index b8389b0ac9..188e829469 100644 --- a/manual/templates/layout.html +++ b/manual/templates/layout.html @@ -62,7 +62,7 @@
- +
From 230ef564eef44b81bc004f1cd21016c3c012adc5 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Tue, 25 Jun 2013 08:20:07 +0200 Subject: [PATCH 17/37] Fix bug #1193763 - Save to device template generates exception for custom column names. --- src/calibre/gui2/dialogs/template_dialog.py | 2 ++ src/calibre/gui2/preferences/save_template.py | 5 +++-- src/calibre/gui2/preferences/saving.py | 3 ++- src/calibre/gui2/preferences/sending.py | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index 2bafc2812a..3db6e37eb0 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -262,6 +262,8 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): self.mi.rating = 4.0 self.mi.tags = [_('Tag 1'), _('Tag 2')] self.mi.languages = ['eng'] + if fm is not None: + self.mi.set_all_user_metadata(fm.custom_field_metadata()) # Remove help icon on title bar icon = self.windowIcon() diff --git a/src/calibre/gui2/preferences/save_template.py b/src/calibre/gui2/preferences/save_template.py index 627c4c7fa9..145e014800 100644 --- a/src/calibre/gui2/preferences/save_template.py +++ b/src/calibre/gui2/preferences/save_template.py @@ -24,7 +24,7 @@ class SaveTemplate(QWidget, Ui_Form): Ui_Form.__init__(self) self.setupUi(self) - def initialize(self, name, default, help): + def initialize(self, name, default, help, field_metadata): variables = sorted(FORMAT_ARG_DESCS.keys()) rows = [] for var in variables: @@ -36,6 +36,7 @@ class SaveTemplate(QWidget, Ui_Form): table = u'%s
'%(u'\n'.join(rows)) self.template_variables.setText(table) + self.field_metadata = field_metadata self.opt_template.initialize(name+'_template_history', default, help) self.opt_template.editTextChanged.connect(self.changed) @@ -44,7 +45,7 @@ class SaveTemplate(QWidget, Ui_Form): self.open_editor.clicked.connect(self.do_open_editor) def do_open_editor(self): - t = TemplateDialog(self, self.opt_template.text()) + t = TemplateDialog(self, self.opt_template.text(), fm=self.field_metadata) t.setWindowTitle(_('Edit template')) if t.exec_(): self.opt_template.set_value(t.rule[1]) diff --git a/src/calibre/gui2/preferences/saving.py b/src/calibre/gui2/preferences/saving.py index bd5fcbb078..e1a235803d 100644 --- a/src/calibre/gui2/preferences/saving.py +++ b/src/calibre/gui2/preferences/saving.py @@ -34,7 +34,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): ConfigWidgetBase.initialize(self) self.save_template.blockSignals(True) self.save_template.initialize('save_to_disk', self.proxy['template'], - self.proxy.help('template')) + self.proxy.help('template'), + self.gui.library_view.model().db.field_metadata) self.save_template.blockSignals(False) def restore_defaults(self): diff --git a/src/calibre/gui2/preferences/sending.py b/src/calibre/gui2/preferences/sending.py index 3fce5cb072..bc46ac500b 100644 --- a/src/calibre/gui2/preferences/sending.py +++ b/src/calibre/gui2/preferences/sending.py @@ -44,7 +44,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): ConfigWidgetBase.initialize(self) self.send_template.blockSignals(True) self.send_template.initialize('send_to_device', self.proxy['send_template'], - self.proxy.help('send_template')) + self.proxy.help('send_template'), + self.gui.library_view.model().db.field_metadata) self.send_template.blockSignals(False) def restore_defaults(self): From 2096dce1cdcea621f314145f71d5ad5a7390a13e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Jun 2013 13:09:51 +0530 Subject: [PATCH 18/37] Move User Manual and staging to the download server --- setup/hosting.py | 113 ++++++++++++++++++++++++++++++++++++++++++++--- setup/upload.py | 11 +++-- 2 files changed, 112 insertions(+), 12 deletions(-) diff --git a/setup/hosting.py b/setup/hosting.py index 76ab3992a0..1e78f4694d 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -7,16 +7,14 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib +import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil from argparse import ArgumentParser, FileType from subprocess import check_call from tempfile import NamedTemporaryFile from collections import OrderedDict -import mechanize -from lxml import html - def login_to_google(username, password): # {{{ + import mechanize br = mechanize.Browser() br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')] @@ -246,6 +244,7 @@ class GoogleCode(Base): # {{{ return login_to_google(self.username, self.gmail_password) def get_files_hosted_by_google_code(self): + from lxml import html self.info('Getting existing files in google code:', self.gc_project) raw = urllib2.urlopen(self.files_list).read() root = html.fromstring(raw) @@ -380,11 +379,111 @@ class SourceForge(Base): # {{{ # }}} +def generate_index(): # {{{ + os.chdir('/srv/download') + releases = set() + for x in os.listdir('.'): + if os.path.isdir(x) and '.' in x: + releases.add(tuple((int(y) for y in x.split('.')))) + rmap = OrderedDict() + for rnum in sorted(releases, reverse=True): + series = rnum[:2] if rnum[0] == 0 else rnum[:1] + if series not in rmap: + rmap[series] = [] + rmap[series].append(rnum) + + template = '''\n {title}

{title}

{msg}

{body} ''' # noqa + style = ''' + body { font-family: sans-serif; background-color: #eee; } + a { text-decoration: none; } + a:visited { color: blue } + a:hover { color: red } + ul { list-style-type: none } + li { padding-bottom: 1ex } + dd li { text-indent: 0; margin: 0 } + dd ul { padding: 0; margin: 0 } + dt { font-weight: bold } + dd { margin-bottom: 2ex } + ''' + body = [] + for series in rmap: + body.append('
  • {0}.x\xa0\xa0\xa0[{1} releases]
    • '.format( # noqa + '.'.join(map(type(''), series)), len(rmap[series]))) + body = '
      {0}
    '.format(' '.join(body)) + index = template.format(title='Previous calibre releases', style=style, msg='Choose a series of calibre releases', body=body) + with open('index.html', 'wb') as f: + f.write(index.encode('utf-8')) + + for series, releases in rmap.iteritems(): + sname = '.'.join(map(type(''), series)) + body = [ + '
  • {0}
    • '.format('.'.join(map(type(''), r))) + for r in releases] + body = '
      {0}
    '.format(' '.join(body)) + index = template.format(title='Previous calibre releases (%s.x)' % sname, style=style, + msg='Choose a calibre release', body=body) + with open('%s.html' % sname, 'wb') as f: + f.write(index.encode('utf-8')) + + for r in releases: + rname = '.'.join(map(type(''), r)) + os.chdir(rname) + try: + body = [] + files = os.listdir('.') + windows = [x for x in files if x.endswith('.msi')] + if windows: + windows = ['
  • {1}
    • '.format( + x, 'Windows 64-bit Installer' if '64bit' in x else 'Windows 32-bit Installer') + for x in windows] + body.append('
    Windows
      {0}
    '.format(' '.join(windows))) + portable = [x for x in files if '-portable-' in x] + if portable: + body.append('
    Calibre Portable
    {1}
    '.format( + portable[0], 'Calibre Portable Installer')) + osx = [x for x in files if x.endswith('.dmg')] + if osx: + body.append('
    Apple Mac
    {1}
    '.format( + osx[0], 'OS X Disk Image (.dmg)')) + linux = [x for x in files if x.endswith('.bz2')] + if linux: + linux = ['
  • {1}
    • '.format( + x, 'Linux 64-bit binary' if 'x86_64' in x else 'Linux 32-bit binary') + for x in linux] + body.append('
    Linux
      {0}
    '.format(' '.join(linux))) + source = [x for x in files if x.endswith('.xz') or x.endswith('.gz')] + if source: + body.append('
    Source Code
    {1}
    '.format( + source[0], 'Source code (all platforms)')) + + body = '
    {0}
    '.format(''.join(body)) + index = template.format(title='calibre release (%s)' % rname, style=style, + msg='', body=body) + with open('index.html', 'wb') as f: + f.write(index.encode('utf-8')) + finally: + os.chdir('..') + +# }}} + def upload_to_servers(files, version): # {{{ - for server, rdir in {'files':'/usr/share/nginx/html'}.iteritems(): + base = '/srv/download/' + dest = os.path.join(base, version) + if not os.path.exists(dest): + os.mkdir(dest) + for src in files: + shutil.copyfile(src, os.path.join(dest, os.path.basename(src))) + generate_index() + + for server, rdir in {'files':'/srv/download/'}.iteritems(): print('Uploading to server:', server) server = '%s.calibre-ebook.com' % server - rdir = '%s/%s/' % (rdir, version) + # Copy the generated index files + print ('Copying generated index') + check_call(['rsync', '-hzr', '-e', 'ssh -x', '--include', '*.html', + '--filter', '-! */', base, 'root@%s:%s' % (server, rdir)]) + # Copy the release files + rdir = '%s%s/' % (rdir, version) for x in files: start = time.time() print ('Uploading', x) @@ -400,6 +499,7 @@ def upload_to_servers(files, version): # {{{ else: break print ('Uploaded in', int(time.time() - start), 'seconds\n\n') + # }}} def upload_to_dbs(files, version): # {{{ @@ -530,3 +630,4 @@ if __name__ == '__main__': # }}} + diff --git a/setup/upload.py b/setup/upload.py index 8a4e467dd0..639a2e98d5 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -19,10 +19,9 @@ from setup import Command, __version__, installer_name, __appname__ PREFIX = "/var/www/calibre-ebook.com" DOWNLOADS = PREFIX+"/htdocs/downloads" BETAS = DOWNLOADS +'/betas' -USER_MANUAL = '/var/www/localhost/htdocs/' HTML2LRF = "calibre/ebooks/lrf/html/demo" TXT2LRF = "src/calibre/ebooks/lrf/txt/demo" -STAGING_HOST = '67.207.135.179' +STAGING_HOST = 'download.calibre-ebook.com' STAGING_USER = 'root' STAGING_DIR = '/root/staging' @@ -141,8 +140,8 @@ class UploadInstallers(Command): # {{{ os.mkdir(backup) try: self.upload_to_staging(tdir, backup, files) - self.upload_to_sourceforge() self.upload_to_calibre() + self.upload_to_sourceforge() self.upload_to_dbs() # self.upload_to_google(opts.replace) finally: @@ -219,9 +218,9 @@ class UploadUserManual(Command): # {{{ for x in glob.glob(self.j(path, '*')): self.build_plugin_example(x) - check_call(' '.join(['rsync', '-z', '-r', '--progress', - 'manual/.build/html/', - 'bugs:%s'%USER_MANUAL]), shell=True) + for host in ('download', 'files'): + check_call(' '.join(['rsync', '-z', '-r', '--progress', + 'manual/.build/html/', '%s:/srv/manual/' % host]), shell=True) # }}} class UploadDemo(Command): # {{{ From e4876b579eba58402b507b58557a8c92db5037f4 Mon Sep 17 00:00:00 2001 From: Hakan Tandogan Date: Tue, 25 Jun 2013 19:47:48 +0200 Subject: [PATCH 19/37] On book merge, merge identifiers as well. In case of conflict, the target wins. --- src/calibre/gui2/actions/edit_metadata.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index e5a9bfbc7d..84b1d367c6 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -399,8 +399,7 @@ class EditMetadataAction(InterfaceAction): if safe_merge: if not confirm('

    '+_( 'Book formats and metadata from the selected books ' - 'will be added to the first selected book (%s). ' - 'ISBN will not be merged.

    ' + 'will be added to the first selected book (%s).
    ' 'The second and subsequently selected books will not ' 'be deleted or changed.

    ' 'Please confirm you want to proceed.')%title @@ -413,7 +412,7 @@ class EditMetadataAction(InterfaceAction): 'Book formats from the selected books will be merged ' 'into the first selected book (%s). ' 'Metadata in the first selected book will not be changed. ' - 'Author, Title, ISBN and all other metadata will not be merged.

    ' + 'Author, Title and all other metadata will not be merged.

    ' 'After merger the second and subsequently ' 'selected books, with any metadata they have will be deleted.

    ' 'All book formats of the first selected book will be kept ' @@ -427,8 +426,7 @@ class EditMetadataAction(InterfaceAction): else: if not confirm('

    '+_( 'Book formats and metadata from the selected books will be merged ' - 'into the first selected book (%s). ' - 'ISBN will not be merged.

    ' + 'into the first selected book (%s).
    ' 'After merger the second and ' 'subsequently selected books will be deleted.

    ' 'All book formats of the first selected book will be kept ' @@ -490,11 +488,13 @@ class EditMetadataAction(InterfaceAction): def merge_metadata(self, dest_id, src_ids): db = self.gui.library_view.model().db dest_mi = db.get_metadata(dest_id, index_is_id=True) + merged_identifiers = db.get_identifiers(dest_id, index_is_id=True) orig_dest_comments = dest_mi.comments dest_cover = db.cover(dest_id, index_is_id=True) had_orig_cover = bool(dest_cover) for src_id in src_ids: src_mi = db.get_metadata(src_id, index_is_id=True) + if src_mi.comments and orig_dest_comments != src_mi.comments: if not dest_mi.comments: dest_mi.comments = src_mi.comments @@ -523,7 +523,15 @@ class EditMetadataAction(InterfaceAction): if not dest_mi.series: dest_mi.series = src_mi.series dest_mi.series_index = src_mi.series_index + + src_identifiers = db.get_identifiers(src_id, index_is_id=True) + src_identifiers.update(merged_identifiers) + merged_identifiers = src_identifiers.copy() + db.set_metadata(dest_id, dest_mi, ignore_errors=False) + + db.set_identifiers(dest_id, merged_identifiers) + if not had_orig_cover and dest_cover: db.set_cover(dest_id, dest_cover) From 7b6a742f2542ba5265898785a5dace388c11e5f5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jun 2013 00:07:00 +0530 Subject: [PATCH 20/37] ... --- src/calibre/gui2/actions/edit_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index babd690384..729de33c7f 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -426,7 +426,7 @@ class EditMetadataAction(InterfaceAction): else: if not confirm('

    '+_( 'Book formats and metadata from the selected books will be merged ' - 'into the first selected book (%s).
    ' + 'into the first selected book (%s).

    ' 'After merger the second and ' 'subsequently selected books will be deleted.

    ' 'All book formats of the first selected book will be kept ' From 30cea5df3a8f5b3c125912854b78699f9cbd4219 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jun 2013 10:40:00 +0530 Subject: [PATCH 21/37] ... --- src/calibre/web/fetch/javascript.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/web/fetch/javascript.py b/src/calibre/web/fetch/javascript.py index 56460c18bf..6e9ef86ff1 100644 --- a/src/calibre/web/fetch/javascript.py +++ b/src/calibre/web/fetch/javascript.py @@ -128,6 +128,8 @@ def download_resources(browser, resource_cache, output_dir): else: img_counter += 1 ext = what(None, raw) or 'jpg' + if ext == 'jpeg': + ext = 'jpg' # Apparently Moon+ cannot handle .jpeg href = 'img_%d.%s' % (img_counter, ext) dest = os.path.join(output_dir, href) resource_cache[h] = dest From 03452d2a038873d8df345512c7433019ada7efa2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jun 2013 14:59:46 +0530 Subject: [PATCH 22/37] Conversion: Add option to embed all referenced fonts Conversion: Add an option to embed all fonts that are referenced in the input document but are not already embedded. This will search your system for the referenced font, and if found, the font will be embedded. Only works if the output format supports font embedding (for example: EPUB or AZW3). --- src/calibre/ebooks/conversion/cli.py | 5 +- src/calibre/ebooks/conversion/plumber.py | 17 + .../ebooks/oeb/transforms/embed_fonts.py | 233 +++++++++++ src/calibre/ebooks/oeb/transforms/flatcss.py | 2 +- src/calibre/ebooks/oeb/transforms/subset.py | 208 +++++----- src/calibre/gui2/convert/look_and_feel.py | 2 +- src/calibre/gui2/convert/look_and_feel.ui | 371 +++++++++--------- 7 files changed, 561 insertions(+), 277 deletions(-) create mode 100644 src/calibre/ebooks/oeb/transforms/embed_fonts.py diff --git a/src/calibre/ebooks/conversion/cli.py b/src/calibre/ebooks/conversion/cli.py index f2e5f4e3c9..a0abebc5fe 100644 --- a/src/calibre/ebooks/conversion/cli.py +++ b/src/calibre/ebooks/conversion/cli.py @@ -136,7 +136,7 @@ def add_pipeline_options(parser, plumber): [ 'base_font_size', 'disable_font_rescaling', 'font_size_mapping', 'embed_font_family', - 'subset_embedded_fonts', + 'subset_embedded_fonts', 'embed_all_fonts', 'line_height', 'minimum_line_height', 'linearize_tables', 'extra_css', 'filter_css', @@ -320,7 +320,7 @@ def main(args=sys.argv): opts.search_replace = read_sr_patterns(opts.search_replace, log) recommendations = [(n.dest, getattr(opts, n.dest), - OptionRecommendation.HIGH) \ + OptionRecommendation.HIGH) for n in parser.options_iter() if n.dest] plumber.merge_ui_recommendations(recommendations) @@ -342,3 +342,4 @@ def main(args=sys.argv): if __name__ == '__main__': sys.exit(main()) + diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 1f459229c8..a96574e904 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -205,6 +205,16 @@ OptionRecommendation(name='embed_font_family', 'with some output formats, principally EPUB and AZW3.') ), +OptionRecommendation(name='embed_all_fonts', + recommended_value=False, level=OptionRecommendation.LOW, + help=_( + 'Embed every font that is referenced in the input document ' + 'but not already embedded. This will search your system for the ' + 'fonts, and if found, they will be embedded. Embedding will only work ' + 'if the format you are converting to supports embedded fonts, such as ' + 'EPUB, AZW3 or PDF.' + )), + OptionRecommendation(name='subset_embedded_fonts', recommended_value=False, level=OptionRecommendation.LOW, help=_( @@ -965,6 +975,9 @@ OptionRecommendation(name='search_replace', if self.for_regex_wizard and hasattr(self.opts, 'no_process'): self.opts.no_process = True self.flush() + if self.opts.embed_all_fonts or self.opts.embed_font_family: + # Start the threaded font scanner now, for performance + from calibre.utils.fonts.scanner import font_scanner # noqa import cssutils, logging cssutils.log.setLevel(logging.WARN) get_types_map() # Ensure the mimetypes module is intialized @@ -1129,6 +1142,10 @@ OptionRecommendation(name='search_replace', RemoveFakeMargins()(self.oeb, self.log, self.opts) RemoveAdobeMargins()(self.oeb, self.log, self.opts) + if self.opts.embed_all_fonts: + from calibre.ebooks.oeb.transforms.embed_fonts import EmbedFonts + EmbedFonts()(self.oeb, self.log, self.opts) + if self.opts.subset_embedded_fonts and self.output_plugin.file_type != 'pdf': from calibre.ebooks.oeb.transforms.subset import SubsetFonts SubsetFonts()(self.oeb, self.log, self.opts) diff --git a/src/calibre/ebooks/oeb/transforms/embed_fonts.py b/src/calibre/ebooks/oeb/transforms/embed_fonts.py new file mode 100644 index 0000000000..027b8af1de --- /dev/null +++ b/src/calibre/ebooks/oeb/transforms/embed_fonts.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +import logging +from collections import defaultdict + +import cssutils +from lxml import etree + +from calibre import guess_type +from calibre.ebooks.oeb.base import XPath, CSS_MIME, XHTML +from calibre.ebooks.oeb.transforms.subset import get_font_properties, find_font_face_rules, elem_style +from calibre.utils.filenames import ascii_filename +from calibre.utils.fonts.scanner import font_scanner, NoFonts + +def used_font(style, embedded_fonts): + ff = [unicode(f) for f in style.get('font-family', []) if unicode(f).lower() not in { + 'serif', 'sansserif', 'sans-serif', 'fantasy', 'cursive', 'monospace'}] + if not ff: + return False, None + lnames = {unicode(x).lower() for x in ff} + + matching_set = [] + + # Filter on font-family + for ef in embedded_fonts: + flnames = {x.lower() for x in ef.get('font-family', [])} + if not lnames.intersection(flnames): + continue + matching_set.append(ef) + if not matching_set: + return True, None + + # Filter on font-stretch + widths = {x:i for i, x in enumerate(('ultra-condensed', + 'extra-condensed', 'condensed', 'semi-condensed', 'normal', + 'semi-expanded', 'expanded', 'extra-expanded', 'ultra-expanded' + ))} + + width = widths[style.get('font-stretch', 'normal')] + for f in matching_set: + f['width'] = widths[style.get('font-stretch', 'normal')] + + min_dist = min(abs(width-f['width']) for f in matching_set) + if min_dist > 0: + return True, None + nearest = [f for f in matching_set if abs(width-f['width']) == + min_dist] + if width <= 4: + lmatches = [f for f in nearest if f['width'] <= width] + else: + lmatches = [f for f in nearest if f['width'] >= width] + matching_set = (lmatches or nearest) + + # Filter on font-style + fs = style.get('font-style', 'normal') + matching_set = [f for f in matching_set if f.get('font-style', 'normal') == fs] + + # Filter on font weight + fw = int(style.get('font-weight', '400')) + matching_set = [f for f in matching_set if f.get('weight', 400) == fw] + + if not matching_set: + return True, None + return True, matching_set[0] + + +class EmbedFonts(object): + + ''' + Embed all referenced fonts, if found on system. Must be called after CSS flattening. + ''' + + def __call__(self, oeb, log, opts): + self.oeb, self.log, self.opts = oeb, log, opts + self.sheet_cache = {} + self.find_style_rules() + self.find_embedded_fonts() + self.parser = cssutils.CSSParser(loglevel=logging.CRITICAL, log=logging.getLogger('calibre.css')) + self.warned = set() + self.warned2 = set() + + for item in oeb.spine: + if not hasattr(item.data, 'xpath'): + continue + sheets = [] + for href in XPath('//h:link[@href and @type="text/css"]/@href')(item.data): + sheet = self.oeb.manifest.hrefs.get(item.abshref(href), None) + if sheet is not None: + sheets.append(sheet) + if sheets: + self.process_item(item, sheets) + + def find_embedded_fonts(self): + ''' + Find all @font-face rules and extract the relevant info from them. + ''' + self.embedded_fonts = [] + for item in self.oeb.manifest: + if not hasattr(item.data, 'cssRules'): + continue + self.embedded_fonts.extend(find_font_face_rules(item, self.oeb)) + + def find_style_rules(self): + ''' + Extract all font related style information from all stylesheets into a + dict mapping classes to font properties specified by that class. All + the heavy lifting has already been done by the CSS flattening code. + ''' + rules = defaultdict(dict) + for item in self.oeb.manifest: + if not hasattr(item.data, 'cssRules'): + continue + for i, rule in enumerate(item.data.cssRules): + if rule.type != rule.STYLE_RULE: + continue + props = {k:v for k,v in + get_font_properties(rule).iteritems() if v} + if not props: + continue + for sel in rule.selectorList: + sel = sel.selectorText + if sel and sel.startswith('.'): + # We dont care about pseudo-selectors as the worst that + # can happen is some extra characters will remain in + # the font + sel = sel.partition(':')[0] + rules[sel[1:]].update(props) + + self.style_rules = dict(rules) + + def get_page_sheet(self): + if self.page_sheet is None: + manifest = self.oeb.manifest + id_, href = manifest.generate('page_css', 'page_styles.css') + self.page_sheet = manifest.add(id_, href, CSS_MIME, data=self.parser.parseString('', validate=False)) + head = self.current_item.xpath('//*[local-name()="head"][1]') + if head: + href = self.current_item.relhref(href) + l = etree.SubElement(head[0], XHTML('link'), + rel='stylesheet', type=CSS_MIME, href=href) + l.tail = '\n' + else: + self.log.warn('No cannot embed font rules') + return self.page_sheet + + def process_item(self, item, sheets): + ff_rules = [] + self.current_item = item + self.page_sheet = None + for sheet in sheets: + if 'page_css' in sheet.id: + ff_rules.extend(find_font_face_rules(sheet, self.oeb)) + self.page_sheet = sheet + + base = {'font-family':['serif'], 'font-weight': '400', + 'font-style':'normal', 'font-stretch':'normal'} + + for body in item.data.xpath('//*[local-name()="body"]'): + self.find_usage_in(body, base, ff_rules) + + def find_usage_in(self, elem, inherited_style, ff_rules): + style = elem_style(self.style_rules, elem.get('class', '') or '', inherited_style) + for child in elem: + self.find_usage_in(child, style, ff_rules) + has_font, existing = used_font(style, ff_rules) + if not has_font: + return + if existing is None: + in_book = used_font(style, self.embedded_fonts)[1] + if in_book is None: + # Try to find the font in the system + added = self.embed_font(style) + if added is not None: + ff_rules.append(added) + self.embedded_fonts.append(added) + else: + # TODO: Create a page rule from the book rule (cannot use it + # directly as paths might be different) + item = in_book['item'] + sheet = self.parser.parseString(in_book['rule'].cssText, validate=False) + rule = sheet.cssRules[0] + page_sheet = self.get_page_sheet() + href = page_sheet.abshref(item.href) + rule.style.setProperty('src', 'url(%s)' % href) + ff_rules.append(find_font_face_rules(sheet, self.oeb)[0]) + page_sheet.data.insertRule(rule, len(page_sheet.data.cssRules)) + + def embed_font(self, style): + ff = [unicode(f) for f in style.get('font-family', []) if unicode(f).lower() not in { + 'serif', 'sansserif', 'sans-serif', 'fantasy', 'cursive', 'monospace'}] + if not ff: + return + ff = ff[0] + if ff in self.warned: + return + try: + fonts = font_scanner.fonts_for_family(ff) + except NoFonts: + self.log.warn('Failed to find fonts for family:', ff, 'not embedding') + self.warned.add(ff) + return + try: + weight = int(style.get('font-weight', '400')) + except (ValueError, TypeError, AttributeError): + w = style['font-weight'] + if w not in self.warned2: + self.log.warn('Invalid weight in font style: %r' % w) + self.warned2.add(w) + return + for f in fonts: + if f['weight'] == weight and f['font-style'] == style.get('font-style', 'normal') and f['font-stretch'] == style.get('font-stretch', 'normal'): + self.log('Embedding font %s from %s' % (f['full_name'], f['path'])) + data = font_scanner.get_font_data(f) + name = f['full_name'] + ext = 'otf' if f['is_otf'] else 'ttf' + name = ascii_filename(name).replace(' ', '-').replace('(', '').replace(')', '') + fid, href = self.oeb.manifest.generate(id=u'font', href=u'fonts/%s.%s'%(name, ext)) + item = self.oeb.manifest.add(fid, href, guess_type('dummy.'+ext)[0], data=data) + item.unload_data_from_memory() + page_sheet = self.get_page_sheet() + href = page_sheet.relhref(item.href) + css = '''@font-face { font-family: "%s"; font-weight: %s; font-style: %s; font-stretch: %s; src: url(%s) }''' % ( + f['font-family'], f['font-weight'], f['font-style'], f['font-stretch'], href) + sheet = self.parser.parseString(css, validate=False) + page_sheet.data.insertRule(sheet.cssRules[0], len(page_sheet.data.cssRules)) + return find_font_face_rules(sheet, self.oeb)[0] + diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index dd2d20333d..9c08934938 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -194,7 +194,7 @@ class CSSFlattener(object): for i, font in enumerate(faces): ext = 'otf' if font['is_otf'] else 'ttf' fid, href = self.oeb.manifest.generate(id=u'font', - href=u'%s.%s'%(ascii_filename(font['full_name']).replace(u' ', u'-'), ext)) + href=u'fonts/%s.%s'%(ascii_filename(font['full_name']).replace(u' ', u'-'), ext)) item = self.oeb.manifest.add(fid, href, guess_type('dummy.'+ext)[0], data=font_scanner.get_font_data(font)) diff --git a/src/calibre/ebooks/oeb/transforms/subset.py b/src/calibre/ebooks/oeb/transforms/subset.py index 744e37b193..96170bd49c 100644 --- a/src/calibre/ebooks/oeb/transforms/subset.py +++ b/src/calibre/ebooks/oeb/transforms/subset.py @@ -12,6 +12,111 @@ from collections import defaultdict from calibre.ebooks.oeb.base import urlnormalize from calibre.utils.fonts.sfnt.subset import subset, NoGlyphs, UnsupportedFont +def get_font_properties(rule, default=None): + ''' + Given a CSS rule, extract normalized font properties from + it. Note that shorthand font property should already have been expanded + by the CSS flattening code. + ''' + props = {} + s = rule.style + for q in ('font-family', 'src', 'font-weight', 'font-stretch', + 'font-style'): + g = 'uri' if q == 'src' else 'value' + try: + val = s.getProperty(q).propertyValue[0] + val = getattr(val, g) + if q == 'font-family': + val = [x.value for x in s.getProperty(q).propertyValue] + if val and val[0] == 'inherit': + val = None + except (IndexError, KeyError, AttributeError, TypeError, ValueError): + val = None if q in {'src', 'font-family'} else default + if q in {'font-weight', 'font-stretch', 'font-style'}: + val = unicode(val).lower() if (val or val == 0) else val + if val == 'inherit': + val = default + if q == 'font-weight': + val = {'normal':'400', 'bold':'700'}.get(val, val) + if val not in {'100', '200', '300', '400', '500', '600', '700', + '800', '900', 'bolder', 'lighter'}: + val = default + if val == 'normal': + val = '400' + elif q == 'font-style': + if val not in {'normal', 'italic', 'oblique'}: + val = default + elif q == 'font-stretch': + if val not in {'normal', 'ultra-condensed', 'extra-condensed', + 'condensed', 'semi-condensed', 'semi-expanded', + 'expanded', 'extra-expanded', 'ultra-expanded'}: + val = default + props[q] = val + return props + + +def find_font_face_rules(sheet, oeb): + ''' + Find all @font-face rules in the given sheet and extract the relevant info from them. + sheet can be either a ManifestItem or a CSSStyleSheet. + ''' + ans = [] + try: + rules = sheet.data.cssRules + except AttributeError: + rules = sheet.cssRules + + for i, rule in enumerate(rules): + if rule.type != rule.FONT_FACE_RULE: + continue + props = get_font_properties(rule, default='normal') + if not props['font-family'] or not props['src']: + continue + + try: + path = sheet.abshref(props['src']) + except AttributeError: + path = props['src'] + ff = oeb.manifest.hrefs.get(urlnormalize(path), None) + if not ff: + continue + props['item'] = ff + if props['font-weight'] in {'bolder', 'lighter'}: + props['font-weight'] = '400' + props['weight'] = int(props['font-weight']) + props['rule'] = rule + props['chars'] = set() + ans.append(props) + + return ans + + +def elem_style(style_rules, cls, inherited_style): + ''' + Find the effective style for the given element. + ''' + classes = cls.split() + style = inherited_style.copy() + for cls in classes: + style.update(style_rules.get(cls, {})) + wt = style.get('font-weight', None) + pwt = inherited_style.get('font-weight', '400') + if wt == 'bolder': + style['font-weight'] = { + '100':'400', + '200':'400', + '300':'400', + '400':'700', + '500':'700', + }.get(pwt, '900') + elif wt == 'lighter': + style['font-weight'] = { + '600':'400', '700':'400', + '800':'700', '900':'700'}.get(pwt, '100') + + return style + + class SubsetFonts(object): ''' @@ -76,72 +181,15 @@ class SubsetFonts(object): self.log('Reduced total font size to %.1f%% of original'% (totals[0]/totals[1] * 100)) - def get_font_properties(self, rule, default=None): - ''' - Given a CSS rule, extract normalized font properties from - it. Note that shorthand font property should already have been expanded - by the CSS flattening code. - ''' - props = {} - s = rule.style - for q in ('font-family', 'src', 'font-weight', 'font-stretch', - 'font-style'): - g = 'uri' if q == 'src' else 'value' - try: - val = s.getProperty(q).propertyValue[0] - val = getattr(val, g) - if q == 'font-family': - val = [x.value for x in s.getProperty(q).propertyValue] - if val and val[0] == 'inherit': - val = None - except (IndexError, KeyError, AttributeError, TypeError, ValueError): - val = None if q in {'src', 'font-family'} else default - if q in {'font-weight', 'font-stretch', 'font-style'}: - val = unicode(val).lower() if (val or val == 0) else val - if val == 'inherit': - val = default - if q == 'font-weight': - val = {'normal':'400', 'bold':'700'}.get(val, val) - if val not in {'100', '200', '300', '400', '500', '600', '700', - '800', '900', 'bolder', 'lighter'}: - val = default - if val == 'normal': val = '400' - elif q == 'font-style': - if val not in {'normal', 'italic', 'oblique'}: - val = default - elif q == 'font-stretch': - if val not in { 'normal', 'ultra-condensed', 'extra-condensed', - 'condensed', 'semi-condensed', 'semi-expanded', - 'expanded', 'extra-expanded', 'ultra-expanded'}: - val = default - props[q] = val - return props - def find_embedded_fonts(self): ''' Find all @font-face rules and extract the relevant info from them. ''' self.embedded_fonts = [] for item in self.oeb.manifest: - if not hasattr(item.data, 'cssRules'): continue - for i, rule in enumerate(item.data.cssRules): - if rule.type != rule.FONT_FACE_RULE: - continue - props = self.get_font_properties(rule, default='normal') - if not props['font-family'] or not props['src']: - continue - - path = item.abshref(props['src']) - ff = self.oeb.manifest.hrefs.get(urlnormalize(path), None) - if not ff: - continue - props['item'] = ff - if props['font-weight'] in {'bolder', 'lighter'}: - props['font-weight'] = '400' - props['weight'] = int(props['font-weight']) - props['chars'] = set() - props['rule'] = rule - self.embedded_fonts.append(props) + if not hasattr(item.data, 'cssRules'): + continue + self.embedded_fonts.extend(find_font_face_rules(item, self.oeb)) def find_style_rules(self): ''' @@ -151,12 +199,13 @@ class SubsetFonts(object): ''' rules = defaultdict(dict) for item in self.oeb.manifest: - if not hasattr(item.data, 'cssRules'): continue + if not hasattr(item.data, 'cssRules'): + continue for i, rule in enumerate(item.data.cssRules): if rule.type != rule.STYLE_RULE: continue props = {k:v for k,v in - self.get_font_properties(rule).iteritems() if v} + get_font_properties(rule).iteritems() if v} if not props: continue for sel in rule.selectorList: @@ -172,41 +221,17 @@ class SubsetFonts(object): def find_font_usage(self): for item in self.oeb.manifest: - if not hasattr(item.data, 'xpath'): continue + if not hasattr(item.data, 'xpath'): + continue for body in item.data.xpath('//*[local-name()="body"]'): base = {'font-family':['serif'], 'font-weight': '400', 'font-style':'normal', 'font-stretch':'normal'} self.find_usage_in(body, base) - def elem_style(self, cls, inherited_style): - ''' - Find the effective style for the given element. - ''' - classes = cls.split() - style = inherited_style.copy() - for cls in classes: - style.update(self.style_rules.get(cls, {})) - wt = style.get('font-weight', None) - pwt = inherited_style.get('font-weight', '400') - if wt == 'bolder': - style['font-weight'] = { - '100':'400', - '200':'400', - '300':'400', - '400':'700', - '500':'700', - }.get(pwt, '900') - elif wt == 'lighter': - style['font-weight'] = { - '600':'400', '700':'400', - '800':'700', '900':'700'}.get(pwt, '100') - - return style - def used_font(self, style): ''' Given a style find the embedded font that matches it. Returns None if - no match is found ( can happen if not family matches). + no match is found (can happen if no family matches). ''' ff = style.get('font-family', []) lnames = {unicode(x).lower() for x in ff} @@ -222,7 +247,7 @@ class SubsetFonts(object): return None # Filter on font-stretch - widths = {x:i for i, x in enumerate(( 'ultra-condensed', + widths = {x:i for i, x in enumerate(('ultra-condensed', 'extra-condensed', 'condensed', 'semi-condensed', 'normal', 'semi-expanded', 'expanded', 'extra-expanded', 'ultra-expanded' ))} @@ -280,7 +305,7 @@ class SubsetFonts(object): return ans def find_usage_in(self, elem, inherited_style): - style = self.elem_style(elem.get('class', '') or '', inherited_style) + style = elem_style(self.style_rules, elem.get('class', '') or '', inherited_style) for child in elem: self.find_usage_in(child, style) font = self.used_font(style) @@ -290,3 +315,4 @@ class SubsetFonts(object): font['chars'] |= chars + diff --git a/src/calibre/gui2/convert/look_and_feel.py b/src/calibre/gui2/convert/look_and_feel.py index 24ee288cc6..a3e364b9ca 100644 --- a/src/calibre/gui2/convert/look_and_feel.py +++ b/src/calibre/gui2/convert/look_and_feel.py @@ -32,7 +32,7 @@ class LookAndFeelWidget(Widget, Ui_Form): Widget.__init__(self, parent, ['change_justification', 'extra_css', 'base_font_size', 'font_size_mapping', 'line_height', 'minimum_line_height', - 'embed_font_family', 'subset_embedded_fonts', + 'embed_font_family', 'embed_all_fonts', 'subset_embedded_fonts', 'smarten_punctuation', 'unsmarten_punctuation', 'disable_font_rescaling', 'insert_blank_line', 'remove_paragraph_spacing', diff --git a/src/calibre/gui2/convert/look_and_feel.ui b/src/calibre/gui2/convert/look_and_feel.ui index 43736fb1f2..e9d9caeed7 100644 --- a/src/calibre/gui2/convert/look_and_feel.ui +++ b/src/calibre/gui2/convert/look_and_feel.ui @@ -14,6 +14,70 @@ Form + + + + Keep &ligatures + + + + + + + &Linearize tables + + + + + + + Base &font size: + + + opt_base_font_size + + + + + + + &Line size: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + opt_insert_blank_line_size + + + + + + + true + + + + + + + Remove &spacing between paragraphs + + + + + + + &Indent size: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + opt_remove_paragraph_spacing_indent_size + + + @@ -24,6 +88,57 @@ + + + + Insert &blank line between paragraphs + + + + + + + em + + + 1 + + + + + + + Text &justification: + + + opt_change_justification + + + + + + + + + + Smarten &punctuation + + + + + + + &Transliterate unicode characters to ASCII + + + + + + + &UnSmarten punctuation + + + @@ -44,51 +159,6 @@ - - - - % - - - 1 - - - 900.000000000000000 - - - - - - - pt - - - 1 - - - 0.000000000000000 - - - 50.000000000000000 - - - 1.000000000000000 - - - 15.000000000000000 - - - - - - - Font size &key: - - - opt_font_size_mapping - - - @@ -133,56 +203,72 @@ - - - - true - - - - - - - Remove &spacing between paragraphs - - - - - - - &Indent size: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - opt_remove_paragraph_spacing_indent_size - - - - - - - <p>When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent. - - - No change - + + - em + % + + + 1 + + + 900.000000000000000 + + + + + + + pt 1 - -0.100000000000000 + 0.000000000000000 + + + 50.000000000000000 - 0.100000000000000 + 1.000000000000000 + + + 15.000000000000000 - + + + + &Disable font size rescaling + + + + + + + + + + Font size &key: + + + opt_font_size_mapping + + + + + + + &Embed font family: + + + opt_embed_font_family + + + + 0 @@ -300,121 +386,42 @@ - - - - Insert &blank line between paragraphs - - - - + + + <p>When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent. + + + No change + em 1 - - - - - - Text &justification: + + -0.100000000000000 - - opt_change_justification + + 0.100000000000000 - - - - - - - Smarten &punctuation - - - - - - - &Transliterate unicode characters to ASCII - - - - - - - &UnSmarten punctuation - - - - - - - Keep &ligatures - - - - - - - &Linearize tables - - - - - - - Base &font size: - - - opt_base_font_size - - - - - - - &Line size: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - opt_insert_blank_line_size - - - - - - - &Embed font family: - - - opt_embed_font_family - - - - - - - &Disable font size rescaling - - - - - - - + &Subset all embedded fonts + + + + &Embed referenced fonts + + + From 63d133ea5c1a3ee8d4949e95ce9cf8e9e9c9d644 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jun 2013 15:46:41 +0530 Subject: [PATCH 23/37] AZW3 Input: Add support for page-progression-direction AZW3 Input: Add support for the page-progression-direction that is used to indicate page turns should happen from right to left. The attribute is passed into EPUB when converting. Fixes #1194766 [Incorrect conversion japanese MOBI](https://bugs.launchpad.net/calibre/+bug/1194766) --- src/calibre/ebooks/metadata/opf2.py | 11 +++++++++++ src/calibre/ebooks/mobi/reader/mobi8.py | 12 +++++++++--- src/calibre/ebooks/mobi/utils.py | 11 ++++++++++- src/calibre/ebooks/oeb/base.py | 3 +++ src/calibre/ebooks/oeb/reader.py | 3 +++ 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index fb80cc8bfe..77e334dd3e 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -1047,6 +1047,14 @@ class OPF(object): # {{{ if raw: return raw.rpartition(':')[-1] + @property + def page_progression_direction(self): + spine = self.XPath('descendant::*[re:match(name(), "spine", "i")][1]')(self.root) + if spine: + for k, v in spine[0].attrib.iteritems(): + if k == 'page-progression-direction' or k.endswith('}page-progression-direction'): + return v + def guess_cover(self): ''' Try to guess a cover. Needed for some old/badly formed OPF files. @@ -1185,6 +1193,7 @@ class OPFCreator(Metadata): ''' Metadata.__init__(self, title='', other=other) self.base_path = os.path.abspath(base_path) + self.page_progression_direction = None if self.application_id is None: self.application_id = str(uuid.uuid4()) if not isinstance(self.toc, TOC): @@ -1356,6 +1365,8 @@ class OPFCreator(Metadata): spine = E.spine() if self.toc is not None: spine.set('toc', 'ncx') + if self.page_progression_direction is not None: + spine.set('page-progression-direction', self.page_progression_direction) if self.spine is not None: for ref in self.spine: if ref.id is not None: diff --git a/src/calibre/ebooks/mobi/reader/mobi8.py b/src/calibre/ebooks/mobi/reader/mobi8.py index aff79d65c2..97d38a9660 100644 --- a/src/calibre/ebooks/mobi/reader/mobi8.py +++ b/src/calibre/ebooks/mobi/reader/mobi8.py @@ -20,7 +20,7 @@ from calibre.ebooks.mobi.reader.ncx import read_ncx, build_toc from calibre.ebooks.mobi.reader.markup import expand_mobi8_markup from calibre.ebooks.metadata.opf2 import Guide, OPFCreator from calibre.ebooks.metadata.toc import TOC -from calibre.ebooks.mobi.utils import read_font_record +from calibre.ebooks.mobi.utils import read_font_record, read_resc_record from calibre.ebooks.oeb.parse_utils import parse_html from calibre.ebooks.oeb.base import XPath, XHTML, xml2text from calibre.utils.imghdr import what @@ -65,6 +65,7 @@ class Mobi8Reader(object): self.mobi6_reader, self.log = mobi6_reader, log self.header = mobi6_reader.book_header self.encrypted_fonts = [] + self.resc_data = {} def __call__(self): self.mobi6_reader.check_for_drm() @@ -389,9 +390,11 @@ class Mobi8Reader(object): data = sec[0] typ = data[:4] href = None - if typ in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n', - b'RESC', b'BOUN', b'FDST', b'DATP', b'AUDI', b'VIDE'}: + if typ in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n', b'BOUN', + b'FDST', b'DATP', b'AUDI', b'VIDE'}: pass # Ignore these records + elif typ == b'RESC': + self.resc_data = read_resc_record(data) elif typ == b'FONT': font = read_font_record(data) href = "fonts/%05d.%s" % (fname_idx, font['ext']) @@ -452,6 +455,9 @@ class Mobi8Reader(object): opf.create_manifest_from_files_in([os.getcwdu()], exclude=exclude) opf.create_spine(spine) opf.set_toc(toc) + ppd = self.resc_data.get('page-progression-direction', None) + if ppd: + opf.page_progression_direction = ppd with open('metadata.opf', 'wb') as of, open('toc.ncx', 'wb') as ncx: opf.render(of, ncx, 'toc.ncx') diff --git a/src/calibre/ebooks/mobi/utils.py b/src/calibre/ebooks/mobi/utils.py index e9bc4f669f..008b33a0ff 100644 --- a/src/calibre/ebooks/mobi/utils.py +++ b/src/calibre/ebooks/mobi/utils.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import struct, string, zlib, os +import struct, string, zlib, os, re from collections import OrderedDict from io import BytesIO @@ -393,6 +393,15 @@ def mobify_image(data): data = im.export('gif') return data +def read_resc_record(data): + ans = {} + match = re.search(br''']*page-progression-direction=['"](.+?)['"]''', data) + if match is not None: + ppd = match.group(1).lower() + if ppd in {b'ltr', b'rtl'}: + ans['page-progression-direction'] = ppd.decode('ascii') + return ans + # Font records {{{ def read_font_record(data, extent=1040): ''' diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index d4b3a2b7ab..29fc27ee3f 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -1210,6 +1210,7 @@ class Spine(object): def __init__(self, oeb): self.oeb = oeb self.items = [] + self.page_progression_direction = None def _linear(self, linear): if isinstance(linear, basestring): @@ -1896,4 +1897,6 @@ class OEBBook(object): attrib={'media-type': PAGE_MAP_MIME}) spine.attrib['page-map'] = id results[PAGE_MAP_MIME] = (href, self.pages.to_page_map()) + if self.spine.page_progression_direction in {'ltr', 'rtl'}: + spine.attrib['page-progression-direction'] = self.spine.page_progression_direction return results diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index eb7e2eca4c..cb10b4ccce 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -330,6 +330,9 @@ class OEBReader(object): if len(spine) == 0: raise OEBError("Spine is empty") self._spine_add_extra() + for val in xpath(opf, '/o2:package/o2:spine/@page-progression-direction'): + if val in {'ltr', 'rtl'}: + spine.page_progression_direction = val def _guide_from_opf(self, opf): guide = self.oeb.guide From f63f142618a503acd4d8597bc97117d69a93840b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jun 2013 16:55:12 +0530 Subject: [PATCH 24/37] PDF Output: Fix add ToC option not being used PDF Output: Fix Table of Contents being added tot he end of the PDF even without the Add Table of Contents option being enabled. Fixes #1194836 [When convert to PDF, it always create TOC at the end](https://bugs.launchpad.net/calibre/+bug/1194836) --- src/calibre/ebooks/pdf/render/from_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/pdf/render/from_html.py b/src/calibre/ebooks/pdf/render/from_html.py index 5b9f58e326..8ea1d8203e 100644 --- a/src/calibre/ebooks/pdf/render/from_html.py +++ b/src/calibre/ebooks/pdf/render/from_html.py @@ -253,7 +253,7 @@ class PDFWriter(QObject): return self.loop.exit(1) try: if not self.render_queue: - if self.toc is not None and len(self.toc) > 0 and not hasattr(self, 'rendered_inline_toc'): + if self.opts.pdf_add_toc and self.toc is not None and len(self.toc) > 0 and not hasattr(self, 'rendered_inline_toc'): return self.render_inline_toc() self.loop.exit() else: From 3743d26d35badcc978a93db0d3d2aa53eae032ee Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 10:14:10 +0530 Subject: [PATCH 25/37] Save dist file sizes for bandwidth calculation Also fix a typo in copying dist files to tdir and backup. --- setup/upload.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/setup/upload.py b/setup/upload.py index 639a2e98d5..dd59067c0c 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -134,6 +134,8 @@ class UploadInstallers(Command): # {{{ available = set(glob.glob('dist/*')) files = {x:installer_description(x) for x in all_possible.intersection(available)} + sizes = {os.path.basename(x):os.path.getsize(x) for x in files} + self.record_sizes(sizes) tdir = mkdtemp() backup = os.path.join('/mnt/external/calibre/%s' % __version__) if not os.path.exists(backup): @@ -147,6 +149,11 @@ class UploadInstallers(Command): # {{{ finally: shutil.rmtree(tdir, ignore_errors=True) + def record_sizes(self, sizes): + print ('\nRecording dist sizes') + args = ['%s:%s:%s' % (__version__, fname, size) for fname, size in sizes.iteritems()] + check_call(['ssh', 'divok', 'dist_sizes'] + args) + def upload_to_staging(self, tdir, backup, files): os.mkdir(tdir+'/dist') hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)), @@ -154,9 +161,9 @@ class UploadInstallers(Command): # {{{ shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py')) for f in files: - for x in (tdir, backup): - dest = os.path.join(x, f) - shutil.copyfile(f, dest) + for x in (tdir+'/dist', backup): + dest = os.path.join(x, os.path.basename(f)) + shutil.copy2(f, x) os.chmod(dest, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH) with open(os.path.join(tdir, 'fmap'), 'wb') as fo: From 87dda89378e8a95663897eba720b0ae04d958d7d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 12:41:46 +0530 Subject: [PATCH 26/37] Add notes on provisioning a file hosting server --- setup/file_hosting_servers.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 setup/file_hosting_servers.rst diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst new file mode 100644 index 0000000000..7121628744 --- /dev/null +++ b/setup/file_hosting_servers.rst @@ -0,0 +1,32 @@ +Provisioning a file hosting server +==================================== + +Create the ssh authorized keys file. + +Edit /etc/ssh/sshd_config and change PermitRootLogin to without-password. +Restart sshd. + +apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools +chsh -s /bin/zsh + +mkdir -p /root/staging /root/work/vim /srv/download /srv/manual + +scp .zshrc .vimrc server: +scp -r ~/work/vim/zsh-syntax-highlighting server:work/vim + +If the server has a backup hard-disk, mount it at /mnt/backup and edit /etc/fstab so that it is auto-mounted. +Then, add the following to crontab +@daily /usr/bin/rsync -ha /srv /mnt/backup +@daily /usr/bin/rsync -ha /etc /mnt/backup + +Nginx +------ + +Copy over /etc/nginx/sites-available/default from another file server. When +copying, remember to use cat instead of cp to preserve hardlinks (the file is a +hardlink to /etc/nginx/sites-enabled/default) + +rsync /srv from another file server + +service nginx start + From c3009256c498893370e0cc6f61bd018abe17a38e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 13:01:52 +0530 Subject: [PATCH 27/37] ToC Editor: Use filenames when generating from files ToC Editor: When generating a ToC from files, if the file has no text, do not skip it. Instead create an entry using the filename of the file. --- src/calibre/ebooks/oeb/polish/toc.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/oeb/polish/toc.py b/src/calibre/ebooks/oeb/polish/toc.py index 8be23bdc38..a364da58f5 100644 --- a/src/calibre/ebooks/oeb/polish/toc.py +++ b/src/calibre/ebooks/oeb/polish/toc.py @@ -281,15 +281,18 @@ def find_text(node): def from_files(container): toc = TOC() - for spinepath in container.spine_items: + for i, spinepath in enumerate(container.spine_items): name = container.abspath_to_name(spinepath) root = container.parsed(name) body = XPath('//h:body')(root) if not body: continue text = find_text(body[0]) - if text: - toc.add(text, name) + if not text: + text = name.rpartition('/')[-1] + if i == 0 and text.rpartition('.')[0].lower() in {'titlepage', 'cover'}: + text = _('Cover') + toc.add(text, name) return toc def add_id(container, name, loc): From f8509fe8260e409c6e9f21d309d7ce8b9fd6a529 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 13:27:31 +0530 Subject: [PATCH 28/37] Log the wait before sending email When waiting before sending email, log the wait. Fixes #1195173 [Feature Request - Multiple books emailed to device](https://bugs.launchpad.net/calibre/+bug/1195173) --- src/calibre/gui2/email.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 9ebb94b00a..9b077fa39f 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -92,7 +92,11 @@ class Sendmail(object): raise worker.exception def sendmail(self, attachment, aname, to, subject, text, log): + logged = False while time.time() - self.last_send_time <= self.rate_limit: + if not logged: + log('Waiting %s seconds before sending, to avoid being marked as spam.\nYou can control this delay via Preferences->Tweaks' % self.rate_limit) + logged = True time.sleep(1) try: opts = email_config().parse() From 86691f22a24c96061b1a78f1c332d58b9b52b6db Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 13:28:06 +0530 Subject: [PATCH 29/37] ... --- src/calibre/gui2/email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 9b077fa39f..52da1909fe 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -94,7 +94,7 @@ class Sendmail(object): def sendmail(self, attachment, aname, to, subject, text, log): logged = False while time.time() - self.last_send_time <= self.rate_limit: - if not logged: + if not logged and self.rate_limit > 0: log('Waiting %s seconds before sending, to avoid being marked as spam.\nYou can control this delay via Preferences->Tweaks' % self.rate_limit) logged = True time.sleep(1) From 32fccdb9010d161f6e2acbb4b1d66cf99162f57b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 13:29:09 +0530 Subject: [PATCH 30/37] pep8 --- src/calibre/gui2/email.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 52da1909fe..f8c7552437 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -32,7 +32,7 @@ class Worker(Thread): self.func, self.args = func, args def run(self): - #time.sleep(1000) + # time.sleep(1000) try: self.func(*self.args) except Exception as e: @@ -46,7 +46,7 @@ class Worker(Thread): class Sendmail(object): MAX_RETRIES = 1 - TIMEOUT = 15 * 60 # seconds + TIMEOUT = 15 * 60 # seconds def __init__(self): self.calculate_rate_limit() @@ -166,7 +166,7 @@ def email_news(mi, remove, get_fmts, done, job_manager): plugboard_email_value = 'email' plugboard_email_formats = ['epub', 'mobi', 'azw3'] -class EmailMixin(object): # {{{ +class EmailMixin(object): # {{{ def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None, do_auto_convert=True, specific_format=None): @@ -208,10 +208,10 @@ class EmailMixin(object): # {{{ if not components: components = [mi.title] subjects.append(os.path.join(*components)) - a = authors_to_string(mi.authors if mi.authors else \ + a = authors_to_string(mi.authors if mi.authors else [_('Unknown')]) - texts.append(_('Attached, you will find the e-book') + \ - '\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' + \ + texts.append(_('Attached, you will find the e-book') + + '\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' + _('in the %s format.') % os.path.splitext(f)[1][1:].upper()) prefix = ascii_filename(t+' - '+a) @@ -231,7 +231,7 @@ class EmailMixin(object): # {{{ auto = [] if _auto_ids != []: for id in _auto_ids: - if specific_format == None: + if specific_format is None: dbfmts = self.library_view.model().db.formats(id, index_is_id=True) formats = [f.lower() for f in (dbfmts.split(',') if dbfmts else [])] @@ -302,8 +302,9 @@ class EmailMixin(object): # {{{ sent_mails = email_news(mi, remove, get_fmts, self.email_sent, self.job_manager) if sent_mails: - self.status_bar.show_message(_('Sent news to')+' '+\ + self.status_bar.show_message(_('Sent news to')+' '+ ', '.join(sent_mails), 3000) # }}} + From 13f31c7839ca98b1663c34a5bf7daef359b21f61 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 17:11:23 +0530 Subject: [PATCH 31/37] ... --- setup/file_hosting_servers.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst index 7121628744..563c7bc64a 100644 --- a/setup/file_hosting_servers.rst +++ b/setup/file_hosting_servers.rst @@ -6,6 +6,11 @@ Create the ssh authorized keys file. Edit /etc/ssh/sshd_config and change PermitRootLogin to without-password. Restart sshd. +hostname whatever +Edit /etc/hosts and put in FQDN in the appropriate places, for example:: + 27.0.1.1 download.calibre-ebook.com download + 46.28.49.116 download.calibre-ebook.com download + apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools chsh -s /bin/zsh @@ -15,9 +20,9 @@ scp .zshrc .vimrc server: scp -r ~/work/vim/zsh-syntax-highlighting server:work/vim If the server has a backup hard-disk, mount it at /mnt/backup and edit /etc/fstab so that it is auto-mounted. -Then, add the following to crontab -@daily /usr/bin/rsync -ha /srv /mnt/backup -@daily /usr/bin/rsync -ha /etc /mnt/backup +Then, add the following to crontab:: + @daily /usr/bin/rsync -ha /srv /mnt/backup + @daily /usr/bin/rsync -ha /etc /mnt/backup Nginx ------ From 836074e37d9b3780093b89548c035cdb29603349 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 17:15:52 +0530 Subject: [PATCH 32/37] ... --- setup/file_hosting_servers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst index 563c7bc64a..8dd0afe098 100644 --- a/setup/file_hosting_servers.rst +++ b/setup/file_hosting_servers.rst @@ -11,7 +11,7 @@ Edit /etc/hosts and put in FQDN in the appropriate places, for example:: 27.0.1.1 download.calibre-ebook.com download 46.28.49.116 download.calibre-ebook.com download -apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools +apt-get install vim nginx zsh python-lxml python-mechanize iotop htop smartmontools mosh chsh -s /bin/zsh mkdir -p /root/staging /root/work/vim /srv/download /srv/manual From 952b95d3ad566dca005863f9403e3a846b5e1e8e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jun 2013 17:34:01 +0530 Subject: [PATCH 33/37] pep8 --- recipes/miradasalsur.recipe | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/recipes/miradasalsur.recipe b/recipes/miradasalsur.recipe index 4794503384..b931fcb1d7 100644 --- a/recipes/miradasalsur.recipe +++ b/recipes/miradasalsur.recipe @@ -4,9 +4,7 @@ sur.infonews.com ''' import datetime -from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe -from calibre.ebooks.BeautifulSoup import Tag class MiradasAlSur(BasicNewsRecipe): title = 'Miradas al Sur' @@ -25,7 +23,7 @@ class MiradasAlSur(BasicNewsRecipe): extra_css = """ body{font-family: Arial,Helvetica,sans-serif} h1{font-family: Georgia,Times,serif} - .field-field-story-author{color: gray; font-size: small} + .field-field-story-author{color: gray; font-size: small} """ conversion_options = { 'comment' : description @@ -34,22 +32,22 @@ class MiradasAlSur(BasicNewsRecipe): , 'language' : language , 'series' : title } - + keep_only_tags = [dict(name='div', attrs={'id':['content-header', 'content-area']})] - remove_tags = [ - dict(name=['link','meta','iframe','embed','object']), + remove_tags = [ + dict(name=['link','meta','iframe','embed','object']), dict(name='form', attrs={'class':'fivestar-widget'}), dict(attrs={'class':lambda x: x and 'terms-inline' in x.split()}) ] feeds = [ - (u'Politica' , u'http://sur.infonews.com/taxonomy/term/1/0/feed' ), - (u'Internacional' , u'http://sur.infonews.com/taxonomy/term/2/0/feed' ), + (u'Politica' , u'http://sur.infonews.com/taxonomy/term/1/0/feed'), + (u'Internacional' , u'http://sur.infonews.com/taxonomy/term/2/0/feed'), (u'Informe Especial' , u'http://sur.infonews.com/taxonomy/term/14/0/feed'), - (u'Delitos y pesquisas', u'http://sur.infonews.com/taxonomy/term/6/0/feed' ), - (u'Lesa Humanidad' , u'http://sur.infonews.com/taxonomy/term/7/0/feed' ), - (u'Cultura' , u'http://sur.infonews.com/taxonomy/term/8/0/feed' ), - (u'Deportes' , u'http://sur.infonews.com/taxonomy/term/9/0/feed' ), + (u'Delitos y pesquisas', u'http://sur.infonews.com/taxonomy/term/6/0/feed'), + (u'Lesa Humanidad' , u'http://sur.infonews.com/taxonomy/term/7/0/feed'), + (u'Cultura' , u'http://sur.infonews.com/taxonomy/term/8/0/feed'), + (u'Deportes' , u'http://sur.infonews.com/taxonomy/term/9/0/feed'), (u'Contratapa' , u'http://sur.infonews.com/taxonomy/term/10/0/feed'), ] @@ -60,10 +58,10 @@ class MiradasAlSur(BasicNewsRecipe): cdate = datetime.date.today() todayweekday = cdate.isoweekday() if (todayweekday != 7): - cdate -= datetime.timedelta(days=todayweekday) - cover_page_url = cdate.strftime('http://sur.infonews.com/ediciones/%Y-%m-%d/tapa'); + cdate -= datetime.timedelta(days=todayweekday) + cover_page_url = cdate.strftime('http://sur.infonews.com/ediciones/%Y-%m-%d/tapa') soup = self.index_to_soup(cover_page_url) cover_item = soup.find('img', attrs={'class':lambda x: x and 'imagecache-tapa_edicion_full' in x.split()}) if cover_item: - cover_url = cover_item['src'] + cover_url = cover_item['src'] return cover_url From 986ab2a0787a538bc5fe0c512da97f982e331dbc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 28 Jun 2013 08:14:25 +0530 Subject: [PATCH 34/37] ebook-convert: Add option to read metadata from OPF --- src/calibre/ebooks/conversion/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/conversion/cli.py b/src/calibre/ebooks/conversion/cli.py index a0abebc5fe..f2795005d8 100644 --- a/src/calibre/ebooks/conversion/cli.py +++ b/src/calibre/ebooks/conversion/cli.py @@ -94,6 +94,8 @@ def option_recommendation_to_cli_option(add_option, rec): if opt.long_switch == 'verbose': attrs['action'] = 'count' attrs.pop('type', '') + if opt.name == 'read_metadata_from_opf': + switches.append('--from-opf') if opt.name in DEFAULT_TRUE_OPTIONS and rec.recommended_value is True: switches = ['--disable-'+opt.long_switch] add_option(Option(*switches, **attrs)) @@ -190,7 +192,7 @@ def add_pipeline_options(parser, plumber): ), 'METADATA' : (_('Options to set metadata in the output'), - plumber.metadata_option_names, + plumber.metadata_option_names + ['read_metadata_from_opf'], ), 'DEBUG': (_('Options to help with debugging the conversion'), [ From 837adb4eabda66083840b3fb15b165946b31376a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 28 Jun 2013 08:50:14 +0530 Subject: [PATCH 35/37] version 0.9.37 --- Changelog.yaml | 44 ++++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index f952b961e2..7439a02986 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,50 @@ # new recipes: # - title: +- version: 0.9.37 + date: 2013-06-28 + + new features: + - title: "Conversion: Add option to embed all referenced fonts" + type: major + description: "Add an option to embed all fonts that are referenced in the input document but are not already embedded. This will search your system for the referenced font, and if found, the font will be embedded. Only works if the output format supports font embedding (for example: EPUB or AZW3). The option is under the Look & Feel section of the conversion dialog." + + - title: "ToC Editor: When generating a ToC from files, if the file has no text, do not skip it. Instead create an entry using the filename of the file." + + - title: "AZW3 Input: Add support for the page-progression-direction that is used to indicate page turns should happen from right to left. The attribute is passed into EPUB when converting." + tickets: [1194766] + + - title: "ebook-convert: Add a --from-opf option to read metadata from OPF files directly, instead of having to run ebook-meta --from-opf after conversion" + + bug fixes: + - title: "PDF Output: Fix Table of Contents being added to the end of the PDF even without the Add Table of Contents option being enabled." + tickets: [1194836] + + - title: "When auto-merging books on add, also merge identifiers." + + - title: "Fix an error when using the Template Editor to create a template that uses custom columns." + tickets: [1193763] + + - title: "LRF Output: Fix " entities in attribute values causing problems" + + - title: "News download: Apply the default page margin conversion settings. Also, when converting to PDF, apply the pdf conversion defaults." + tickets: [1193912] + + - title: "Fix a regression that broke scanning for books on all devices that used the Aluratek Color driver." + tickets: [1192940] + + - title: "fetch-ebbok-metadata: Fix --opf argument erroneously requiring a value" + + - title: "When waiting before sending email, log the wait." + tickets: [1195173] + + improved recipes: + - taz.de (RSS) + - Miradas al sur + - Frontline + - La Nacion (Costa Rica) + + - version: 0.9.36 date: 2013-06-21 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 6834a4e66d..a4edca6bd5 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 9, 36) +numeric_version = (0, 9, 37) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " From 6579327a6d99635411ee8a7dcf1d143f5fb5a789 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 28 Jun 2013 12:26:56 +0530 Subject: [PATCH 36/37] Various minor fixes in the publish process --- setup/file_hosting_servers.rst | 10 ++++++++++ setup/hosting.py | 8 ++++++-- setup/upload.py | 2 -- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/setup/file_hosting_servers.rst b/setup/file_hosting_servers.rst index 8dd0afe098..261241e24d 100644 --- a/setup/file_hosting_servers.rst +++ b/setup/file_hosting_servers.rst @@ -35,3 +35,13 @@ rsync /srv from another file server service nginx start +Services +--------- + +SSH into sourceforge and downloadbestsoftware so that their host keys are +stored. + + ssh -oStrictHostKeyChecking=no kovid@www.downloadbestsoft-mirror1.com + ssh -oStrictHostKeyChecking=no kovidgoyal,calibre@frs.sourceforge.net + ssh -oStrictHostKeyChecking=no files.calibre-ebook.com (and whatever other mirrors are present) + diff --git a/setup/hosting.py b/setup/hosting.py index 1e78f4694d..d97373cdbc 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -473,14 +473,18 @@ def upload_to_servers(files, version): # {{{ os.mkdir(dest) for src in files: shutil.copyfile(src, os.path.join(dest, os.path.basename(src))) - generate_index() + cwd = os.getcwd() + try: + generate_index() + finally: + os.chdir(cwd) for server, rdir in {'files':'/srv/download/'}.iteritems(): print('Uploading to server:', server) server = '%s.calibre-ebook.com' % server # Copy the generated index files print ('Copying generated index') - check_call(['rsync', '-hzr', '-e', 'ssh -x', '--include', '*.html', + check_call(['rsync', '-hza', '-e', 'ssh -x', '--include', '*.html', '--filter', '-! */', base, 'root@%s:%s' % (server, rdir)]) # Copy the release files rdir = '%s%s/' % (rdir, version) diff --git a/setup/upload.py b/setup/upload.py index dd59067c0c..0475773e01 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -255,8 +255,6 @@ class UploadToServer(Command): # {{{ description = 'Upload miscellaneous data to calibre server' def run(self, opts): - check_call('ssh divok rm -f %s/calibre-\*.tar.xz'%DOWNLOADS, shell=True) - # check_call('scp dist/calibre-*.tar.xz divok:%s/'%DOWNLOADS, shell=True) check_call('gpg --armor --detach-sign dist/calibre-*.tar.xz', shell=True) check_call('scp dist/calibre-*.tar.xz.asc divok:%s/signatures/'%DOWNLOADS, From 54a9a7c98e7bbf995201fc6f323ad45cb0ca20f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 28 Jun 2013 12:44:44 +0530 Subject: [PATCH 37/37] pep8 and small perf improvement for human_readable() --- src/calibre/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 5e940efcd9..07ad906247 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -310,9 +310,9 @@ def get_parsed_proxy(typ='http', debug=True): proxy = proxies.get(typ, None) if proxy: pattern = re.compile(( - '(?:ptype://)?' \ - '(?:(?P\w+):(?P.*)@)?' \ - '(?P[\w\-\.]+)' \ + '(?:ptype://)?' + '(?:(?P\w+):(?P.*)@)?' + '(?P[\w\-\.]+)' '(?::(?P\d+))?').replace('ptype', typ) ) @@ -535,7 +535,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252', ent = match.group(1) if ent in exceptions: return '&'+ent+';' - if ent in {'apos', 'squot'}: # squot is generated by some broken CMS software + if ent in {'apos', 'squot'}: # squot is generated by some broken CMS software return check("'") if ent == 'hellips': ent = 'hellip' @@ -565,7 +565,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252', return '&'+ent+';' _ent_pat = re.compile(r'&(\S+?);') -xml_entity_to_unicode = partial(entity_to_unicode, result_exceptions = { +xml_entity_to_unicode = partial(entity_to_unicode, result_exceptions={ '"' : '"', "'" : ''', '<' : '<', @@ -670,8 +670,8 @@ def human_readable(size, sep=' '): """ Convert a size in bytes into a human readable form """ divisor, suffix = 1, "B" for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')): - if size < 1024**(i+1): - divisor, suffix = 1024**(i), candidate + if size < (1 << ((i + 1) * 10)): + divisor, suffix = (1 << (i * 10)), candidate break size = str(float(size)/divisor) if size.find(".") > -1: