Download list of available piper voices during build

This commit is contained in:
Kovid Goyal 2024-09-02 11:08:51 +05:30
parent c347ecd628
commit 4b565c124b
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
6 changed files with 85 additions and 6 deletions

1
.gitignore vendored
View File

@ -34,6 +34,7 @@
/resources/fonts/liberation
/resources/mozilla-ca-certs.pem
/resources/user-agent-data.json
/resources/piper-voices.json
/icons/icns/*.iconset
/setup/installer/windows/calibre/build.log
/setup/pyqt_enums

View File

@ -20,7 +20,7 @@ __all__ = [
'upload_user_manual', 'upload_demo', 'reupload',
'stage1', 'stage2', 'stage3', 'stage4', 'stage5', 'publish', 'publish_betas', 'publish_preview',
'linux', 'linux64', 'linuxarm64', 'win', 'win64', 'osx', 'build_dep',
'export_packages', 'hyphenation', 'liberation_fonts', 'stylelint', 'xwin',
'export_packages', 'hyphenation', 'piper_voices', 'liberation_fonts', 'stylelint', 'xwin',
]
from setup.installers import OSX, BuildDep, ExportPackages, ExtDev, Linux, Linux64, LinuxArm64, Win, Win64
@ -32,8 +32,8 @@ extdev = ExtDev()
build_dep = BuildDep()
export_packages = ExportPackages()
from setup.translations import ISO639, ISO3166, POT, GetTranslations, Translations
from setup.iso_codes import iso_data
from setup.translations import ISO639, ISO3166, POT, GetTranslations, Translations
pot = POT()
translations = Translations()
@ -57,6 +57,10 @@ from setup.hyphenation import Hyphenation
hyphenation = Hyphenation()
from setup.piper import PiperVoices
piper_voices = PiperVoices()
from setup.liberation import LiberationFonts
liberation_fonts = LiberationFonts()

70
setup/piper.py Normal file
View File

@ -0,0 +1,70 @@
#!/usr/bin/env python
# License: GPLv3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
import json
import os
import re
from contextlib import suppress
from setup.revendor import ReVendor
class PiperVoices(ReVendor):
description = 'Download the list of Piper voices'
NAME = 'piper_voices'
TAR_NAME = 'piper voice list'
VERSION = 'master'
DOWNLOAD_URL = f'https://raw.githubusercontent.com/rhasspy/piper/{VERSION}/VOICES.md'
CAN_USE_SYSTEM_VERSION = False
@property
def output_file_path(self) -> str:
return os.path.join(self.RESOURCES, 'piper-voices.json')
def run(self, opts):
url = opts.path_to_piper_voices
if url:
with open(opts.path_to_piper_voices) as f:
src = f.read()
else:
url = opts.piper_voices_url
src = self.download_securely(url).decode('utf-8')
lang_map = {}
current_lang = current_voice = ''
lang_pat = re.compile(r'`(.+?)`')
model_pat = re.compile(r'\[model\]\((.+?)\)')
config_pat = re.compile(r'\[config\]\((.+?)\)')
for line in src.splitlines():
if line.startswith('* '):
if m := lang_pat.search(line):
current_lang = m.group(1)
lang_map[current_lang] = {}
current_voice = ''
else:
line = line.strip()
if not line.startswith('*'):
continue
if '[model]' in line:
if current_lang and current_voice:
qual_map = lang_map[current_lang][current_voice]
quality = line.partition('-')[0].strip().lstrip('*').strip()
model = config = ''
if m := model_pat.search(line):
model = m.group(1)
if m := config_pat.search(line):
config = m.group(1)
if not quality or not model or not config:
raise SystemExit('Failed to parse piper voice model definition from:\n' + line)
qual_map[quality] = {'model': model, 'config': config}
else:
current_voice = line.partition(' ')[-1].strip()
lang_map[current_lang][current_voice] = {}
if not lang_map:
raise SystemExit(f'Failed to read any piper voices from: {url}')
with open(self.output_file_path, 'w') as f:
json.dump(lang_map, f, indent=2, sort_keys=False)
def clean(self):
with suppress(FileNotFoundError):
os.remove(self.output_file_path)

View File

@ -213,7 +213,7 @@ class RapydScript(Command): # {{{
class Resources(Command): # {{{
description = 'Compile various needed calibre resources'
sub_commands = ['kakasi', 'liberation_fonts', 'mathjax', 'rapydscript', 'hyphenation']
sub_commands = ['kakasi', 'liberation_fonts', 'mathjax', 'rapydscript', 'hyphenation', 'piper_voices']
def run(self, opts):
from calibre.utils.serialize import msgpack_dumps

View File

@ -23,17 +23,20 @@ class ReVendor(Command):
parser.add_option('--system-%s' % self.NAME, default=False, action='store_true',
help='Treat %s as system copy and symlink instead of copy' % self.TAR_NAME)
def download_vendor_release(self, tdir, url):
self.info('Downloading %s:' % self.TAR_NAME, url)
def download_securely(self, url: str) -> bytes:
num = 5 if is_ci else 1
for i in range(num):
try:
raw = download_securely(url)
return download_securely(url)
except Exception as err:
if i == num - 1:
raise
self.info(f'Download failed with error "{err}" sleeping and retrying...')
time.sleep(2)
def download_vendor_release(self, tdir, url):
self.info('Downloading %s:' % self.TAR_NAME, url)
raw = self.download_securely(url)
with tarfile.open(fileobj=BytesIO(raw)) as tf:
tf.extractall(tdir)
if len(os.listdir(tdir)) == 1:

View File

@ -42,6 +42,7 @@ class Quality(Enum):
High: int = auto()
Medium: int = auto()
Low: int = auto()
ExtraLow: int = auto()
class Voice(NamedTuple):