Download list of available piper voices during build

.gitignore

@@ -34,6 +34,7 @@
 /resources/fonts/liberation
 /resources/mozilla-ca-certs.pem
 /resources/user-agent-data.json
+/resources/piper-voices.json
 /icons/icns/*.iconset
 /setup/installer/windows/calibre/build.log
 /setup/pyqt_enums

setup/commands.py

@@ -20,7 +20,7 @@ __all__ = [
     'upload_user_manual', 'upload_demo', 'reupload',
     'stage1', 'stage2', 'stage3', 'stage4', 'stage5', 'publish', 'publish_betas', 'publish_preview',
     'linux', 'linux64', 'linuxarm64', 'win', 'win64', 'osx', 'build_dep',
-    'export_packages', 'hyphenation', 'liberation_fonts', 'stylelint', 'xwin',
+    'export_packages', 'hyphenation', 'piper_voices', 'liberation_fonts', 'stylelint', 'xwin',
 ]
 
 from setup.installers import OSX, BuildDep, ExportPackages, ExtDev, Linux, Linux64, LinuxArm64, Win, Win64
@@ -32,8 +32,8 @@ extdev = ExtDev()
 build_dep = BuildDep()
 export_packages = ExportPackages()
 
-from setup.translations import ISO639, ISO3166, POT, GetTranslations, Translations
 from setup.iso_codes import iso_data
+from setup.translations import ISO639, ISO3166, POT, GetTranslations, Translations
 
 pot = POT()
 translations = Translations()
@@ -57,6 +57,10 @@ from setup.hyphenation import Hyphenation
 
 hyphenation = Hyphenation()
 
+from setup.piper import PiperVoices
+
+piper_voices = PiperVoices()
+
 from setup.liberation import LiberationFonts
 
 liberation_fonts = LiberationFonts()

setup/piper.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# License: GPLv3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
+
+import json
+import os
+import re
+from contextlib import suppress
+
+from setup.revendor import ReVendor
+
+
+class PiperVoices(ReVendor):
+
+    description = 'Download the list of Piper voices'
+    NAME = 'piper_voices'
+    TAR_NAME = 'piper voice list'
+    VERSION = 'master'
+    DOWNLOAD_URL = f'https://raw.githubusercontent.com/rhasspy/piper/{VERSION}/VOICES.md'
+    CAN_USE_SYSTEM_VERSION = False
+
+    @property
+    def output_file_path(self) -> str:
+        return os.path.join(self.RESOURCES, 'piper-voices.json')
+
+    def run(self, opts):
+        url = opts.path_to_piper_voices
+        if url:
+            with open(opts.path_to_piper_voices) as f:
+                src = f.read()
+        else:
+            url = opts.piper_voices_url
+            src = self.download_securely(url).decode('utf-8')
+        lang_map = {}
+        current_lang = current_voice = ''
+        lang_pat = re.compile(r'`(.+?)`')
+        model_pat = re.compile(r'\[model\]\((.+?)\)')
+        config_pat = re.compile(r'\[config\]\((.+?)\)')
+        for line in src.splitlines():
+            if line.startswith('* '):
+                if m := lang_pat.search(line):
+                    current_lang = m.group(1)
+                    lang_map[current_lang] = {}
+                    current_voice = ''
+            else:
+                line = line.strip()
+                if not line.startswith('*'):
+                    continue
+                if '[model]' in line:
+                    if current_lang and current_voice:
+                        qual_map = lang_map[current_lang][current_voice]
+                        quality = line.partition('-')[0].strip().lstrip('*').strip()
+                        model = config = ''
+                        if m := model_pat.search(line):
+                            model = m.group(1)
+                        if m := config_pat.search(line):
+                            config = m.group(1)
+                        if not quality or not model or not config:
+                            raise SystemExit('Failed to parse piper voice model definition from:\n' + line)
+                        qual_map[quality] = {'model': model, 'config': config}
+                else:
+                    current_voice = line.partition(' ')[-1].strip()
+                    lang_map[current_lang][current_voice] = {}
+        if not lang_map:
+            raise SystemExit(f'Failed to read any piper voices from: {url}')
+        with open(self.output_file_path, 'w') as f:
+            json.dump(lang_map, f, indent=2, sort_keys=False)
+
+    def clean(self):
+        with suppress(FileNotFoundError):
+            os.remove(self.output_file_path)

setup/resources.py

@@ -213,7 +213,7 @@ class RapydScript(Command):  # {{{
 class Resources(Command):  # {{{
 
     description = 'Compile various needed calibre resources'
-    sub_commands = ['kakasi', 'liberation_fonts', 'mathjax', 'rapydscript', 'hyphenation']
+    sub_commands = ['kakasi', 'liberation_fonts', 'mathjax', 'rapydscript', 'hyphenation', 'piper_voices']
 
     def run(self, opts):
         from calibre.utils.serialize import msgpack_dumps

setup/revendor.py

@@ -23,17 +23,20 @@ class ReVendor(Command):
             parser.add_option('--system-%s' % self.NAME, default=False, action='store_true',
                     help='Treat %s as system copy and symlink instead of copy' % self.TAR_NAME)
 
-    def download_vendor_release(self, tdir, url):
-        self.info('Downloading %s:' % self.TAR_NAME, url)
+    def download_securely(self, url: str) -> bytes:
         num = 5 if is_ci else 1
         for i in range(num):
             try:
-                raw = download_securely(url)
+                return download_securely(url)
             except Exception as err:
                 if i == num - 1:
                     raise
                 self.info(f'Download failed with error "{err}" sleeping and retrying...')
                 time.sleep(2)
+
+    def download_vendor_release(self, tdir, url):
+        self.info('Downloading %s:' % self.TAR_NAME, url)
+        raw = self.download_securely(url)
         with tarfile.open(fileobj=BytesIO(raw)) as tf:
             tf.extractall(tdir)
             if len(os.listdir(tdir)) == 1:

src/calibre/gui2/tts2/types.py

@@ -42,6 +42,7 @@ class Quality(Enum):
     High: int = auto()
     Medium: int = auto()
     Low: int = auto()
+    ExtraLow: int = auto()
 
 
 class Voice(NamedTuple):