Remove the in src copy of the ISO lang and country code data

Instead download it when building the resources
This commit is contained in:
Kovid Goyal 2023-07-13 14:17:40 +05:30
parent 5a74da0443
commit a209713677
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
4 changed files with 31 additions and 51036 deletions

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

27
setup/iso_codes.py Normal file
View File

@ -0,0 +1,27 @@
#!/usr/bin/env python
# License: GPLv3 Copyright: 2023, Kovid Goyal <kovid at kovidgoyal.net>
from setup import download_securely
import zipfile
from io import BytesIO
class ISOData:
URL = 'https://salsa.debian.org/iso-codes-team/iso-codes/-/archive/main/iso-codes-main.zip'
def __init__(self):
self._zip_data = None
@property
def zip_data(self):
if self._zip_data is None:
self._zip_data = BytesIO(download_securely(self.URL))
return self._zip_data
def db_data(self, name: str) -> bytes:
with zipfile.ZipFile(self.zip_data) as zf:
with zf.open(f'iso-codes-main/data/{name}') as f:
return f.read()
iso_data = ISOData()

View File

@ -12,6 +12,7 @@ from functools import partial
from setup import Command, __appname__, __version__, require_git_master, build_cache_dir, edit_file, dump_json, is_ci
from setup.parallel_build import batched_parallel_jobs
from setup.iso_codes import iso_data
from polyglot.builtins import codepoint_to_chr, iteritems
@ -92,11 +93,7 @@ class POT(Command): # {{{
def get_iso639_strings(self):
self.info('Generating translation template for iso639')
src = self.j(self.d(self.SRC), 'setup', 'iso_639-3.json')
if not os.path.exists(src):
raise Exception(src + ' does not exist')
with open(src, 'rb') as f:
root = json.load(f)
root = json.loads(iso_data.db_data('iso_639-3.json'))
entries = root['639-3']
ans = []
@ -780,19 +777,12 @@ class ISO639(Command): # {{{
'iso639.calibre_msgpack')
def run(self, opts):
src = self.j(self.d(self.SRC), 'setup', 'iso_639-3.json')
if not os.path.exists(src):
raise Exception(src + ' does not exist')
dest = self.DEST
base = self.d(dest)
if not os.path.exists(base):
os.makedirs(base)
if not self.newer(dest, [src, __file__]):
self.info('Packed code is up to date')
return
self.info('Packing ISO-639 codes to', dest)
with open(src, 'rb') as f:
root = json.load(f)
root = json.loads(iso_data.db_data('iso_639-3.json'))
entries = root['639-3']
by_2 = {}
by_3 = {}
@ -847,19 +837,12 @@ class ISO3166(ISO639): # {{{
'iso3166.calibre_msgpack')
def run(self, opts):
src = self.j(self.d(self.SRC), 'setup', 'iso_3166-1.json')
if not os.path.exists(src):
raise Exception(src + ' does not exist')
dest = self.DEST
base = self.d(dest)
if not os.path.exists(base):
os.makedirs(base)
if not self.newer(dest, [src, __file__]):
self.info('Packed code is up to date')
return
self.info('Packing ISO-3166 codes to', dest)
with open(src, 'rb') as f:
db = json.load(f)
db = json.loads(iso_data.db_data('iso_3166-1.json'))
codes = set()
three_map = {}
name_map = {}