Switch to using more modern language names/translations from the iso-codes package, rather than pycountry

COPYRIGHT

@@ -9,6 +9,12 @@ License: GPL-2 or later
  The full text of the GPL is distributed as in
  /usr/share/common-licenses/GPL-2 on Debian systems.
 
+Files: setup/iso_639/*
+Copyright: Various
+License: LGPL 2.1
+ The full text of the LGPL is distributed as in
+ /usr/share/common-licenses/LGPL-2.1 on Debian systems.
+
 Files: src/calibre/ebooks/BeautifulSoup.py
 Copyright: Copyright (c) 2004-2007, Leonard Richardson
 License: BSD

setup/iso_639/README

@@ -0,0 +1,3 @@
+These files are taken from the iso-codes package, licensed under the LGPL 2.1
+
+All the files are from the iso_639_3 sub-directory.

setup/translations.py

@@ -7,7 +7,6 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, tempfile, shutil, subprocess, glob, re, time, textwrap
-from distutils import sysconfig
 from functools import partial
 
 from setup import Command, __appname__, __version__
@@ -142,19 +141,18 @@ class Translations(POT): # {{{
                 os.makedirs(base)
             self.info('\tCompiling translations for', locale)
             subprocess.check_call(['msgfmt', '-o', dest, f])
-            if locale in ('en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', 'ltg', 'nds', 'te', 'yi'):
-                continue
-            pycountry = self.j(sysconfig.get_python_lib(), 'pycountry',
-                    'locales', locale, 'LC_MESSAGES')
-            if os.path.exists(pycountry):
-                iso639 = self.j(pycountry, 'iso639.mo')
-                dest = self.j(self.d(dest), self.b(iso639))
-                if self.newer(dest, iso639) and os.path.exists(iso639):
+            iscpo = {'bn':'bn_IN', 'zh_HK':'zh_CN'}.get(locale, locale)
+            iso639 = self.j(self.d(self.SRC), 'setup', 'iso_639',
+                    '%s.po'%iscpo)
+
+            if os.path.exists(iso639):
+                dest = self.j(self.d(dest), 'iso639.mo')
+                if self.newer(dest, iso639):
                     self.info('\tCopying ISO 639 translations')
-                    shutil.copy2(iso639, dest)
-            else:
-                self.warn('No ISO 639 translations for locale:', locale,
-                '\nDo you have pycountry installed?')
+                    subprocess.check_call(['msgfmt', '-o', dest, iso639])
+            elif locale not in ('en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc',
+                    'ltg', 'nds', 'te', 'yi', 'fo', 'sq', 'ast', 'ml'):
+                self.warn('No ISO 639 translations for locale:', locale)
 
         self.write_stats()
         self.freeze_locales()
@@ -212,7 +210,7 @@ class Translations(POT): # {{{
 
 # }}}
 
-class GetTranslations(Translations):
+class GetTranslations(Translations): # {{{
 
     description = 'Get updated translations from Launchpad'
     BRANCH = 'lp:~kovid/calibre/translations'
@@ -273,24 +271,25 @@ class GetTranslations(Translations):
                 subprocess.check_call(['bzr', 'commit', '-m',
                     'IGN:Translation corrections', cls.PATH])
 
+# }}}
 
-class ISO639(Command):
+class ISO639(Command): # {{{
 
     description = 'Compile translations for ISO 639 codes'
     DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization',
             'iso639.pickle')
 
     def run(self, opts):
-        src = self.j(self.d(self.SRC), 'setup', 'iso639.xml')
+        src = self.j(self.d(self.SRC), 'setup', 'iso_639')
         if not os.path.exists(src):
             raise Exception(src + ' does not exist')
         dest = self.DEST
-        if not self.newer(dest, src):
+        if not self.newer(dest, [src, __file__]):
             self.info('Pickled code is up to date')
             return
         self.info('Pickling ISO-639 codes to', dest)
         from lxml import etree
-        root = etree.fromstring(open(src, 'rb').read())
+        root = etree.fromstring(open(self.j(src, 'iso_639_3.xml'), 'rb').read())
         by_2 = {}
         by_3b = {}
         by_3t = {}
@@ -298,12 +297,16 @@ class ISO639(Command):
         m3to2 = {}
         m3bto3t = {}
         nm = {}
-        codes2, codes3t, codes3b = set([]), set([]), set([])
-        for x in root.xpath('//iso_639_entry'):
+        codes2, codes3t, codes3b = set(), set(), set()
+        for x in root.xpath('//iso_639_3_entry'):
+            two = x.get('part1_code', None)
+            threet = x.get('id')
+            threeb = x.get('part2_code', None)
+            if threeb is None:
+                # Only recognize langauges in ISO-639-2
+                continue
             name = x.get('name')
-            two = x.get('iso_639_1_code', None)
-            threeb = x.get('iso_639_2B_code')
-            threet = x.get('iso_639_2T_code')
+
             if two is not None:
                 by_2[two] = name
                 codes2.add(two)
@@ -313,13 +316,10 @@ class ISO639(Command):
             by_3t[threet] = name
             if threeb != threet:
                 m3bto3t[threeb] = threet
-            codes3b.add(x.get('iso_639_2B_code'))
-            codes3t.add(x.get('iso_639_2T_code'))
+            codes3b.add(threeb)
+            codes3t.add(threet)
             base_name = name.lower()
             nm[base_name] = threet
-            simple_name = base_name.partition(';')[0].strip()
-            if simple_name not in nm:
-                nm[simple_name] = threet
 
         from cPickle import dump
         x = {'by_2':by_2, 'by_3b':by_3b, 'by_3t':by_3t, 'codes2':codes2,
@@ -331,4 +331,5 @@ class ISO639(Command):
         if os.path.exists(self.DEST):
             os.remove(self.DEST)
 
+# }}}