Translation framework for languages now works

setup/translations.py

@@ -11,6 +11,7 @@ from distutils import sysconfig
 
 from setup import Command, __appname__
 from setup.pygettext import main as pygettext
+from setup.build_environment import pyqt
 
 class POT(Command):
 
@@ -77,6 +78,17 @@ class Translations(POT):
             else:
                 self.warn('No ISO 639 translations for locale:', locale)
 
+        base = os.path.join(pyqt.qt_data_dir, 'translations')
+        qt_translations = glob.glob(os.path.join(base, 'qt_*.qm'))
+        if not qt_translations:
+            raise Exception('Could not find qt translations')
+        for f in qt_translations:
+            locale = self.s(self.b(f))[0][3:]
+            dest = self.j(self.DEST, locale, 'LC_MESSAGES', 'qt.qm')
+            if self.e(self.d(dest)) and self.newer(dest, f):
+                self.info('\tCopying Qt translation for locale:', locale)
+                shutil.copy2(f, dest)
+
         self.write_stats()
 
     @property
@@ -113,7 +125,8 @@ class Translations(POT):
         for f in self.po_files():
             l, d = self.mo_file(f)
             i = self.j(self.d(d), 'iso639.mo')
-            for x in (i, d):
+            j = self.j(self.d(d), 'qt.qm')
+            for x in (i, j, d):
                 if os.path.exists(x):
                     os.remove(x)
 

src/calibre/ebooks/html/input.py

@@ -22,7 +22,7 @@ from calibre.ebooks.chardet import xml_to_unicode
 from calibre.customize.conversion import OptionRecommendation
 from calibre.constants import islinux
 from calibre import unicode_path
-from calibre.startup import get_lang
+from calibre.utils.localization import get_lang
 
 class Link(object):
     '''

src/calibre/ebooks/oeb/reader.py

@@ -27,7 +27,7 @@ from calibre.ebooks.oeb.base import namespace, barename, XPath, xpath, \
                                     OEBError, OEBBook, DirContainer
 from calibre.ebooks.oeb.writer import OEBWriter
 from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
-from calibre.startup import get_lang
+from calibre.utils.localization import get_lang
 from calibre.ptempfile import TemporaryDirectory
 from calibre.constants import __appname__, __version__
 

src/calibre/gui2/__init__.py

@@ -10,9 +10,8 @@ from PyQt4.QtGui import QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \
 ORG_NAME = 'KovidsBrain'
 APP_UID  = 'libprs500'
 from calibre import islinux, iswindows, isosx
-from calibre.startup import get_lang
 from calibre.utils.config import Config, ConfigProxy, dynamic
-import calibre.resources as resources
+from calibre.utils.localization import set_qt_translator
 from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
 from calibre.ebooks.metadata import MetaInformation
 
@@ -541,12 +540,9 @@ class Application(QApplication):
         global gui_thread
         gui_thread = QThread.currentThread()
         self.translator = QTranslator(self)
-        lang = get_lang()
+        if set_qt_translator(self.translator):
-        if lang:
+            print 1111111
-            data = getattr(resources, 'qt_'+lang, None)
+            self.installTranslator(self.translator)
-            if data:
-                self.translator.loadFromData(data)
-                self.installTranslator(self.translator)
 
 def is_ok_to_use_qt():
     global gui_thread

src/calibre/gui2/dialogs/config/__init__.py

@@ -390,19 +390,16 @@ class ConfigDialog(QDialog, Ui_Dialog):
 
         self.cover_browse.setValue(config['cover_flow_queue_length'])
         self.systray_notifications.setChecked(not config['disable_tray_notification'])
-        from calibre.translations.compiled import translations
+        from calibre.utils.localization import available_translations, \
-        from calibre.translations import language_codes
+            get_language, get_lang
-        from calibre.startup import get_lang
         lang = get_lang()
-        if lang is not None and language_codes.has_key(lang):
+        if lang is None or lang not in available_translations():
-            self.language.addItem(language_codes[lang], QVariant(lang))
-        else:
             lang = 'en'
-            self.language.addItem('English', QVariant('en'))
+        self.language.addItem(get_language(lang), QVariant(lang))
-        items = [(l, language_codes[l]) for l in translations.keys() \
+        items = [(l, get_language(l)) for l in available_translations() \
                  if l != lang]
         if lang != 'en':
-            items.append(('en', 'English'))
+            items.append(('en', get_language('en')))
         items.sort(cmp=lambda x, y: cmp(x[1], y[1]))
         for item in items:
             self.language.addItem(item[1], QVariant(item[0]))

src/calibre/gui2/dialogs/scheduler.py

@@ -19,6 +19,7 @@ from calibre.gui2.search_box import SearchBox2
 from calibre.web.feeds.recipes import recipes, recipe_modules, compile_recipe
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.utils.pyparsing import ParseException
+from calibre.utils.localization import get_language
 from calibre.gui2 import NONE, error_dialog, config as gconf
 from calibre.utils.config import DynamicConfig
 from calibre.ptempfile import PersistentTemporaryFile
@@ -32,7 +33,7 @@ class Recipe(object):
         self.id                 = id
         self.title              = getattr(recipe_class, 'title', None)
         self.description        = getattr(recipe_class, 'description', None)
-        self.language           = getattr(recipe_class, 'language', _('Unknown'))
+        self.language           = getattr(recipe_class, 'language', 'und')
         self.last_downloaded    = datetime.fromordinal(1)
         self.downloading        = False
         self.builtin            = builtin
@@ -121,7 +122,7 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
 
         self.category_map = {}
         for r in self.recipes:
-            category = getattr(r, 'language', _('Unknown'))
+            category = get_language(getattr(r, 'language', 'und'))
             if not r.builtin:
                 category = _('Custom')
             if r.schedule is not None:

src/calibre/startup.py

@@ -6,8 +6,7 @@ __docformat__ = 'restructuredtext en'
 Perform various initialization tasks.
 '''
 
-import locale, sys, os, re, cStringIO
+import locale, sys, os
-from gettext import GNUTranslations
 
 # Default translation is NOOP
 import __builtin__
@@ -18,8 +17,6 @@ __builtin__.__dict__['_'] = lambda s: s
 __builtin__.__dict__['__'] = lambda s: s
 
 from calibre.constants import iswindows, preferred_encoding, plugins
-from calibre.utils.config import prefs
-from calibre.translations.msgfmt import make
 
 _run_once = False
 if not _run_once:
@@ -33,45 +30,9 @@ if not _run_once:
 
     ################################################################################
     # Setup translations
+    from calibre.utils.localization import set_translators
 
-    def get_lang():
+    set_translators()
-        lang = prefs['language']
-        if lang is not None:
-            return lang
-        lang = locale.getdefaultlocale(['LANGUAGE', 'LC_ALL', 'LC_CTYPE',
-                                       'LC_MESSAGES', 'LANG'])[0]
-        if lang is None and os.environ.has_key('LANG'): # Needed for OS X
-            try:
-                lang = os.environ['LANG']
-            except:
-                pass
-        if lang:
-            match = re.match('[a-z]{2,3}', lang)
-            if match:
-                lang = match.group()
-        return lang
-
-    def set_translator():
-        # To test different translations invoke as
-        # LC_ALL=de_DE.utf8 program
-        try:
-            from calibre.translations.compiled import translations
-        except:
-            return
-        lang = get_lang()
-        if lang:
-            buf = None
-            if os.access(lang+'.po', os.R_OK):
-                buf = cStringIO.StringIO()
-                make(lang+'.po', buf)
-                buf = cStringIO.StringIO(buf.getvalue())
-            elif translations.has_key(lang):
-                buf = cStringIO.StringIO(translations[lang])
-            if buf is not None:
-                t = GNUTranslations(buf)
-                t.install(unicode=True)
-
-    set_translator()
 
     ################################################################################
     # Initialize locale

src/calibre/utils/localization.py

@@ -6,13 +6,115 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os
+import os, locale, re, cStringIO, cPickle
+from gettext import GNUTranslations
 
 _available_translations = None
 
 def available_translations():
     global _available_translations
     if _available_translations is None:
-        base = P('resources/localization/locales')
+        stats = P('localization/stats.pickle')
-        _available_translations = os.listdir(base)
+        stats = cPickle.load(open(stats, 'rb'))
+        _available_translations = [x for x in stats if stats[x] > 0.1]
     return _available_translations
+
+def get_lang():
+    'Try to figure out what language to display the interface in'
+    from calibre.utils.config import prefs
+    lang = prefs['language']
+    lang = os.environ.get('CALIBRE_OVERRIDE_LANG', lang)
+    if lang is not None:
+        return lang
+    lang = locale.getdefaultlocale(['LANGUAGE', 'LC_ALL', 'LC_CTYPE',
+                                    'LC_MESSAGES', 'LANG'])[0]
+    if lang is None and os.environ.has_key('LANG'): # Needed for OS X
+        try:
+            lang = os.environ['LANG']
+        except:
+            pass
+    if lang:
+        match = re.match('[a-z]{2,3}(_[A-Z]{2}){0,1}', lang)
+        if match:
+            lang = match.group()
+    if lang == 'zh':
+        lang = 'zh_CN'
+    return lang
+
+def messages_path(lang):
+    return P('localization/locales/%s/LC_MESSAGES'%lang)
+
+def set_translators():
+    # To test different translations invoke as
+    # CALIBRE_OVERRIDE_LANG=de_DE.utf8 program
+    lang = get_lang()
+    if lang:
+        translations = available_translations()
+        buf = iso639 = None
+        if os.access(lang+'.po', os.R_OK):
+            from calibre.translations.msgfmt import make
+            buf = cStringIO.StringIO()
+            make(lang+'.po', buf)
+            buf = cStringIO.StringIO(buf.getvalue())
+
+        hlang = None
+        if lang in available_translations():
+            hlang = lang
+        else:
+            xlang = lang.split('_')[0]
+            if xlang in available_translations():
+                hlang = xlang
+        if hlang is not None:
+            if buf is None:
+                buf = open(os.path.join(messages_path(hlang),
+                    'messages.mo'), 'rb')
+            iso639 = open(os.path.join(messages_path(hlang),
+                    'iso639.mo'), 'rb')
+
+        if buf is not None:
+            t = GNUTranslations(buf)
+            if iso639 is not None:
+                iso639 = GNUTranslations(iso639)
+                t.add_fallback(iso639)
+            t.install(unicode=True)
+
+_iso639 = None
+_extra_lang_codes = {
+        'pt_BR' : _('Brazilian Portuguese'),
+        'en_GB' : _('English (UK)'),
+        'zh_CN' : _('Simplified Chinese'),
+        'zh_HK' : _('Chinese (HK)'),
+        'zh_TW' : _('Traditional Chinese'),
+        'en'    : _('English (US)'),
+        'und'   : _('Unknown')
+        }
+
+def get_language(lang):
+    global _iso639
+    if lang in _extra_lang_codes:
+        return _extra_lang_codes[lang]
+    if _iso639 is None:
+        _iso639 = cPickle.load(open(P('localization/iso639.pickle'), 'rb'))
+    ans = lang
+    lang = lang.split('_')[0].lower()
+    if len(lang) == 2:
+        ans = _iso639['by_2'].get(lang, ans)
+    elif len(lang) == 3:
+        if lang in _iso639['by_3b']:
+            ans = _iso639['by_3b'][lang]
+        else:
+            ans = _iso639['by_3t'].get(lang, ans)
+    return _(ans)
+
+
+def set_qt_translator(translator):
+    lang = get_lang()
+    if lang is not None:
+        if lang == 'nds':
+            lang = 'de'
+        for x in (lang, lang.split('_')[0]):
+            p = os.path.join(messages_path(x), 'qt.qm')
+            if os.path.exists(p):
+                return translator.loadFromData(open(p, 'rb').read())
+    return False
+

src/calibre/web/feeds/recipes/recipe_ars_technica.py

@@ -10,7 +10,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 
 class ArsTechnica2(BasicNewsRecipe):
     title                 = u'Ars Technica'
-    language              = _('English')
+    language              = 'en'
     __author__            = 'Darko Miletic and Sujata Raman'
     description           = 'The art of technology'
     publisher             = 'Ars Technica'