Support for setting per book language

setup/translations.py

@@ -291,6 +291,8 @@ class ISO639(Command):
         by_3t = {}
         m2to3 = {}
         m3to2 = {}
+        m3bto3t = {}
+        nm = {}
         codes2, codes3t, codes3b = set([]), set([]), set([])
         for x in root.xpath('//iso_639_entry'):
             name = x.get('name')
@@ -304,12 +306,15 @@ class ISO639(Command):
                 m3to2[threeb] = m3to2[threet] = two
             by_3b[threeb] = name
             by_3t[threet] = name
+            if threeb != threet:
+                m3bto3t[threeb] = threet
             codes3b.add(x.get('iso_639_2B_code'))
             codes3t.add(x.get('iso_639_2T_code'))
+            nm[name.lower().partition(';')[0].strip()] = threet
 
         from cPickle import dump
         x = {'by_2':by_2, 'by_3b':by_3b, 'by_3t':by_3t, 'codes2':codes2,
                 'codes3b':codes3b, 'codes3t':codes3t, '2to3':m2to3,
-                '3to2':m3to2}
+                '3to2':m3to2, '3bto3t':m3bto3t, 'name_map':nm}
         dump(x, open(dest, 'wb'), -1)
 

src/calibre/ebooks/metadata/book/__init__.py

@@ -47,8 +47,7 @@ PUBLICATION_METADATA_FIELDS = frozenset([
     # If None, means book
     'publication_type',
     'uuid',             # A UUID usually of type 4
-    'language',         # the primary language of this book
+    'languages',        # ordered list of languages in this publication
-    'languages',        # ordered list
     'publisher',        # Simple string, no special semantics
     # Absolute path to image file encoded in filesystem_encoding
     'cover',
@@ -109,7 +108,7 @@ STANDARD_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
 # Metadata fields that smart update must do special processing to copy.
 SC_FIELDS_NOT_COPIED =     frozenset(['title', 'title_sort', 'authors',
                                       'author_sort', 'author_sort_map',
-                                      'cover_data', 'tags', 'language',
+                                      'cover_data', 'tags', 'languages',
                                       'identifiers'])
 
 # Metadata fields that smart update should copy only if the source is not None

src/calibre/ebooks/metadata/book/base.py

@@ -102,6 +102,7 @@ class Metadata(object):
         @param other: None or a metadata object
         '''
         _data = copy.deepcopy(NULL_VALUES)
+        _data.pop('language')
         object.__setattr__(self, '_data', _data)
         if other is not None:
             self.smart_update(other)
@@ -136,6 +137,11 @@ class Metadata(object):
         _data = object.__getattribute__(self, '_data')
         if field in TOP_LEVEL_IDENTIFIERS:
             return _data.get('identifiers').get(field, None)
+        if field == 'language':
+            try:
+                return _data.get('languages', [])[0]
+            except:
+                return NULL_VALUES['language']
         if field in STANDARD_METADATA_FIELDS:
             return _data.get(field, None)
         try:
@@ -175,6 +181,11 @@ class Metadata(object):
             if not val:
                 val = copy.copy(NULL_VALUES.get('identifiers', None))
             self.set_identifiers(val)
+        elif field == 'language':
+            langs = []
+            if val and val.lower() != 'und':
+                langs = [val]
+            _data['languages'] = langs
         elif field in STANDARD_METADATA_FIELDS:
             if val is None:
                 val = copy.copy(NULL_VALUES.get(field, None))
@@ -553,9 +564,9 @@ class Metadata(object):
                 for attr in TOP_LEVEL_IDENTIFIERS:
                     copy_not_none(self, other, attr)
 
-        other_lang = getattr(other, 'language', None)
+        other_lang = getattr(other, 'languages', [])
-        if other_lang and other_lang.lower() != 'und':
+        if other_lang and other_lang != ['und']:
-            self.language = other_lang
+            self.languages = list(other_lang)
         if not getattr(self, 'series', None):
             self.series_index = None
 
@@ -706,8 +717,8 @@ class Metadata(object):
             fmt('Tags', u', '.join([unicode(t) for t in self.tags]))
         if self.series:
             fmt('Series', self.series + ' #%s'%self.format_series_index())
-        if not self.is_null('language'):
+        if not self.is_null('languages'):
-            fmt('Language', self.language)
+            fmt('Languages', ', '.join(self.languages))
         if self.rating is not None:
             fmt('Rating', self.rating)
         if self.timestamp is not None:
@@ -719,6 +730,8 @@ class Metadata(object):
         if self.identifiers:
             fmt('Identifiers', u', '.join(['%s:%s'%(k, v) for k, v in
                 self.identifiers.iteritems()]))
+        if self.languages:
+            fmt('Languages', u', '.join(self.languages))
         if self.comments:
             fmt('Comments', self.comments)
 
@@ -743,7 +756,7 @@ class Metadata(object):
         ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
         if self.series:
             ans += [(_('Series'), unicode(self.series) + ' #%s'%self.format_series_index())]
-        ans += [(_('Language'), unicode(self.language))]
+        ans += [(_('Languages'), u', '.join(self.languages))]
         if self.timestamp is not None:
             ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
         if self.pubdate is not None:

src/calibre/ebooks/metadata/opf2.py

@@ -19,7 +19,7 @@ from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.metadata import string_to_authors, MetaInformation, check_isbn
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.date import parse_date, isoformat
-from calibre.utils.localization import get_lang
+from calibre.utils.localization import get_lang, canonicalize_lang
 from calibre import prints, guess_type
 from calibre.utils.cleantext import clean_ascii_chars
 from calibre.utils.config import tweaks
@@ -515,6 +515,7 @@ class OPF(object): # {{{
                             '(re:match(@opf:scheme, "calibre|libprs500", "i") or re:match(@scheme, "calibre|libprs500", "i"))]')
     uuid_id_path    = XPath('descendant::*[re:match(name(), "identifier", "i") and '+
                             '(re:match(@opf:scheme, "uuid", "i") or re:match(@scheme, "uuid", "i"))]')
+    languages_path  = XPath('descendant::*[local-name()="language"]')
 
     manifest_path   = XPath('descendant::*[re:match(name(), "manifest", "i")]/*[re:match(name(), "item", "i")]')
     manifest_ppath  = XPath('descendant::*[re:match(name(), "manifest", "i")]')
@@ -523,7 +524,6 @@ class OPF(object): # {{{
 
     title           = MetadataField('title', formatter=lambda x: re.sub(r'\s+', ' ', x))
     publisher       = MetadataField('publisher')
-    language        = MetadataField('language')
     comments        = MetadataField('description')
     category        = MetadataField('type')
     rights          = MetadataField('rights')
@@ -930,6 +930,44 @@ class OPF(object): # {{{
         return property(fget=fget, fset=fset)
 
 
+    @dynamic_property
+    def language(self):
+
+        def fget(self):
+            ans = self.languages
+            if ans:
+                return ans[0]
+
+        def fset(self, val):
+            self.languages = [val]
+
+        return property(fget=fget, fset=fset)
+
+
+    @dynamic_property
+    def languages(self):
+
+        def fget(self):
+            ans = []
+            for match in self.languages_path(self.metadata):
+                t = self.get_text(match)
+                if t and t.strip():
+                    l = canonicalize_lang(t.strip())
+                    if l:
+                        ans.append(l)
+            return ans
+
+        def fset(self, val):
+            matches = self.languages_path(self.metadata)
+            for x in matches:
+                x.getparent().remove(x)
+
+            for lang in val:
+                l = self.create_metadata_element('language')
+                self.set_text(l, unicode(lang))
+
+        return property(fget=fget, fset=fset)
+
 
     @dynamic_property
     def book_producer(self):
@@ -1052,9 +1090,9 @@ class OPF(object): # {{{
             val = getattr(mi, attr, None)
             if val is not None and val != [] and val != (None, None):
                 setattr(self, attr, val)
-        lang = getattr(mi, 'language', None)
+        langs = getattr(mi, 'languages', [])
-        if lang and lang != 'und':
+        if langs and langs != ['und']:
-            self.language = lang
+            self.languages = langs
         temp = self.to_book_metadata()
         temp.smart_update(mi, replace_metadata=replace_metadata)
         self._user_metadata_ = temp.get_all_user_metadata(True)
@@ -1202,9 +1240,10 @@ class OPFCreator(Metadata):
             dc_attrs={'id':__appname__+'_id'}))
         if getattr(self, 'pubdate', None) is not None:
             a(DC_ELEM('date', self.pubdate.isoformat()))
-        lang = self.language
+        langs = self.languages
-        if not lang or lang.lower() == 'und':
+        if not langs or langs == ['und']:
-            lang = get_lang().replace('_', '-')
+            langs = [get_lang().replace('_', '-').partition('-')[0]]
+        for lang in langs:
             a(DC_ELEM('language', lang))
         if self.comments:
             a(DC_ELEM('description', self.comments))
@@ -1288,8 +1327,8 @@ def metadata_to_opf(mi, as_string=True):
         mi.book_producer = __appname__ + ' (%s) '%__version__ + \
             '[http://calibre-ebook.com]'
 
-    if not mi.language:
+    if not mi.languages:
-        mi.language = 'UND'
+        mi.languages = ['UND']
 
     root = etree.fromstring(textwrap.dedent(
     '''
@@ -1339,8 +1378,10 @@ def metadata_to_opf(mi, as_string=True):
         factory(DC('identifier'), val, scheme=icu_upper(key))
     if mi.rights:
         factory(DC('rights'), mi.rights)
-    factory(DC('language'), mi.language if mi.language and mi.language.lower()
+    for lang in mi.languages:
-            != 'und' else get_lang().replace('_', '-'))
+        if not lang or lang.lower() == 'und':
+            lang = get_lang().replace('_', '-').partition('-')[0]
+        factory(DC('language'), lang)
     if mi.tags:
         for tag in mi.tags:
             factory(DC('subject'), tag)

src/calibre/gui2/__init__.py

@@ -94,7 +94,7 @@ gprefs.defaults['book_display_fields'] = [
         ('path', True), ('publisher', False), ('rating', False),
         ('author_sort', False), ('sort', False), ('timestamp', False),
         ('uuid', False), ('comments', True), ('id', False), ('pubdate', False),
-        ('last_modified', False), ('size', False),
+        ('last_modified', False), ('size', False), ('languages', False),
         ]
 gprefs.defaults['default_author_link'] = 'http://en.wikipedia.org/w/index.php?search={author}'
 gprefs.defaults['preserve_date_on_ctl'] = True

src/calibre/gui2/book_details.py

@@ -24,6 +24,7 @@ from calibre.gui2 import (config, open_local_file, open_url, pixmap_to_data,
 from calibre.utils.icu import sort_key
 from calibre.utils.formatter import EvalFormatter
 from calibre.utils.date import is_date_undefined
+from calibre.utils.localization import calibre_langcode_to_name
 
 def render_html(mi, css, vertical, widget, all_fields=False): # {{{
     table = render_data(mi, all_fields=all_fields,
@@ -152,6 +153,12 @@ def render_data(mi, use_roman_numbers=True, all_fields=False):
                     authors.append(aut)
             ans.append((field, u'<td class="title">%s</td><td>%s</td>'%(name,
                 u' & '.join(authors))))
+        elif field == 'languages':
+            if not mi.languages:
+                continue
+            names = filter(None, map(calibre_langcode_to_name, mi.languages))
+            ans.append((field, u'<td class="title">%s</td><td>%s</td>'%(name,
+                u', '.join(names))))
         else:
             val = mi.format_field(field)[-1]
             if val is None:

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -134,7 +134,7 @@ class MyBlockingBusy(QDialog): # {{{
             do_autonumber, do_remove_format, remove_format, do_swap_ta, \
             do_remove_conv, do_auto_author, series, do_series_restart, \
             series_start_value, do_title_case, cover_action, clear_series, \
-            pubdate, adddate, do_title_sort = self.args
+            pubdate, adddate, do_title_sort, languages, clear_languages = self.args
 
 
         # first loop: do author and title. These will commit at the end of each
@@ -238,6 +238,12 @@ class MyBlockingBusy(QDialog): # {{{
 
             if do_remove_conv:
                 self.db.delete_conversion_options(id, 'PIPE', commit=False)
+
+            if clear_languages:
+                self.db.set_languages(id, [], notify=False, commit=False)
+            elif languages:
+                self.db.set_languages(id, languages, notify=False, commit=False)
+
         elif self.current_phase == 3:
             # both of these are fast enough to just do them all
             for w in self.cc_widgets:
@@ -329,6 +335,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         geom = gprefs.get('bulk_metadata_window_geometry', None)
         if geom is not None:
             self.restoreGeometry(bytes(geom))
+        self.languages.setEditText('')
         self.exec_()
 
     def save_state(self, *args):
@@ -352,6 +359,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             self.do_again = True
             self.accept()
 
+    # S&R {{{
     def prepare_search_and_replace(self):
         self.search_for.initialize('bulk_edit_search_for')
         self.replace_with.initialize('bulk_edit_replace_with')
@@ -796,6 +804,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             # permanent. Make sure it really is.
             self.db.commit()
             self.model.refresh_ids(list(books_to_refresh))
+    # }}}
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
@@ -919,6 +928,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         do_auto_author = self.auto_author_sort.isChecked()
         do_title_case = self.change_title_to_title_case.isChecked()
         do_title_sort = self.update_title_sort.isChecked()
+        clear_languages = self.clear_languages.isChecked()
+        languages = self.languages.lang_codes
         pubdate = adddate = None
         if self.apply_pubdate.isChecked():
             pubdate = qt_to_dt(self.pubdate.date())
@@ -937,7 +948,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
                 do_autonumber, do_remove_format, remove_format, do_swap_ta,
                 do_remove_conv, do_auto_author, series, do_series_restart,
                 series_start_value, do_title_case, cover_action, clear_series,
-                pubdate, adddate, do_title_sort)
+                pubdate, adddate, do_title_sort, languages, clear_languages)
 
         bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.')
                 %len(self.ids), args, self.db, self.ids,

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -443,7 +443,7 @@ from the value in the box</string>
              </property>
             </widget>
            </item>
-           <item row="11" column="0">
+           <item row="13" column="0">
             <widget class="QLabel" name="label_5">
              <property name="text">
               <string>Remove &amp;format:</string>
@@ -453,7 +453,7 @@ from the value in the box</string>
              </property>
             </widget>
            </item>
-           <item row="11" column="1">
+           <item row="13" column="1">
             <widget class="QComboBox" name="remove_format">
              <property name="maximumSize">
               <size>
@@ -463,7 +463,7 @@ from the value in the box</string>
              </property>
             </widget>
            </item>
-           <item row="12" column="0">
+           <item row="14" column="0">
             <spacer name="verticalSpacer">
              <property name="orientation">
               <enum>Qt::Vertical</enum>
@@ -479,7 +479,7 @@ from the value in the box</string>
              </property>
             </spacer>
            </item>
-           <item row="13" column="0" colspan="3">
+           <item row="15" column="0" colspan="3">
             <layout class="QHBoxLayout" name="horizontalLayout_3">
              <item>
               <widget class="QCheckBox" name="change_title_to_title_case">
@@ -529,7 +529,7 @@ Future conversion of these books will use the default settings.</string>
              </item>
             </layout>
            </item>
-           <item row="14" column="0" colspan="3">
+           <item row="16" column="0" colspan="3">
             <widget class="QGroupBox" name="groupBox">
              <property name="title">
               <string>Change &amp;cover</string>
@@ -559,7 +559,7 @@ Future conversion of these books will use the default settings.</string>
              </layout>
             </widget>
            </item>
-           <item row="15" column="0">
+           <item row="17" column="0">
             <spacer name="verticalSpacer_2">
              <property name="orientation">
               <enum>Qt::Vertical</enum>
@@ -572,6 +572,29 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </spacer>
            </item>
+           <item row="11" column="0">
+            <widget class="QLabel" name="label_11">
+             <property name="text">
+              <string>&amp;Languages:</string>
+             </property>
+             <property name="alignment">
+              <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+             </property>
+             <property name="buddy">
+              <cstring>languages</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="11" column="1">
+            <widget class="LanguagesEdit" name="languages"/>
+           </item>
+           <item row="11" column="2">
+            <widget class="QCheckBox" name="clear_languages">
+             <property name="text">
+              <string>Remove &amp;all</string>
+             </property>
+            </widget>
+           </item>
           </layout>
          </widget>
          <widget class="QWidget" name="tab">
@@ -1145,6 +1168,11 @@ not multiple and the destination field is multiple</string>
    <extends>QLineEdit</extends>
    <header>widgets.h</header>
   </customwidget>
+  <customwidget>
+   <class>LanguagesEdit</class>
+   <extends>QComboBox</extends>
+   <header>calibre/gui2/languages.h</header>
+  </customwidget>
  </customwidgets>
  <tabstops>
   <tabstop>authors</tabstop>

src/calibre/gui2/languages.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.gui2.complete import MultiCompleteComboBox
+from calibre.utils.localization import lang_map
+from calibre.utils.icu import sort_key
+
+class LanguagesEdit(MultiCompleteComboBox):
+
+    def __init__(self, parent=None):
+        MultiCompleteComboBox.__init__(self, parent)
+
+        self._lang_map = lang_map()
+        self._rmap = {v:k for k,v in self._lang_map.iteritems()}
+
+        all_items = sorted(self._lang_map.itervalues(),
+            key=sort_key)
+        self.update_items_cache(all_items)
+        for item in all_items:
+            self.addItem(item)
+
+    @dynamic_property
+    def lang_codes(self):
+
+        def fget(self):
+            vals = [x.strip() for x in
+                    unicode(self.lineEdit().text()).split(',')]
+            ans = []
+            for name in vals:
+                if name:
+                    code = self._rmap.get(name, None)
+                    if code is not None:
+                        ans.append(code)
+            return ans
+
+        def fset(self, lang_codes):
+            ans = []
+            for lc in lang_codes:
+                name = self._lang_map.get(lc, None)
+                if name is not None:
+                    ans.append(name)
+            self.setEditText(', '.join(ans))
+
+        return property(fget=fget, fset=fset)
+
+    def validate(self):
+        vals = [x.strip() for x in
+                    unicode(self.lineEdit().text()).split(',')]
+        bad = []
+        for name in vals:
+            if name:
+                code = self._rmap.get(name, None)
+                if code is None:
+                    bad.append(name)
+        return bad
+

src/calibre/gui2/library/delegates.py

@@ -23,6 +23,7 @@ from calibre.utils.formatter import validation_formatter
 from calibre.utils.icu import sort_key
 from calibre.gui2.dialogs.comments_dialog import CommentsDialog
 from calibre.gui2.dialogs.template_dialog import TemplateDialog
+from calibre.gui2.languages import LanguagesEdit
 
 
 class RatingDelegate(QStyledItemDelegate): # {{{
@@ -155,7 +156,7 @@ class TextDelegate(QStyledItemDelegate): # {{{
     def __init__(self, parent):
         '''
         Delegate for text data. If auto_complete_function needs to return a list
-        of text items to auto-complete with. The funciton is None no
+        of text items to auto-complete with. If the function is None no
         auto-complete will be used.
         '''
         QStyledItemDelegate.__init__(self, parent)
@@ -229,6 +230,20 @@ class CompleteDelegate(QStyledItemDelegate): # {{{
             QStyledItemDelegate.setModelData(self, editor, model, index)
 # }}}
 
+class LanguagesDelegate(QStyledItemDelegate): # {{{
+
+    def createEditor(self, parent, option, index):
+        editor = LanguagesEdit(parent)
+        ct = index.data(Qt.DisplayRole).toString()
+        editor.setEditText(ct)
+        editor.lineEdit().selectAll()
+        return editor
+
+    def setModelData(self, editor, model, index):
+        val = ','.join(editor.lang_codes)
+        model.setData(index, QVariant(val), Qt.EditRole)
+# }}}
+
 class CcDateDelegate(QStyledItemDelegate): # {{{
     '''
     Delegate for custom columns dates. Because this delegate stores the

src/calibre/gui2/library/models.py

@@ -25,6 +25,7 @@ from calibre.library.caches import (_match, CONTAINS_MATCH, EQUALS_MATCH,
 from calibre import strftime, isbytestring
 from calibre.constants import filesystem_encoding, DEBUG
 from calibre.gui2.library import DEFAULT_SORT
+from calibre.utils.localization import calibre_langcode_to_name
 
 def human_readable(size, precision=1):
     """ Convert a size in bytes into megabytes """
@@ -64,6 +65,7 @@ class BooksModel(QAbstractTableModel): # {{{
                         'tags'      : _("Tags"),
                         'series'    : ngettext("Series", 'Series', 1),
                         'last_modified' : _('Modified'),
+                        'languages' : _('Languages'),
     }
 
     def __init__(self, parent=None, buffer=40):
@@ -71,7 +73,8 @@ class BooksModel(QAbstractTableModel): # {{{
         self.db = None
         self.book_on_device = None
         self.editable_cols = ['title', 'authors', 'rating', 'publisher',
-                              'tags', 'series', 'timestamp', 'pubdate']
+                              'tags', 'series', 'timestamp', 'pubdate',
+                              'languages']
         self.default_image = default_image()
         self.sorted_on = DEFAULT_SORT
         self.sort_history = [self.sorted_on]
@@ -540,6 +543,13 @@ class BooksModel(QAbstractTableModel): # {{{
             else:
                 return None
 
+        def languages(r, idx=-1):
+            lc = self.db.data[r][idx]
+            if lc:
+                langs = [calibre_langcode_to_name(l.strip()) for l in lc.split(',')]
+                return QVariant(', '.join(langs))
+            return None
+
         def tags(r, idx=-1):
             tags = self.db.data[r][idx]
             if tags:
@@ -641,6 +651,8 @@ class BooksModel(QAbstractTableModel): # {{{
                                 siix=self.db.field_metadata['series_index']['rec_index']),
                    'ondevice' : functools.partial(text_type,
                                 idx=self.db.field_metadata['ondevice']['rec_index'], mult=None),
+                   'languages': functools.partial(languages,
+                                idx=self.db.field_metadata['languages']['rec_index']),
                    }
 
         self.dc_decorator = {
@@ -884,6 +896,9 @@ class BooksModel(QAbstractTableModel): # {{{
                     if val.isNull() or not val.isValid():
                         return False
                     self.db.set_pubdate(id, qt_to_dt(val, as_utc=False))
+                elif column == 'languages':
+                    val = val.split(',')
+                    self.db.set_languages(id, val)
                 else:
                     books_to_refresh |= self.db.set(row, column, val,
                                                     allow_case_change=True)

src/calibre/gui2/library/views.py

@@ -8,14 +8,14 @@ __docformat__ = 'restructuredtext en'
 import os
 from functools import partial
 
-from PyQt4.Qt import QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, \
+from PyQt4.Qt import (QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal,
-    QModelIndex, QIcon, QItemSelection, QMimeData, QDrag, QApplication, \
+    QModelIndex, QIcon, QItemSelection, QMimeData, QDrag, QApplication,
-    QPoint, QPixmap, QUrl, QImage, QPainter, QColor, QRect
+    QPoint, QPixmap, QUrl, QImage, QPainter, QColor, QRect)
 
-from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \
+from calibre.gui2.library.delegates import (RatingDelegate, PubDateDelegate,
-    TextDelegate, DateDelegate, CompleteDelegate, CcTextDelegate, \
+    TextDelegate, DateDelegate, CompleteDelegate, CcTextDelegate,
-    CcBoolDelegate, CcCommentsDelegate, CcDateDelegate, CcTemplateDelegate, \
+    CcBoolDelegate, CcCommentsDelegate, CcDateDelegate, CcTemplateDelegate,
-    CcEnumDelegate, CcNumberDelegate
+    CcEnumDelegate, CcNumberDelegate, LanguagesDelegate)
 from calibre.gui2.library.models import BooksModel, DeviceBooksModel
 from calibre.utils.config import tweaks, prefs
 from calibre.gui2 import error_dialog, gprefs
@@ -85,6 +85,7 @@ class BooksView(QTableView): # {{{
         self.pubdate_delegate = PubDateDelegate(self)
         self.last_modified_delegate = DateDelegate(self,
                 tweak_name='gui_last_modified_display_format')
+        self.languages_delegate = LanguagesDelegate(self)
         self.tags_delegate = CompleteDelegate(self, ',', 'all_tags')
         self.authors_delegate = CompleteDelegate(self, '&', 'all_author_names', True)
         self.cc_names_delegate = CompleteDelegate(self, '&', 'all_custom', True)
@@ -306,6 +307,7 @@ class BooksView(QTableView): # {{{
         state['hidden_columns'] = [cm[i] for i in  range(h.count())
                 if h.isSectionHidden(i) and cm[i] != 'ondevice']
         state['last_modified_injected'] = True
+        state['languages_injected'] = True
         state['sort_history'] = \
             self.cleanup_sort_history(self.model().sort_history)
         state['column_positions'] = {}
@@ -390,7 +392,7 @@ class BooksView(QTableView): # {{{
 
     def get_default_state(self):
         old_state = {
-                'hidden_columns': ['last_modified'],
+                'hidden_columns': ['last_modified', 'languages'],
                 'sort_history':[DEFAULT_SORT],
                 'column_positions': {},
                 'column_sizes': {},
@@ -399,6 +401,7 @@ class BooksView(QTableView): # {{{
                     'timestamp':'center',
                     'pubdate':'center'},
                 'last_modified_injected': True,
+                'languages_injected': True,
                 }
         h = self.column_header
         cm = self.column_map
@@ -430,11 +433,20 @@ class BooksView(QTableView): # {{{
                     if ans is not None:
                         db.prefs[name] = ans
                 else:
+                    injected = False
                     if not ans.get('last_modified_injected', False):
+                        injected = True
                         ans['last_modified_injected'] = True
                         hc = ans.get('hidden_columns', [])
                         if 'last_modified' not in hc:
                             hc.append('last_modified')
+                    if not ans.get('languages_injected', False):
+                        injected = True
+                        ans['languages_injected'] = True
+                        hc = ans.get('hidden_columns', [])
+                        if 'languages' not in hc:
+                            hc.append('languages')
+                    if injected:
                         db.prefs[name] = ans
         return ans
 
@@ -501,7 +513,7 @@ class BooksView(QTableView): # {{{
         for i in range(self.model().columnCount(None)):
             if self.itemDelegateForColumn(i) in (self.rating_delegate,
                     self.timestamp_delegate, self.pubdate_delegate,
-                    self.last_modified_delegate):
+                    self.last_modified_delegate, self.languages_delegate):
                 self.setItemDelegateForColumn(i, self.itemDelegate())
 
         cm = self.column_map

src/calibre/gui2/metadata/basic_widgets.py

@@ -34,6 +34,7 @@ from calibre.library.comments import comments_to_html
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.utils.icu import strcmp
 from calibre.ptempfile import PersistentTemporaryFile
+from calibre.gui2.languages import LanguagesEdit as LE
 
 def save_dialog(parent, title, msg, det_msg=''):
     d = QMessageBox(parent)
@@ -1133,6 +1134,43 @@ class TagsEdit(MultiCompleteLineEdit): # {{{
 
 # }}}
 
+class LanguagesEdit(LE): # {{{
+
+    LABEL = _('&Languages:')
+    TOOLTIP = _('A comma separated list of languages for this book')
+
+    def __init__(self, *args, **kwargs):
+        LE.__init__(self, *args, **kwargs)
+        self.setToolTip(self.TOOLTIP)
+
+    @dynamic_property
+    def current_val(self):
+        def fget(self): return self.lang_codes
+        def fset(self, val): self.lang_codes = val
+        return property(fget=fget, fset=fset)
+
+    def initialize(self, db, id_):
+        lc = []
+        langs = db.languages(id_, index_is_id=True)
+        if langs:
+            lc = [x.strip() for x in langs.split(',')]
+        self.current_val = self.original_val = lc
+
+    def commit(self, db, id_):
+        bad = self.validate()
+        if bad:
+            error_dialog(self, _('Unknown language'),
+                    ngettext('The language %s is not recognized',
+                        'The languages %s are not recognized', len(bad))%(
+                            ', '.join(bad)),
+                    show=True)
+            return False
+        cv = self.current_val
+        if cv != self.original_val:
+            db.set_languages(id_, cv)
+        return True
+# }}}
+
 class IdentifiersEdit(QLineEdit): # {{{
     LABEL = _('I&ds:')
     BASE_TT = _('Edit the identifiers for this book. '

src/calibre/gui2/metadata/single.py

@@ -20,7 +20,7 @@ from calibre.gui2 import ResizableDialog, error_dialog, gprefs, pixmap_to_data
 from calibre.gui2.metadata.basic_widgets import (TitleEdit, AuthorsEdit,
     AuthorSortEdit, TitleSortEdit, SeriesEdit, SeriesIndexEdit, IdentifiersEdit,
     RatingEdit, PublisherEdit, TagsEdit, FormatsManager, Cover, CommentsEdit,
-    BuddyLabel, DateEdit, PubdateEdit)
+    BuddyLabel, DateEdit, PubdateEdit, LanguagesEdit)
 from calibre.gui2.metadata.single_download import FullFetch
 from calibre.gui2.custom_column_widgets import populate_metadata_page
 from calibre.utils.config import tweaks
@@ -183,6 +183,9 @@ class MetadataSingleDialogBase(ResizableDialog):
         self.publisher = PublisherEdit(self)
         self.basic_metadata_widgets.append(self.publisher)
 
+        self.languages = LanguagesEdit(self)
+        self.basic_metadata_widgets.append(self.languages)
+
         self.timestamp = DateEdit(self)
         self.pubdate = PubdateEdit(self)
         self.basic_metadata_widgets.extend([self.timestamp, self.pubdate])
@@ -610,11 +613,13 @@ class MetadataSingleDialog(MetadataSingleDialogBase): # {{{
         create_row2(5, self.pubdate, self.pubdate.clear_button)
         sto(self.pubdate.clear_button, self.publisher)
         create_row2(6, self.publisher)
+        sto(self.publisher, self.languages)
+        create_row2(7, self.languages)
         self.tabs[0].spc_two = QSpacerItem(10, 10, QSizePolicy.Expanding,
                 QSizePolicy.Expanding)
-        l.addItem(self.tabs[0].spc_two, 8, 0, 1, 3)
+        l.addItem(self.tabs[0].spc_two, 9, 0, 1, 3)
-        l.addWidget(self.fetch_metadata_button, 9, 0, 1, 2)
+        l.addWidget(self.fetch_metadata_button, 10, 0, 1, 2)
-        l.addWidget(self.config_metadata_button, 9, 2, 1, 1)
+        l.addWidget(self.config_metadata_button, 10, 2, 1, 1)
 
         self.tabs[0].gb2 = gb = QGroupBox(_('Co&mments'), self)
         gb.l = l = QVBoxLayout()
@@ -717,16 +722,17 @@ class MetadataSingleDialogAlt1(MetadataSingleDialogBase): # {{{
         create_row(7, self.rating, self.pubdate)
         create_row(8, self.pubdate, self.publisher,
                    button=self.pubdate.clear_button, icon='trash.png')
-        create_row(9, self.publisher, self.timestamp)
+        create_row(9, self.publisher, self.languages)
-        create_row(10, self.timestamp, self.identifiers,
+        create_row(10, self.languages, self.timestamp)
+        create_row(11, self.timestamp, self.identifiers,
                    button=self.timestamp.clear_button, icon='trash.png')
-        create_row(11, self.identifiers, self.comments,
+        create_row(12, self.identifiers, self.comments,
                    button=self.clear_identifiers_button, icon='trash.png')
         sto(self.clear_identifiers_button, self.swap_title_author_button)
         sto(self.swap_title_author_button, self.manage_authors_button)
         sto(self.manage_authors_button, self.paste_isbn_button)
         tl.addItem(QSpacerItem(1, 1, QSizePolicy.Fixed, QSizePolicy.Expanding),
-                   12, 1, 1 ,1)
+                   13, 1, 1 ,1)
 
         w = getattr(self, 'custom_metadata_widgets_parent', None)
         if w is not None:
@@ -852,16 +858,17 @@ class MetadataSingleDialogAlt2(MetadataSingleDialogBase): # {{{
         create_row(7, self.rating, self.pubdate)
         create_row(8, self.pubdate, self.publisher,
                    button=self.pubdate.clear_button, icon='trash.png')
-        create_row(9, self.publisher, self.timestamp)
+        create_row(9, self.publisher, self.languages)
-        create_row(10, self.timestamp, self.identifiers,
+        create_row(10, self.languages, self.timestamp)
+        create_row(11, self.timestamp, self.identifiers,
                    button=self.timestamp.clear_button, icon='trash.png')
-        create_row(11, self.identifiers, self.comments,
+        create_row(12, self.identifiers, self.comments,
                    button=self.clear_identifiers_button, icon='trash.png')
         sto(self.clear_identifiers_button, self.swap_title_author_button)
         sto(self.swap_title_author_button, self.manage_authors_button)
         sto(self.manage_authors_button, self.paste_isbn_button)
         tl.addItem(QSpacerItem(1, 1, QSizePolicy.Fixed, QSizePolicy.Expanding),
-                   12, 1, 1 ,1)
+                   13, 1, 1 ,1)
 
         # Custom metadata in col 1
         w = getattr(self, 'custom_metadata_widgets_parent', None)

src/calibre/library/caches.py

@@ -15,6 +15,7 @@ from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import parse_date, now, UNDEFINED_DATE
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.utils.pyparsing import ParseException
+from calibre.utils.localization import canonicalize_lang
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre import prints
@@ -721,9 +722,13 @@ class ResultCache(SearchQueryParser): # {{{
                 if loc == db_col['authors']:
                     ### DB stores authors with commas changed to bars, so change query
                     if matchkind == REGEXP_MATCH:
-                        q = query.replace(',', r'\|');
+                        q = query.replace(',', r'\|')
                     else:
-                        q = query.replace(',', '|');
+                        q = query.replace(',', '|')
+                elif loc == db_col['languages']:
+                    q = canonicalize_lang(query)
+                    if q is None:
+                        q = query
                 else:
                     q = query
 

src/calibre/library/database2.py

@@ -39,6 +39,8 @@ from calibre.utils.magick.draw import save_cover_data_to
 from calibre.utils.recycle_bin import delete_file, delete_tree
 from calibre.utils.formatter_functions import load_user_template_functions
 from calibre.db.errors import NoSuchFormat
+from calibre.utils.localization import (canonicalize_lang,
+        calibre_langcode_to_name)
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 SPOOL_SIZE = 30*1024*1024
@@ -372,6 +374,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 'aum_sortconcat(link.id, authors.name, authors.sort, authors.link)'),
             'last_modified',
             '(SELECT identifiers_concat(type, val) FROM identifiers WHERE identifiers.book=books.id) identifiers',
+            ('languages', 'languages', 'lang_code',
+                'sortconcat(link.id, languages.lang_code)'),
             ]
         lines = []
         for col in columns:
@@ -390,7 +394,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
              'size':4, 'rating':5, 'tags':6, 'comments':7, 'series':8,
              'publisher':9, 'series_index':10, 'sort':11, 'author_sort':12,
              'formats':13, 'path':14, 'pubdate':15, 'uuid':16, 'cover':17,
-             'au_map':18, 'last_modified':19, 'identifiers':20}
+             'au_map':18, 'last_modified':19, 'identifiers':20, 'languages':21}
 
         for k,v in self.FIELD_MAP.iteritems():
             self.field_metadata.set_field_record_index(k, v, prefer_custom=False)
@@ -469,7 +473,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 'author_sort', 'authors', 'comment', 'comments',
                 'publisher', 'rating', 'series', 'series_index', 'tags',
                 'title', 'timestamp', 'uuid', 'pubdate', 'ondevice',
-                'metadata_last_modified',
+                'metadata_last_modified', 'languages',
                 ):
             fm = {'comment':'comments', 'metadata_last_modified':
                     'last_modified'}.get(prop, prop)
@@ -930,6 +934,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         tags = row[fm['tags']]
         if tags:
             mi.tags = [i.strip() for i in tags.split(',')]
+        languages = row[fm['languages']]
+        if languages:
+            mi.languages = [i.strip() for i in languages.split(',')]
         mi.series = row[fm['series']]
         if mi.series:
             mi.series_index = row[fm['series_index']]
@@ -1390,7 +1397,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 ('authors', 'authors', 'author'),
                 ('publishers', 'publishers', 'publisher'),
                 ('tags', 'tags', 'tag'),
-                ('series', 'series', 'series')
+                ('series', 'series', 'series'),
+                ('languages', 'languages', 'lang_code'),
                 ]:
             doit(ltable, table, ltable_col)
 
@@ -1507,6 +1515,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                         'series'   : self.get_series_with_ids,
                         'publisher': self.get_publishers_with_ids,
                         'tags'     : self.get_tags_with_ids,
+                        'languages': self.get_languages_with_ids,
                         'rating'   : self.get_ratings_with_ids,
                     }
                 func = funcs.get(category, None)
@@ -1521,6 +1530,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 for l in list:
                     (id, val, sort_val) = (l[0], l[1], l[2])
                     tids[category][val] = (id, sort_val)
+            elif category == 'languages':
+                for l in list:
+                    id, val = l[0], calibre_langcode_to_name(l[1])
+                    tids[category][l[1]] = (id, val)
             elif cat['datatype'] == 'series':
                 for l in list:
                     (id, val) = (l[0], l[1])
@@ -1620,6 +1633,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                 item.rt += rating
                                 item.rc += 1
                         except:
+                            prints(tid_cat, val)
                             prints('get_categories: item', val, 'is not in', cat, 'list!')
 
         #print 'end phase "books":', time.clock() - last, 'seconds'
@@ -1684,6 +1698,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 # Clean up the authors strings to human-readable form
                 formatter = (lambda x: x.replace('|', ','))
                 items = [v for v in tcategories[category].values() if v.c > 0]
+            elif category == 'languages':
+                # Use a human readable language string
+                formatter = calibre_langcode_to_name
+                items = [v for v in tcategories[category].values() if v.c > 0]
             else:
                 formatter = (lambda x:unicode(x))
                 items = [v for v in tcategories[category].values() if v.c > 0]
@@ -2043,6 +2061,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if should_replace_field('comments'):
             doit(self.set_comment, id, mi.comments, notify=False, commit=False)
 
+        if should_replace_field('languages'):
+            doit(self.set_languages, id, mi.languages, notify=False, commit=False)
+
         # Setting series_index to zero is acceptable
         if mi.series_index is not None:
             doit(self.set_series_index, id, mi.series_index, notify=False,
@@ -2265,6 +2286,37 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if notify:
             self.notify('metadata', [id])
 
+    def set_languages(self, book_id, languages, notify=True, commit=True):
+        self.conn.execute(
+            'DELETE FROM books_languages_link WHERE book=?', (book_id,))
+        self.conn.execute('''DELETE FROM languages WHERE (SELECT COUNT(id)
+                                 FROM books_languages_link WHERE
+                                 lang_code=languages.id) < 1''')
+
+        books_to_refresh = set([book_id])
+        final_languages = []
+        for l in languages:
+            lc = canonicalize_lang(l)
+            if not lc or lc in final_languages or lc in ('und', 'zxx', 'mis',
+                    'mul'):
+                continue
+            final_languages.append(lc)
+            lc_id = self.conn.get('SELECT id FROM languages WHERE lang_code=?',
+                    (lc,), all=False)
+            if lc_id is None:
+                lc_id = self.conn.execute('''INSERT INTO languages(lang_code)
+                                           VALUES (?)''', (lc,)).lastrowid
+            self.conn.execute('''INSERT INTO books_languages_link(book, lang_code)
+                                     VALUES (?,?)''', (book_id, lc_id))
+        self.dirtied(books_to_refresh, commit=False)
+        if commit:
+            self.conn.commit()
+        self.data.set(book_id, self.FIELD_MAP['languages'],
+                u','.join(final_languages), row_is_id=True)
+        if notify:
+            self.notify('metadata', [book_id])
+        return books_to_refresh
+
     def set_timestamp(self, id, dt, notify=True, commit=True):
         if dt:
             self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id))
@@ -2363,6 +2415,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             return []
         return result
 
+    def get_languages_with_ids(self):
+        result = self.conn.get('SELECT id,lang_code FROM languages')
+        if not result:
+            return []
+        return result
+
     def rename_tag(self, old_id, new_name):
         # It is possible that new_name is in fact a set of names. Split it on
         # comma to find out. If it is, then rename the first one and append the

src/calibre/library/field_metadata.py

@@ -17,7 +17,7 @@ class TagsIcons(dict):
 
     category_icons = ['authors', 'series', 'formats', 'publisher', 'rating',
                       'news',    'tags',   'custom:', 'user:',     'search',
-                      'identifiers',       'gst']
+                      'identifiers', 'languages', 'gst']
     def __init__(self, icon_dict):
         for a in self.category_icons:
             if a not in icon_dict:
@@ -37,6 +37,7 @@ category_icon_map = {
                     'search'     : 'search.png',
                     'identifiers': 'identifiers.png',
                     'gst'        : 'catalog.png',
+                    'languages'  : 'languages.png',
             }
 
 
@@ -114,6 +115,21 @@ class FieldMetadata(dict):
                            'is_custom':False,
                            'is_category':True,
                            'is_csp': False}),
+            ('languages', {'table':'languages',
+                           'column':'lang_code',
+                           'link_column':'lang_code',
+                           'category_sort':'lang_code',
+                           'datatype':'text',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': ',',
+                                          'list_to_ui': ', '},
+                           'kind':'field',
+                           'name':_('Languages'),
+                           'search_terms':['languages', 'language'],
+                           'is_custom':False,
+                           'is_category':True,
+                           'is_csp': False}),
+
             ('series',    {'table':'series',
                            'column':'name',
                            'link_column':'series',

src/calibre/utils/localization.py

@@ -192,6 +192,74 @@ def get_language(lang):
             ans = iso639['by_3t'].get(lang, ans)
     return translate(ans)
 
+def calibre_langcode_to_name(lc, localize=True):
+    iso639 = _load_iso639()
+    translate = _ if localize else lambda x: x
+    try:
+        return translate(iso639['by_3t'][lc])
+    except:
+        pass
+    return lc
+
+def canonicalize_lang(raw):
+    if not raw:
+        return None
+    if not isinstance(raw, unicode):
+        raw = raw.decode('utf-8', 'ignore')
+    raw = raw.lower().strip()
+    if not raw:
+        return None
+    raw = raw.replace('_', '-').partition('-')[0].strip()
+    if not raw:
+        return None
+    iso639 = _load_iso639()
+    m2to3 = iso639['2to3']
+
+    if len(raw) == 2:
+        ans = m2to3.get(raw, None)
+        if ans is not None:
+            return ans
+    elif len(raw) == 3:
+        if raw in iso639['by_3t']:
+            return raw
+        if raw in iso639['3bto3t']:
+            return iso639['3bto3t'][raw]
+
+    return iso639['name_map'].get(raw, None)
+
+_lang_map = None
+
+def lang_map():
+    ' Return mapping of ISO 639 3 letter codes to localized language names '
+    iso639 = _load_iso639()
+    translate = _
+    global _lang_map
+    if _lang_map is None:
+        _lang_map = {k:translate(v) for k, v in iso639['by_3t'].iteritems()}
+    return _lang_map
+
+def langnames_to_langcodes(names):
+    '''
+    Given a list of localized language names return a mapping of the names to 3
+    letter ISO 639 language codes. If a name is not recognized, it is mapped to
+    None.
+    '''
+    iso639 = _load_iso639()
+    translate = _
+    ans = {}
+    names = set(names)
+    for k, v in iso639['by_3t'].iteritems():
+        tv = translate(v)
+        if tv in names:
+            names.remove(tv)
+            ans[tv] = k
+        if not names:
+            break
+    for x in names:
+        ans[x] = None
+
+    return ans
+
 _udc = None
 
 def get_udc():