Merge from trunk

recipes/le_monde.recipe

@@ -48,7 +48,7 @@ class LeMonde(BasicNewsRecipe):
             if alink.string is not None:
                tstr = alink.string
                alink.replaceWith(tstr)
-        return soup
+        return self.adeify_images(soup)
 
     preprocess_regexps = [
         (re.compile(r'([0-9])%'), lambda m: m.group(1) + ' %'),

resources/default_tweaks.py

@@ -118,6 +118,7 @@ sort_columns_at_startup = None
 # timestamp default if not set: dd MMM yyyy
 gui_pubdate_display_format = 'MMM yyyy'
 gui_timestamp_display_format = 'dd MMM yyyy'
+gui_last_modified_display_format = 'dd MMM yyyy'
 
 #: Control sorting of titles and series in the library display
 # Control title and series sorting in the library view. If set to

src/calibre/devices/apple/driver.py

@@ -460,7 +460,7 @@ class ITUNES(DriverBase):
 
                             cached_books[this_book.path] = {
                              'title':book.Name,
-                             'author':book.artist().split(' & '),
+                             'author':book.Artist.split(' & '),
                              'lib_book':library_books[this_book.path] if this_book.path in library_books else None,
                              'uuid': book.Composer,
                              'format': 'pdf' if book.KindAsString.startswith('PDF') else 'epub'

src/calibre/ebooks/metadata/mobi.py

@@ -400,7 +400,8 @@ class MetadataUpdater(object):
         if getattr(self, 'exth', None) is None:
             raise MobiError('No existing EXTH record. Cannot update metadata.')
 
-        self.record0[92:96] = iana2mobi(mi.language)
+        if not mi.is_null('language'):
+            self.record0[92:96] = iana2mobi(mi.language)
         self.create_exth(exth=exth, new_title=mi.title)
 
         # Fetch updated timestamp, cover_record, thumbnail_record

src/calibre/ebooks/metadata/worker.py

@@ -222,7 +222,7 @@ class SaveWorker(Thread):
                         if isbytestring(fpath):
                             fpath = fpath.decode(filesystem_encoding)
                         formats[fmt.lower()] = fpath
-            data[i] = [opf, cpath, formats]
+            data[i] = [opf, cpath, formats, mi.last_modified.isoformat()]
         return data
 
     def run(self):

src/calibre/ebooks/pdb/plucker/reader.py

@@ -16,6 +16,7 @@ from calibre import CurrentDir
 from calibre.ebooks.pdb.formatreader import FormatReader
 from calibre.ptempfile import TemporaryFile
 from calibre.utils.magick import Image, create_canvas
+from calibre.ebooks.compression.palmdoc import decompress_doc
 
 DATATYPE_PHTML = 0
 DATATYPE_PHTML_COMPRESSED = 1

src/calibre/gui2/book_details.py

@@ -118,8 +118,9 @@ def render_data(mi, use_roman_numbers=True, all_fields=False):
             links = [u'<a href="%s" title="%s:%s">%s</a>' % (url, id_typ, id_val, name)
                     for name, id_typ, id_val, url in urls]
             links = u', '.join(links)
-            ans.append((field, u'<td class="title">%s</td><td>%s</td>'%(
+            if links:
-                _('Ids')+':', links)))
+                ans.append((field, u'<td class="title">%s</td><td>%s</td>'%(
+                    _('Ids')+':', links)))
         else:
             val = mi.format_field(field)[-1]
             if val is None:

src/calibre/gui2/library/delegates.py

@@ -97,18 +97,25 @@ class RatingDelegate(QStyledItemDelegate): # {{{
 
 class DateDelegate(QStyledItemDelegate): # {{{
 
+    def __init__(self, parent, tweak_name='gui_timestamp_display_format',
+            default_format='dd MMM yyyy', editor_format='dd MMM yyyy'):
+        QStyledItemDelegate.__init__(self, parent)
+        self.tweak_name = tweak_name
+        self.default_format = default_format
+        self.editor_format = editor_format
+
     def displayText(self, val, locale):
         d = val.toDate()
         if d <= UNDEFINED_QDATE:
             return ''
-        format = tweaks['gui_timestamp_display_format']
+        format = tweaks[self.tweak_name]
         if format is None:
-            format = 'dd MMM yyyy'
+            format = self.default_format
         return format_date(d.toPyDate(), format)
 
     def createEditor(self, parent, option, index):
         qde = QStyledItemDelegate.createEditor(self, parent, option, index)
-        qde.setDisplayFormat('dd MMM yyyy')
+        qde.setDisplayFormat(self.editor_format)
         qde.setMinimumDate(UNDEFINED_QDATE)
         qde.setSpecialValueText(_('Undefined'))
         qde.setCalendarPopup(True)

src/calibre/gui2/library/models.py

@@ -70,6 +70,7 @@ class BooksModel(QAbstractTableModel): # {{{
                         'publisher' : _("Publisher"),
                         'tags'      : _("Tags"),
                         'series'    : _("Series"),
+                        'last_modified' : _('Modified'),
     }
 
     def __init__(self, parent=None, buffer=40):
@@ -620,6 +621,8 @@ class BooksModel(QAbstractTableModel): # {{{
                                 idx=self.db.field_metadata['timestamp']['rec_index']),
                    'pubdate'  : functools.partial(datetime_type,
                                 idx=self.db.field_metadata['pubdate']['rec_index']),
+                   'last_modified': functools.partial(datetime_type,
+                                idx=self.db.field_metadata['last_modified']['rec_index']),
                    'rating'   : functools.partial(rating_type,
                                 idx=self.db.field_metadata['rating']['rec_index']),
                    'publisher': functools.partial(text_type,

src/calibre/gui2/library/views.py

@@ -76,6 +76,8 @@ class BooksView(QTableView): # {{{
         self.rating_delegate = RatingDelegate(self)
         self.timestamp_delegate = DateDelegate(self)
         self.pubdate_delegate = PubDateDelegate(self)
+        self.last_modified_delegate = DateDelegate(self,
+                tweak_name='gui_last_modified_display_format')
         self.tags_delegate = CompleteDelegate(self, ',', 'all_tags')
         self.authors_delegate = CompleteDelegate(self, '&', 'all_author_names', True)
         self.cc_names_delegate = CompleteDelegate(self, '&', 'all_custom', True)
@@ -296,6 +298,7 @@ class BooksView(QTableView): # {{{
         state = {}
         state['hidden_columns'] = [cm[i] for i in  range(h.count())
                 if h.isSectionHidden(i) and cm[i] != 'ondevice']
+        state['last_modified_injected'] = True
         state['sort_history'] = \
             self.cleanup_sort_history(self.model().sort_history)
         state['column_positions'] = {}
@@ -380,7 +383,7 @@ class BooksView(QTableView): # {{{
 
     def get_default_state(self):
         old_state = {
-                'hidden_columns': [],
+                'hidden_columns': ['last_modified'],
                 'sort_history':[DEFAULT_SORT],
                 'column_positions': {},
                 'column_sizes': {},
@@ -388,6 +391,7 @@ class BooksView(QTableView): # {{{
                     'size':'center',
                     'timestamp':'center',
                     'pubdate':'center'},
+                'last_modified_injected': True,
                 }
         h = self.column_header
         cm = self.column_map
@@ -398,7 +402,7 @@ class BooksView(QTableView): # {{{
                 old_state['column_sizes'][name] = \
                     min(350, max(self.sizeHintForColumn(i),
                         h.sectionSizeHint(i)))
-                if name == 'timestamp':
+                if name in ('timestamp', 'last_modified'):
                     old_state['column_sizes'][name] += 12
         return old_state
 
@@ -418,6 +422,13 @@ class BooksView(QTableView): # {{{
                         pass
                     if ans is not None:
                         db.prefs[name] = ans
+                else:
+                    if not ans.get('last_modified_injected', False):
+                        ans['last_modified_injected'] = True
+                        hc = ans.get('hidden_columns', [])
+                        if 'last_modified' not in hc:
+                            hc.append('last_modified')
+                        db.prefs[name] = ans
         return ans
 
 
@@ -459,7 +470,8 @@ class BooksView(QTableView): # {{{
     def database_changed(self, db):
         for i in range(self.model().columnCount(None)):
             if self.itemDelegateForColumn(i) in (self.rating_delegate,
-                    self.timestamp_delegate, self.pubdate_delegate):
+                    self.timestamp_delegate, self.pubdate_delegate,
+                    self.last_modified_delegate):
                 self.setItemDelegateForColumn(i, self.itemDelegate())
 
         cm = self.column_map

src/calibre/gui2/metadata/basic_widgets.py

@@ -280,11 +280,16 @@ class AuthorSortEdit(EnLineEdit):
         aus = self.current_val
         meth = tweaks['author_sort_copy_method']
         if aus:
-            ln, _, rest = aus.partition(',')
+            ans = []
-            if rest:
+            for one in [a.strip() for a in aus.split('&')]:
-                if meth in ('invert', 'nocomma', 'comma'):
+                if not one:
-                    aus = rest.strip() + ' ' + ln.strip()
+                    continue
-            self.authors_edit.current_val = [aus]
+                ln, _, rest = one.partition(',')
+                if rest:
+                    if meth in ('invert', 'nocomma', 'comma'):
+                        one = rest.strip() + ' ' + ln.strip()
+                ans.append(one)
+            self.authors_edit.current_val = ans
 
     def auto_generate(self, *args):
         au = unicode(self.authors_edit.text())
@@ -937,7 +942,11 @@ class IdentifiersEdit(QLineEdit): # {{{
             ans = {}
             for x in parts:
                 c = x.split(':')
-                if len(c) == 2:
+                if len(c) > 1:
+                    if c[0] == 'isbn':
+                        v = check_isbn(c[1])
+                        if v is not None:
+                            c[1] = v
                     ans[c[0]] = c[1]
             return ans
         def fset(self, val):
@@ -948,6 +957,11 @@ class IdentifiersEdit(QLineEdit): # {{{
                 if x == 'isbn':
                     x = '00isbn'
                 return x
+            for k in list(val):
+                if k == 'isbn':
+                    v = check_isbn(k)
+                    if v is not None:
+                        val[k] = v
             ids = sorted(val.iteritems(), key=keygen)
             txt = ', '.join(['%s:%s'%(k, v) for k, v in ids])
             self.setText(txt.strip())
@@ -955,8 +969,8 @@ class IdentifiersEdit(QLineEdit): # {{{
         return property(fget=fget, fset=fset)
 
     def initialize(self, db, id_):
-        self.current_val = db.get_identifiers(id_, index_is_id=True)
+        self.original_val = db.get_identifiers(id_, index_is_id=True)
-        self.original_val = self.current_val
+        self.current_val = self.original_val
 
     def commit(self, db, id_):
         if self.original_val != self.current_val:

src/calibre/gui2/metadata/config.py

@@ -41,8 +41,11 @@ class FieldsModel(FM): # {{{
         self.reset()
 
     def commit(self):
-        val = [k for k, v in self.overrides.iteritems() if v == Qt.Unchecked]
+        ignored_fields = set([x for x in self.prefs['ignore_fields'] if x not in
-        self.prefs['ignore_fields'] = val
+            self.overrides])
+        changed = set([k for k, v in self.overrides.iteritems() if v ==
+            Qt.Unchecked])
+        self.prefs['ignore_fields'] = list(ignored_fields.union(changed))
 
 # }}}
 

src/calibre/gui2/preferences/create_custom_column.py

@@ -63,7 +63,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         self.shortcuts.linkActivated.connect(self.shortcut_activated)
         text = '<p>'+_('Quick create:')
         for col, name in [('isbn', _('ISBN')), ('formats', _('Formats')),
-                ('last_modified', _('Modified Date')), ('yesno', _('Yes/No')),
+                ('yesno', _('Yes/No')),
                 ('tags', _('Tags')), ('series', _('Series')), ('rating',
                     _('Rating')), ('people', _("People's names"))]:
             text += ' <a href="col:%s">%s</a>,'%(col, name)
@@ -150,7 +150,6 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             'tags': _('My Tags'),
             'series': _('My Series'),
             'rating': _('My Rating'),
-            'last_modified':_('Modified Date'),
             'people': _('People')}[which])
         self.is_names.setChecked(which == 'people')
         if self.composite_box.isVisible():
@@ -158,9 +157,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                 {
                     'isbn': '{identifiers:select(isbn)}',
                     'formats': '{formats}',
-                    'last_modified':'''{last_modified:'format_date($, "dd MMM yyyy")'}'''
                     }[which])
-            self.composite_sort_by.setCurrentIndex(2 if which == 'last_modified' else 0)
+            self.composite_sort_by.setCurrentIndex(0)
 
     def datatype_changed(self, *args):
         try:

src/calibre/gui2/preferences/metadata_sources.py

@@ -209,8 +209,11 @@ class FieldsModel(QAbstractListModel): # {{{
         return ret
 
     def commit(self):
-        val = [k for k, v in self.overrides.iteritems() if v == Qt.Unchecked]
+        ignored_fields = set([x for x in msprefs['ignore_fields'] if x not in
-        msprefs['ignore_fields'] = val
+            self.overrides])
+        changed = set([k for k, v in self.overrides.iteritems() if v ==
+            Qt.Unchecked])
+        msprefs['ignore_fields'] = list(ignored_fields.union(changed))
 
 
 # }}}

src/calibre/library/caches.py

@@ -406,11 +406,9 @@ class ResultCache(SearchQueryParser): # {{{
         if val_func is None:
             loc = self.field_metadata[location]['rec_index']
             val_func = lambda item, loc=loc: item[loc]
-        dt = self.field_metadata[location]['datatype']
-
         q = ''
-        val_func = lambda item, loc=loc: item[loc]
         cast = adjust = lambda x: x
+        dt = self.field_metadata[location]['datatype']
 
         if query == 'false':
             if dt == 'rating' or location == 'cover':

src/calibre/library/field_metadata.py

@@ -368,7 +368,8 @@ class FieldMetadata(dict):
                         'date_format': tweaks['gui_timestamp_display_format']}
         self._tb_cats['pubdate']['display'] = {
                         'date_format': tweaks['gui_pubdate_display_format']}
-        self._tb_cats['last_modified']['display'] = {'date_format': 'iso'}
+        self._tb_cats['last_modified']['display'] = {
+                        'date_format': tweaks['gui_last_modified_display_format']}
         self.custom_field_prefix = '#'
         self.get = self._tb_cats.get
 

src/calibre/library/save_to_disk.py

@@ -17,6 +17,7 @@ from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.constants import preferred_encoding
 from calibre.ebooks.metadata import fmt_sidx
 from calibre.ebooks.metadata import title_sort
+from calibre.utils.date import parse_date
 from calibre import strftime, prints, sanitize_file_name_unicode
 
 plugboard_any_device_value = 'any device'
@@ -42,6 +43,8 @@ FORMAT_ARG_DESCS = dict(
         publisher=_('The publisher'),
         timestamp=_('The date'),
         pubdate=_('The published date'),
+        last_modified=_('The date when the metadata for this book record'
+            ' was last modified'),
         id=_('The calibre internal id')
         )
 
@@ -191,6 +194,9 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
         format_args['timestamp'] = strftime(timefmt, mi.timestamp.timetuple())
     if hasattr(mi.pubdate, 'timetuple'):
         format_args['pubdate'] = strftime(timefmt, mi.pubdate.timetuple())
+    if hasattr(mi, 'last_modified') and hasattr(mi.last_modified, 'timetuple'):
+        format_args['last_modified'] = strftime(timefmt, mi.last_modified.timetuple())
+
     format_args['id'] = str(id)
     # Now format the custom fields
     custom_metadata = mi.get_all_user_metadata(make_copy=False)
@@ -373,10 +379,14 @@ def save_serialized_to_disk(ids, data, plugboards, root, opts, callback):
     root, opts, length = _sanitize_args(root, opts)
     failures = []
     for x in ids:
-        opf, cover, format_map = data[x]
+        opf, cover, format_map, last_modified = data[x]
         if isinstance(opf, unicode):
             opf = opf.encode('utf-8')
         mi = OPF(cStringIO.StringIO(opf)).to_book_metadata()
+        try:
+            mi.last_modified = parse_date(last_modified)
+        except:
+            pass
         tb = ''
         try:
             failed, id, title = do_save_book_to_disk(x, mi, cover, plugboards,

src/calibre/manual/gui.rst

@@ -365,6 +365,8 @@ Dates and numeric fields support the relational operators ``=`` (equals), ``>``
 Rating fields are considered to be numeric. For example, the search ``rating:>=3`` will find all books rated 3
 or higher.
 
+You can search for the number of items in multiple-valued fields such as tags). These searches begin with the character ``#``, then use the same syntax as numeric fields. For example, to find all books with more than 4 tags, use ``tags:#>4``. To find all books with exactly 10 tags, use ``tags:#=10``.
+
 Series indices are searchable. For the standard series, the search name is 'series_index'. For
 custom series columns, use the column search name followed by _index. For example, to search the indices for a
 custom series column named ``#my_series``, you would use the search name ``#my_series_index``.