merge from trunk

resources/default_tweaks.py

@@ -39,7 +39,7 @@ completer_append_separator = False
 
 # The algorithm used to copy author to author_sort
 # Possible values are:
-#  invert: use "fn ln" -> "ln, fn" (the original algorithm)
+#  invert: use "fn ln" -> "ln, fn" (the default algorithm)
 #  copy  : copy author to author_sort without modification
 #  comma : use 'copy' if there is a ',' in the name, otherwise use 'invert'
 #  nocomma : "fn ln" -> "ln fn" (without the comma)

resources/recipes/new_yorker.recipe

@@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe):
                         ,dict(attrs={'id':['show-header','show-footer'] })
                      ]
     remove_attributes = ['lang']
-    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')]
+    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/mrss/feeds/everything.xml')]
 
     def print_version(self, url):
-        return 'http://www.newyorker.com' + url + '?printable=true'
+        return url + '?printable=true'
 
     def image_url_processor(self, baseurl, url):
         return url.strip()

src/calibre/customize/builtins.py

@@ -325,6 +325,17 @@ class TXTMetadataReader(MetadataReaderPlugin):
         from calibre.ebooks.metadata.txt import get_metadata
         return get_metadata(stream)
 
+class TXTZMetadataReader(MetadataReaderPlugin):
+
+    name        = 'Read TXTZ metadata'
+    file_types  = set(['txtz'])
+    description = _('Read metadata from %s files') % 'TXTZ'
+    author      = 'John Schember'
+
+    def get_metadata(self, stream, ftype):
+        from calibre.ebooks.metadata.txtz import get_metadata
+        return get_metadata(stream)
+
 class ZipMetadataReader(MetadataReaderPlugin):
 
     name = 'Read ZIP metadata'
@@ -412,6 +423,17 @@ class TOPAZMetadataWriter(MetadataWriterPlugin):
         from calibre.ebooks.metadata.topaz import set_metadata
         set_metadata(stream, mi)
 
+class TXTZMetadataWriter(MetadataWriterPlugin):
+
+    name        = 'Set TXTZ metadata'
+    file_types  = set(['txtz'])
+    description = _('Set metadata from %s files') % 'TXTZ'
+    author      = 'John Schember'
+
+    def set_metadata(self, stream, mi, type):
+        from calibre.ebooks.metadata.txtz import set_metadata
+        set_metadata(stream, mi)
+
 # }}}
 
 from calibre.ebooks.comic.input import ComicInput
@@ -446,6 +468,7 @@ from calibre.ebooks.rb.output import RBOutput
 from calibre.ebooks.rtf.output import RTFOutput
 from calibre.ebooks.tcr.output import TCROutput
 from calibre.ebooks.txt.output import TXTOutput
+from calibre.ebooks.txt.output import TXTZOutput
 from calibre.ebooks.html.output import HTMLOutput
 from calibre.ebooks.snb.output import SNBOutput
 
@@ -531,6 +554,7 @@ plugins += [
     RTFOutput,
     TCROutput,
     TXTOutput,
+    TXTZOutput,
     HTMLOutput,
     SNBOutput,
 ]

src/calibre/devices/android/driver.py

@@ -19,10 +19,15 @@ class ANDROID(USBMS):
 
     VENDOR_ID   = {
             # HTC
-            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100,
+            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226],
-                0x0227, 0x0226], 0x0ff9
+                       0x0c01 : [0x100, 0x0227, 0x0226],
-                : [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226],
+                       0x0ff9 : [0x0100, 0x0227, 0x0226],
-                0xc92 : [0x100], 0xc97: [0x226], 0xc99 : [0x0100]},
+                       0x0c87 : [0x0100, 0x0227, 0x0226],
+                       0xc92  : [0x100],
+                       0xc97  : [0x226],
+                       0xc99  : [0x0100],
+                       0xca3  : [0x100],
+            },
 
             # Eken
             0x040d : { 0x8510 : [0x0001], 0x0851 : [0x1] },

src/calibre/devices/eb600/driver.py

@@ -172,10 +172,10 @@ class INVESBOOK(EB600):
     gui_name = 'Inves Book 600'
 
     FORMATS = ['epub', 'mobi', 'prc', 'fb2', 'html', 'pdf', 'rtf', 'txt']
+    BCD         = [0x110, 0x323]
 
-    VENDOR_NAME = 'INVES_E6'
+    VENDOR_NAME = ['INVES_E6', 'INVES-WI']
-    WINDOWS_MAIN_MEM = '00INVES_E600'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['00INVES_E600', 'INVES-WIBOOK']
-    WINDOWS_CARD_A_MEM = '00INVES_E600'
 
 class BOOQ(EB600):
     name = 'Booq Device Interface'

src/calibre/ebooks/__init__.py

@@ -25,7 +25,7 @@ class DRMError(ValueError):
 class ParserError(ValueError):
     pass
 
-BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'htm', 'xhtm',
+BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'htm', 'xhtm',
                    'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc',
                    'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
                    'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'mbp', 'tan', 'snb']

src/calibre/ebooks/metadata/txt.py

@@ -1,16 +1,20 @@
-'''Read meta information from TXT files'''
+# -*- coding: utf-8 -*-
-
-from __future__ import with_statement
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 
+'''
+Read meta information from TXT files
+'''
+
 import re
 
 from calibre.ebooks.metadata import MetaInformation
 
 def get_metadata(stream, extract_cover=True):
-    """ Return metadata as a L{MetaInfo} object """
+    '''
+    Return metadata as a L{MetaInfo} object
+    '''
     mi = MetaInformation(_('Unknown'), [_('Unknown')])
     stream.seek(0)
 

src/calibre/ebooks/metadata/txtz.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+
+'''
+Read meta information from TXT files
+'''
+
+import os
+
+from cStringIO import StringIO
+
+from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf
+from calibre.ptempfile import TemporaryDirectory
+from calibre.utils.zipfile import ZipFile, safe_replace
+
+def get_metadata(stream, extract_cover=True):
+    '''
+    Return metadata as a L{MetaInfo} object
+    '''
+    mi = MetaInformation(_('Unknown'), [_('Unknown')])
+    stream.seek(0)
+
+    with TemporaryDirectory('_untxtz_mdata') as tdir:
+        try:
+            zf = ZipFile(stream)
+            zf.extract('metadata.opf', tdir)
+            with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff:
+                mi = OPF(opff).to_book_metadata()
+        except:
+            return mi
+    return mi
+
+def set_metadata(stream, mi):
+    opf = StringIO(metadata_to_opf(mi))
+    safe_replace(stream, 'metadata.opf', opf)

src/calibre/ebooks/txt/input.py

@@ -4,23 +4,27 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import glob
 import os
 
+from calibre import _ent_pat, xml_entity_to_unicode
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
 from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator
 from calibre.ebooks.chardet import detect
 from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \
     separate_paragraphs_single_line, separate_paragraphs_print_formatted, \
     preserve_spaces, detect_paragraph_type, detect_formatting_type, \
-    normalize_line_endings, convert_textile, remove_indents, block_to_single_line
+    normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \
-from calibre import _ent_pat, xml_entity_to_unicode
+    separate_hard_scene_breaks
+from calibre.ptempfile import TemporaryDirectory
+from calibre.utils.zipfile import ZipFile
 
 class TXTInput(InputFormatPlugin):
 
     name        = 'TXT Input'
     author      = 'John Schember'
     description = 'Convert TXT files to HTML'
-    file_types  = set(['txt'])
+    file_types  = set(['txt', 'txtz'])
 
     options = set([
         OptionRecommendation(name='paragraph_type', recommended_value='auto',
@@ -57,10 +61,23 @@ class TXTInput(InputFormatPlugin):
     def convert(self, stream, options, file_ext, log,
                 accelerators):
         self.log = log
+        txt = ''
         log.debug('Reading text from file...')
         length = 0
 
-        txt = stream.read()
+        # Extract content from zip archive.
+        if file_ext == 'txtz':
+            log.debug('De-compressing content to temporary directory...')
+            with TemporaryDirectory('_untxtz') as tdir:
+                zf = ZipFile(stream)
+                zf.extractall(tdir)
+
+                txts = glob.glob(os.path.join(tdir, '*.txt'))
+                for t in txts:
+                    with open(t, 'rb') as tf:
+                        txt += tf.read()
+        else:
+            txt = stream.read()
 
         # Get the encoding of the document.
         if options.input_encoding:
@@ -98,6 +115,7 @@ class TXTInput(InputFormatPlugin):
         if options.formatting_type == 'heuristic':
             setattr(options, 'enable_heuristics', True)
             setattr(options, 'unwrap_lines', False)
+            setattr(options, 'smarten_punctuation', True)
 
         # Reformat paragraphs to block formatting based on the detected type.
         # We don't check for block because the processor assumes block.
@@ -105,6 +123,7 @@ class TXTInput(InputFormatPlugin):
         if options.paragraph_type == 'single':
             txt = separate_paragraphs_single_line(txt)
         elif options.paragraph_type == 'print':
+            txt = separate_hard_scene_breaks(txt)
             txt = separate_paragraphs_print_formatted(txt)
             txt = block_to_single_line(txt)
         elif options.paragraph_type == 'unformatted':
@@ -116,6 +135,7 @@ class TXTInput(InputFormatPlugin):
             txt = preprocessor.punctuation_unwrap(length, txt, 'txt')
             txt = separate_paragraphs_single_line(txt)
         else:
+            txt = separate_hard_scene_breaks(txt)
             txt = block_to_single_line(txt)
 
         if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False):
@@ -175,4 +195,11 @@ class TXTInput(InputFormatPlugin):
                 {})
         options.debug_pipeline = odi
         os.remove(htmlfile.name)
+        
+        # Set metadata from file.
+        from calibre.customize.ui import get_file_type_metadata
+        from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata
+        mi = get_file_type_metadata(stream, file_ext)
+        meta_info_to_oeb_metadata(mi, oeb.metadata, log)
+        
         return oeb

src/calibre/ebooks/txt/markdownml.py

@@ -35,11 +35,9 @@ class MarkdownMLizer(object):
             html = unicode(etree.tostring(item.data, encoding=unicode))
             
             if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
             if not self.opts.keep_image_references:
-                html = re.sub(r'<\s*img[^>]*>', '', html)
+                html = re.sub(r'<\s*img[^>]*>', '', html)\
-                html = re.sub(r'<\s*img\s*>', '', html)
             
             text = html2text(html)
         

src/calibre/ebooks/txt/output.py

@@ -5,11 +5,18 @@ __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
 import os
+import shutil
+
+from lxml import etree
 
 from calibre.customize.conversion import OutputFormatPlugin, \
     OptionRecommendation
+from calibre.ebooks.oeb.base import OEB_IMAGES 
 from calibre.ebooks.txt.txtml import TXTMLizer
 from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
+from calibre.ptempfile import TemporaryDirectory, TemporaryFile
+from calibre.utils.cleantext import clean_ascii_chars
+from calibre.utils.zipfile import ZipFile
 
 class TXTOutput(OutputFormatPlugin):
 
@@ -73,6 +80,7 @@ class TXTOutput(OutputFormatPlugin):
             writer = TXTMLizer(log)
 
         txt = writer.extract_content(oeb_book, opts)
+        txt = clean_ascii_chars(txt)
 
         log.debug('\tReplacing newlines with selected type...')
         txt = specified_newlines(TxtNewlines(opts.newline).newline, txt)
@@ -93,3 +101,32 @@ class TXTOutput(OutputFormatPlugin):
         if close:
             out_stream.close()
 
+
+class TXTZOutput(TXTOutput):
+    
+    name = 'TXTZ Output'
+    author = 'John Schember'
+    file_type = 'txtz'
+
+    def convert(self, oeb_book, output_path, input_plugin, opts, log):
+        with TemporaryDirectory('_txtz_output') as tdir:
+            # TXT
+            with TemporaryFile('index.txt') as tf:
+                TXTOutput.convert(self, oeb_book, tf, input_plugin, opts, log)
+                shutil.copy(tf, os.path.join(tdir, 'index.txt'))
+
+            # Images
+            for item in oeb_book.manifest:
+                if item.media_type in OEB_IMAGES:
+                    path = os.path.join(tdir, os.path.dirname(item.href))
+                    if not os.path.exists(path):
+                        os.makedirs(path)
+                    with open(os.path.join(tdir, item.href), 'wb') as imgf:
+                        imgf.write(item.data)
+            
+            # Metadata
+            with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: 
+                mdataf.write(etree.tostring(oeb_book.metadata.to_opf1()))
+            
+            txtz = ZipFile(output_path, 'w')
+            txtz.add_dir(tdir)

src/calibre/ebooks/txt/processor.py

@@ -29,8 +29,7 @@ def clean_txt(txt):
     txt = '\n'.join([line.rstrip() for line in txt.splitlines()])
     
     # Replace whitespace at the beginning of the line with  
-    txt = re.sub('(?m)(?P<space>^[ ]+)(?=.)', lambda mo: '&nbsp;' * mo.groups('space').count(' '), txt)
+    txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', '&nbsp;' * 4, txt)
-    txt = re.sub('(?m)(?P<space>^[\t]+)(?=.)', lambda mo: '&nbsp;' * 4 * mo.groups('space').count('\t'), txt)
 
     # Condense redundant spaces
     txt = re.sub('[ ]{2,}', ' ', txt)
@@ -121,6 +120,15 @@ def separate_paragraphs_print_formatted(txt):
     txt = re.sub(u'(?miu)^(?P<indent>\t+|[ ]{2,})(?=.)', lambda mo: '\n%s' % mo.group('indent'), txt)
     return txt
 
+def separate_hard_scene_breaks(txt):
+    def sep_break(line):
+        if len(line.strip()) > 0:
+            return '\n%s\n' % line
+        else:
+            return line
+    txt = re.sub(u'(?miu)^[ \t-=~\/]+$', lambda mo: sep_break(mo.group()), txt)
+    return txt
+
 def block_to_single_line(txt):
     txt = re.sub(r'(?<=.)\n(?=.)', ' ', txt)
     return txt
@@ -221,9 +229,9 @@ def detect_formatting_type(txt):
     markdown_count += len(re.findall('(?mu)^=+$', txt))
     markdown_count += len(re.findall('(?mu)^-+$', txt))
     # Images
-    markdown_count += len(re.findall('(?u)!\[.*?\]\(.+?\)', txt))
+    markdown_count += len(re.findall('(?u)!\[.*?\](\[|\()', txt))
     # Links
-    markdown_count += len(re.findall('(?u)(^|(?P<pre>[^!]))\[.*?\]\([^)]+\)', txt))
+    markdown_count += len(re.findall('(?u)^|[^!]\[.*?\](\[|\()', txt))
 
     # Check for textile
     # Headings
@@ -231,9 +239,9 @@ def detect_formatting_type(txt):
     # Block quote.
     textile_count += len(re.findall(r'(?mu)^bq\.', txt))
     # Images
-    textile_count += len(re.findall(r'\![^\s]+(?=.*?/)(:[^\s]+)*', txt))
+    textile_count += len(re.findall(r'(?mu)(?<=\!)\S+(?=\!)', txt))
     # Links
-    textile_count += len(re.findall(r'"(?=".*?\()(\(.+?\))*[^\(]+?(\(.+?\))*":[^\s]+', txt))
+    textile_count += len(re.findall(r'"[^"]*":\S+', txt))
 
     # Decide if either markdown or textile is used in the text
     # based on the number of unique formatting elements found.

src/calibre/ebooks/txt/textileml.py

@@ -36,13 +36,12 @@ class TextileMLizer(object):
             html = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
 
             if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
             if not self.opts.keep_image_references:
                 html = re.sub(r'<\s*img[^>]*>', '', html)
-                html = re.sub(r'<\s*img\s*>', '', html)
 
             text = html2textile(html)
+            text = text.replace('%', '')
 
             # Ensure the section ends with at least two new line characters.
             # This is to prevent the last paragraph from a section being

src/calibre/gui2/__init__.py

@@ -50,6 +50,7 @@ gprefs.defaults['action-layout-context-menu-device'] = (
 
 gprefs.defaults['show_splash_screen'] = True
 gprefs.defaults['toolbar_icon_size'] = 'medium'
+gprefs.defaults['automerge'] = 'ignore'
 gprefs.defaults['toolbar_text'] = 'auto'
 gprefs.defaults['show_child_bar'] = False
 gprefs.defaults['font'] = None

src/calibre/gui2/actions/add.py

@@ -244,8 +244,8 @@ class AddAction(InterfaceAction):
                     x.decode(preferred_encoding, 'replace') for x in
                     self._adder.merged_books])
             info_dialog(self.gui, _('Merged some books'),
-                    _('Some duplicates were found and merged into the '
+                    _('The following duplicate books were found and incoming book formats were '
-                        'following existing books:'), det_msg=books, show=True)
+                        'processed and merged into your Calibre database according to your automerge settings:'), det_msg=books, show=True)
         if getattr(self._adder, 'critical', None):
             det_msg = []
             for name, log in self._adder.critical.items():

src/calibre/gui2/add.py

@@ -8,7 +8,7 @@ from functools import partial
 from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer
 
 from calibre.gui2.dialogs.progress import ProgressDialog
-from calibre.gui2 import question_dialog, error_dialog, info_dialog
+from calibre.gui2 import question_dialog, error_dialog, info_dialog, gprefs
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.metadata import MetaInformation
 from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG
@@ -179,23 +179,47 @@ class DBAdder(QObject): # {{{
                     cover = f.read()
             orig_formats = formats
             formats = [f for f in formats if not f.lower().endswith('.opf')]
-            if prefs['add_formats_to_existing']:
+            if prefs['add_formats_to_existing']: #automerge is on
                 identical_book_list = self.db.find_identical_books(mi)
-
+                if identical_book_list:  # books with same author and nearly same title exist in db
-                if identical_book_list: # books with same author and nearly same title exist in db
                     self.merged_books.add(mi.title)
+                    seen_fmts = set([])
+
                     for identical_book in identical_book_list:
-                        self.add_formats(identical_book, formats, replace=False)
+                        ib_fmts = self.db.formats(identical_book, index_is_id=True)
+                        if ib_fmts:
+                            seen_fmts |= set(ib_fmts.split(','))
+                        replace = gprefs['automerge'] == 'overwrite'
+                        self.add_formats(identical_book, formats,
+                                replace=replace)
+                    if gprefs['automerge'] == 'new record':
+                        incoming_fmts = \
+                            set([os.path.splitext(path)[-1].replace('.',
+                                '').upper() for path in formats])
+                        if incoming_fmts.intersection(seen_fmts):
+                            # There was at least one duplicate format
+                            # so create a new record and put the
+                            # incoming formats into it
+                            # We should arguably put only the duplicate
+                            # formats, but no real harm is done by having
+                            # all formats
+                            id_ = self.db.create_book_entry(mi, cover=cover,
+                                    add_duplicates=True)
+                            self.number_of_books_added += 1
+                            self.add_formats(id_, formats)
+
                 else:
-                    id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
+                    # books with same author and nearly same title do not exist in db
+                    id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
                     self.number_of_books_added += 1
-                    self.add_formats(id, formats)
+                    self.add_formats(id_, formats)
-            else:
+
-                id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
+            else: #automerge is off
-                if id is None:
+                id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
+                if id_ is None:
                     self.duplicates.append((mi, cover, orig_formats))
                 else:
-                    self.add_formats(id, formats)
+                    self.add_formats(id_, formats)
                     self.number_of_books_added += 1
         else:
             self.names.append(name)

src/calibre/gui2/dialogs/metadata_single.py

@@ -616,6 +616,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         self.original_series_name = unicode(self.series.text()).strip()
         if len(db.custom_column_label_map) == 0:
             self.central_widget.tabBar().setVisible(False)
+            self.central_widget.setTabEnabled(1, False)
         else:
             self.create_custom_column_editors()
         self.generate_cover_button.clicked.connect(self.generate_cover)
@@ -780,8 +781,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                     _('You have changed the tags. In order to use the tags'
                        ' editor, you must either discard or apply these '
                        'changes. Apply changes?'), show_copy_button=False):
-                self.books_to_refresh |= self.apply_tags(commit=True, notify=True,
+                self.books_to_refresh |= self.apply_tags(commit=True,
-                                                         allow_case_change=True)
+                        notify=True)
                 self.original_tags = unicode(self.tags.text())
             else:
                 self.tags.setText(self.original_tags)

src/calibre/gui2/dialogs/tag_list_editor.py

@@ -1,7 +1,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-from PyQt4.QtCore import SIGNAL, Qt
+from PyQt4.QtCore import Qt, QString
 from PyQt4.QtGui import QDialog, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
@@ -11,30 +11,38 @@ class ListWidgetItem(QListWidgetItem):
 
     def __init__(self, txt):
         QListWidgetItem.__init__(self, txt)
-        self.old_value = txt
+        self.initial_value = QString(txt)
-        self.cur_value = txt
+        self.current_value = QString(txt)
+        self.previous_value = QString(txt)
 
     def data(self, role):
         if role == Qt.DisplayRole:
-            if self.old_value != self.cur_value:
+            if self.initial_value != self.current_value:
-                return _('%s (was %s)')%(self.cur_value, self.old_value)
+                return _('%s (was %s)')%(self.current_value, self.initial_value)
             else:
-                return self.cur_value
+                return self.current_value
         elif role == Qt.EditRole:
-            return self.cur_value
+            return self.current_value
         else:
             return QListWidgetItem.data(self, role)
 
     def setData(self, role, data):
         if role == Qt.EditRole:
-            self.cur_value = data.toString()
+            self.previous_value = self.current_value
+            self.current_value = data.toString()
         QListWidgetItem.setData(self, role, data)
 
     def text(self):
-        return self.cur_value
+        return self.current_value
+
+    def initial_text(self):
+        return self.initial_value
+
+    def previous_text(self):
+        return self.previous_value
 
     def setText(self, txt):
-        self.cur_value = txt
+        self.current_value = txt
         QListWidgetItem.setText(txt)
 
 class TagListEditor(QDialog, Ui_TagListEditor):
@@ -49,7 +57,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         self.setWindowIcon(icon)
 
         self.to_rename = {}
-        self.to_delete = []
+        self.to_delete = set([])
         self.all_tags = {}
 
         for k,v in data:
@@ -57,6 +65,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         for tag in sorted(self.all_tags.keys(), key=key):
             item = ListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
+            item.setFlags (item.flags() | Qt.ItemIsEditable)
             self.available_tags.addItem(item)
 
         if tag_to_match is not None:
@@ -64,23 +73,20 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             if len(items) == 1:
                 self.available_tags.setCurrentItem(items[0])
 
-        self.connect(self.delete_button,  SIGNAL('clicked()'), self.delete_tags)
+        self.delete_button.clicked.connect(self.delete_tags)
-        self.connect(self.rename_button,  SIGNAL('clicked()'), self.rename_tag)
+        self.rename_button.clicked.connect(self.rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemDoubleClicked(QListWidgetItem *)'), self._rename_tag)
+        self.available_tags.itemDoubleClicked.connect(self._rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemChanged(QListWidgetItem *)'), self.finish_editing)
+        self.available_tags.itemChanged.connect(self.finish_editing)
 
     def finish_editing(self, item):
         if not item.text():
                 error_dialog(self, _('Item is blank'),
                              _('An item cannot be set to nothing. Delete it instead.')).exec_()
-                item.setText(self.item_before_editing.text())
+                item.setText(item.previous_text())
                 return
-        if item.text() != self.item_before_editing.text():
+        if item.text() != item.initial_text():
-            (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
+            id_ = item.data(Qt.UserRole).toInt()[0]
-            if item.text() not in self.to_rename:
+            self.to_rename[id_] = unicode(item.text())
-                self.to_rename[item.text()] = [id]
-            else:
-                self.to_rename[item.text()].append(id)
 
     def rename_tag(self):
         item = self.available_tags.currentItem()
@@ -91,8 +97,6 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             error_dialog(self, _('No item selected'),
                          _('You must select one item from the list of Available items.')).exec_()
             return
-        self.item_before_editing = item.clone()
-        item.setFlags (item.flags() | Qt.ItemIsEditable);
         self.available_tags.editItem(item)
 
     def delete_tags(self, item=None):
@@ -108,7 +112,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         row = self.available_tags.row(deletes[0])
         for item in deletes:
             (id,ign) = item.data(Qt.UserRole).toInt()
-            self.to_delete.append(id)
+            self.to_delete.add(id)
             self.available_tags.takeItem(self.available_tags.row(item))
 
         if row >= self.available_tags.count():

src/calibre/gui2/metadata/single.py

@@ -197,7 +197,7 @@ class MetadataSingleDialogBase(ResizableDialog):
         self.books_to_refresh = set([])
         for widget in self.basic_metadata_widgets:
             widget.initialize(self.db, id_)
-        for widget in self.custom_metadata_widgets:
+        for widget in getattr(self, 'custom_metadata_widgets', []):
             widget.initialize(id_)
         # Commented out as it doesn't play nice with Next, Prev buttons
         #self.fetch_metadata_button.setFocus(Qt.OtherFocusReason)

src/calibre/gui2/preferences/adding.py

@@ -12,6 +12,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
 from calibre.gui2.preferences.adding_ui import Ui_Form
 from calibre.utils.config import prefs
 from calibre.gui2.widgets import FilenamePattern
+from calibre.gui2 import gprefs
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
@@ -23,18 +24,23 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('read_file_metadata', prefs)
         r('swap_author_names', prefs)
         r('add_formats_to_existing', prefs)
+        choices = [
+                (_('Ignore duplicate incoming formats'), 'ignore'),
+                (_('Overwrite existing duplicate formats'), 'overwrite'),
+                (_('Create new record for each duplicate format'), 'new record')]
+        r('automerge', gprefs, choices=choices)
         r('new_book_tags', prefs, setting=CommaSeparatedList)
 
         self.filename_pattern = FilenamePattern(self)
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)
         self.filename_pattern.changed_signal.connect(self.changed_signal.emit)
 
-
     def initialize(self):
         ConfigWidgetBase.initialize(self)
         self.filename_pattern.blockSignals(True)
         self.filename_pattern.initialize()
         self.filename_pattern.blockSignals(False)
+        self.opt_automerge.setEnabled(self.opt_add_formats_to_existing.isChecked())
 
     def restore_defaults(self):
         ConfigWidgetBase.restore_defaults(self)

src/calibre/gui2/preferences/adding.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>750</width>
+    <width>753</width>
     <height>339</height>
    </rect>
   </property>
@@ -58,16 +58,33 @@
      </item>
     </layout>
    </item>
-   <item row="2" column="0" colspan="2">
+   <item row="2" column="0">
     <widget class="QCheckBox" name="opt_add_formats_to_existing">
      <property name="toolTip">
-      <string>If an existing book with a similar title and author is found that does not have the format being added, the format is added
+      <string>Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
-to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored.
+existing book records. The box to the right controls what happens when an existing record already has
+the incoming format. Note that this option also affects the Copy to library action.
 
 Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc. Author match is exact.</string>
      </property>
      <property name="text">
-      <string>If books with similar titles and authors found, &amp;merge the new files automatically</string>
+      <string>&amp;Automerge added books if they already exist in the calibre library:</string>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="1">
+    <widget class="QComboBox" name="opt_automerge">
+     <property name="toolTip">
+      <string>Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+existing book records. This box controls what happens when an existing record already has
+the incoming format: 
+
+Ignore duplicate incoming files - means that existing files in your calibre library will not be replaced
+Overwrite existing duplicate files - means that existing files in your calibre library will be replaced
+Create new record for each duplicate file - means that a new book entry will be created for each duplicate file
+
+Title matching ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc.
+Author matching is exact.</string>
      </property>
     </widget>
    </item>
@@ -113,5 +130,22 @@ Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;,
   </layout>
  </widget>
  <resources/>
- <connections/>
+ <connections>
+  <connection>
+   <sender>opt_add_formats_to_existing</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>opt_automerge</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>406</x>
+     <y>83</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>457</x>
+     <y>83</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
 </ui>

src/calibre/gui2/preferences/plugins.py

@@ -329,7 +329,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     return error_dialog(self, _('Must restart'),
                             _('You must restart calibre before you can'
                                 ' configure the <b>%s</b> plugin')%plugin.name, show=True)
-                if plugin.do_user_config():
+                if plugin.do_user_config(self.gui):
                     self._plugin_model.refresh_plugin(plugin)
             elif op == 'remove':
                 msg = _('Plugin <b>{0}</b> successfully removed').format(plugin.name)

src/calibre/gui2/tag_view.py

@@ -1259,9 +1259,8 @@ class TagBrowserMixin(object): # {{{
             if rename_func:
                 for item in to_delete:
                     delete_func(item)
-                for text in to_rename:
+                for old_id in to_rename:
-                        for old_id in to_rename[text]:
+                    rename_func(old_id, new_name=unicode(to_rename[old_id]))
-                            rename_func(old_id, new_name=unicode(text))
 
             # Clean up the library view
             self.do_tag_item_renamed()

src/calibre/library/server/content.py

@@ -124,8 +124,7 @@ class ContentServer(object):
             cherrypy.request.headers.get('Want-OPDS-Catalog', 919) != 919 or \
             ua.startswith('Stanza')
 
-        # A better search would be great
+        want_mobile = self.is_mobile_browser(ua)
-        want_mobile = self.MOBILE_UA.search(ua) is not None
         if self.opts.develop and not want_mobile:
             cherrypy.log('User agent: '+ua)
 

src/calibre/library/server/mobile.py

@@ -169,6 +169,10 @@ class MobileServer(object):
 
     MOBILE_UA = re.compile('(?i)(?:iPhone|Opera Mini|NetFront|webOS|Mobile|Android|imode|DoCoMo|Minimo|Blackberry|MIDP|Symbian|HD2|Kindle)')
 
+    def is_mobile_browser(self, ua):
+        match = self.MOBILE_UA.search(ua)
+        return match is not None and 'iPad' not in ua
+
     def add_routes(self, connect):
         connect('mobile', '/mobile', self.mobile)
         connect('mobile_css', '/mobile/style.css', self.mobile_css)

src/calibre/manual/faq.rst

@@ -316,6 +316,27 @@ When you first run |app|, it will ask you for a folder in which to store your bo
 
 Metadata about the books is stored in the file ``metadata.db`` at the top level of the library folder This file is is a sqlite database. When backing up your library make sure you copy the entire folder and all its sub-folders.
 
+How does |app| manage author names and sorting?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Author names are complex, especially across cultures. |app| has a very flexible strategy for managing author names. The first thing to understand is that books and authors are separate entities in |app|. A book can have more than one author, and an author can have more than one book. You can manage the authors of a book by the edit metadata dialog. You can manage individual authors by right clicking on the author in the Tag Browser on the left of the main |app| screen and selecting :guilabel:`Manage authors`. Using this dialog you can change the name of an author and also how that name is sorted. This will automatically change the name of the author in all the books of that author. When a book has multiple authors, separate their names using the & character.
+
+Now coming to author name sorting:
+
+    * When a new author is added to |app| (this happens whenever a book by a new author is added), |app| automatically computes a sort string for both the book and the author.
+    * Authors in the Tag Browser are sorted by the sort value for the **authors**. Remember that this is different from the Author sort field for a book. 
+    * By default, this sort algorithm assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort value.
+    * You can change this algorithm by going to Preferences->Tweaks and setting the :guilabel:`author_sort_copy_method` tweak.
+    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`, then pushing the `Recalculate all author sort values` button. Do this after you have set the author_sort_copy_method tweak to what you want.
+    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata, check the `Automatically set author sort` checkbox, then press OK.)
+    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author. Therefore, ensure that the individual author sort values are correct before recalculating the books' author sort values.
+    * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
+
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values for both authors and books as described above.
+
+Note that you can set an individual author's sort value to whatever you want using :guilabel:`Manage authors`. This is useful when dealing with names that |app| will not get right, such as complex multi-part names like Miguel de Cervantes Saavedra or when dealing with Asian names like Sun Tzu.
+
+
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

src/calibre/utils/localization.py

@@ -104,6 +104,7 @@ _extra_lang_codes = {
         'en_IN' : _('English (India)'),
         'en_TH' : _('English (Thailand)'),
         'en_CY' : _('English (Cyprus)'),
+        'en_CZ' : _('English (Czechoslovakia)'),
         'en_PK' : _('English (Pakistan)'),
         'en_HR' : _('English (Croatia)'),
         'en_IL' : _('English (Israel)'),