diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index f1abfbe7ea..893c8b6b6a 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -39,7 +39,7 @@ completer_append_separator = False # The algorithm used to copy author to author_sort # Possible values are: -# invert: use "fn ln" -> "ln, fn" (the original algorithm) +# invert: use "fn ln" -> "ln, fn" (the default algorithm) # copy : copy author to author_sort without modification # comma : use 'copy' if there is a ',' in the name, otherwise use 'invert' # nocomma : "fn ln" -> "ln fn" (without the comma) diff --git a/resources/recipes/new_yorker.recipe b/resources/recipes/new_yorker.recipe index d69a4df24f..9eeb8b31ee 100644 --- a/resources/recipes/new_yorker.recipe +++ b/resources/recipes/new_yorker.recipe @@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe): ,dict(attrs={'id':['show-header','show-footer'] }) ] remove_attributes = ['lang'] - feeds = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')] + feeds = [(u'The New Yorker', u'http://www.newyorker.com/services/mrss/feeds/everything.xml')] def print_version(self, url): - return 'http://www.newyorker.com' + url + '?printable=true' + return url + '?printable=true' def image_url_processor(self, baseurl, url): return url.strip() diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index e0367515bc..32c512fe39 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -325,6 +325,17 @@ class TXTMetadataReader(MetadataReaderPlugin): from calibre.ebooks.metadata.txt import get_metadata return get_metadata(stream) +class TXTZMetadataReader(MetadataReaderPlugin): + + name = 'Read TXTZ metadata' + file_types = set(['txtz']) + description = _('Read metadata from %s files') % 'TXTZ' + author = 'John Schember' + + def get_metadata(self, stream, ftype): + from calibre.ebooks.metadata.txtz import get_metadata + return get_metadata(stream) + class ZipMetadataReader(MetadataReaderPlugin): name = 'Read ZIP metadata' @@ -412,6 +423,17 @@ class TOPAZMetadataWriter(MetadataWriterPlugin): from calibre.ebooks.metadata.topaz import set_metadata set_metadata(stream, mi) +class TXTZMetadataWriter(MetadataWriterPlugin): + + name = 'Set TXTZ metadata' + file_types = set(['txtz']) + description = _('Set metadata from %s files') % 'TXTZ' + author = 'John Schember' + + def set_metadata(self, stream, mi, type): + from calibre.ebooks.metadata.txtz import set_metadata + set_metadata(stream, mi) + # }}} from calibre.ebooks.comic.input import ComicInput @@ -446,6 +468,7 @@ from calibre.ebooks.rb.output import RBOutput from calibre.ebooks.rtf.output import RTFOutput from calibre.ebooks.tcr.output import TCROutput from calibre.ebooks.txt.output import TXTOutput +from calibre.ebooks.txt.output import TXTZOutput from calibre.ebooks.html.output import HTMLOutput from calibre.ebooks.snb.output import SNBOutput @@ -531,6 +554,7 @@ plugins += [ RTFOutput, TCROutput, TXTOutput, + TXTZOutput, HTMLOutput, SNBOutput, ] diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 5912e40a69..11d636791b 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -19,10 +19,15 @@ class ANDROID(USBMS): VENDOR_ID = { # HTC - 0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100, - 0x0227, 0x0226], 0x0ff9 - : [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226], - 0xc92 : [0x100], 0xc97: [0x226], 0xc99 : [0x0100]}, + 0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], + 0x0c01 : [0x100, 0x0227, 0x0226], + 0x0ff9 : [0x0100, 0x0227, 0x0226], + 0x0c87 : [0x0100, 0x0227, 0x0226], + 0xc92 : [0x100], + 0xc97 : [0x226], + 0xc99 : [0x0100], + 0xca3 : [0x100], + }, # Eken 0x040d : { 0x8510 : [0x0001], 0x0851 : [0x1] }, diff --git a/src/calibre/devices/eb600/driver.py b/src/calibre/devices/eb600/driver.py index e38f72aea5..5374c6c4e2 100644 --- a/src/calibre/devices/eb600/driver.py +++ b/src/calibre/devices/eb600/driver.py @@ -172,10 +172,10 @@ class INVESBOOK(EB600): gui_name = 'Inves Book 600' FORMATS = ['epub', 'mobi', 'prc', 'fb2', 'html', 'pdf', 'rtf', 'txt'] + BCD = [0x110, 0x323] - VENDOR_NAME = 'INVES_E6' - WINDOWS_MAIN_MEM = '00INVES_E600' - WINDOWS_CARD_A_MEM = '00INVES_E600' + VENDOR_NAME = ['INVES_E6', 'INVES-WI'] + WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['00INVES_E600', 'INVES-WIBOOK'] class BOOQ(EB600): name = 'Booq Device Interface' diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py index 4dc97f43ed..49604ae682 100644 --- a/src/calibre/ebooks/__init__.py +++ b/src/calibre/ebooks/__init__.py @@ -25,7 +25,7 @@ class DRMError(ValueError): class ParserError(ValueError): pass -BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'htm', 'xhtm', +BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'htm', 'xhtm', 'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc', 'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip', 'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'mbp', 'tan', 'snb'] diff --git a/src/calibre/ebooks/metadata/txt.py b/src/calibre/ebooks/metadata/txt.py index 79713774e3..70d3c72ae0 100644 --- a/src/calibre/ebooks/metadata/txt.py +++ b/src/calibre/ebooks/metadata/txt.py @@ -1,16 +1,20 @@ -'''Read meta information from TXT files''' - -from __future__ import with_statement +# -*- coding: utf-8 -*- __license__ = 'GPL v3' __copyright__ = '2009, John Schember ' +''' +Read meta information from TXT files +''' + import re from calibre.ebooks.metadata import MetaInformation def get_metadata(stream, extract_cover=True): - """ Return metadata as a L{MetaInfo} object """ + ''' + Return metadata as a L{MetaInfo} object + ''' mi = MetaInformation(_('Unknown'), [_('Unknown')]) stream.seek(0) diff --git a/src/calibre/ebooks/metadata/txtz.py b/src/calibre/ebooks/metadata/txtz.py new file mode 100644 index 0000000000..ae6efb4838 --- /dev/null +++ b/src/calibre/ebooks/metadata/txtz.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL v3' +__copyright__ = '2011, John Schember ' + +''' +Read meta information from TXT files +''' + +import os + +from cStringIO import StringIO + +from calibre.ebooks.metadata import MetaInformation +from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf +from calibre.ptempfile import TemporaryDirectory +from calibre.utils.zipfile import ZipFile, safe_replace + +def get_metadata(stream, extract_cover=True): + ''' + Return metadata as a L{MetaInfo} object + ''' + mi = MetaInformation(_('Unknown'), [_('Unknown')]) + stream.seek(0) + + with TemporaryDirectory('_untxtz_mdata') as tdir: + try: + zf = ZipFile(stream) + zf.extract('metadata.opf', tdir) + with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff: + mi = OPF(opff).to_book_metadata() + except: + return mi + return mi + +def set_metadata(stream, mi): + opf = StringIO(metadata_to_opf(mi)) + safe_replace(stream, 'metadata.opf', opf) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index e240205222..8ab1524b02 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -4,23 +4,27 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' +import glob import os +from calibre import _ent_pat, xml_entity_to_unicode from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator from calibre.ebooks.chardet import detect from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \ separate_paragraphs_single_line, separate_paragraphs_print_formatted, \ preserve_spaces, detect_paragraph_type, detect_formatting_type, \ - normalize_line_endings, convert_textile, remove_indents, block_to_single_line -from calibre import _ent_pat, xml_entity_to_unicode + normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \ + separate_hard_scene_breaks +from calibre.ptempfile import TemporaryDirectory +from calibre.utils.zipfile import ZipFile class TXTInput(InputFormatPlugin): name = 'TXT Input' author = 'John Schember' description = 'Convert TXT files to HTML' - file_types = set(['txt']) + file_types = set(['txt', 'txtz']) options = set([ OptionRecommendation(name='paragraph_type', recommended_value='auto', @@ -57,10 +61,23 @@ class TXTInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): self.log = log + txt = '' log.debug('Reading text from file...') length = 0 - txt = stream.read() + # Extract content from zip archive. + if file_ext == 'txtz': + log.debug('De-compressing content to temporary directory...') + with TemporaryDirectory('_untxtz') as tdir: + zf = ZipFile(stream) + zf.extractall(tdir) + + txts = glob.glob(os.path.join(tdir, '*.txt')) + for t in txts: + with open(t, 'rb') as tf: + txt += tf.read() + else: + txt = stream.read() # Get the encoding of the document. if options.input_encoding: @@ -98,6 +115,7 @@ class TXTInput(InputFormatPlugin): if options.formatting_type == 'heuristic': setattr(options, 'enable_heuristics', True) setattr(options, 'unwrap_lines', False) + setattr(options, 'smarten_punctuation', True) # Reformat paragraphs to block formatting based on the detected type. # We don't check for block because the processor assumes block. @@ -105,6 +123,7 @@ class TXTInput(InputFormatPlugin): if options.paragraph_type == 'single': txt = separate_paragraphs_single_line(txt) elif options.paragraph_type == 'print': + txt = separate_hard_scene_breaks(txt) txt = separate_paragraphs_print_formatted(txt) txt = block_to_single_line(txt) elif options.paragraph_type == 'unformatted': @@ -116,6 +135,7 @@ class TXTInput(InputFormatPlugin): txt = preprocessor.punctuation_unwrap(length, txt, 'txt') txt = separate_paragraphs_single_line(txt) else: + txt = separate_hard_scene_breaks(txt) txt = block_to_single_line(txt) if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): @@ -175,4 +195,11 @@ class TXTInput(InputFormatPlugin): {}) options.debug_pipeline = odi os.remove(htmlfile.name) + + # Set metadata from file. + from calibre.customize.ui import get_file_type_metadata + from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata + mi = get_file_type_metadata(stream, file_ext) + meta_info_to_oeb_metadata(mi, oeb.metadata, log) + return oeb diff --git a/src/calibre/ebooks/txt/markdownml.py b/src/calibre/ebooks/txt/markdownml.py index 116561f355..c179378049 100644 --- a/src/calibre/ebooks/txt/markdownml.py +++ b/src/calibre/ebooks/txt/markdownml.py @@ -35,11 +35,9 @@ class MarkdownMLizer(object): html = unicode(etree.tostring(item.data, encoding=unicode)) if not self.opts.keep_links: - html = re.sub(r'<\s*a[^>]*>', '', html) - html = re.sub(r'<\s*/\s*a\s*>', '', html) + html = re.sub(r'<\s*/*\s*a[^>]*>', '', html) if not self.opts.keep_image_references: - html = re.sub(r'<\s*img[^>]*>', '', html) - html = re.sub(r'<\s*img\s*>', '', html) + html = re.sub(r'<\s*img[^>]*>', '', html)\ text = html2text(html) diff --git a/src/calibre/ebooks/txt/output.py b/src/calibre/ebooks/txt/output.py index b73a6e8908..d021cbbba6 100644 --- a/src/calibre/ebooks/txt/output.py +++ b/src/calibre/ebooks/txt/output.py @@ -5,11 +5,18 @@ __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' import os +import shutil + +from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation +from calibre.ebooks.oeb.base import OEB_IMAGES from calibre.ebooks.txt.txtml import TXTMLizer from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines +from calibre.ptempfile import TemporaryDirectory, TemporaryFile +from calibre.utils.cleantext import clean_ascii_chars +from calibre.utils.zipfile import ZipFile class TXTOutput(OutputFormatPlugin): @@ -73,6 +80,7 @@ class TXTOutput(OutputFormatPlugin): writer = TXTMLizer(log) txt = writer.extract_content(oeb_book, opts) + txt = clean_ascii_chars(txt) log.debug('\tReplacing newlines with selected type...') txt = specified_newlines(TxtNewlines(opts.newline).newline, txt) @@ -93,3 +101,32 @@ class TXTOutput(OutputFormatPlugin): if close: out_stream.close() + +class TXTZOutput(TXTOutput): + + name = 'TXTZ Output' + author = 'John Schember' + file_type = 'txtz' + + def convert(self, oeb_book, output_path, input_plugin, opts, log): + with TemporaryDirectory('_txtz_output') as tdir: + # TXT + with TemporaryFile('index.txt') as tf: + TXTOutput.convert(self, oeb_book, tf, input_plugin, opts, log) + shutil.copy(tf, os.path.join(tdir, 'index.txt')) + + # Images + for item in oeb_book.manifest: + if item.media_type in OEB_IMAGES: + path = os.path.join(tdir, os.path.dirname(item.href)) + if not os.path.exists(path): + os.makedirs(path) + with open(os.path.join(tdir, item.href), 'wb') as imgf: + imgf.write(item.data) + + # Metadata + with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: + mdataf.write(etree.tostring(oeb_book.metadata.to_opf1())) + + txtz = ZipFile(output_path, 'w') + txtz.add_dir(tdir) diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index f7b6cce234..55213381c9 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -29,8 +29,7 @@ def clean_txt(txt): txt = '\n'.join([line.rstrip() for line in txt.splitlines()]) # Replace whitespace at the beginning of the line with   - txt = re.sub('(?m)(?P^[ ]+)(?=.)', lambda mo: ' ' * mo.groups('space').count(' '), txt) - txt = re.sub('(?m)(?P^[\t]+)(?=.)', lambda mo: ' ' * 4 * mo.groups('space').count('\t'), txt) + txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', ' ' * 4, txt) # Condense redundant spaces txt = re.sub('[ ]{2,}', ' ', txt) @@ -121,6 +120,15 @@ def separate_paragraphs_print_formatted(txt): txt = re.sub(u'(?miu)^(?P\t+|[ ]{2,})(?=.)', lambda mo: '\n%s' % mo.group('indent'), txt) return txt +def separate_hard_scene_breaks(txt): + def sep_break(line): + if len(line.strip()) > 0: + return '\n%s\n' % line + else: + return line + txt = re.sub(u'(?miu)^[ \t-=~\/]+$', lambda mo: sep_break(mo.group()), txt) + return txt + def block_to_single_line(txt): txt = re.sub(r'(?<=.)\n(?=.)', ' ', txt) return txt @@ -221,9 +229,9 @@ def detect_formatting_type(txt): markdown_count += len(re.findall('(?mu)^=+$', txt)) markdown_count += len(re.findall('(?mu)^-+$', txt)) # Images - markdown_count += len(re.findall('(?u)!\[.*?\]\(.+?\)', txt)) + markdown_count += len(re.findall('(?u)!\[.*?\](\[|\()', txt)) # Links - markdown_count += len(re.findall('(?u)(^|(?P
[^!]))\[.*?\]\([^)]+\)', txt))
+    markdown_count += len(re.findall('(?u)^|[^!]\[.*?\](\[|\()', txt))
 
     # Check for textile
     # Headings
@@ -231,9 +239,9 @@ def detect_formatting_type(txt):
     # Block quote.
     textile_count += len(re.findall(r'(?mu)^bq\.', txt))
     # Images
-    textile_count += len(re.findall(r'\![^\s]+(?=.*?/)(:[^\s]+)*', txt))
+    textile_count += len(re.findall(r'(?mu)(?<=\!)\S+(?=\!)', txt))
     # Links
-    textile_count += len(re.findall(r'"(?=".*?\()(\(.+?\))*[^\(]+?(\(.+?\))*":[^\s]+', txt))
+    textile_count += len(re.findall(r'"[^"]*":\S+', txt))
 
     # Decide if either markdown or textile is used in the text
     # based on the number of unique formatting elements found.
diff --git a/src/calibre/ebooks/txt/textileml.py b/src/calibre/ebooks/txt/textileml.py
index 94834d8e79..284e4846d9 100644
--- a/src/calibre/ebooks/txt/textileml.py
+++ b/src/calibre/ebooks/txt/textileml.py
@@ -36,13 +36,12 @@ class TextileMLizer(object):
             html = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
 
             if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
             if not self.opts.keep_image_references:
                 html = re.sub(r'<\s*img[^>]*>', '', html)
-                html = re.sub(r'<\s*img\s*>', '', html)
 
             text = html2textile(html)
+            text = text.replace('%', '')
 
             # Ensure the section ends with at least two new line characters.
             # This is to prevent the last paragraph from a section being
diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index 9150172fc1..92a68fa840 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -50,6 +50,7 @@ gprefs.defaults['action-layout-context-menu-device'] = (
 
 gprefs.defaults['show_splash_screen'] = True
 gprefs.defaults['toolbar_icon_size'] = 'medium'
+gprefs.defaults['automerge'] = 'ignore'
 gprefs.defaults['toolbar_text'] = 'auto'
 gprefs.defaults['show_child_bar'] = False
 gprefs.defaults['font'] = None
diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py
index 4236a63340..25127d3635 100644
--- a/src/calibre/gui2/actions/add.py
+++ b/src/calibre/gui2/actions/add.py
@@ -244,8 +244,8 @@ class AddAction(InterfaceAction):
                     x.decode(preferred_encoding, 'replace') for x in
                     self._adder.merged_books])
             info_dialog(self.gui, _('Merged some books'),
-                    _('Some duplicates were found and merged into the '
-                        'following existing books:'), det_msg=books, show=True)
+                    _('The following duplicate books were found and incoming book formats were '
+                        'processed and merged into your Calibre database according to your automerge settings:'), det_msg=books, show=True)
         if getattr(self._adder, 'critical', None):
             det_msg = []
             for name, log in self._adder.critical.items():
diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py
index 026fabea07..f40cf0ff75 100644
--- a/src/calibre/gui2/add.py
+++ b/src/calibre/gui2/add.py
@@ -8,7 +8,7 @@ from functools import partial
 from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer
 
 from calibre.gui2.dialogs.progress import ProgressDialog
-from calibre.gui2 import question_dialog, error_dialog, info_dialog
+from calibre.gui2 import question_dialog, error_dialog, info_dialog, gprefs
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.metadata import MetaInformation
 from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG
@@ -179,23 +179,47 @@ class DBAdder(QObject): # {{{
                     cover = f.read()
             orig_formats = formats
             formats = [f for f in formats if not f.lower().endswith('.opf')]
-            if prefs['add_formats_to_existing']:
+            if prefs['add_formats_to_existing']: #automerge is on
                 identical_book_list = self.db.find_identical_books(mi)
-
-                if identical_book_list: # books with same author and nearly same title exist in db
+                if identical_book_list:  # books with same author and nearly same title exist in db
                     self.merged_books.add(mi.title)
+                    seen_fmts = set([])
+
                     for identical_book in identical_book_list:
-                        self.add_formats(identical_book, formats, replace=False)
+                        ib_fmts = self.db.formats(identical_book, index_is_id=True)
+                        if ib_fmts:
+                            seen_fmts |= set(ib_fmts.split(','))
+                        replace = gprefs['automerge'] == 'overwrite'
+                        self.add_formats(identical_book, formats,
+                                replace=replace)
+                    if gprefs['automerge'] == 'new record':
+                        incoming_fmts = \
+                            set([os.path.splitext(path)[-1].replace('.',
+                                '').upper() for path in formats])
+                        if incoming_fmts.intersection(seen_fmts):
+                            # There was at least one duplicate format
+                            # so create a new record and put the
+                            # incoming formats into it
+                            # We should arguably put only the duplicate
+                            # formats, but no real harm is done by having
+                            # all formats
+                            id_ = self.db.create_book_entry(mi, cover=cover,
+                                    add_duplicates=True)
+                            self.number_of_books_added += 1
+                            self.add_formats(id_, formats)
+
                 else:
-                    id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
+                    # books with same author and nearly same title do not exist in db
+                    id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
                     self.number_of_books_added += 1
-                    self.add_formats(id, formats)
-            else:
-                id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
-                if id is None:
+                    self.add_formats(id_, formats)
+
+            else: #automerge is off
+                id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
+                if id_ is None:
                     self.duplicates.append((mi, cover, orig_formats))
                 else:
-                    self.add_formats(id, formats)
+                    self.add_formats(id_, formats)
                     self.number_of_books_added += 1
         else:
             self.names.append(name)
diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py
index 3e711edd2d..52d263fe36 100644
--- a/src/calibre/gui2/dialogs/metadata_single.py
+++ b/src/calibre/gui2/dialogs/metadata_single.py
@@ -616,6 +616,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         self.original_series_name = unicode(self.series.text()).strip()
         if len(db.custom_column_label_map) == 0:
             self.central_widget.tabBar().setVisible(False)
+            self.central_widget.setTabEnabled(1, False)
         else:
             self.create_custom_column_editors()
         self.generate_cover_button.clicked.connect(self.generate_cover)
@@ -780,8 +781,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                     _('You have changed the tags. In order to use the tags'
                        ' editor, you must either discard or apply these '
                        'changes. Apply changes?'), show_copy_button=False):
-                self.books_to_refresh |= self.apply_tags(commit=True, notify=True,
-                                                         allow_case_change=True)
+                self.books_to_refresh |= self.apply_tags(commit=True,
+                        notify=True)
                 self.original_tags = unicode(self.tags.text())
             else:
                 self.tags.setText(self.original_tags)
diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py
index ced0e9a505..9694a9a459 100644
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@@ -1,7 +1,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal '
 
-from PyQt4.QtCore import SIGNAL, Qt
+from PyQt4.QtCore import Qt, QString
 from PyQt4.QtGui import QDialog, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
@@ -11,30 +11,38 @@ class ListWidgetItem(QListWidgetItem):
 
     def __init__(self, txt):
         QListWidgetItem.__init__(self, txt)
-        self.old_value = txt
-        self.cur_value = txt
+        self.initial_value = QString(txt)
+        self.current_value = QString(txt)
+        self.previous_value = QString(txt)
 
     def data(self, role):
         if role == Qt.DisplayRole:
-            if self.old_value != self.cur_value:
-                return _('%s (was %s)')%(self.cur_value, self.old_value)
+            if self.initial_value != self.current_value:
+                return _('%s (was %s)')%(self.current_value, self.initial_value)
             else:
-                return self.cur_value
+                return self.current_value
         elif role == Qt.EditRole:
-            return self.cur_value
+            return self.current_value
         else:
             return QListWidgetItem.data(self, role)
 
     def setData(self, role, data):
         if role == Qt.EditRole:
-            self.cur_value = data.toString()
+            self.previous_value = self.current_value
+            self.current_value = data.toString()
         QListWidgetItem.setData(self, role, data)
 
     def text(self):
-        return self.cur_value
+        return self.current_value
+
+    def initial_text(self):
+        return self.initial_value
+
+    def previous_text(self):
+        return self.previous_value
 
     def setText(self, txt):
-        self.cur_value = txt
+        self.current_value = txt
         QListWidgetItem.setText(txt)
 
 class TagListEditor(QDialog, Ui_TagListEditor):
@@ -49,7 +57,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         self.setWindowIcon(icon)
 
         self.to_rename = {}
-        self.to_delete = []
+        self.to_delete = set([])
         self.all_tags = {}
 
         for k,v in data:
@@ -57,6 +65,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         for tag in sorted(self.all_tags.keys(), key=key):
             item = ListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
+            item.setFlags (item.flags() | Qt.ItemIsEditable)
             self.available_tags.addItem(item)
 
         if tag_to_match is not None:
@@ -64,23 +73,20 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             if len(items) == 1:
                 self.available_tags.setCurrentItem(items[0])
 
-        self.connect(self.delete_button,  SIGNAL('clicked()'), self.delete_tags)
-        self.connect(self.rename_button,  SIGNAL('clicked()'), self.rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemDoubleClicked(QListWidgetItem *)'), self._rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemChanged(QListWidgetItem *)'), self.finish_editing)
+        self.delete_button.clicked.connect(self.delete_tags)
+        self.rename_button.clicked.connect(self.rename_tag)
+        self.available_tags.itemDoubleClicked.connect(self._rename_tag)
+        self.available_tags.itemChanged.connect(self.finish_editing)
 
     def finish_editing(self, item):
         if not item.text():
                 error_dialog(self, _('Item is blank'),
                              _('An item cannot be set to nothing. Delete it instead.')).exec_()
-                item.setText(self.item_before_editing.text())
+                item.setText(item.previous_text())
                 return
-        if item.text() != self.item_before_editing.text():
-            (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
-            if item.text() not in self.to_rename:
-                self.to_rename[item.text()] = [id]
-            else:
-                self.to_rename[item.text()].append(id)
+        if item.text() != item.initial_text():
+            id_ = item.data(Qt.UserRole).toInt()[0]
+            self.to_rename[id_] = unicode(item.text())
 
     def rename_tag(self):
         item = self.available_tags.currentItem()
@@ -91,8 +97,6 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             error_dialog(self, _('No item selected'),
                          _('You must select one item from the list of Available items.')).exec_()
             return
-        self.item_before_editing = item.clone()
-        item.setFlags (item.flags() | Qt.ItemIsEditable);
         self.available_tags.editItem(item)
 
     def delete_tags(self, item=None):
@@ -108,7 +112,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         row = self.available_tags.row(deletes[0])
         for item in deletes:
             (id,ign) = item.data(Qt.UserRole).toInt()
-            self.to_delete.append(id)
+            self.to_delete.add(id)
             self.available_tags.takeItem(self.available_tags.row(item))
 
         if row >= self.available_tags.count():
diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py
index 1be954155c..0fa5c746e7 100644
--- a/src/calibre/gui2/metadata/single.py
+++ b/src/calibre/gui2/metadata/single.py
@@ -197,7 +197,7 @@ class MetadataSingleDialogBase(ResizableDialog):
         self.books_to_refresh = set([])
         for widget in self.basic_metadata_widgets:
             widget.initialize(self.db, id_)
-        for widget in self.custom_metadata_widgets:
+        for widget in getattr(self, 'custom_metadata_widgets', []):
             widget.initialize(id_)
         # Commented out as it doesn't play nice with Next, Prev buttons
         #self.fetch_metadata_button.setFocus(Qt.OtherFocusReason)
diff --git a/src/calibre/gui2/preferences/adding.py b/src/calibre/gui2/preferences/adding.py
index e919d53b64..b4c4ce846a 100644
--- a/src/calibre/gui2/preferences/adding.py
+++ b/src/calibre/gui2/preferences/adding.py
@@ -12,6 +12,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
 from calibre.gui2.preferences.adding_ui import Ui_Form
 from calibre.utils.config import prefs
 from calibre.gui2.widgets import FilenamePattern
+from calibre.gui2 import gprefs
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
@@ -23,18 +24,23 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('read_file_metadata', prefs)
         r('swap_author_names', prefs)
         r('add_formats_to_existing', prefs)
+        choices = [
+                (_('Ignore duplicate incoming formats'), 'ignore'),
+                (_('Overwrite existing duplicate formats'), 'overwrite'),
+                (_('Create new record for each duplicate format'), 'new record')]
+        r('automerge', gprefs, choices=choices)
         r('new_book_tags', prefs, setting=CommaSeparatedList)
 
         self.filename_pattern = FilenamePattern(self)
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)
         self.filename_pattern.changed_signal.connect(self.changed_signal.emit)
 
-
     def initialize(self):
         ConfigWidgetBase.initialize(self)
         self.filename_pattern.blockSignals(True)
         self.filename_pattern.initialize()
         self.filename_pattern.blockSignals(False)
+        self.opt_automerge.setEnabled(self.opt_add_formats_to_existing.isChecked())
 
     def restore_defaults(self):
         ConfigWidgetBase.restore_defaults(self)
diff --git a/src/calibre/gui2/preferences/adding.ui b/src/calibre/gui2/preferences/adding.ui
index 75e6c466f0..f9a2c74444 100644
--- a/src/calibre/gui2/preferences/adding.ui
+++ b/src/calibre/gui2/preferences/adding.ui
@@ -6,7 +6,7 @@
    
     0
     0
-    750
+    753
     339
    
   
@@ -58,16 +58,33 @@
      
     
    
-   
+   
     
      
-      If an existing book with a similar title and author is found that does not have the format being added, the format is added
-to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored.
+      Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+existing book records. The box to the right controls what happens when an existing record already has
+the incoming format. Note that this option also affects the Copy to library action.
 
 Title match ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author match is exact.
      
      
-      If books with similar titles and authors found, &merge the new files automatically
+      &Automerge added books if they already exist in the calibre library:
+     
+    
+   
+   
+    
+     
+      Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+existing book records. This box controls what happens when an existing record already has
+the incoming format: 
+
+Ignore duplicate incoming files - means that existing files in your calibre library will not be replaced
+Overwrite existing duplicate files - means that existing files in your calibre library will be replaced
+Create new record for each duplicate file - means that a new book entry will be created for each duplicate file
+
+Title matching ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc.
+Author matching is exact.
      
     
    
@@ -113,5 +130,22 @@ Title match ignores leading indefinite articles ("the", "a",
   
  
  
- 
+ 
+  
+   opt_add_formats_to_existing
+   toggled(bool)
+   opt_automerge
+   setEnabled(bool)
+   
+    
+     406
+     83
+    
+    
+     457
+     83
+    
+   
+  
+ 
 
diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py
index 8f77a03c24..4b83df71c7 100644
--- a/src/calibre/gui2/preferences/plugins.py
+++ b/src/calibre/gui2/preferences/plugins.py
@@ -329,7 +329,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     return error_dialog(self, _('Must restart'),
                             _('You must restart calibre before you can'
                                 ' configure the %s plugin')%plugin.name, show=True)
-                if plugin.do_user_config():
+                if plugin.do_user_config(self.gui):
                     self._plugin_model.refresh_plugin(plugin)
             elif op == 'remove':
                 msg = _('Plugin {0} successfully removed').format(plugin.name)
diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index 041f0a715e..fd3530d333 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -1259,9 +1259,8 @@ class TagBrowserMixin(object): # {{{
             if rename_func:
                 for item in to_delete:
                     delete_func(item)
-                for text in to_rename:
-                        for old_id in to_rename[text]:
-                            rename_func(old_id, new_name=unicode(text))
+                for old_id in to_rename:
+                    rename_func(old_id, new_name=unicode(to_rename[old_id]))
 
             # Clean up the library view
             self.do_tag_item_renamed()
diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py
index 8af70d5675..11ea2b951e 100644
--- a/src/calibre/library/server/content.py
+++ b/src/calibre/library/server/content.py
@@ -124,8 +124,7 @@ class ContentServer(object):
             cherrypy.request.headers.get('Want-OPDS-Catalog', 919) != 919 or \
             ua.startswith('Stanza')
 
-        # A better search would be great
-        want_mobile = self.MOBILE_UA.search(ua) is not None
+        want_mobile = self.is_mobile_browser(ua)
         if self.opts.develop and not want_mobile:
             cherrypy.log('User agent: '+ua)
 
diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py
index 0992e6c30b..1bf9f549bc 100644
--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@@ -169,6 +169,10 @@ class MobileServer(object):
 
     MOBILE_UA = re.compile('(?i)(?:iPhone|Opera Mini|NetFront|webOS|Mobile|Android|imode|DoCoMo|Minimo|Blackberry|MIDP|Symbian|HD2|Kindle)')
 
+    def is_mobile_browser(self, ua):
+        match = self.MOBILE_UA.search(ua)
+        return match is not None and 'iPad' not in ua
+
     def add_routes(self, connect):
         connect('mobile', '/mobile', self.mobile)
         connect('mobile_css', '/mobile/style.css', self.mobile_css)
diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 18c53ade5d..cdae20ea3b 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -316,6 +316,27 @@ When you first run |app|, it will ask you for a folder in which to store your bo
 
 Metadata about the books is stored in the file ``metadata.db`` at the top level of the library folder This file is is a sqlite database. When backing up your library make sure you copy the entire folder and all its sub-folders.
 
+How does |app| manage author names and sorting?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Author names are complex, especially across cultures. |app| has a very flexible strategy for managing author names. The first thing to understand is that books and authors are separate entities in |app|. A book can have more than one author, and an author can have more than one book. You can manage the authors of a book by the edit metadata dialog. You can manage individual authors by right clicking on the author in the Tag Browser on the left of the main |app| screen and selecting :guilabel:`Manage authors`. Using this dialog you can change the name of an author and also how that name is sorted. This will automatically change the name of the author in all the books of that author. When a book has multiple authors, separate their names using the & character.
+
+Now coming to author name sorting:
+
+    * When a new author is added to |app| (this happens whenever a book by a new author is added), |app| automatically computes a sort string for both the book and the author.
+    * Authors in the Tag Browser are sorted by the sort value for the **authors**. Remember that this is different from the Author sort field for a book. 
+    * By default, this sort algorithm assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort value.
+    * You can change this algorithm by going to Preferences->Tweaks and setting the :guilabel:`author_sort_copy_method` tweak.
+    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`, then pushing the `Recalculate all author sort values` button. Do this after you have set the author_sort_copy_method tweak to what you want.
+    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata, check the `Automatically set author sort` checkbox, then press OK.)
+    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author. Therefore, ensure that the individual author sort values are correct before recalculating the books' author sort values.
+    * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
+
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values for both authors and books as described above.
+
+Note that you can set an individual author's sort value to whatever you want using :guilabel:`Manage authors`. This is useful when dealing with names that |app| will not get right, such as complex multi-part names like Miguel de Cervantes Saavedra or when dealing with Asian names like Sun Tzu.
+
+
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py
index 037a147e28..97356df081 100644
--- a/src/calibre/utils/localization.py
+++ b/src/calibre/utils/localization.py
@@ -104,6 +104,7 @@ _extra_lang_codes = {
         'en_IN' : _('English (India)'),
         'en_TH' : _('English (Thailand)'),
         'en_CY' : _('English (Cyprus)'),
+        'en_CZ' : _('English (Czechoslovakia)'),
         'en_PK' : _('English (Pakistan)'),
         'en_HR' : _('English (Croatia)'),
         'en_IL' : _('English (Israel)'),