Merge from trunk

resources/default_tweaks.py

@@ -24,6 +24,7 @@ series_index_auto_increment = 'next'
 #  invert: use "fn ln" -> "ln, fn" (the original algorithm)
 #  copy  : copy author to author_sort without modification
 #  comma : use 'copy' if there is a ',' in the name, otherwise use 'invert'
+#  nocomma : "fn ln" -> "ln fn" (without the comma)
 author_sort_copy_method = 'invert'
 
 

src/calibre/customize/builtins.py

@@ -657,9 +657,14 @@ class ActionEditCollections(InterfaceActionBase):
     name = 'Edit Collections'
     actual_plugin = 'calibre.gui2.actions.edit_collections:EditCollectionsAction'
 
+class ActionCopyToLibrary(InterfaceActionBase):
+    name = 'Copy To Library'
+    actual_plugin = 'calibre.gui2.actions.copy_to_library:CopyToLibraryAction'
+
 plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
         ActionConvert, ActionDelete, ActionEditMetadata, ActionView,
         ActionFetchNews, ActionSaveToDisk, ActionShowBookDetails,
         ActionRestart, ActionOpenFolder, ActionConnectShare,
         ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks,
-        ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary]
+        ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary,
+        ActionCopyToLibrary]

src/calibre/devices/android/driver.py

@@ -55,9 +55,9 @@ class ANDROID(USBMS):
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
-            '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD',
+            '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID']
-    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD',
+    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID']
 
     OSX_MAIN_MEM = 'HTC Android Phone Media'

src/calibre/devices/hanlin/driver.py

@@ -80,6 +80,13 @@ class HANLINV3(USBMS):
             drives['carda'] = main
         return drives
 
+class SPECTRA(HANLINV3):
+
+    name = 'Spectra'
+    gui_name = 'Spectra'
+    PRODUCT_ID  = [0xa4a5]
+
+    FORMATS = ['epub', 'mobi', 'fb2', 'lit', 'prc', 'djvu', 'pdf', 'rtf', 'txt']
 
 class HANLINV5(HANLINV3):
     name           = 'Hanlin V5 driver'

src/calibre/ebooks/metadata/__init__.py

@@ -38,7 +38,7 @@ def author_to_author_sort(author):
     author = _bracket_pat.sub('', author).strip()
     tokens = author.split()
     tokens = tokens[-1:] + tokens[:-1]
-    if len(tokens) > 1:
+    if len(tokens) > 1 and method != 'nocomma':
         tokens[0] += ','
     return ' '.join(tokens)
 

src/calibre/gui2/actions/copy_to_library.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import QMenu
+
+from calibre.gui2.actions import InterfaceAction
+
+class CopyToLibraryAction(InterfaceAction):
+
+    name = 'Copy To Library'
+    action_spec = (_('Copy to library'), 'lt.png',
+            _('Copy selected books to the specified library'), None)
+
+    def genesis(self):
+        self.menu = QMenu(self.gui)
+
+

src/calibre/gui2/actions/edit_metadata.py

@@ -174,8 +174,14 @@ class EditMetadataAction(InterfaceAction):
                     _('No books selected'))
             d.exec_()
             return
-        if MetadataBulkDialog(self.gui, rows,
+        # Prevent the TagView from updating due to signals from the database
-                self.gui.library_view.model().db).changed:
+        self.gui.tags_view.blockSignals(True)
+        try:
+            changed = MetadataBulkDialog(self.gui, rows,
+                self.gui.library_view.model().db).changed
+        finally:
+            self.gui.tags_view.blockSignals(False)
+        if changed:
             self.gui.library_view.model().resort(reset=False)
             self.gui.library_view.model().research()
             self.gui.tags_view.recount()

src/calibre/gui2/custom_column_widgets.py

@@ -11,7 +11,7 @@ from functools import partial
 from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \
         QDate, QGroupBox, QVBoxLayout, QPlainTextEdit, QSizePolicy, \
         QSpacerItem, QIcon, QCheckBox, QWidget, QHBoxLayout, SIGNAL, \
-        QPushButton
+        QPushButton, QCoreApplication
 
 from calibre.utils.date import qt_to_dt, now
 from calibre.gui2.widgets import TagsLineEdit, EnComboBox
@@ -406,6 +406,7 @@ class BulkBase(Base):
     def commit(self, book_ids, notify=False):
         if self.process_each_book():
             for book_id in book_ids:
+                QCoreApplication.processEvents()
                 val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
                 new_val = self.getter(val)
                 if set(val) != new_val:
@@ -415,6 +416,7 @@ class BulkBase(Base):
             val = self.normalize_ui_val(val)
             if val != self.initial_val:
                 for book_id in book_ids:
+                    QCoreApplication.processEvents()
                     self.db.set_custom(book_id, val, num=self.col_id, notify=notify)
 
 class BulkBool(BulkBase, Bool):
@@ -433,6 +435,7 @@ class BulkDateTime(BulkBase, DateTime):
     pass
 
 class BulkSeries(BulkBase):
+
     def setup_ui(self, parent):
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
         values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
@@ -458,6 +461,7 @@ class BulkSeries(BulkBase):
         update_indices = self.idx_widget.checkState()
         if val != '':
             for book_id in book_ids:
+                QCoreApplication.processEvents()
                 if update_indices:
                     if tweaks['series_index_auto_increment'] == 'next':
                         s_index = self.db.get_next_cc_series_num_for\

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -3,14 +3,15 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 '''Dialog to edit metadata in bulk'''
 
-from PyQt4.QtCore import SIGNAL, QObject
+from PyQt4.Qt import SIGNAL, QObject, QDialog, QGridLayout, \
-from PyQt4.QtGui import QDialog, QGridLayout
+    QCoreApplication
 
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.ebooks.metadata import string_to_authors, \
     authors_to_string
 from calibre.gui2.custom_column_widgets import populate_metadata_page
+from calibre.gui2.dialogs.progress import ProgressDialog
 
 class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
@@ -25,10 +26,10 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
                 len(rows))
         self.write_series = False
         self.changed = False
-        QObject.connect(self.button_box, SIGNAL("accepted()"), self.sync)
 
-        self.tags.update_tags_cache(self.db.all_tags())
+        all_tags = self.db.all_tags()
-        self.remove_tags.update_tags_cache(self.db.all_tags())
+        self.tags.update_tags_cache(all_tags)
+        self.remove_tags.update_tags_cache(all_tags)
 
         self.initialize_combos()
 
@@ -102,43 +103,40 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             self.tags.update_tags_cache(self.db.all_tags())
             self.remove_tags.update_tags_cache(self.db.all_tags())
 
-    def sync(self):
+    def accept(self):
-        for id in self.ids:
+        if len(self.ids) < 1:
+            return QDialog.accept(self)
+
+        pd = ProgressDialog(_('Working'),
+                _('Applying changes to %d books. This may take a while.')%len(self.ids),
+                0, 0, self, cancelable=False)
+        pd.setModal(True)
+        pd.show()
+        def upd():
+            QCoreApplication.processEvents()
+
+        try:
+            remove = unicode(self.remove_tags.text()).strip().split(',')
+            add = unicode(self.tags.text()).strip().split(',')
             au = unicode(self.authors.text())
-            if au:
-                au = string_to_authors(au)
-                self.db.set_authors(id, au, notify=False)
-            if self.auto_author_sort.isChecked():
-                x = self.db.author_sort_from_book(id, index_is_id=True)
-                if x:
-                    self.db.set_author_sort(id, x, notify=False)
             aus = unicode(self.author_sort.text())
-            if aus and self.author_sort.isEnabled():
+            do_aus = self.author_sort.isEnabled()
-                self.db.set_author_sort(id, aus, notify=False)
+            rating = self.rating.value()
-            if self.rating.value() != -1:
-                self.db.set_rating(id, 2*self.rating.value(), notify=False)
             pub = unicode(self.publisher.text())
-            if pub:
+            do_series = self.write_series
-                self.db.set_publisher(id, pub, notify=False)
-            remove_tags = unicode(self.remove_tags.text()).strip()
-            if remove_tags:
-                remove_tags = [i.strip() for i in remove_tags.split(',')]
-                self.db.unapply_tags(id, remove_tags, notify=False)
-            tags = unicode(self.tags.text()).strip()
-            if tags:
-                tags = map(lambda x: x.strip(), tags.split(','))
-                self.db.set_tags(id, tags, append=True, notify=False)
-            if self.write_series:
             series = unicode(self.series.currentText()).strip()
-                next = self.db.get_next_series_num_for(series)
+            do_autonumber = self.autonumber_series.isChecked()
-                self.db.set_series(id, series, notify=False)
+            do_remove_format = self.remove_format.currentIndex() > -1
-                num = next if self.autonumber_series.isChecked() and series else 1.0
+            remove_format = unicode(self.remove_format.currentText())
-                self.db.set_series_index(id, num, notify=False)
+            do_swap_ta = self.swap_title_and_author.isChecked()
+            do_remove_conv = self.remove_conversion_settings.isChecked()
+            do_auto_author = self.auto_author_sort.isChecked()
 
-            if self.remove_format.currentIndex() > -1:
+            upd()
-                self.db.remove_format(id, unicode(self.remove_format.currentText()), index_is_id=True, notify=False)
+            self.changed = bool(self.ids)
-
+            for id in self.ids:
-            if self.swap_title_and_author.isChecked():
+                upd()
+                if do_swap_ta:
                     title = self.db.title(id, index_is_id=True)
                     aum = self.db.authors(id, index_is_id=True)
                     if aum:
@@ -148,13 +146,55 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
                     if title:
                         new_authors = string_to_authors(title)
                         self.db.set_authors(id, new_authors, notify=False)
+                upd()
 
-            if self.remove_conversion_settings.isChecked():
+                if au:
+                    self.db.set_authors(id, string_to_authors(au), notify=False)
+                upd()
+
+                if do_auto_author:
+                    x = self.db.author_sort_from_book(id, index_is_id=True)
+                    if x:
+                        self.db.set_author_sort(id, x, notify=False)
+                upd()
+
+                if aus and do_aus:
+                    self.db.set_author_sort(id, aus, notify=False)
+                upd()
+
+                if rating != -1:
+                    self.db.set_rating(id, 2*rating, notify=False)
+
+                if pub:
+                    self.db.set_publisher(id, pub, notify=False)
+                upd()
+
+                if do_series:
+                    next = self.db.get_next_series_num_for(series)
+                    self.db.set_series(id, series, notify=False)
+                    num = next if do_autonumber and series else 1.0
+                    self.db.set_series_index(id, num, notify=False)
+                upd()
+
+                if do_remove_format:
+                    self.db.remove_format(id, remove_format, index_is_id=True, notify=False)
+                upd()
+
+                if do_remove_conv:
                     self.db.delete_conversion_options(id, 'PIPE')
 
-            self.changed = True
+            upd()
             for w in getattr(self, 'custom_column_widgets', []):
                 w.commit(self.ids)
+            self.db.bulk_modify_tags(self.ids, add=add, remove=remove,
+                    notify=False)
+            upd()
+
+
+        finally:
+            pd.hide()
+
+        return QDialog.accept(self)
 
 
     def series_changed(self):

src/calibre/gui2/dialogs/progress.py

@@ -13,7 +13,8 @@ class ProgressDialog(QDialog, Ui_Dialog):
 
     canceled_signal = pyqtSignal()
 
-    def __init__(self, title, msg='', min=0, max=99, parent=None):
+    def __init__(self, title, msg='', min=0, max=99, parent=None,
+            cancelable=True):
         QDialog.__init__(self, parent)
         self.setupUi(self)
         self.setWindowTitle(title)
@@ -26,6 +27,9 @@ class ProgressDialog(QDialog, Ui_Dialog):
         self.canceled = False
 
         self.button_box.rejected.connect(self._canceled)
+        if not cancelable:
+            self.button_box.setVisible(False)
+        self.cancelable = cancelable
 
     def set_msg(self, msg=''):
         self.message.setText(msg)
@@ -54,8 +58,14 @@ class ProgressDialog(QDialog, Ui_Dialog):
         self.title.setText(_('Aborting...'))
         self.canceled_signal.emit()
 
+    def reject(self):
+        if not self.cancelable:
+            return
+        QDialog.reject(self)
+
     def keyPressEvent(self, ev):
         if ev.key() == Qt.Key_Escape:
+            if self.cancelable:
                 self._canceled()
         else:
             QDialog.keyPressEvent(self, ev)

src/calibre/library/database2.py

@@ -26,7 +26,7 @@ from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
 from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.customize.ui import run_plugins_on_import
-
+from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
 from calibre.utils.config import prefs, tweaks
@@ -116,6 +116,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         # so that various code taht connects directly will not complain about
         # missing functions
         self.books_list_filter = self.conn.create_dynamic_filter('books_list_filter')
+        # Store temporary tables in memory
+        self.conn.execute('pragma temp_store=2')
+        self.conn.commit()
 
     @classmethod
     def exists_at(cls, path):
@@ -1369,6 +1372,80 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             return set([])
         return set([r[0] for r in result])
 
+    @classmethod
+    def cleanup_tags(cls, tags):
+        tags = [x.strip() for x in tags if x.strip()]
+        tags = [x.decode(preferred_encoding, 'replace') \
+                    if isbytestring(x) else x for x in tags]
+        tags = [u' '.join(x.split()) for x in tags]
+        ans, seen = [], set([])
+        for tag in tags:
+            if tag.lower() not in seen:
+                seen.add(tag.lower())
+                ans.append(tag)
+        return ans
+
+    def bulk_modify_tags(self, ids, add=[], remove=[], notify=False):
+        add = self.cleanup_tags(add)
+        remove = self.cleanup_tags(remove)
+        remove = set(remove) - set(add)
+        if not ids or (not add and not remove):
+            return
+
+        # Add tags that do not already exist into the tag table
+        all_tags = self.all_tags()
+        lt = [t.lower() for t in all_tags]
+        new_tags = [t for t in add if t.lower() not in lt]
+        if new_tags:
+            self.conn.executemany('INSERT INTO tags(name) VALUES (?)', [(x,) for x in
+                new_tags])
+
+        # Create the temporary tables to store the ids for books and tags
+        # to be operated on
+        tables = ('temp_bulk_tag_edit_books', 'temp_bulk_tag_edit_add',
+                    'temp_bulk_tag_edit_remove')
+        drops = '\n'.join(['DROP TABLE IF EXISTS %s;'%t for t in tables])
+        creates = '\n'.join(['CREATE TEMP TABLE %s(id INTEGER PRIMARY KEY);'%t
+                for t in tables])
+        self.conn.executescript(drops + creates)
+
+        # Populate the books temp table
+        self.conn.executemany(
+            'INSERT INTO temp_bulk_tag_edit_books VALUES (?)',
+                [(x,) for x in ids])
+
+        # Populate the add/remove tags temp tables
+        for table, tags in enumerate([add, remove]):
+            if not tags:
+                continue
+            table = tables[table+1]
+            insert = ('INSERT INTO %s(id) SELECT tags.id FROM tags WHERE name=?'
+                     ' COLLATE PYNOCASE LIMIT 1')
+            self.conn.executemany(insert%table, [(x,) for x in tags])
+
+        if remove:
+            self.conn.execute(
+              '''DELETE FROM books_tags_link WHERE
+                    book IN (SELECT id FROM %s) AND
+                    tag IN (SELECT id FROM %s)'''
+              % (tables[0], tables[2]))
+
+        if add:
+            self.conn.execute(
+            '''
+            INSERT INTO books_tags_link(book, tag) SELECT {0}.id, {1}.id FROM
+            {0}, {1}
+            '''.format(tables[0], tables[1])
+            )
+        self.conn.executescript(drops)
+        self.conn.commit()
+
+        for x in ids:
+            tags = u','.join(self.get_tags(x))
+            self.data.set(x, self.FIELD_MAP['tags'], tags, row_is_id=True)
+        if notify:
+            self.notify('metadata', ids)
+
     def set_tags(self, id, tags, append=False, notify=True):
         '''
         @param tags: list of strings
@@ -1378,10 +1455,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             self.conn.execute('DELETE FROM books_tags_link WHERE book=?', (id,))
             self.conn.execute('DELETE FROM tags WHERE (SELECT COUNT(id) FROM books_tags_link WHERE tag=tags.id) < 1')
         otags = self.get_tags(id)
-        tags = [x.strip() for x in tags if x.strip()]
+        tags = self.cleanup_tags(tags)
-        tags = [x.decode(preferred_encoding, 'replace') if not isinstance(x,
-            unicode) else x for x in tags]
-        tags = [u' '.join(x.split()) for x in tags]
         for tag in (set(tags)-otags):
             tag = tag.strip()
             if not tag:
@@ -1407,7 +1481,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 self.conn.execute('INSERT INTO books_tags_link(book, tag) VALUES (?,?)',
                               (id, tid))
         self.conn.commit()
-        tags = ','.join(self.get_tags(id))
+        tags = u','.join(self.get_tags(id))
         self.data.set(id, self.FIELD_MAP['tags'], tags, row_is_id=True)
         if notify:
             self.notify('metadata', [id])

src/calibre/library/sqlite.py

@@ -13,10 +13,12 @@ from threading import Thread
 from Queue import Queue
 from threading import RLock
 from datetime import datetime
+from functools import partial
 
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.utils.config import tweaks
 from calibre.utils.date import parse_date, isoformat
+from calibre import isbytestring
 
 global_lock = RLock()
 
@@ -98,6 +100,19 @@ def _author_to_author_sort(x):
     if not x: return ''
     return author_to_author_sort(x.replace('|', ','))
 
+def pynocase(one, two, encoding='utf-8'):
+    if isbytestring(one):
+        try:
+            one = one.decode(encoding, 'replace')
+        except:
+            pass
+    if isbytestring(two):
+        try:
+            two = two.decode(encoding, 'replace')
+        except:
+            pass
+    return cmp(one.lower(), two.lower())
+
 class DBThread(Thread):
 
     CLOSE = '-------close---------'
@@ -115,10 +130,13 @@ class DBThread(Thread):
     def connect(self):
         self.conn = sqlite.connect(self.path, factory=Connection,
                                    detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES)
+        encoding = self.conn.execute('pragma encoding').fetchone()[0]
         self.conn.row_factory = sqlite.Row if self.row_factory else  lambda cursor, row : list(row)
         self.conn.create_aggregate('concat', 1, Concatenate)
         self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
         self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)
+        self.conn.create_collation('PYNOCASE', partial(pynocase,
+            encoding=encoding))
         if tweaks['title_series_sorting'] == 'strictly_alphabetic':
             self.conn.create_function('title_sort', 1, lambda x:x)
         else: