UI for exporting calibre data

src/calibre/db/cache.py

@@ -2101,7 +2101,7 @@ class Cache(object):
                 report_progress(i+1, len(book_ids), mi)
 
     @read_api
-    def export_library(self, library_key, exporter, progress=None):
+    def export_library(self, library_key, exporter, progress=None, abort=None):
         from binascii import hexlify
         key_prefix = hexlify(library_key)
         book_ids = self._all_book_ids()
@@ -2118,6 +2118,8 @@ class Cache(object):
         os.remove(pt.name)
         metadata = {'format_data':format_metadata, 'metadata.db':dbkey, 'total':total}
         for i, book_id in enumerate(book_ids):
+            if abort is not None and abort.is_set():
+                return
             if progress is not None:
                 progress(self._field_for('title', book_id), i + 1, total)
             format_metadata[book_id] = {}
@@ -2137,12 +2139,14 @@ class Cache(object):
         if progress is not None:
             progress(_('Completed'), total, total)
 
-def import_library(library_key, importer, library_path, progress=None):
+def import_library(library_key, importer, library_path, progress=None, abort=None):
     from calibre.db.backend import DB
     metadata = importer.metadata[library_key]
     total = metadata['total']
     if progress is not None:
         progress('metadata.db', 0, total)
+    if abort is not None and abort.is_set():
+        return
     with open(os.path.join(library_path, 'metadata.db'), 'wb') as f:
         src = importer.start_file(metadata['metadata.db'], 'metadata.db for ' + library_path)
         shutil.copyfileobj(src, f)
@@ -2151,6 +2155,8 @@ def import_library(library_key, importer, library_path, progress=None):
     cache.init()
     format_data = {int(book_id):data for book_id, data in metadata['format_data'].iteritems()}
     for i, (book_id, fmt_key_map) in enumerate(format_data.iteritems()):
+        if abort is not None and abort.is_set():
+            return
         title = cache._field_for('title', book_id)
         if progress is not None:
             progress(title, i + 1, total)

src/calibre/gui2/actions/choose_library.py

@@ -10,11 +10,10 @@ from functools import partial
 
 from PyQt5.Qt import (QMenu, Qt, QInputDialog, QToolButton, QDialog,
         QDialogButtonBox, QGridLayout, QLabel, QLineEdit, QIcon, QSize,
-        QCoreApplication, pyqtSignal, QVBoxLayout, QTimer)
+        QCoreApplication, pyqtSignal, QVBoxLayout, QTimer, QAction)
 
 from calibre import isbytestring, sanitize_file_name_unicode
-from calibre.constants import (filesystem_encoding, iswindows,
+from calibre.constants import (filesystem_encoding, iswindows, get_portable_base, isportable)
-        get_portable_base)
 from calibre.utils.config import prefs, tweaks
 from calibre.utils.icu import sort_key
 from calibre.gui2 import (gprefs, warning_dialog, Dispatcher, error_dialog,
@@ -208,6 +207,8 @@ class ChooseLibraryAction(InterfaceAction):
     def genesis(self):
         self.count_changed(0)
         self.action_choose = self.menuless_qaction
+        self.action_exim = ac = QAction(_('Export/Import all calibre data'), self.gui)
+        ac.triggered.connect(self.exim_data)
 
         self.stats = LibraryUsageStats()
         self.popup_type = (QToolButton.InstantPopup if len(self.stats.stats) > 1 else
@@ -233,6 +234,7 @@ class ChooseLibraryAction(InterfaceAction):
             self.choose_menu.addAction(ac)
             self.delete_menu = QMenu(_('Remove library'))
             self.delete_menu_action = self.choose_menu.addMenu(self.delete_menu)
+            self.choose_menu.addAction(self.action_exim)
         else:
             self.choose_menu.addAction(ac)
 
@@ -281,6 +283,17 @@ class ChooseLibraryAction(InterfaceAction):
     def pick_random(self, *args):
         self.gui.iactions['Pick Random Book'].pick_random()
 
+    def exim_data(self):
+        if isportable:
+            return error_dialog(self.gui, _('Cannot export/import'), _(
+                'You are running calibre portable, all calibre data is already in the'
+                ' calibre portable folder. Export/Import is unavailable.'), show=True)
+        if self.gui.job_manager.has_jobs():
+            return error_dialog(self.gui, _('Cannot Export/Import'),
+                    _('Cannot export/import data while there are running jobs.'), show=True)
+        from calibre.gui2.dialogs.exim import EximDialog
+        EximDialog(parent=self.gui).exec_()
+
     def library_name(self):
         db = self.gui.library_view.model().db
         path = db.library_path

src/calibre/gui2/dialogs/exim.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+from functools import partial
+from threading import Thread, Event
+import os, stat
+
+from PyQt5.Qt import (
+    QSize, QStackedLayout, QWidget, QVBoxLayout, QLabel, QPushButton,
+    QListWidget, QListWidgetItem, QIcon, Qt, pyqtSignal, QGridLayout,
+    QProgressBar, QDialog, QDialogButtonBox
+)
+
+from calibre import human_readable
+from calibre.gui2 import choose_dir, error_dialog
+from calibre.gui2.widgets2 import Dialog
+from calibre.utils.exim import all_known_libraries, export
+from calibre.utils.icu import numeric_sort_key
+
+def disk_usage(path_to_dir, abort=None):
+    stack = [path_to_dir]
+    ans = 0
+    while stack:
+        bdir = stack.pop()
+        try:
+            for child in os.listdir(bdir):
+                cpath = os.path.join(bdir, child)
+                if abort is not None and abort.is_set():
+                    return -1
+                r = os.lstat(cpath)
+                if stat.S_ISDIR(r.st_mode):
+                    stack.append(cpath)
+                ans += r.st_size
+        except EnvironmentError:
+            pass
+    return ans
+
+class RunAction(QDialog):
+
+    update_current_signal = pyqtSignal(object, object, object)
+    update_overall_signal = pyqtSignal(object, object, object)
+    finish_signal = pyqtSignal()
+
+    def __init__(self, title, err_msg, action, parent=None):
+        QDialog.__init__(self, parent)
+        self.setWindowTitle(_('Working please wait...'))
+        self.title, self.action, self.tb, self.err_msg = title, action, None, err_msg
+        self.abort = Event()
+        self.setup_ui()
+        t = Thread(name='ExImWorker', target=self.run_action)
+        t.daemon = True
+        t.start()
+
+    def setup_ui(self):
+        self.l = l = QGridLayout(self)
+        self.bb = QDialogButtonBox(self)
+        self.bb.setStandardButtons(self.bb.Cancel)
+        self.bb.rejected.connect(self.reject)
+
+        self.la1 = la = QLabel('<h2>' + self.title)
+        l.addWidget(la, 0, 0, 1, -1)
+        self.la2 = la = QLabel(_('Total:'))
+        l.addWidget(la, l.rowCount(), 0)
+        self.overall = p = QProgressBar(self)
+        p.setMinimum(0), p.setValue(0), p.setMaximum(0)
+        p.setMinimumWidth(450)
+        l.addWidget(p, l.rowCount()-1, 1)
+        self.omsg = la = QLabel(self)
+        la.setMaximumWidth(450)
+        l.addWidget(la, l.rowCount(), 1)
+        self.la3 = la = QLabel(_('Current:'))
+        l.addWidget(la, l.rowCount(), 0)
+        self.current = p = QProgressBar(self)
+        p.setMinimum(0), p.setValue(0), p.setMaximum(0)
+        l.addWidget(p, l.rowCount()-1, 1)
+        self.cmsg = la = QLabel(self)
+        la.setMaximumWidth(450)
+        l.addWidget(la, l.rowCount(), 1)
+        l.addWidget(self.bb, l.rowCount(), 0, 1, -1)
+        self.update_current_signal.connect(self.update_current, type=Qt.QueuedConnection)
+        self.update_overall_signal.connect(self.update_overall, type=Qt.QueuedConnection)
+        self.finish_signal.connect(self.finish_processing, type=Qt.QueuedConnection)
+
+    def update_overall(self, msg, count, total):
+        self.overall.setMaximum(total), self.overall.setValue(count)
+        self.omsg.setText(msg)
+
+    def update_current(self, msg, count, total):
+        self.current.setMaximum(total), self.current.setValue(count)
+        self.cmsg.setText(msg)
+
+    def reject(self):
+        self.abort.set()
+        self.bb.button(self.bb.Cancel).setEnabled(False)
+
+    def finish_processing(self):
+        if self.abort.is_set():
+            return QDialog.reject(self)
+        if self.tb is not None:
+            error_dialog(self, _('Failed'), self.err_msg + ' ' + _('Click "Show Details" for more information.'),
+                            det_msg=self.tb, show=True)
+        self.accept()
+
+    def run_action(self):
+        try:
+            self.action(abort=self.abort, progress1=self.update_overall_signal.emit, progress2=self.update_current_signal.emit)
+        except Exception:
+            import traceback
+            self.tb = traceback.format_exc()
+        self.finish_signal.emit()
+
+class EximDialog(Dialog):
+
+    update_disk_usage = pyqtSignal(object, object)
+
+    def __init__(self, parent=None, initial_panel=None):
+        self.initial_panel = initial_panel
+        self.abort_disk_usage = Event()
+        Dialog.__init__(self, _('Export/Import all calibre data'), 'exim-calibre', parent=parent)
+
+    def sizeHint(self):
+        return QSize(800, 600)
+
+    def setup_ui(self):
+        self.l = l = QVBoxLayout(self)
+        self.stack = s = QStackedLayout()
+        l.addLayout(s)
+        l.addWidget(self.bb)
+        self.welcome = w = QWidget(self)
+        s.addWidget(w)
+        w.l = l = QVBoxLayout(w)
+        w.la = la = QLabel('<p>' + _(
+            'You can export all calibre data, including your books, settings and plugins'
+            ' into a single directory. Then, you can use this tool to re-import all that'
+            ' data into a different calibre install, for example, on another computer.') + '<p>' +
+        _(
+            'This is a simple way to move your calibre installation with all its data to'
+            ' a new computer, or to replicate your current setup on a second computer.'
+        ))
+        la.setWordWrap(True)
+        l.addWidget(la)
+        l.addSpacing(20)
+        self.exp_button = b = QPushButton(_('&Export all your calibre data'))
+        b.clicked.connect(partial(self.show_panel, 'export'))
+        l.addWidget(b), l.addSpacing(20)
+        self.imp_button = b = QPushButton(_('&Import previously exported data'))
+        b.clicked.connect(partial(self.show_panel, 'import'))
+        l.addWidget(b), l.addStretch(20)
+
+        self.setup_export_panel()
+        self.show_panel(self.initial_panel)
+
+    def export_lib_text(self, lpath, size=None):
+        return _('{0} [Size: {1}]\nin {2}').format(
+            os.path.basename(lpath), ('' if size < 0 else human_readable(size))
+            if size is not None else _('Calculating...'), os.path.dirname(lpath))
+
+    def setup_export_panel(self):
+        self.export_panel = w = QWidget(self)
+        self.stack.addWidget(w)
+        w.l = l = QVBoxLayout(w)
+        w.la = la = QLabel(_('Select which libraries you want to export below'))
+        la.setWordWrap(True), l.addWidget(la)
+        self.lib_list = ll = QListWidget(self)
+        l.addWidget(ll)
+        ll.setSelectionMode(ll.ExtendedSelection)
+        ll.setStyleSheet('QListView::item { padding: 5px }')
+        ll.setAlternatingRowColors(True)
+        lpaths = all_known_libraries()
+        for lpath in sorted(lpaths, key=lambda x:numeric_sort_key(os.path.basename(x))):
+            i = QListWidgetItem(self.export_lib_text(lpath), ll)
+            i.setData(Qt.UserRole, lpath)
+            i.setData(Qt.UserRole+1, lpaths[lpath])
+            i.setIcon(QIcon(I('lt.png')))
+            i.setSelected(True)
+        self.update_disk_usage.connect((
+            lambda i, sz: self.lib_list.item(i).setText(self.export_lib_text(self.lib_list.item(i).data(Qt.UserRole), sz))), type=Qt.QueuedConnection)
+        t = Thread(name='GetLibSizes', target=self.get_lib_sizes)
+        t.daemon = True
+        t.start()
+
+    def get_lib_sizes(self):
+        for i in xrange(self.lib_list.count()):
+            path = self.lib_list.item(i).data(Qt.UserRole)
+            try:
+                sz = disk_usage(path, abort=self.abort_disk_usage)
+            except Exception:
+                import traceback
+                traceback.print_exc()
+            self.update_disk_usage.emit(i, sz)
+
+    def show_panel(self, which):
+        self.validate = self.run_action = lambda : True
+        if which is None:
+            self.bb.setStandardButtons(self.bb.Cancel)
+        else:
+            if which == 'export':
+                self.validate = self.validate_export
+                self.run_action = self.run_export_action
+            self.bb.setStandardButtons(self.bb.Ok | self.bb.Cancel)
+        self.stack.setCurrentIndex({'export':1, 'import':2}.get(which, 0))
+
+    def validate_export(self):
+        path = choose_dir(self, 'export-calibre-dir', _('Choose a directory to export to'))
+        if not path:
+            return False
+        if os.listdir(path):
+            error_dialog(self, _('Export dir not empty'), _(
+                'The directory you choose to export the data to must be empty.'), show=True)
+            return False
+        self.export_dir = path
+        return True
+
+    def run_export_action(self):
+        from calibre.gui2.ui import get_gui
+        library_paths = {i.data(Qt.UserRole):i.data(Qt.UserRole+1) for i in self.lib_list.selectedItems()}
+        dbmap = {}
+        gui = get_gui()
+        if gui is not None:
+            db = gui.current_db.new_api
+            dbmap[db.library_path] = db
+        return RunAction(_('Exporting all calibre data...'), _(
+            'Failed to export data.'), partial(export, self.export_dir, library_paths=library_paths, dbmap=dbmap),
+                      parent=self).exec_() == Dialog.Accepted
+
+    def accept(self):
+        if not self.validate():
+            return
+        self.abort_disk_usage.set()
+        if self.run_action():
+            Dialog.accept(self)
+
+    def reject(self):
+        self.abort_disk_usage.set()
+        Dialog.reject(self)
+
+if __name__ == '__main__':
+    from calibre.gui2 import Application
+    app = Application([])
+    d = EximDialog(initial_panel='export')
+    d.exec_()
+    del app

src/calibre/utils/exim.py

@@ -165,7 +165,7 @@ def all_known_libraries():
                 added[path] = lus.get(path, 1)
     return added
 
-def export(destdir, library_paths=None, dbmap=None, progress1=None, progress2=None):
+def export(destdir, library_paths=None, dbmap=None, progress1=None, progress2=None, abort=None):
     from calibre.db.cache import Cache
     from calibre.db.backend import DB
     if library_paths is None:
@@ -174,10 +174,12 @@ def export(destdir, library_paths=None, dbmap=None, progress1=None, progress2=No
     dbmap = {os.path.normace(os.path.abspath(k)):v for k, v in dbmap.iteritems()}
     exporter = Exporter(destdir)
     exporter.metadata['libraries'] = libraries = {}
-    total = len(library_paths) + 2
+    total = len(library_paths) + 1
     for i, (lpath, count) in enumerate(library_paths.iteritems()):
+        if abort is not None and abort.is_set():
+            return
         if progress1 is not None:
-            progress1(lpath, i + 1, total)
+            progress1(lpath, i, total)
         key = os.path.normcase(os.path.abspath(lpath))
         db, closedb = dbmap.get(lpath), False
         if db is None:
@@ -186,12 +188,14 @@ def export(destdir, library_paths=None, dbmap=None, progress1=None, progress2=No
             closedb = True
         else:
             db = db.new_api
-        db.export_library(key, exporter, progress=progress2)
+        db.export_library(key, exporter, progress=progress2, abort=abort)
         if closedb:
             db.close()
         libraries[key] = count
     if progress1 is not None:
         progress1(_('Settings and plugins'), total-1, total)
+    if abort is not None and abort.is_set():
+        return
     exporter.export_dir(config_dir, 'config_dir')
     exporter.commit()
     if progress1 is not None:
@@ -305,14 +309,16 @@ class Importer(object):
         gprefs = JSONConfig('gui', base_path=base_dir)
         gprefs['library_usage_stats'] = dict(library_usage_stats)
 
-def import_data(importer, library_path_map, config_location=None, progress1=None, progress2=None):
+def import_data(importer, library_path_map, config_location=None, progress1=None, progress2=None, abort=None):
     from calibre.db.cache import import_library
     config_location = config_location or config_dir
-    total = len(library_path_map) + 2
+    total = len(library_path_map) + 1
     library_usage_stats = Counter()
     for i, (library_key, dest) in enumerate(library_path_map.iteritems()):
+        if abort is not None and abort.is_set():
+            return
         if progress1 is not None:
-            progress1(dest, i + 1, total)
+            progress1(dest, i, total)
         try:
             os.makedirs(dest)
         except EnvironmentError as err:
@@ -320,11 +326,13 @@ def import_data(importer, library_path_map, config_location=None, progress1=None
                 raise
         if not os.path.isdir(dest):
             raise ValueError('%s is not a directory' % dest)
-        import_library(library_key, importer, dest, progress=progress2).close()
+        import_library(library_key, importer, dest, progress=progress2, abort=abort).close()
         library_usage_stats[dest] = importer.metadata['libraries'].get(library_key, 1)
     if progress1 is not None:
         progress1(_('Settings and plugins'), total - 1, total)
 
+    if abort is not None and abort.is_set():
+        return
     base_dir = tempfile.mkdtemp(dir=os.path.dirname(config_location))
     importer.export_config(base_dir, library_usage_stats)
     if os.path.exists(config_location):