From 7d67e2d27375cabfe81a97c173a0710cc804b9fd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 17 Dec 2015 10:52:46 +0530 Subject: [PATCH] UI for exporting calibre data --- src/calibre/db/cache.py | 10 +- src/calibre/gui2/actions/choose_library.py | 19 +- src/calibre/gui2/dialogs/exim.py | 245 +++++++++++++++++++++ src/calibre/utils/exim.py | 24 +- 4 files changed, 285 insertions(+), 13 deletions(-) create mode 100644 src/calibre/gui2/dialogs/exim.py diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index bcc31806eb..0ebada3f02 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -2101,7 +2101,7 @@ class Cache(object): report_progress(i+1, len(book_ids), mi) @read_api - def export_library(self, library_key, exporter, progress=None): + def export_library(self, library_key, exporter, progress=None, abort=None): from binascii import hexlify key_prefix = hexlify(library_key) book_ids = self._all_book_ids() @@ -2118,6 +2118,8 @@ class Cache(object): os.remove(pt.name) metadata = {'format_data':format_metadata, 'metadata.db':dbkey, 'total':total} for i, book_id in enumerate(book_ids): + if abort is not None and abort.is_set(): + return if progress is not None: progress(self._field_for('title', book_id), i + 1, total) format_metadata[book_id] = {} @@ -2137,12 +2139,14 @@ class Cache(object): if progress is not None: progress(_('Completed'), total, total) -def import_library(library_key, importer, library_path, progress=None): +def import_library(library_key, importer, library_path, progress=None, abort=None): from calibre.db.backend import DB metadata = importer.metadata[library_key] total = metadata['total'] if progress is not None: progress('metadata.db', 0, total) + if abort is not None and abort.is_set(): + return with open(os.path.join(library_path, 'metadata.db'), 'wb') as f: src = importer.start_file(metadata['metadata.db'], 'metadata.db for ' + library_path) shutil.copyfileobj(src, f) @@ -2151,6 +2155,8 @@ def import_library(library_key, importer, library_path, progress=None): cache.init() format_data = {int(book_id):data for book_id, data in metadata['format_data'].iteritems()} for i, (book_id, fmt_key_map) in enumerate(format_data.iteritems()): + if abort is not None and abort.is_set(): + return title = cache._field_for('title', book_id) if progress is not None: progress(title, i + 1, total) diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index eb4d39fb0f..324ee26d94 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -10,11 +10,10 @@ from functools import partial from PyQt5.Qt import (QMenu, Qt, QInputDialog, QToolButton, QDialog, QDialogButtonBox, QGridLayout, QLabel, QLineEdit, QIcon, QSize, - QCoreApplication, pyqtSignal, QVBoxLayout, QTimer) + QCoreApplication, pyqtSignal, QVBoxLayout, QTimer, QAction) from calibre import isbytestring, sanitize_file_name_unicode -from calibre.constants import (filesystem_encoding, iswindows, - get_portable_base) +from calibre.constants import (filesystem_encoding, iswindows, get_portable_base, isportable) from calibre.utils.config import prefs, tweaks from calibre.utils.icu import sort_key from calibre.gui2 import (gprefs, warning_dialog, Dispatcher, error_dialog, @@ -208,6 +207,8 @@ class ChooseLibraryAction(InterfaceAction): def genesis(self): self.count_changed(0) self.action_choose = self.menuless_qaction + self.action_exim = ac = QAction(_('Export/Import all calibre data'), self.gui) + ac.triggered.connect(self.exim_data) self.stats = LibraryUsageStats() self.popup_type = (QToolButton.InstantPopup if len(self.stats.stats) > 1 else @@ -233,6 +234,7 @@ class ChooseLibraryAction(InterfaceAction): self.choose_menu.addAction(ac) self.delete_menu = QMenu(_('Remove library')) self.delete_menu_action = self.choose_menu.addMenu(self.delete_menu) + self.choose_menu.addAction(self.action_exim) else: self.choose_menu.addAction(ac) @@ -281,6 +283,17 @@ class ChooseLibraryAction(InterfaceAction): def pick_random(self, *args): self.gui.iactions['Pick Random Book'].pick_random() + def exim_data(self): + if isportable: + return error_dialog(self.gui, _('Cannot export/import'), _( + 'You are running calibre portable, all calibre data is already in the' + ' calibre portable folder. Export/Import is unavailable.'), show=True) + if self.gui.job_manager.has_jobs(): + return error_dialog(self.gui, _('Cannot Export/Import'), + _('Cannot export/import data while there are running jobs.'), show=True) + from calibre.gui2.dialogs.exim import EximDialog + EximDialog(parent=self.gui).exec_() + def library_name(self): db = self.gui.library_view.model().db path = db.library_path diff --git a/src/calibre/gui2/dialogs/exim.py b/src/calibre/gui2/dialogs/exim.py new file mode 100644 index 0000000000..f170bac11c --- /dev/null +++ b/src/calibre/gui2/dialogs/exim.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPLv3 Copyright: 2015, Kovid Goyal + +from __future__ import (unicode_literals, division, absolute_import, + print_function) +from functools import partial +from threading import Thread, Event +import os, stat + +from PyQt5.Qt import ( + QSize, QStackedLayout, QWidget, QVBoxLayout, QLabel, QPushButton, + QListWidget, QListWidgetItem, QIcon, Qt, pyqtSignal, QGridLayout, + QProgressBar, QDialog, QDialogButtonBox +) + +from calibre import human_readable +from calibre.gui2 import choose_dir, error_dialog +from calibre.gui2.widgets2 import Dialog +from calibre.utils.exim import all_known_libraries, export +from calibre.utils.icu import numeric_sort_key + +def disk_usage(path_to_dir, abort=None): + stack = [path_to_dir] + ans = 0 + while stack: + bdir = stack.pop() + try: + for child in os.listdir(bdir): + cpath = os.path.join(bdir, child) + if abort is not None and abort.is_set(): + return -1 + r = os.lstat(cpath) + if stat.S_ISDIR(r.st_mode): + stack.append(cpath) + ans += r.st_size + except EnvironmentError: + pass + return ans + +class RunAction(QDialog): + + update_current_signal = pyqtSignal(object, object, object) + update_overall_signal = pyqtSignal(object, object, object) + finish_signal = pyqtSignal() + + def __init__(self, title, err_msg, action, parent=None): + QDialog.__init__(self, parent) + self.setWindowTitle(_('Working please wait...')) + self.title, self.action, self.tb, self.err_msg = title, action, None, err_msg + self.abort = Event() + self.setup_ui() + t = Thread(name='ExImWorker', target=self.run_action) + t.daemon = True + t.start() + + def setup_ui(self): + self.l = l = QGridLayout(self) + self.bb = QDialogButtonBox(self) + self.bb.setStandardButtons(self.bb.Cancel) + self.bb.rejected.connect(self.reject) + + self.la1 = la = QLabel('

' + self.title) + l.addWidget(la, 0, 0, 1, -1) + self.la2 = la = QLabel(_('Total:')) + l.addWidget(la, l.rowCount(), 0) + self.overall = p = QProgressBar(self) + p.setMinimum(0), p.setValue(0), p.setMaximum(0) + p.setMinimumWidth(450) + l.addWidget(p, l.rowCount()-1, 1) + self.omsg = la = QLabel(self) + la.setMaximumWidth(450) + l.addWidget(la, l.rowCount(), 1) + self.la3 = la = QLabel(_('Current:')) + l.addWidget(la, l.rowCount(), 0) + self.current = p = QProgressBar(self) + p.setMinimum(0), p.setValue(0), p.setMaximum(0) + l.addWidget(p, l.rowCount()-1, 1) + self.cmsg = la = QLabel(self) + la.setMaximumWidth(450) + l.addWidget(la, l.rowCount(), 1) + l.addWidget(self.bb, l.rowCount(), 0, 1, -1) + self.update_current_signal.connect(self.update_current, type=Qt.QueuedConnection) + self.update_overall_signal.connect(self.update_overall, type=Qt.QueuedConnection) + self.finish_signal.connect(self.finish_processing, type=Qt.QueuedConnection) + + def update_overall(self, msg, count, total): + self.overall.setMaximum(total), self.overall.setValue(count) + self.omsg.setText(msg) + + def update_current(self, msg, count, total): + self.current.setMaximum(total), self.current.setValue(count) + self.cmsg.setText(msg) + + def reject(self): + self.abort.set() + self.bb.button(self.bb.Cancel).setEnabled(False) + + def finish_processing(self): + if self.abort.is_set(): + return QDialog.reject(self) + if self.tb is not None: + error_dialog(self, _('Failed'), self.err_msg + ' ' + _('Click "Show Details" for more information.'), + det_msg=self.tb, show=True) + self.accept() + + def run_action(self): + try: + self.action(abort=self.abort, progress1=self.update_overall_signal.emit, progress2=self.update_current_signal.emit) + except Exception: + import traceback + self.tb = traceback.format_exc() + self.finish_signal.emit() + +class EximDialog(Dialog): + + update_disk_usage = pyqtSignal(object, object) + + def __init__(self, parent=None, initial_panel=None): + self.initial_panel = initial_panel + self.abort_disk_usage = Event() + Dialog.__init__(self, _('Export/Import all calibre data'), 'exim-calibre', parent=parent) + + def sizeHint(self): + return QSize(800, 600) + + def setup_ui(self): + self.l = l = QVBoxLayout(self) + self.stack = s = QStackedLayout() + l.addLayout(s) + l.addWidget(self.bb) + self.welcome = w = QWidget(self) + s.addWidget(w) + w.l = l = QVBoxLayout(w) + w.la = la = QLabel('

' + _( + 'You can export all calibre data, including your books, settings and plugins' + ' into a single directory. Then, you can use this tool to re-import all that' + ' data into a different calibre install, for example, on another computer.') + '

' + + _( + 'This is a simple way to move your calibre installation with all its data to' + ' a new computer, or to replicate your current setup on a second computer.' + )) + la.setWordWrap(True) + l.addWidget(la) + l.addSpacing(20) + self.exp_button = b = QPushButton(_('&Export all your calibre data')) + b.clicked.connect(partial(self.show_panel, 'export')) + l.addWidget(b), l.addSpacing(20) + self.imp_button = b = QPushButton(_('&Import previously exported data')) + b.clicked.connect(partial(self.show_panel, 'import')) + l.addWidget(b), l.addStretch(20) + + self.setup_export_panel() + self.show_panel(self.initial_panel) + + def export_lib_text(self, lpath, size=None): + return _('{0} [Size: {1}]\nin {2}').format( + os.path.basename(lpath), ('' if size < 0 else human_readable(size)) + if size is not None else _('Calculating...'), os.path.dirname(lpath)) + + def setup_export_panel(self): + self.export_panel = w = QWidget(self) + self.stack.addWidget(w) + w.l = l = QVBoxLayout(w) + w.la = la = QLabel(_('Select which libraries you want to export below')) + la.setWordWrap(True), l.addWidget(la) + self.lib_list = ll = QListWidget(self) + l.addWidget(ll) + ll.setSelectionMode(ll.ExtendedSelection) + ll.setStyleSheet('QListView::item { padding: 5px }') + ll.setAlternatingRowColors(True) + lpaths = all_known_libraries() + for lpath in sorted(lpaths, key=lambda x:numeric_sort_key(os.path.basename(x))): + i = QListWidgetItem(self.export_lib_text(lpath), ll) + i.setData(Qt.UserRole, lpath) + i.setData(Qt.UserRole+1, lpaths[lpath]) + i.setIcon(QIcon(I('lt.png'))) + i.setSelected(True) + self.update_disk_usage.connect(( + lambda i, sz: self.lib_list.item(i).setText(self.export_lib_text(self.lib_list.item(i).data(Qt.UserRole), sz))), type=Qt.QueuedConnection) + t = Thread(name='GetLibSizes', target=self.get_lib_sizes) + t.daemon = True + t.start() + + def get_lib_sizes(self): + for i in xrange(self.lib_list.count()): + path = self.lib_list.item(i).data(Qt.UserRole) + try: + sz = disk_usage(path, abort=self.abort_disk_usage) + except Exception: + import traceback + traceback.print_exc() + self.update_disk_usage.emit(i, sz) + + def show_panel(self, which): + self.validate = self.run_action = lambda : True + if which is None: + self.bb.setStandardButtons(self.bb.Cancel) + else: + if which == 'export': + self.validate = self.validate_export + self.run_action = self.run_export_action + self.bb.setStandardButtons(self.bb.Ok | self.bb.Cancel) + self.stack.setCurrentIndex({'export':1, 'import':2}.get(which, 0)) + + def validate_export(self): + path = choose_dir(self, 'export-calibre-dir', _('Choose a directory to export to')) + if not path: + return False + if os.listdir(path): + error_dialog(self, _('Export dir not empty'), _( + 'The directory you choose to export the data to must be empty.'), show=True) + return False + self.export_dir = path + return True + + def run_export_action(self): + from calibre.gui2.ui import get_gui + library_paths = {i.data(Qt.UserRole):i.data(Qt.UserRole+1) for i in self.lib_list.selectedItems()} + dbmap = {} + gui = get_gui() + if gui is not None: + db = gui.current_db.new_api + dbmap[db.library_path] = db + return RunAction(_('Exporting all calibre data...'), _( + 'Failed to export data.'), partial(export, self.export_dir, library_paths=library_paths, dbmap=dbmap), + parent=self).exec_() == Dialog.Accepted + + def accept(self): + if not self.validate(): + return + self.abort_disk_usage.set() + if self.run_action(): + Dialog.accept(self) + + def reject(self): + self.abort_disk_usage.set() + Dialog.reject(self) + +if __name__ == '__main__': + from calibre.gui2 import Application + app = Application([]) + d = EximDialog(initial_panel='export') + d.exec_() + del app diff --git a/src/calibre/utils/exim.py b/src/calibre/utils/exim.py index 2965545913..b43c07a171 100644 --- a/src/calibre/utils/exim.py +++ b/src/calibre/utils/exim.py @@ -165,7 +165,7 @@ def all_known_libraries(): added[path] = lus.get(path, 1) return added -def export(destdir, library_paths=None, dbmap=None, progress1=None, progress2=None): +def export(destdir, library_paths=None, dbmap=None, progress1=None, progress2=None, abort=None): from calibre.db.cache import Cache from calibre.db.backend import DB if library_paths is None: @@ -174,10 +174,12 @@ def export(destdir, library_paths=None, dbmap=None, progress1=None, progress2=No dbmap = {os.path.normace(os.path.abspath(k)):v for k, v in dbmap.iteritems()} exporter = Exporter(destdir) exporter.metadata['libraries'] = libraries = {} - total = len(library_paths) + 2 + total = len(library_paths) + 1 for i, (lpath, count) in enumerate(library_paths.iteritems()): + if abort is not None and abort.is_set(): + return if progress1 is not None: - progress1(lpath, i + 1, total) + progress1(lpath, i, total) key = os.path.normcase(os.path.abspath(lpath)) db, closedb = dbmap.get(lpath), False if db is None: @@ -186,12 +188,14 @@ def export(destdir, library_paths=None, dbmap=None, progress1=None, progress2=No closedb = True else: db = db.new_api - db.export_library(key, exporter, progress=progress2) + db.export_library(key, exporter, progress=progress2, abort=abort) if closedb: db.close() libraries[key] = count if progress1 is not None: progress1(_('Settings and plugins'), total-1, total) + if abort is not None and abort.is_set(): + return exporter.export_dir(config_dir, 'config_dir') exporter.commit() if progress1 is not None: @@ -305,14 +309,16 @@ class Importer(object): gprefs = JSONConfig('gui', base_path=base_dir) gprefs['library_usage_stats'] = dict(library_usage_stats) -def import_data(importer, library_path_map, config_location=None, progress1=None, progress2=None): +def import_data(importer, library_path_map, config_location=None, progress1=None, progress2=None, abort=None): from calibre.db.cache import import_library config_location = config_location or config_dir - total = len(library_path_map) + 2 + total = len(library_path_map) + 1 library_usage_stats = Counter() for i, (library_key, dest) in enumerate(library_path_map.iteritems()): + if abort is not None and abort.is_set(): + return if progress1 is not None: - progress1(dest, i + 1, total) + progress1(dest, i, total) try: os.makedirs(dest) except EnvironmentError as err: @@ -320,11 +326,13 @@ def import_data(importer, library_path_map, config_location=None, progress1=None raise if not os.path.isdir(dest): raise ValueError('%s is not a directory' % dest) - import_library(library_key, importer, dest, progress=progress2).close() + import_library(library_key, importer, dest, progress=progress2, abort=abort).close() library_usage_stats[dest] = importer.metadata['libraries'].get(library_key, 1) if progress1 is not None: progress1(_('Settings and plugins'), total - 1, total) + if abort is not None and abort.is_set(): + return base_dir = tempfile.mkdtemp(dir=os.path.dirname(config_location)) importer.export_config(base_dir, library_usage_stats) if os.path.exists(config_location):