diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index 7d8855de14..26b0b6c59b 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -217,3 +217,18 @@ def opf_metadata(opfpath): import traceback traceback.print_exc() pass + +def forked_read_metadata(path, tdir): + from calibre.ebooks.metadata.opf2 import metadata_to_opf + with open(path, 'rb') as f: + fmt = os.path.splitext(path)[1][1:].lower() + mi = get_metadata(f, fmt) + if mi.cover_data and mi.cover_data[1]: + with open(os.path.join(tdir, 'cover.jpg'), 'wb') as f: + f.write(mi.cover_data[1]) + mi.cover_data = (None, None) + mi.cover = 'cover.jpg' + opf = metadata_to_opf(mi) + with open(os.path.join(tdir, 'metadata.opf'), 'wb') as f: + f.write(opf) + diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 727f28b228..35cc249acb 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -101,6 +101,7 @@ gprefs.defaults['preserve_date_on_ctl'] = True gprefs.defaults['cb_fullscreen'] = False gprefs.defaults['worker_max_time'] = 0 gprefs.defaults['show_files_after_save'] = True +gprefs.defaults['auto_add_path'] = None # }}} NONE = QVariant() #: Null value to return from the data function of item models diff --git a/src/calibre/gui2/auto_add.py b/src/calibre/gui2/auto_add.py new file mode 100644 index 0000000000..71d2b8ecd0 --- /dev/null +++ b/src/calibre/gui2/auto_add.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, tempfile, shutil +from threading import Thread, Event + +from PyQt4.Qt import (QFileSystemWatcher, QObject, Qt, pyqtSignal, QTimer) + +from calibre import prints +from calibre.ptempfile import PersistentTemporaryDirectory +from calibre.ebooks import BOOK_EXTENSIONS + +class Worker(Thread): + + def __init__(self, path, callback): + Thread.__init__(self) + self.daemon = True + self.keep_running = True + self.wake_up = Event() + self.path, self.callback = path, callback + self.staging = set() + self.be = frozenset(BOOK_EXTENSIONS) + + def run(self): + self.tdir = PersistentTemporaryDirectory('_auto_adder') + while self.keep_running: + self.wake_up.wait() + self.wake_up.clear() + if not self.keep_running: + break + try: + self.auto_add() + except: + import traceback + traceback.print_exc() + + def auto_add(self): + from calibre.utils.ipc.simple_worker import fork_job + from calibre.ebooks.metadata.opf2 import metadata_to_opf + from calibre.ebooks.metadata.meta import metadata_from_filename + + files = [x for x in os.listdir(self.path) if x not in self.staging + and os.path.isfile(os.path.join(self.path, x)) and + os.access(os.path.join(self.path, x), os.R_OK|os.W_OK) and + os.path.splitext(x)[1][1:].lower() in self.be] + data = {} + for fname in files: + f = os.path.join(self.path, fname) + tdir = tempfile.mkdtemp(dir=self.tdir) + try: + fork_job('calibre.ebooks.metadata.meta', + 'forked_read_metadata', (f, tdir), no_output=True) + except: + import traceback + traceback.print_exc() + + opfpath = os.path.join(tdir, 'metadata.opf') + try: + if os.stat(opfpath).st_size < 30: + raise Exception('metadata reading failed') + except: + mi = metadata_from_filename(fname) + with open(opfpath, 'wb') as f: + f.write(metadata_to_opf(mi)) + self.staging.add(fname) + data[fname] = tdir + if data: + self.callback(data) + + +class AutoAdder(QObject): + + metadata_read = pyqtSignal(object) + + def __init__(self, path, parent): + QObject.__init__(self, parent) + if path and os.path.isdir(path) and os.access(path, os.R_OK|os.W_OK): + self.watcher = QFileSystemWatcher(self) + self.worker = Worker(path, self.metadata_read.emit) + self.watcher.directoryChanged.connect(self.dir_changed, + type=Qt.QueuedConnection) + self.metadata_read.connect(self.add_to_db, + type=Qt.QueuedConnection) + QTimer.singleShot(2000, self.initialize) + elif path: + prints(path, + 'is not a valid directory to watch for new ebooks, ignoring') + + def initialize(self): + try: + if os.listdir(self.worker.path): + self.dir_changed() + except: + pass + self.watcher.addPath(self.worker.path) + + def dir_changed(self, *args): + if os.path.isdir(self.worker.path) and os.access(self.worker.path, + os.R_OK|os.W_OK): + if not self.worker.is_alive(): + self.worker.start() + self.worker.wake_up.set() + + def stop(self): + if hasattr(self, 'worker'): + self.worker.keep_running = False + self.worker.wake_up.set() + + def wait(self): + if hasattr(self, 'worker'): + self.worker.join() + + def add_to_db(self, data): + from calibre.ebooks.metadata.opf2 import OPF + + gui = self.parent() + if gui is None: + return + m = gui.library_view.model() + count = 0 + + for fname, tdir in data.iteritems(): + paths = [os.path.join(self.worker.path, fname)] + mi = os.path.join(tdir, 'metadata.opf') + if not os.access(mi, os.R_OK): + continue + mi = [OPF(open(mi, 'rb'), tdir, + populate_spine=False).to_book_metadata()] + m.add_books(paths, [os.path.splitext(fname)[1][1:].upper()], mi, + add_duplicates=True) + try: + os.remove(os.path.join(self.worker.path, fname)) + try: + self.worker.staging.remove(fname) + except KeyError: + pass + shutil.rmtree(tdir) + except: + pass + count += 1 + + if count > 0: + m.books_added(count) + gui.status_bar.show_message(_( + 'Added %d book(s) automatically from %s') % + (count, self.worker.path), 2000) + if hasattr(gui, 'db_images'): + gui.db_images.reset() + + diff --git a/src/calibre/gui2/preferences/adding.py b/src/calibre/gui2/preferences/adding.py index 3965cd2afd..f7552b429a 100644 --- a/src/calibre/gui2/preferences/adding.py +++ b/src/calibre/gui2/preferences/adding.py @@ -5,14 +5,14 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' - +import os from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \ - CommaSeparatedList + CommaSeparatedList, AbortCommit from calibre.gui2.preferences.adding_ui import Ui_Form from calibre.utils.config import prefs from calibre.gui2.widgets import FilenamePattern -from calibre.gui2 import gprefs +from calibre.gui2 import gprefs, choose_dir, error_dialog, question_dialog class ConfigWidget(ConfigWidgetBase, Ui_Form): @@ -31,10 +31,18 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): (_('Create new record for each duplicate format'), 'new record')] r('automerge', gprefs, choices=choices) r('new_book_tags', prefs, setting=CommaSeparatedList) + r('auto_add_path', gprefs, restart_required=True) self.filename_pattern = FilenamePattern(self) self.metadata_box.layout().insertWidget(0, self.filename_pattern) self.filename_pattern.changed_signal.connect(self.changed_signal.emit) + self.auto_add_browse_button.clicked.connect(self.choose_aa_path) + + def choose_aa_path(self): + path = choose_dir(self, 'auto add path choose', + _('Choose a folder')) + if path: + self.opt_auto_add_path.setText(path) def initialize(self): ConfigWidgetBase.initialize(self) @@ -48,6 +56,25 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.filename_pattern.initialize(defaults=True) def commit(self): + path = unicode(self.opt_auto_add_path.text()).strip() + if path != gprefs['auto_add_path']: + if path: + if not os.path.isdir(path): + error_dialog(self, _('Invalid folder'), + _('You must specify an existing folder as your ' + 'auto-add folder. %s does not exist.')%path, + show=True) + raise AbortCommit('invalid auto-add folder') + if not os.access(path, os.R_OK|os.W_OK): + error_dialog(self, _('Invalid folder'), + _('You do not have read/write permissions for ' + 'the folder: %s')%path, show=True) + raise AbortCommit('invalid auto-add folder') + if not question_dialog(self, _('Are you sure'), + _('WARNING: Any files you place in %s will be ' + 'automatically deleted after being added to ' + 'calibre. Are you sure?')%path): + return pattern = self.filename_pattern.commit() prefs['filename_pattern'] = pattern return ConfigWidgetBase.commit(self) diff --git a/src/calibre/gui2/preferences/adding.ui b/src/calibre/gui2/preferences/adding.ui index dae050b7ea..9b7401447c 100644 --- a/src/calibre/gui2/preferences/adding.ui +++ b/src/calibre/gui2/preferences/adding.ui @@ -7,75 +7,92 @@ 0 0 753 - 339 + 547 Form - - - - Here you can control how calibre will read metadata from the files you add to it. calibre can either read metadata from the contents of the file, or from the filename. + + + + 0 - - true - - - - - - - Read &metadata from file contents rather than file name - - - - - - - - - Qt::Horizontal - - - - 40 - 20 - - - - - - - - Swap the firstname and lastname of the author. This affects only metadata read from file names. - - - &Swap author firstname and lastname - - - - - - - - - Automerge: If books with similar titles and authors found, merge the incoming formats automatically into + + + The Add &Process + + + + + + Here you can control how calibre will read metadata from the files you add to it. calibre can either read metadata from the contents of the file, or from the filename. + + + true + + + + + + + Read &metadata from file contents rather than file name + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Swap the firstname and lastname of the author. This affects only metadata read from file names. + + + &Swap author firstname and lastname + + + + + + + + + When using the "&Copy to library" action to copy books between libraries, preserve the date + + + + + + + Automerge: If books with similar titles and authors found, merge the incoming formats automatically into existing book records. The box to the right controls what happens when an existing record already has the incoming format. Note that this option also affects the Copy to library action. Title match ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author match is exact. - - - &Automerge added books if they already exist in the calibre library: - - - - - - - Automerge: If books with similar titles and authors found, merge the incoming formats automatically into + + + &Automerge added books if they already exist in the calibre library: + + + + + + + Automerge: If books with similar titles and authors found, merge the incoming formats automatically into existing book records. This box controls what happens when an existing record already has the incoming format: @@ -85,58 +102,119 @@ Create new record for each duplicate file - means that a new book entry will be Title matching ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author matching is exact. - - - - - - - &Tags to apply when adding a book: - - - opt_new_book_tags - - - - - - - A comma-separated list of tags that will be applied to books added to the library - - - - - - - &Configure metadata from file name - - - - - - Qt::Vertical - - - - 20 - 363 - - - - - - - - - - - When using the "&Copy to library" action to copy books between libraries, preserve the date - + + + + + + + &Tags to apply when adding a book: + + + opt_new_book_tags + + + + + + + A comma-separated list of tags that will be applied to books added to the library + + + + + + + &Configure metadata from file name + + + + + + Qt::Vertical + + + + 20 + 363 + + + + + + + + + + + + &Automatic Adding + + + + + + Specify a folder. Any files you put into this folder will be automatically added to calibre (restart required). + + + true + + + + + + + + + Folder to auto-add files from + + + + + + + Browse for folder + + + ... + + + + :/images/document_open.png:/images/document_open.png + + + + + + + + + <b>WARNING:</b> Files in the above folder will be deleted after being added to calibre. + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + - + + + opt_add_formats_to_existing diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index f34d8cbad1..116283a010 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -41,6 +41,7 @@ from calibre.gui2.search_box import SearchBoxMixin, SavedSearchBoxMixin from calibre.gui2.search_restriction_mixin import SearchRestrictionMixin from calibre.gui2.tag_browser.ui import TagBrowserMixin from calibre.gui2.keyboard import Manager +from calibre.gui2.auto_add import AutoAdder from calibre.library.sqlite import sqlite, DatabaseException class Listener(Thread): # {{{ @@ -349,6 +350,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.device_manager.set_current_library_uuid(db.library_id) self.keyboard.finalize() + self.auto_adder = AutoAdder(gprefs['auto_add_path'], self) # Collect cycles now gc.collect() @@ -697,6 +699,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ while self.spare_servers: self.spare_servers.pop().close() self.device_manager.keep_going = False + self.auto_adder.stop() mb = self.library_view.model().metadata_backup if mb is not None: mb.stop() diff --git a/src/calibre/utils/ipc/launch.py b/src/calibre/utils/ipc/launch.py index 982c05b065..508e302708 100644 --- a/src/calibre/utils/ipc/launch.py +++ b/src/calibre/utils/ipc/launch.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' import subprocess, os, sys, time, binascii, cPickle from functools import partial -from calibre.constants import iswindows, isosx, isfrozen +from calibre.constants import iswindows, isosx, isfrozen, filesystem_encoding from calibre.utils.config import prefs from calibre.ptempfile import PersistentTemporaryFile, base_dir @@ -87,12 +87,21 @@ class Worker(object): env = {} for key in os.environ: try: - env[key] = os.environ[key] + val = os.environ[key] + if isinstance(val, unicode): + # On windows subprocess cannot handle unicode env vars + try: + val = val.encode(filesystem_encoding) + except ValueError: + val = val.encode('utf-8') + if isinstance(key, unicode): + key = key.encode('ascii') + env[key] = val except: pass - env['CALIBRE_WORKER'] = '1' + env[b'CALIBRE_WORKER'] = b'1' td = binascii.hexlify(cPickle.dumps(base_dir())) - env['CALIBRE_WORKER_TEMP_DIR'] = td + env[b'CALIBRE_WORKER_TEMP_DIR'] = bytes(td) env.update(self._env) return env @@ -137,7 +146,19 @@ class Worker(object): def __init__(self, env, gui=False): self._env = {} self.gui = gui - self._env.update(env) + # Windows cannot handle unicode env vars + for k, v in env.iteritems(): + try: + if isinstance(k, unicode): + k = k.encode('ascii') + if isinstance(v, unicode): + try: + v = v.encode(filesystem_encoding) + except: + v = v.encode('utf-8') + self._env[k] = v + except: + pass def __call__(self, redirect_output=True, cwd=None, priority=None): '''