Fix #920249 ([Enhancement] Ability to Automatically Add eBook by Dropping It into a Folder)

src/calibre/ebooks/metadata/meta.py

@@ -217,3 +217,18 @@ def opf_metadata(opfpath):
         import traceback
         traceback.print_exc()
         pass
+
+def forked_read_metadata(path, tdir):
+    from calibre.ebooks.metadata.opf2 import metadata_to_opf
+    with open(path, 'rb') as f:
+        fmt = os.path.splitext(path)[1][1:].lower()
+        mi = get_metadata(f, fmt)
+    if mi.cover_data and mi.cover_data[1]:
+        with open(os.path.join(tdir, 'cover.jpg'), 'wb') as f:
+            f.write(mi.cover_data[1])
+        mi.cover_data = (None, None)
+        mi.cover = 'cover.jpg'
+    opf = metadata_to_opf(mi)
+    with open(os.path.join(tdir, 'metadata.opf'), 'wb') as f:
+        f.write(opf)
+

src/calibre/gui2/__init__.py

@@ -101,6 +101,7 @@ gprefs.defaults['preserve_date_on_ctl'] = True
 gprefs.defaults['cb_fullscreen'] = False
 gprefs.defaults['worker_max_time'] = 0
 gprefs.defaults['show_files_after_save'] = True
+gprefs.defaults['auto_add_path'] = None
 # }}}
 
 NONE = QVariant() #: Null value to return from the data function of item models

src/calibre/gui2/auto_add.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, tempfile, shutil
+from threading import Thread, Event
+
+from PyQt4.Qt import (QFileSystemWatcher, QObject, Qt, pyqtSignal, QTimer)
+
+from calibre import prints
+from calibre.ptempfile import PersistentTemporaryDirectory
+from calibre.ebooks import BOOK_EXTENSIONS
+
+class Worker(Thread):
+
+    def __init__(self, path, callback):
+        Thread.__init__(self)
+        self.daemon = True
+        self.keep_running = True
+        self.wake_up = Event()
+        self.path, self.callback = path, callback
+        self.staging = set()
+        self.be = frozenset(BOOK_EXTENSIONS)
+
+    def run(self):
+        self.tdir = PersistentTemporaryDirectory('_auto_adder')
+        while self.keep_running:
+            self.wake_up.wait()
+            self.wake_up.clear()
+            if not self.keep_running:
+                break
+            try:
+                self.auto_add()
+            except:
+                import traceback
+                traceback.print_exc()
+
+    def auto_add(self):
+        from calibre.utils.ipc.simple_worker import fork_job
+        from calibre.ebooks.metadata.opf2 import metadata_to_opf
+        from calibre.ebooks.metadata.meta import metadata_from_filename
+
+        files = [x for x in os.listdir(self.path) if x not in self.staging
+                and os.path.isfile(os.path.join(self.path, x)) and
+                os.access(os.path.join(self.path, x), os.R_OK|os.W_OK) and
+                os.path.splitext(x)[1][1:].lower() in self.be]
+        data = {}
+        for fname in files:
+            f = os.path.join(self.path, fname)
+            tdir = tempfile.mkdtemp(dir=self.tdir)
+            try:
+                fork_job('calibre.ebooks.metadata.meta',
+                        'forked_read_metadata', (f, tdir), no_output=True)
+            except:
+                import traceback
+                traceback.print_exc()
+
+            opfpath = os.path.join(tdir, 'metadata.opf')
+            try:
+                if os.stat(opfpath).st_size < 30:
+                    raise Exception('metadata reading failed')
+            except:
+                mi = metadata_from_filename(fname)
+                with open(opfpath, 'wb') as f:
+                    f.write(metadata_to_opf(mi))
+            self.staging.add(fname)
+            data[fname] = tdir
+        if data:
+            self.callback(data)
+
+
+class AutoAdder(QObject):
+
+    metadata_read = pyqtSignal(object)
+
+    def __init__(self, path, parent):
+        QObject.__init__(self, parent)
+        if path and os.path.isdir(path) and os.access(path, os.R_OK|os.W_OK):
+            self.watcher = QFileSystemWatcher(self)
+            self.worker = Worker(path, self.metadata_read.emit)
+            self.watcher.directoryChanged.connect(self.dir_changed,
+                    type=Qt.QueuedConnection)
+            self.metadata_read.connect(self.add_to_db,
+                    type=Qt.QueuedConnection)
+            QTimer.singleShot(2000, self.initialize)
+        elif path:
+            prints(path,
+                'is not a valid directory to watch for new ebooks, ignoring')
+
+    def initialize(self):
+        try:
+            if os.listdir(self.worker.path):
+                self.dir_changed()
+        except:
+            pass
+        self.watcher.addPath(self.worker.path)
+
+    def dir_changed(self, *args):
+        if os.path.isdir(self.worker.path) and os.access(self.worker.path,
+                os.R_OK|os.W_OK):
+            if not self.worker.is_alive():
+                self.worker.start()
+            self.worker.wake_up.set()
+
+    def stop(self):
+        if hasattr(self, 'worker'):
+            self.worker.keep_running = False
+            self.worker.wake_up.set()
+
+    def wait(self):
+        if hasattr(self, 'worker'):
+            self.worker.join()
+
+    def add_to_db(self, data):
+        from calibre.ebooks.metadata.opf2 import OPF
+
+        gui = self.parent()
+        if gui is None:
+            return
+        m = gui.library_view.model()
+        count = 0
+
+        for fname, tdir in data.iteritems():
+            paths = [os.path.join(self.worker.path, fname)]
+            mi = os.path.join(tdir, 'metadata.opf')
+            if not os.access(mi, os.R_OK):
+                continue
+            mi = [OPF(open(mi, 'rb'), tdir,
+                    populate_spine=False).to_book_metadata()]
+            m.add_books(paths, [os.path.splitext(fname)[1][1:].upper()], mi,
+                    add_duplicates=True)
+            try:
+                os.remove(os.path.join(self.worker.path, fname))
+                try:
+                    self.worker.staging.remove(fname)
+                except KeyError:
+                    pass
+                shutil.rmtree(tdir)
+            except:
+                pass
+            count += 1
+
+        if count > 0:
+            m.books_added(count)
+            gui.status_bar.show_message(_(
+                'Added %d book(s) automatically from %s') %
+                (count, self.worker.path), 2000)
+            if hasattr(gui, 'db_images'):
+                gui.db_images.reset()
+
+

src/calibre/gui2/preferences/adding.py

@@ -5,14 +5,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-
+import os
 
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
-    CommaSeparatedList
+    CommaSeparatedList, AbortCommit
 from calibre.gui2.preferences.adding_ui import Ui_Form
 from calibre.utils.config import prefs
 from calibre.gui2.widgets import FilenamePattern
-from calibre.gui2 import gprefs
+from calibre.gui2 import gprefs, choose_dir, error_dialog, question_dialog
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
@@ -31,10 +31,18 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                 (_('Create new record for each duplicate format'), 'new record')]
         r('automerge', gprefs, choices=choices)
         r('new_book_tags', prefs, setting=CommaSeparatedList)
+        r('auto_add_path', gprefs, restart_required=True)
 
         self.filename_pattern = FilenamePattern(self)
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)
         self.filename_pattern.changed_signal.connect(self.changed_signal.emit)
+        self.auto_add_browse_button.clicked.connect(self.choose_aa_path)
+
+    def choose_aa_path(self):
+        path = choose_dir(self, 'auto add path choose',
+                _('Choose a folder'))
+        if path:
+            self.opt_auto_add_path.setText(path)
 
     def initialize(self):
         ConfigWidgetBase.initialize(self)
@@ -48,6 +56,25 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.filename_pattern.initialize(defaults=True)
 
     def commit(self):
+        path = unicode(self.opt_auto_add_path.text()).strip()
+        if path != gprefs['auto_add_path']:
+            if path:
+                if not os.path.isdir(path):
+                    error_dialog(self, _('Invalid folder'),
+                            _('You must specify an existing folder as your '
+                                'auto-add folder. %s does not exist.')%path,
+                            show=True)
+                    raise AbortCommit('invalid auto-add folder')
+                if not os.access(path, os.R_OK|os.W_OK):
+                    error_dialog(self, _('Invalid folder'),
+                            _('You do not have read/write permissions for '
+                                'the folder: %s')%path, show=True)
+                    raise AbortCommit('invalid auto-add folder')
+                if not question_dialog(self, _('Are you sure'),
+                        _('<b>WARNING:</b> Any files you place in %s will be '
+                            'automatically deleted after being added to '
+                            'calibre. Are you sure?')%path):
+                    return
         pattern = self.filename_pattern.commit()
         prefs['filename_pattern'] = pattern
         return ConfigWidgetBase.commit(self)

src/calibre/gui2/preferences/adding.ui

@@ -7,75 +7,92 @@
     <x>0</x>
     <y>0</y>
     <width>753</width>
-    <height>339</height>
+    <height>547</height>
    </rect>
   </property>
   <property name="windowTitle">
    <string>Form</string>
   </property>
   <layout class="QGridLayout" name="gridLayout">
-   <item row="0" column="0" colspan="2">
+   <item row="0" column="0">
-    <widget class="QLabel" name="label_6">
+    <widget class="QTabWidget" name="tabWidget">
-     <property name="text">
+     <property name="currentIndex">
-      <string>Here you can control how calibre will read metadata from the files you add to it. calibre can either read metadata from the contents of the file, or from the filename.</string>
+      <number>0</number>
      </property>
-     <property name="wordWrap">
+     <widget class="QWidget" name="tab_3">
-      <bool>true</bool>
+      <attribute name="title">
-     </property>
+       <string>The Add &amp;Process</string>
-    </widget>
+      </attribute>
-   </item>
+      <layout class="QGridLayout" name="gridLayout_2">
-   <item row="1" column="0">
+       <item row="0" column="0" colspan="3">
-    <widget class="QCheckBox" name="opt_read_file_metadata">
+        <widget class="QLabel" name="label_6">
-     <property name="text">
+         <property name="text">
-      <string>Read &amp;metadata from file contents rather than file name</string>
+          <string>Here you can control how calibre will read metadata from the files you add to it. calibre can either read metadata from the contents of the file, or from the filename.</string>
-     </property>
+         </property>
-    </widget>
+         <property name="wordWrap">
-   </item>
+          <bool>true</bool>
-   <item row="1" column="1">
+         </property>
-    <layout class="QHBoxLayout" name="horizontalLayout">
+        </widget>
-     <item>
+       </item>
-      <spacer name="horizontalSpacer">
+       <item row="1" column="0">
-       <property name="orientation">
+        <widget class="QCheckBox" name="opt_read_file_metadata">
-        <enum>Qt::Horizontal</enum>
+         <property name="text">
-       </property>
+          <string>Read &amp;metadata from file contents rather than file name</string>
-       <property name="sizeHint" stdset="0">
+         </property>
-        <size>
+        </widget>
-         <width>40</width>
+       </item>
-         <height>20</height>
+       <item row="1" column="1" colspan="2">
-        </size>
+        <layout class="QHBoxLayout" name="horizontalLayout">
-       </property>
+         <item>
-      </spacer>
+          <spacer name="horizontalSpacer">
-     </item>
+           <property name="orientation">
-     <item>
+            <enum>Qt::Horizontal</enum>
-      <widget class="QCheckBox" name="opt_swap_author_names">
+           </property>
-       <property name="toolTip">
+           <property name="sizeHint" stdset="0">
-        <string>Swap the firstname and lastname of the author. This affects only metadata read from file names.</string>
+            <size>
-       </property>
+             <width>40</width>
-       <property name="text">
+             <height>20</height>
-        <string>&amp;Swap author firstname and lastname</string>
+            </size>
-       </property>
+           </property>
-      </widget>
+          </spacer>
-     </item>
+         </item>
-    </layout>
+         <item>
-   </item>
+          <widget class="QCheckBox" name="opt_swap_author_names">
-   <item row="3" column="0">
+           <property name="toolTip">
-    <widget class="QCheckBox" name="opt_add_formats_to_existing">
+            <string>Swap the firstname and lastname of the author. This affects only metadata read from file names.</string>
-     <property name="toolTip">
+           </property>
-      <string>Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+           <property name="text">
+            <string>&amp;Swap author firstname and lastname</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item row="2" column="0" colspan="3">
+        <widget class="QCheckBox" name="opt_preserve_date_on_ctl">
+         <property name="text">
+          <string>When using the  &quot;&amp;Copy to library&quot; action to copy books between libraries, preserve the date</string>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="0" colspan="2">
+        <widget class="QCheckBox" name="opt_add_formats_to_existing">
+         <property name="toolTip">
+          <string>Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
 existing book records. The box to the right controls what happens when an existing record already has
 the incoming format. Note that this option also affects the Copy to library action.
 
 Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc. Author match is exact.</string>
-     </property>
+         </property>
-     <property name="text">
+         <property name="text">
-      <string>&amp;Automerge added books if they already exist in the calibre library:</string>
+          <string>&amp;Automerge added books if they already exist in the calibre library:</string>
-     </property>
+         </property>
-    </widget>
+        </widget>
-   </item>
+       </item>
-   <item row="3" column="1">
+       <item row="3" column="2">
-    <widget class="QComboBox" name="opt_automerge">
+        <widget class="QComboBox" name="opt_automerge">
-     <property name="toolTip">
+         <property name="toolTip">
-      <string>Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+          <string>Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
 existing book records. This box controls what happens when an existing record already has
 the incoming format: 
 
@@ -85,58 +102,119 @@ Create new record for each duplicate file - means that a new book entry will be
 
 Title matching ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc.
 Author matching is exact.</string>
-     </property>
+         </property>
-    </widget>
+        </widget>
-   </item>
+       </item>
-   <item row="4" column="0">
+       <item row="4" column="0">
-    <widget class="QLabel" name="label_230">
+        <widget class="QLabel" name="label_230">
-     <property name="text">
+         <property name="text">
-      <string>&amp;Tags to apply when adding a book:</string>
+          <string>&amp;Tags to apply when adding a book:</string>
-     </property>
+         </property>
-     <property name="buddy">
+         <property name="buddy">
-      <cstring>opt_new_book_tags</cstring>
+          <cstring>opt_new_book_tags</cstring>
-     </property>
+         </property>
-    </widget>
+        </widget>
-   </item>
+       </item>
-   <item row="4" column="1">
+       <item row="4" column="2">
-    <widget class="QLineEdit" name="opt_new_book_tags">
+        <widget class="QLineEdit" name="opt_new_book_tags">
-     <property name="toolTip">
+         <property name="toolTip">
-      <string>A comma-separated list of tags that will be applied to books added to the library</string>
+          <string>A comma-separated list of tags that will be applied to books added to the library</string>
-     </property>
+         </property>
-    </widget>
+        </widget>
-   </item>
+       </item>
-   <item row="5" column="0" colspan="2">
+       <item row="5" column="0" colspan="3">
-    <widget class="QGroupBox" name="metadata_box">
+        <widget class="QGroupBox" name="metadata_box">
-     <property name="title">
+         <property name="title">
-      <string>&amp;Configure metadata from file name</string>
+          <string>&amp;Configure metadata from file name</string>
-     </property>
+         </property>
-     <layout class="QVBoxLayout" name="verticalLayout">
+         <layout class="QVBoxLayout" name="verticalLayout">
-      <item>
+          <item>
-       <spacer name="verticalSpacer">
+           <spacer name="verticalSpacer">
-        <property name="orientation">
+            <property name="orientation">
-         <enum>Qt::Vertical</enum>
+             <enum>Qt::Vertical</enum>
-        </property>
+            </property>
-        <property name="sizeHint" stdset="0">
+            <property name="sizeHint" stdset="0">
-         <size>
+             <size>
-          <width>20</width>
+              <width>20</width>
-          <height>363</height>
+              <height>363</height>
-         </size>
+             </size>
-        </property>
+            </property>
-       </spacer>
+           </spacer>
-      </item>
+          </item>
-     </layout>
+         </layout>
-    </widget>
+        </widget>
-   </item>
+       </item>
-   <item row="2" column="0" colspan="2">
+      </layout>
-    <widget class="QCheckBox" name="opt_preserve_date_on_ctl">
+     </widget>
-     <property name="text">
+     <widget class="QWidget" name="tab_4">
-      <string>When using the  &quot;&amp;Copy to library&quot; action to copy books between libraries, preserve the date</string>
+      <attribute name="title">
-     </property>
+       <string>&amp;Automatic Adding</string>
+      </attribute>
+      <layout class="QVBoxLayout" name="verticalLayout_2">
+       <item>
+        <widget class="QLabel" name="label">
+         <property name="text">
+          <string>Specify a folder. Any files you put into this folder will be automatically added to calibre (restart required).</string>
+         </property>
+         <property name="wordWrap">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <layout class="QHBoxLayout" name="horizontalLayout_2">
+         <item>
+          <widget class="QLineEdit" name="opt_auto_add_path">
+           <property name="placeholderText">
+            <string>Folder to auto-add files from</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QToolButton" name="auto_add_browse_button">
+           <property name="toolTip">
+            <string>Browse for folder</string>
+           </property>
+           <property name="text">
+            <string>...</string>
+           </property>
+           <property name="icon">
+            <iconset resource="../../../work/calibre/resources/images.qrc">
+             <normaloff>:/images/document_open.png</normaloff>:/images/document_open.png</iconset>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item>
+        <widget class="QLabel" name="label_2">
+         <property name="text">
+          <string>&lt;b&gt;WARNING:&lt;/b&gt; Files in the above folder will be deleted after being added to calibre.</string>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <spacer name="verticalSpacer_2">
+         <property name="orientation">
+          <enum>Qt::Vertical</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>20</width>
+           <height>40</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+      </layout>
+     </widget>
     </widget>
    </item>
   </layout>
  </widget>
- <resources/>
+ <resources>
+  <include location="../../../work/calibre/resources/images.qrc"/>
+ </resources>
  <connections>
   <connection>
    <sender>opt_add_formats_to_existing</sender>

src/calibre/gui2/ui.py

@@ -41,6 +41,7 @@ from calibre.gui2.search_box import SearchBoxMixin, SavedSearchBoxMixin
 from calibre.gui2.search_restriction_mixin import SearchRestrictionMixin
 from calibre.gui2.tag_browser.ui import TagBrowserMixin
 from calibre.gui2.keyboard import Manager
+from calibre.gui2.auto_add import AutoAdder
 from calibre.library.sqlite import sqlite, DatabaseException
 
 class Listener(Thread): # {{{
@@ -349,6 +350,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.device_manager.set_current_library_uuid(db.library_id)
 
         self.keyboard.finalize()
+        self.auto_adder = AutoAdder(gprefs['auto_add_path'], self)
 
         # Collect cycles now
         gc.collect()
@@ -697,6 +699,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         while self.spare_servers:
             self.spare_servers.pop().close()
         self.device_manager.keep_going = False
+        self.auto_adder.stop()
         mb = self.library_view.model().metadata_backup
         if mb is not None:
             mb.stop()

src/calibre/utils/ipc/launch.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 import subprocess, os, sys, time, binascii, cPickle
 from functools import partial
 
-from calibre.constants import iswindows, isosx, isfrozen
+from calibre.constants import iswindows, isosx, isfrozen, filesystem_encoding
 from calibre.utils.config import prefs
 from calibre.ptempfile import PersistentTemporaryFile, base_dir
 
@@ -87,12 +87,21 @@ class Worker(object):
         env = {}
         for key in os.environ:
             try:
-                env[key] = os.environ[key]
+                val = os.environ[key]
+                if isinstance(val, unicode):
+                    # On windows subprocess cannot handle unicode env vars
+                    try:
+                        val = val.encode(filesystem_encoding)
+                    except ValueError:
+                        val = val.encode('utf-8')
+                if isinstance(key, unicode):
+                    key = key.encode('ascii')
+                env[key] = val
             except:
                 pass
-        env['CALIBRE_WORKER'] = '1'
+        env[b'CALIBRE_WORKER'] = b'1'
         td = binascii.hexlify(cPickle.dumps(base_dir()))
-        env['CALIBRE_WORKER_TEMP_DIR'] = td
+        env[b'CALIBRE_WORKER_TEMP_DIR'] = bytes(td)
         env.update(self._env)
         return env
 
@@ -137,7 +146,19 @@ class Worker(object):
     def __init__(self, env, gui=False):
         self._env = {}
         self.gui = gui
-        self._env.update(env)
+        # Windows cannot handle unicode env vars
+        for k, v in env.iteritems():
+            try:
+                if isinstance(k, unicode):
+                    k = k.encode('ascii')
+                if isinstance(v, unicode):
+                    try:
+                        v = v.encode(filesystem_encoding)
+                    except:
+                        v = v.encode('utf-8')
+                self._env[k] = v
+            except:
+                pass
 
     def __call__(self, redirect_output=True, cwd=None, priority=None):
         '''