calibre is now able to download social metadata like tags/rating/reviews etc. in addition to normal metadata and covers. Currently it uses Amazon as the only source for social metadata, but the download system supports plugins for the addition of more sources in the future. Fixes #2860 (Amazon Tag Scrapper)

src/calibre/ebooks/metadata/fetch.py

@@ -201,11 +201,13 @@ def get_social_metadata(mi, verbose=0):
         mi.tags += list(tags)
         mi.tags = list(sorted(list(set(mi.tags))))
     if comments:
-        mi.comments = ''
+        if not mi.comments or len(mi.comments) < len(comments):
-        for x in comments:
+            mi.comments = ''
-            mi.comments += '\n\n'+x
+            for x in comments:
+                mi.comments += '\n\n'+x
 
-    return [(x.name, x.exception, x.tb) for x in fetchers]
+    return [(x.name, x.exception, x.tb) for x in fetchers if x.exception is not
+            None]
 
 
 

src/calibre/gui2/__init__.py

@@ -83,6 +83,8 @@ def _config():
         help='Search history for the recipe scheduler')
     c.add_opt('worker_limit', default=6,
             help=_('Maximum number of waiting worker processes'))
+    c.add_opt('get_social_metadata', default=True,
+            help=_('Download social metadata (tags/rating/etc.)'))
 
     return ConfigProxy(c)
 

src/calibre/gui2/dialogs/config/__init__.py

@@ -455,6 +455,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
         self.opt_worker_limit.setValue(config['worker_limit'])
         self.connect(self.button_open_config_dir, SIGNAL('clicked()'),
                 self.open_config_dir)
+        self.opt_get_social_metadata.setChecked(config['get_social_metadata'])
 
     def open_config_dir(self):
         from calibre.utils.config import config_dir
@@ -740,6 +741,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
         config['delete_news_from_library_on_upload'] = self.delete_news.isChecked()
         config['upload_news_to_device'] = self.sync_news.isChecked()
         config['search_as_you_type'] = self.search_as_you_type.isChecked()
+        config['get_social_metadata'] = self.opt_get_social_metadata.isChecked()
         fmts = []
         for i in range(self.viewer.count()):
             if self.viewer.item(i).checkState() == Qt.Checked:

src/calibre/gui2/dialogs/config/config.ui

@@ -90,7 +90,7 @@
           <x>0</x>
           <y>0</y>
           <width>524</width>
-          <height>682</height>
+          <height>680</height>
          </rect>
         </property>
         <layout class="QGridLayout" name="gridLayout_7">
@@ -164,6 +164,13 @@
                </property>
               </widget>
              </item>
+             <item>
+              <widget class="QCheckBox" name="opt_get_social_metadata">
+               <property name="text">
+                <string>Download &amp;social metadata (tags/ratings/etc.) by default</string>
+               </property>
+              </widget>
+             </item>
              <item>
               <layout class="QGridLayout" name="gridLayout_2">
                <item row="1" column="0">

src/calibre/gui2/dialogs/config/social.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import QDialog, QDialogButtonBox, Qt, QLabel, QVBoxLayout, \
+        SIGNAL, QThread
+
+from calibre.ebooks.metadata import MetaInformation
+
+class Worker(QThread):
+
+    def __init__(self, mi, parent):
+        QThread.__init__(self, parent)
+        self.mi = MetaInformation(mi)
+        self.exceptions = []
+
+    def run(self):
+        from calibre.ebooks.metadata.fetch import get_social_metadata
+        self.exceptions = get_social_metadata(self.mi)
+
+class SocialMetadata(QDialog):
+
+    def __init__(self, mi, parent):
+        QDialog.__init__(self, parent)
+
+        self.bbox = QDialogButtonBox(QDialogButtonBox.Ok, Qt.Horizontal, self)
+        self.mi = mi
+        self.layout = QVBoxLayout(self)
+        self.label = QLabel(_('Downloading social metadata, please wait...'), self)
+        self.label.setWordWrap(True)
+        self.layout.addWidget(self.label)
+        self.layout.addWidget(self.bbox)
+
+        self.worker = Worker(mi, self)
+        self.connect(self.worker, SIGNAL('finished()'), self.accept)
+        self.connect(self.bbox, SIGNAL('rejected()'), self.reject)
+        self.worker.start()
+
+    def reject(self):
+        self.disconnect(self.worker, SIGNAL('finished()'), self.accept)
+        QDialog.reject(self)
+
+    def accept(self):
+        self.mi.tags = self.worker.mi.tags
+        self.mi.rating = self.worker.mi.rating
+        self.mi.comments = self.worker.mi.comments
+        QDialog.accept(self)
+
+    @property
+    def exceptions(self):
+        return self.worker.exceptions

src/calibre/gui2/dialogs/fetch_metadata.py

@@ -11,7 +11,7 @@ from PyQt4.QtCore import Qt, QObject, SIGNAL, QVariant, QThread, \
 from PyQt4.QtGui import QDialog, QItemSelectionModel
 
 from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata
-from calibre.gui2 import error_dialog, NONE, info_dialog
+from calibre.gui2 import error_dialog, NONE, info_dialog, config
 from calibre.gui2.widgets import ProgressIndicator
 from calibre import strftime
 from calibre.customize.ui import get_isbndb_key, set_isbndb_key
@@ -115,6 +115,7 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
                      self.show_summary)
         self.matches.setMouseTracking(True)
         self.fetch_metadata()
+        self.opt_get_social_metadata.setChecked(config['get_social_metadata'])
 
 
     def show_summary(self, current, *args):

src/calibre/gui2/dialogs/fetch_metadata.ui

@@ -1,10 +1,11 @@
-<ui version="4.0" >
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
  <class>FetchMetadata</class>
- <widget class="QDialog" name="FetchMetadata" >
+ <widget class="QDialog" name="FetchMetadata">
-  <property name="windowModality" >
+  <property name="windowModality">
    <enum>Qt::WindowModal</enum>
   </property>
-  <property name="geometry" >
+  <property name="geometry">
    <rect>
     <x>0</x>
     <y>0</y>
@@ -12,48 +13,48 @@
     <height>642</height>
    </rect>
   </property>
-  <property name="windowTitle" >
+  <property name="windowTitle">
    <string>Fetch metadata</string>
   </property>
-  <property name="windowIcon" >
+  <property name="windowIcon">
-   <iconset resource="../../../../resources/images.qrc" >
+   <iconset resource="../../../../resources/images.qrc">
     <normaloff>:/images/metadata.svg</normaloff>:/images/metadata.svg</iconset>
   </property>
-  <layout class="QVBoxLayout" >
+  <layout class="QVBoxLayout">
    <item>
-    <widget class="QLabel" name="tlabel" >
+    <widget class="QLabel" name="tlabel">
-     <property name="text" >
+     <property name="text">
-      <string>&lt;p>calibre can find metadata for your books from two locations: &lt;b>Google Books&lt;/b> and &lt;b>isbndb.com&lt;/b>. &lt;p>To use isbndb.com you must sign up for a &lt;a href="http://www.isbndb.com">free account&lt;/a> and enter your access key below.</string>
+      <string>&lt;p&gt;calibre can find metadata for your books from two locations: &lt;b&gt;Google Books&lt;/b&gt; and &lt;b&gt;isbndb.com&lt;/b&gt;. &lt;p&gt;To use isbndb.com you must sign up for a &lt;a href=&quot;http://www.isbndb.com&quot;&gt;free account&lt;/a&gt; and enter your access key below.</string>
      </property>
-     <property name="alignment" >
+     <property name="alignment">
       <set>Qt::AlignCenter</set>
      </property>
-     <property name="wordWrap" >
+     <property name="wordWrap">
       <bool>true</bool>
      </property>
-     <property name="openExternalLinks" >
+     <property name="openExternalLinks">
       <bool>true</bool>
      </property>
     </widget>
    </item>
    <item>
-    <layout class="QHBoxLayout" >
+    <layout class="QHBoxLayout">
      <item>
-      <widget class="QLabel" name="label_2" >
+      <widget class="QLabel" name="label_2">
-       <property name="text" >
+       <property name="text">
         <string>&amp;Access Key:</string>
        </property>
-       <property name="buddy" >
+       <property name="buddy">
         <cstring>key</cstring>
        </property>
       </widget>
      </item>
      <item>
-      <widget class="QLineEdit" name="key" />
+      <widget class="QLineEdit" name="key"/>
      </item>
      <item>
-      <widget class="QPushButton" name="fetch" >
+      <widget class="QPushButton" name="fetch">
-       <property name="text" >
+       <property name="text">
         <string>Fetch</string>
        </property>
       </widget>
@@ -61,56 +62,63 @@
     </layout>
    </item>
    <item>
-    <widget class="QLabel" name="warning" >
+    <widget class="QLabel" name="warning">
-     <property name="text" >
+     <property name="text">
       <string/>
      </property>
-     <property name="wordWrap" >
+     <property name="wordWrap">
       <bool>true</bool>
      </property>
     </widget>
    </item>
    <item>
-    <widget class="QGroupBox" name="groupBox" >
+    <widget class="QGroupBox" name="groupBox">
-     <property name="title" >
+     <property name="title">
       <string>Matches</string>
      </property>
-     <layout class="QVBoxLayout" >
+     <layout class="QVBoxLayout">
       <item>
-       <widget class="QLabel" name="label_3" >
+       <widget class="QLabel" name="label_3">
-        <property name="text" >
+        <property name="text">
          <string>Select the book that most closely matches your copy from the list below</string>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="QTableView" name="matches" >
+       <widget class="QTableView" name="matches">
-        <property name="sizePolicy" >
+        <property name="sizePolicy">
-         <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
+         <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
           <horstretch>0</horstretch>
           <verstretch>1</verstretch>
          </sizepolicy>
         </property>
-        <property name="alternatingRowColors" >
+        <property name="alternatingRowColors">
          <bool>true</bool>
         </property>
-        <property name="selectionMode" >
+        <property name="selectionMode">
          <enum>QAbstractItemView::SingleSelection</enum>
         </property>
-        <property name="selectionBehavior" >
+        <property name="selectionBehavior">
          <enum>QAbstractItemView::SelectRows</enum>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="QTextBrowser" name="summary" />
+       <widget class="QTextBrowser" name="summary"/>
       </item>
      </layout>
     </widget>
    </item>
    <item>
-    <widget class="QDialogButtonBox" name="buttonBox" >
+    <widget class="QCheckBox" name="opt_get_social_metadata">
-     <property name="standardButtons" >
+     <property name="text">
+      <string>Download &amp;social metadata (tags/rating/etc.) for the selected book</string>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="standardButtons">
       <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
      </property>
     </widget>
@@ -118,7 +126,7 @@
   </layout>
  </widget>
  <resources>
-  <include location="../../../../resources/images.qrc" />
+  <include location="../../../../resources/images.qrc"/>
  </resources>
  <connections>
   <connection>
@@ -127,11 +135,11 @@
    <receiver>FetchMetadata</receiver>
    <slot>accept()</slot>
    <hints>
-    <hint type="sourcelabel" >
+    <hint type="sourcelabel">
      <x>460</x>
      <y>599</y>
     </hint>
-    <hint type="destinationlabel" >
+    <hint type="destinationlabel">
      <x>657</x>
      <y>530</y>
     </hint>
@@ -143,11 +151,11 @@
    <receiver>FetchMetadata</receiver>
    <slot>reject()</slot>
    <hints>
-    <hint type="sourcelabel" >
+    <hint type="sourcelabel">
      <x>417</x>
      <y>599</y>
     </hint>
-    <hint type="destinationlabel" >
+    <hint type="destinationlabel">
      <x>0</x>
      <y>491</y>
     </hint>

src/calibre/gui2/dialogs/metadata_single.py

@@ -16,7 +16,8 @@ from PyQt4.Qt import SIGNAL, QObject, QCoreApplication, Qt, QTimer, QThread, QDa
     QPixmap, QListWidgetItem, QDialog
 
 from calibre.gui2 import qstring_to_unicode, error_dialog, file_icon_provider, \
-                           choose_files, choose_images, ResizableDialog
+                           choose_files, choose_images, ResizableDialog, \
+                           warning_dialog
 from calibre.gui2.dialogs.metadata_single_ui import Ui_MetadataSingleDialog
 from calibre.gui2.dialogs.fetch_metadata import FetchMetadata
 from calibre.gui2.dialogs.tag_editor import TagEditor
@@ -28,6 +29,7 @@ from calibre import islinux
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.utils.config import prefs
 from calibre.customize.ui import run_plugins_on_import, get_isbndb_key
+from calibre.gui2.dialogs.config.social import SocialMetadata
 
 class CoverFetcher(QThread):
 
@@ -541,6 +543,15 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                 if d.exec_() == QDialog.Accepted:
                     book = d.selected_book()
                     if book:
+                        if d.opt_get_social_metadata.isChecked():
+                            d2 = SocialMetadata(book, self)
+                            d2.exec_()
+                            if d2.exceptions:
+                                det = '\n'.join([x[0]+'\n\n'+x[-1]+'\n\n\n' for
+                                    x in d2.exceptions])
+                                warning_dialog(self, _('There were errors'),
+                                       _('There were errors downloading social metadata'),
+                                       det_msg=det, show=True)
                         self.title.setText(book.title)
                         self.authors.setText(authors_to_string(book.authors))
                         if book.author_sort: self.author_sort.setText(book.author_sort)
@@ -552,14 +563,18 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                             self.pubdate.setDate(QDate(d.year, d.month, d.day))
                         summ = book.comments
                         if summ:
-                            prefix = qstring_to_unicode(self.comments.toPlainText())
+                            prefix = unicode(self.comments.toPlainText())
                             if prefix:
                                 prefix += '\n'
                             self.comments.setText(prefix + summ)
+                        if book.rating is not None:
+                            self.rating.setValue(int(book.rating))
+                        if book.tags:
+                            self.tags.setText(', '.join(book.tags))
         else:
             error_dialog(self, _('Cannot fetch metadata'),
                          _('You must specify at least one of ISBN, Title, '
-                           'Authors or Publisher'))
+                           'Authors or Publisher'), show=True)
 
     def enable_series_index(self, *args):
         self.series_index.setEnabled(True)

src/calibre/gui2/main.py

@@ -1039,7 +1039,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         ids = [db.id(row.row()) for row in rows]
         from calibre.gui2.metadata import DownloadMetadata
         self._download_book_metadata = DownloadMetadata(db, ids,
-                get_covers=covers, set_metadata=set_metadata)
+                get_covers=covers, set_metadata=set_metadata,
+                get_social_metadata=config['get_social_metadata'])
         self._download_book_metadata.start()
         x = _('covers') if covers and not set_metadata else _('metadata')
         self.progress_indicator.start(

src/calibre/gui2/metadata.py

@@ -11,7 +11,7 @@ from threading import Thread
 from Queue import Queue, Empty
 
 
-from calibre.ebooks.metadata.fetch import search
+from calibre.ebooks.metadata.fetch import search, get_social_metadata
 from calibre.ebooks.metadata.library_thing import cover_from_isbn
 from calibre.customize.ui import get_isbndb_key
 
@@ -45,12 +45,15 @@ class Worker(Thread):
 
 class DownloadMetadata(Thread):
 
-    def __init__(self, db, ids, get_covers, set_metadata=True):
+    def __init__(self, db, ids, get_covers, set_metadata=True,
+            get_social_metadata=True):
         Thread.__init__(self)
         self.setDaemon(True)
         self.metadata = {}
         self.covers   = {}
         self.set_metadata = set_metadata
+        self.get_social_metadata = get_social_metadata
+        self.social_metadata_exceptions = []
         self.db = db
         self.updated = set([])
         self.get_covers = get_covers
@@ -95,6 +98,8 @@ class DownloadMetadata(Thread):
                     if fmi.isbn and self.get_covers:
                         self.worker.jobs.put(fmi.isbn)
                     mi.smart_update(fmi)
+                    if mi.isbn and self.get_social_metadata:
+                        self.social_metadata_exceptions = get_social_metadata(mi)
                 else:
                     self.failures[id] = (mi.title,
                         _('No matches found for this book'))

src/calibre/manual/custom.py

@@ -40,6 +40,9 @@ Command Line Interface
 
 .. image:: ../images/cli.png
 
+On OS X you have to go to Preferences->Advanced and click install command line
+tools to make the command line tools available.  On other platforms, just start
+a terminal and type the command.
 
 Documented Commands
 --------------------