Merge from trunk

recipes/kommersant.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2010-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 www.kommersant.ru
 '''
@@ -20,7 +20,13 @@ class Kommersant_ru(BasicNewsRecipe):
     language              = 'ru'
     publication_type      = 'newspaper'
     masthead_url          = 'http://www.kommersant.ru/CorpPics/logo_daily_1.gif'
-    extra_css             = ' @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: Arial, sans1, sans-serif} span#ctl00_ContentPlaceHolderStyle_LabelSubTitle{margin-bottom: 1em; display: block} .author{margin-bottom: 1em; display: block} .paragraph{margin-bottom: 1em; display: block} .vvodka{font-weight: bold; margin-bottom: 1em} '
+    extra_css             = """ 
+	                          @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)}
+	                          body{font-family: Tahoma, Arial, Helvetica, sans1, sans-serif}
+							  .title{font-size: x-large; font-weight: bold; margin-bottom: 1em}
+							  .subtitle{font-size: large; margin-bottom: 1em}
+							  .document_vvodka{font-weight: bold; margin-bottom: 1em}
+							"""
 
     conversion_options = {
                           'comment'          : description
@@ -29,14 +35,11 @@ class Kommersant_ru(BasicNewsRecipe):
                         , 'language'         : language
                         }
 
-    keep_only_tags = [
+    keep_only_tags = [dict(attrs={'class':['document','document_vvodka','document_text','document_authors vblock']})]
-                         dict(attrs={'id':'ctl00_ContentPlaceHolderStyle_PanelHeader'})
+    remove_tags    = [dict(name=['iframe','object','link','img','base','meta'])]
-                        ,dict(attrs={'class':['vvodka','paragraph','author']})
-                     ]
-    remove_tags        = [dict(name=['iframe','object','link','img','base'])]
 
     feeds       = [(u'Articles', u'http://feeds.kommersant.ru/RSS_Export/RU/daily.xml')]
 
     def print_version(self, url):	    
-        return url.replace('doc-rss.aspx','doc.aspx') + '&print=true'
+        return url.replace('/doc-rss/','/Doc/') + '/Print'
 		

src/calibre/__init__.py

@@ -217,8 +217,19 @@ def filename_to_utf8(name):
     return name.decode(codec, 'replace').encode('utf8')
 
 def extract(path, dir):
-    ext = os.path.splitext(path)[1][1:].lower()
     extractor = None
+    # First use the file header to identify its type
+    with open(path, 'rb') as f:
+        id_ = f.read(3)
+    if id_ == b'Rar':
+        from calibre.libunrar import extract as rarextract
+        extractor = rarextract
+    elif id_.startswith(b'PK'):
+        from calibre.libunzip import extract as zipextract
+        extractor = zipextract
+    if extractor is None:
+        # Fallback to file extension
+        ext = os.path.splitext(path)[1][1:].lower()
         if ext in ['zip', 'cbz', 'epub', 'oebzip']:
             from calibre.libunzip import extract as zipextract
             extractor = zipextract

src/calibre/customize/builtins.py

@@ -10,6 +10,7 @@ from calibre.constants import numeric_version
 from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.ebooks.oeb.base import OEB_IMAGES
+from calibre.utils.config import test_eight_code
 
 # To archive plugins {{{
 class HTML2ZIP(FileTypePlugin):
@@ -166,6 +167,14 @@ class ComicMetadataReader(MetadataReaderPlugin):
     description = _('Extract cover from comic files')
 
     def get_metadata(self, stream, ftype):
+        if hasattr(stream, 'seek') and hasattr(stream, 'tell'):
+            pos = stream.tell()
+            id_ = stream.read(3)
+            stream.seek(pos)
+            if id_ == b'Rar':
+                ftype = 'cbr'
+            elif id.startswith(b'PK'):
+                ftype = 'cbz'
         if ftype == 'cbr':
             from calibre.libunrar import extract_first_alphabetically as extract_first
             extract_first
@@ -604,20 +613,34 @@ from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
 from calibre.devices.kobo.driver import KOBO
 from calibre.devices.bambook.driver import BAMBOOK
 
+from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX
+from calibre.ebooks.epub.fix.unmanifested import Unmanifested
+from calibre.ebooks.epub.fix.epubcheck import Epubcheck
+
+plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested,
+        Epubcheck, ]
+
+if test_eight_code:
+# New metadata download plugins {{{
+    from calibre.ebooks.metadata.sources.google import GoogleBooks
+    from calibre.ebooks.metadata.sources.amazon import Amazon
+    from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary
+
+    plugins += [GoogleBooks, Amazon, OpenLibrary]
+
+# }}}
+else:
     from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \
         KentDistrictLibrary
     from calibre.ebooks.metadata.douban import DoubanBooks
     from calibre.ebooks.metadata.nicebooks import NiceBooks, NiceBooksCovers
     from calibre.ebooks.metadata.covers import OpenLibraryCovers, \
             AmazonCovers, DoubanCovers
-from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX
-from calibre.ebooks.epub.fix.unmanifested import Unmanifested
-from calibre.ebooks.epub.fix.epubcheck import Epubcheck
 
-plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, GoogleBooks, ISBNDB, Amazon,
+    plugins += [GoogleBooks, ISBNDB, Amazon,
-        KentDistrictLibrary, DoubanBooks, NiceBooks, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested,
+        OpenLibraryCovers, AmazonCovers, DoubanCovers,
-        Epubcheck, OpenLibraryCovers, AmazonCovers, DoubanCovers,
+        NiceBooksCovers, KentDistrictLibrary, DoubanBooks, NiceBooks]
-        NiceBooksCovers]
+
 plugins += [
     ComicInput,
     EPUBInput,
@@ -1055,11 +1078,4 @@ plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions,
 
 #}}}
 
-# New metadata download plugins {{{
-from calibre.ebooks.metadata.sources.google import GoogleBooks
-from calibre.ebooks.metadata.sources.amazon import Amazon
-from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary
 
-plugins += [GoogleBooks, Amazon, OpenLibrary]
-
-# }}}

src/calibre/devices/android/driver.py

@@ -36,7 +36,9 @@ class ANDROID(USBMS):
             # Motorola
             0x22b8 : { 0x41d9 : [0x216], 0x2d61 : [0x100], 0x2d67 : [0x100],
                        0x41db : [0x216], 0x4285 : [0x216], 0x42a3 : [0x216],
-                       0x4286 : [0x216], 0x42b3 : [0x216], 0x42b4 : [0x216] },
+                       0x4286 : [0x216], 0x42b3 : [0x216], 0x42b4 : [0x216],
+                       0x7086 : [0x0226],
+                     },
 
             # Sony Ericsson
             0xfce : { 0xd12e : [0x0100]},
@@ -101,7 +103,8 @@ class ANDROID(USBMS):
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
             'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
             'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD',
-            '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2']
+            '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2',
+            'MB860']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7']

src/calibre/ebooks/htmlz/oeb2html.py

@@ -12,10 +12,13 @@ Transform OEB content into a single (more or less) HTML file.
 
 import os
 
-from urlparse import urlparse
+from functools import partial
+from lxml import html
+from urlparse import urldefrag
 
 from calibre import prepare_string_for_xml
-from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace
+from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace,\
+    OEB_IMAGES, XLINK, rewrite_links
 from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.utils.logging import default_log
 
@@ -40,6 +43,8 @@ class OEB2HTML(object):
         self.opts = opts
         self.links = {}
         self.images = {}
+        self.base_hrefs = [item.href for item in oeb_book.spine]
+        self.map_resources(oeb_book)
 
         return self.mlize_spine(oeb_book)
 
@@ -47,6 +52,8 @@ class OEB2HTML(object):
         output = [u'<html><body><head><meta http-equiv="Content-Type" content="text/html;charset=utf-8" /></head>']
         for item in oeb_book.spine:
             self.log.debug('Converting %s to HTML...' % item.href)
+            self.rewrite_ids(item.data, item)
+            rewrite_links(item.data, partial(self.rewrite_link, page=item))
             stylizer = Stylizer(item.data, item.href, oeb_book, self.opts)
             output += self.dump_text(item.data.find(XHTML('body')), stylizer, item)
             output.append('\n\n')
@@ -56,43 +63,61 @@ class OEB2HTML(object):
     def dump_text(self, elem, stylizer, page):
         raise NotImplementedError
 
-    def get_link_id(self, href, aid):
+    def get_link_id(self, href, id=''):
-        aid = '%s#%s' % (href, aid)
+        if id:
-        if aid not in self.links:
+            href += '#%s' % id
-            self.links[aid] = 'calibre_link-%s' % len(self.links.keys())
+        if href not in self.links:
-        return self.links[aid]
+            self.links[href] = '#calibre_link-%s' % len(self.links.keys())
+        return self.links[href]
 
-    def rewrite_link(self, tag, attribs, page):
+    def map_resources(self, oeb_book):
-        # Rewrite ids.
+        for item in oeb_book.manifest:
-        if 'id' in attribs:
+            if item.media_type in OEB_IMAGES:
-            attribs['id'] = self.get_link_id(page.href, attribs['id'])
+                if item.href not in self.images:
-        # Rewrite links.
+                    ext = os.path.splitext(item.href)[1]
-        if tag == 'a' and 'href' in attribs:
-            href = page.abshref(attribs['href'])
-            if self.url_is_relative(href):
-                id = ''
-                if '#' in href:
-                    href, n, id = href.partition('#')
-                href = '#%s' % self.get_link_id(href, id)
-                attribs['href'] = href
-        return attribs
-
-    def rewrite_image(self, tag, attribs, page):
-        if tag == 'img':
-            src = attribs.get('src', None)
-            if src:
-                src = page.abshref(src)
-                if src not in self.images:
-                    ext = os.path.splitext(src)[1]
                     fname = '%s%s' % (len(self.images), ext)
                     fname = fname.zfill(10)
-                    self.images[src] = fname
+                    self.images[item.href] = fname
-                attribs['src'] = 'images/%s' % self.images[src]
+            if item in oeb_book.spine:
-        return attribs
+                self.get_link_id(item.href)
+                root = item.data.find(XHTML('body'))
+                link_attrs = set(html.defs.link_attrs)
+                link_attrs.add(XLINK('href'))
+                for el in root.iter():
+                    attribs = el.attrib
+                    try:
+                        if not isinstance(el.tag, basestring):
+                            continue
+                    except:
+                        continue
+                    for attr in attribs:
+                        if attr in link_attrs:
+                            href = item.abshref(attribs[attr])
+                            href, id = urldefrag(href)
+                            if href in self.base_hrefs:
+                                self.get_link_id(href, id)
 
-    def url_is_relative(self, url):
+    def rewrite_link(self, url, page=None):
-        o = urlparse(url)
+        if not page:
-        return False if o.scheme else True
+            return url
+        abs_url = page.abshref(url)
+        if abs_url in self.images:
+            return 'images/%s' % self.images[abs_url]
+        if abs_url in self.links:
+            return self.links[abs_url]
+        return url
+
+    def rewrite_ids(self, root, page):
+        for el in root.iter():
+            try:
+                tag = el.tag
+            except UnicodeDecodeError:
+                continue
+            if tag == XHTML('body'):
+                el.attrib['id'] = self.get_link_id(page.href)[1:]
+                continue
+            if 'id' in el.attrib:
+                el.attrib['id'] = self.get_link_id(page.href, el.attrib['id'])[1:]
 
     def get_css(self, oeb_book):
         css = u''
@@ -130,12 +155,8 @@ class OEB2HTMLNoCSSizer(OEB2HTML):
         tag = barename(elem.tag)
         attribs = elem.attrib
 
-        attribs = self.rewrite_link(tag, attribs, page)
-        attribs = self.rewrite_image(tag, attribs, page)
-
         if tag == 'body':
             tag = 'div'
-            attribs['id'] = self.get_link_id(page.href, '')
         tags.append(tag)
 
         # Ignore anything that is set to not be displayed.
@@ -218,13 +239,9 @@ class OEB2HTMLInlineCSSizer(OEB2HTML):
         tag = barename(elem.tag)
         attribs = elem.attrib
 
-        attribs = self.rewrite_link(tag, attribs, page)
-        attribs = self.rewrite_image(tag, attribs, page)
-
         style_a = '%s' % style
         if tag == 'body':
             tag = 'div'
-            attribs['id'] = self.get_link_id(page.href, '')
             if not style['page-break-before'] == 'always':
                 style_a = 'page-break-before: always;' + ' ' if style_a else '' + style_a
         tags.append(tag)
@@ -279,6 +296,8 @@ class OEB2HTMLClassCSSizer(OEB2HTML):
         output = []
         for item in oeb_book.spine:
             self.log.debug('Converting %s to HTML...' % item.href)
+            self.rewrite_ids(item.data, item)
+            rewrite_links(item.data, partial(self.rewrite_link, page=item))
             stylizer = Stylizer(item.data, item.href, oeb_book, self.opts)
             output += self.dump_text(item.data.find(XHTML('body')), stylizer, item)
             output.append('\n\n')
@@ -306,17 +325,12 @@ class OEB2HTMLClassCSSizer(OEB2HTML):
 
         # Setup our variables.
         text = ['']
-        #style = stylizer.style(elem)
         tags = []
         tag = barename(elem.tag)
         attribs = elem.attrib
 
-        attribs = self.rewrite_link(tag, attribs, page)
-        attribs = self.rewrite_image(tag, attribs, page)
-
         if tag == 'body':
             tag = 'div'
-            attribs['id'] = self.get_link_id(page.href, '')
         tags.append(tag)
 
         # Remove attributes we won't want.

src/calibre/ebooks/metadata/sources/amazon.py

@@ -279,7 +279,7 @@ class Worker(Thread): # Get details {{{
 
 class Amazon(Source):
 
-    name = 'Amazon Metadata'
+    name = 'Amazon Store'
     description = _('Downloads metadata from Amazon')
 
     capabilities = frozenset(['identify', 'cover'])

src/calibre/ebooks/metadata/sources/base.py

@@ -167,6 +167,13 @@ class Source(Plugin):
 
     # Configuration {{{
 
+    def is_configured(self):
+        '''
+        Return False if your plugin needs to be configured before it can be
+        used. For example, it might need a username/password/API key.
+        '''
+        return True
+
     @property
     def prefs(self):
         if self._config_obj is None:

src/calibre/ebooks/metadata/sources/cli.py

@@ -19,9 +19,13 @@ from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.ebooks.metadata.sources.base import create_log
 from calibre.ebooks.metadata.sources.identify import identify
 from calibre.ebooks.metadata.sources.covers import download_cover
-
+from calibre.utils.config import test_eight_code
 
 def option_parser():
+    if not test_eight_code:
+        from calibre.ebooks.metadata.fetch import option_parser
+        return option_parser()
+
     parser = OptionParser(textwrap.dedent(
         '''\
         %prog [options]
@@ -44,6 +48,9 @@ def option_parser():
     return parser
 
 def main(args=sys.argv):
+    if not test_eight_code:
+        from calibre.ebooks.metadata.fetch import main
+        return main(args)
     parser = option_parser()
     opts, args = parser.parse_args(args)
 

src/calibre/ebooks/metadata/sources/covers.py

@@ -76,7 +76,7 @@ def run_download(log, results, abort,
         (plugin, width, height, fmt, bytes)
 
     '''
-    plugins = list(metadata_plugins(['cover']))
+    plugins = [p for p in metadata_plugins(['cover']) if p.is_configured()]
 
     rq = Queue()
     workers = [Worker(p, abort, title, authors, identifiers, timeout, rq) for p

src/calibre/ebooks/metadata/sources/identify.py

@@ -250,7 +250,7 @@ def merge_identify_results(result_map, log):
 def identify(log, abort, # {{{
         title=None, authors=None, identifiers={}, timeout=30):
     start_time = time.time()
-    plugins = list(metadata_plugins(['identify']))
+    plugins = [p for p in metadata_plugins(['identify']) if p.is_configured()]
 
     kwargs = {
             'title': title,

src/calibre/ebooks/metadata/sources/isbndb.py

@@ -37,4 +37,7 @@ class ISBNDB(Source):
 
         self.isbndb_key = prefs['isbndb_key']
 
+    def is_configured(self):
+        return self.isbndb_key is not None
+
 

src/calibre/gui2/actions/edit_metadata.py

@@ -17,6 +17,7 @@ from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.actions import InterfaceAction
 from calibre.utils.icu import sort_key
+from calibre.utils.config import test_eight_code
 
 class EditMetadataAction(InterfaceAction):
 
@@ -133,8 +134,6 @@ class EditMetadataAction(InterfaceAction):
 
         row_list = [r.row() for r in rows]
         current_row = 0
-        changed = set([])
-        db = self.gui.library_view.model().db
 
         if len(row_list) == 1:
             cr = row_list[0]
@@ -142,6 +141,24 @@ class EditMetadataAction(InterfaceAction):
                 list(range(self.gui.library_view.model().rowCount(QModelIndex())))
             current_row = row_list.index(cr)
 
+        if test_eight_code:
+            changed = self.do_edit_metadata(row_list, current_row)
+        else:
+            changed = self.do_edit_metadata_old(row_list, current_row)
+
+        if changed:
+            self.gui.library_view.model().refresh_ids(list(changed))
+            current = self.gui.library_view.currentIndex()
+            m = self.gui.library_view.model()
+            if self.gui.cover_flow:
+                self.gui.cover_flow.dataChanged()
+            m.current_changed(current, previous)
+            self.gui.tags_view.recount()
+
+    def do_edit_metadata_old(self, row_list, current_row):
+        changed = set([])
+        db = self.gui.library_view.model().db
+
         while True:
             prev = next_ = None
             if current_row > 0:
@@ -167,15 +184,28 @@ class EditMetadataAction(InterfaceAction):
             self.gui.library_view.set_current_row(current_row)
             self.gui.library_view.scroll_to_row(current_row)
 
+    def do_edit_metadata(self, row_list, current_row):
+        from calibre.gui2.metadata.single import edit_metadata
+        db = self.gui.library_view.model().db
+        changed, rows_to_refresh = edit_metadata(db, row_list, current_row,
+                parent=self.gui, view_slot=self.view_format_callback,
+                set_current_callback=self.set_current_callback)
+        return changed
+
+    def set_current_callback(self, id_):
+        db = self.gui.library_view.model().db
+        current_row = db.row(id_)
+        self.gui.library_view.set_current_row(current_row)
+        self.gui.library_view.scroll_to_row(current_row)
+
+    def view_format_callback(self, id_, fmt):
+        view = self.gui.iactions['View']
+        if id_ is None:
+            view._view_file(fmt)
+        else:
+            db = self.gui.library_view.model().db
+            view.view_format(db.row(id_), fmt)
 
-        if changed:
-            self.gui.library_view.model().refresh_ids(list(changed))
-            current = self.gui.library_view.currentIndex()
-            m = self.gui.library_view.model()
-            if self.gui.cover_flow:
-                self.gui.cover_flow.dataChanged()
-            m.current_changed(current, previous)
-            self.gui.tags_view.recount()
 
     def edit_bulk_metadata(self, checked):
         '''

src/calibre/gui2/metadata/basic_widgets.py

@@ -428,7 +428,7 @@ class Format(QListWidgetItem): # {{{
         if timestamp is not None:
             ts = timestamp.astimezone(local_tz)
             t = strftime('%a, %d %b %Y [%H:%M:%S]', ts.timetuple())
-            text = _('Last modified: %s')%t
+            text = _('Last modified: %s\n\nDouble click to view')%t
             self.setToolTip(text)
             self.setStatusTip(text)
 
@@ -577,8 +577,7 @@ class FormatsManager(QWidget): # {{{
             self.changed = True
 
     def show_format(self, item, *args):
-        fmt = item.ext
+        self.dialog.do_view_format(item.path, item.ext)
-        self.dialog.view_format.emit(fmt)
 
     def get_selected_format_metadata(self, db, id_):
         old = prefs['read_file_metadata']

src/calibre/gui2/metadata/single.py

@@ -26,15 +26,15 @@ from calibre.utils.config import tweaks
 
 class MetadataSingleDialogBase(ResizableDialog):
 
-    view_format = pyqtSignal(object)
+    view_format = pyqtSignal(object, object)
     cc_two_column = tweaks['metadata_single_use_2_cols_for_custom_fields']
     one_line_comments_toolbar = False
 
     def __init__(self, db, parent=None):
         self.db = db
-        self.changed = set([])
+        self.changed = set()
-        self.books_to_refresh = set([])
+        self.books_to_refresh = set()
-        self.rows_to_refresh = set([])
+        self.rows_to_refresh = set()
         ResizableDialog.__init__(self, parent)
 
     def setupUi(self, *args): # {{{
@@ -194,6 +194,13 @@ class MetadataSingleDialogBase(ResizableDialog):
                 pass # Do something
     # }}}
 
+    def do_view_format(self, path, fmt):
+        if path:
+            self.view_format.emit(None, path)
+        else:
+            self.view_format.emit(self.book_id, fmt)
+
+
     def do_layout(self):
         raise NotImplementedError()
 
@@ -204,6 +211,8 @@ class MetadataSingleDialogBase(ResizableDialog):
             widget.initialize(self.db, id_)
         for widget in getattr(self, 'custom_metadata_widgets', []):
             widget.initialize(id_)
+        if callable(self.set_current_callback):
+            self.set_current_callback(id_)
         # Commented out as it doesn't play nice with Next, Prev buttons
         #self.fetch_metadata_button.setFocus(Qt.OtherFocusReason)
 
@@ -339,11 +348,13 @@ class MetadataSingleDialogBase(ResizableDialog):
         gprefs['metasingle_window_geometry3'] = bytearray(self.saveGeometry())
 
     # Dialog use methods {{{
-    def start(self, row_list, current_row, view_slot=None):
+    def start(self, row_list, current_row, view_slot=None,
+            set_current_callback=None):
         self.row_list = row_list
         self.current_row = current_row
         if view_slot is not None:
             self.view_format.connect(view_slot)
+        self.set_current_callback = set_current_callback
         self.do_one(apply_changes=False)
         ret = self.exec_()
         self.break_cycles()
@@ -375,6 +386,7 @@ class MetadataSingleDialogBase(ResizableDialog):
     def break_cycles(self):
         # Break any reference cycles that could prevent python
         # from garbage collecting this dialog
+        self.set_current_callback = self.db = None
         def disconnect(signal):
             try:
                 signal.disconnect()
@@ -643,9 +655,11 @@ class MetadataSingleDialogAlt(MetadataSingleDialogBase): # {{{
 # }}}
 
 
-def edit_metadata(db, row_list, current_row, parent=None, view_slot=None):
+def edit_metadata(db, row_list, current_row, parent=None, view_slot=None,
+        set_current_callback=None):
     d = MetadataSingleDialog(db, parent)
-    d.start(row_list, current_row, view_slot=view_slot)
+    d.start(row_list, current_row, view_slot=view_slot,
+            set_current_callback=set_current_callback)
     return d.changed, d.rows_to_refresh
 
 if __name__ == '__main__':

src/calibre/gui2/metadata/single_download.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from threading import Thread, Event
+
+from PyQt4.Qt import (QStyledItemDelegate, QTextDocument, QRectF, QIcon, Qt,
+        QStyle, QApplication, QDialog, QVBoxLayout, QLabel, QDialogButtonBox,
+        QStackedWidget, QWidget, QTableView, QGridLayout, QFontInfo, QPalette)
+from PyQt4.QtWebKit import QWebView
+
+from calibre.customize.ui import metadata_plugins
+from calibre.ebooks.metadata import authors_to_string
+from calibre.utils.logging import ThreadSafeLog, UnicodeHTMLStream
+from calibre.ebooks.metadata.sources.identify import identify
+
+class Log(ThreadSafeLog): # {{{
+
+    def __init__(self):
+        ThreadSafeLog.__init__(self, level=self.DEBUG)
+        self.outputs = [UnicodeHTMLStream()]
+
+    def clear(self):
+        self.outputs[0].clear()
+# }}}
+
+class RichTextDelegate(QStyledItemDelegate): # {{{
+
+    def __init__(self, parent=None):
+        QStyledItemDelegate.__init__(self, parent)
+
+    def to_doc(self, index):
+        doc = QTextDocument()
+        doc.setHtml(index.data().toString())
+        return doc
+
+    def sizeHint(self, option, index):
+        ans = self.to_doc(index).size().toSize()
+        ans.setHeight(ans.height()+10)
+        return ans
+
+    def paint(self, painter, option, index):
+        painter.save()
+        painter.setClipRect(QRectF(option.rect))
+        if hasattr(QStyle, 'CE_ItemViewItem'):
+            QApplication.style().drawControl(QStyle.CE_ItemViewItem, option, painter)
+        elif option.state & QStyle.State_Selected:
+            painter.fillRect(option.rect, option.palette.highlight())
+        painter.translate(option.rect.topLeft())
+        self.to_doc(index).drawContents(painter)
+        painter.restore()
+# }}}
+
+class ResultsView(QTableView):
+
+    def __init__(self, parent=None):
+        QTableView.__init__(self, parent)
+
+class Comments(QWebView): # {{{
+
+    def __init__(self, parent=None):
+        QWebView.__init__(self, parent)
+        self.setAcceptDrops(False)
+        self.setMaximumWidth(270)
+        self.setMinimumWidth(270)
+
+        palette = self.palette()
+        palette.setBrush(QPalette.Base, Qt.transparent)
+        self.page().setPalette(palette)
+        self.setAttribute(Qt.WA_OpaquePaintEvent, False)
+
+    def turnoff_scrollbar(self, *args):
+        self.page().mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
+
+    def show_data(self, html):
+        def color_to_string(col):
+            ans = '#000000'
+            if col.isValid():
+                col = col.toRgb()
+                if col.isValid():
+                    ans = unicode(col.name())
+            return ans
+
+        f = QFontInfo(QApplication.font(self.parent())).pixelSize()
+        c = color_to_string(QApplication.palette().color(QPalette.Normal,
+                        QPalette.WindowText))
+        templ = '''\
+        <html>
+            <head>
+            <style type="text/css">
+                body, td {background-color: transparent; font-size: %dpx; color: %s }
+                a { text-decoration: none; color: blue }
+                div.description { margin-top: 0; padding-top: 0; text-indent: 0 }
+                table { margin-bottom: 0; padding-bottom: 0; }
+            </style>
+            </head>
+            <body>
+            <div class="description">
+            %%s
+            </div>
+            </body>
+        <html>
+        '''%(f, c)
+        self.setHtml(templ%html)
+# }}}
+
+class IdentifyWorker(Thread):
+
+    def __init__(self, log, abort, title, authors, identifiers):
+        Thread.__init__(self)
+        self.daemon = True
+
+        self.log, self.abort = log, abort
+        self.title, self.authors, self.identifiers = (title, authors,
+                identifiers)
+
+        self.results = []
+        self.error = None
+
+    def run(self):
+        try:
+            self.results = identify(self.log, self.abort, title=self.title,
+                    authors=self.authors, identifiers=self.identifiers)
+            for i, result in enumerate(self.results):
+                result.gui_rank = i
+        except:
+            import traceback
+            self.error = traceback.format_exc()
+
+class IdentifyWidget(QWidget):
+
+    def __init__(self, log, parent=None):
+        QWidget.__init__(self, parent)
+        self.log = log
+        self.abort = Event()
+
+        self.l = l = QGridLayout()
+        self.setLayout(l)
+
+        names = ['<b>'+p.name+'</b>' for p in metadata_plugins(['identify']) if
+                p.is_configured()]
+        self.top = QLabel('<p>'+_('calibre is downloading metadata from: ') +
+            ', '.join(names))
+        self.top.setWordWrap(True)
+        l.addWidget(self.top, 0, 0)
+
+        self.results_view = ResultsView(self)
+        l.addWidget(self.results_view, 1, 0)
+
+        self.comments_view = Comments(self)
+        l.addWidget(self.comments_view, 1, 1)
+
+        self.query = QLabel('download starting...')
+        f = self.query.font()
+        f.setPointSize(f.pointSize()-2)
+        self.query.setFont(f)
+        self.query.setWordWrap(True)
+        l.addWidget(self.query, 2, 0, 1, 2)
+
+        self.comments_view.show_data('<h2>'+_('Downloading')+
+                '<br><span id="dots">.</span></h2>'+
+                '''
+                <script type="text/javascript">
+                window.onload=function(){
+                    var dotspan = document.getElementById('dots');
+                    window.setInterval(function(){
+                        if(dotspan.textContent == '............'){
+                        dotspan.textContent = '.';
+                        }
+                        else{
+                        dotspan.textContent += '.';
+                        }
+                    }, 400);
+                }
+                </script>
+                ''')
+
+    def start(self, title=None, authors=None, identifiers={}):
+        self.log.clear()
+        self.log('Starting download')
+        parts = []
+        if title:
+            parts.append('title:'+title)
+        if authors:
+            parts.append('authors:'+authors_to_string(authors))
+        if identifiers:
+            x = ', '.join('%s:%s'%(k, v) for k, v in identifiers)
+            parts.append(x)
+        self.query.setText(_('Query: ')+'; '.join(parts))
+        self.log(unicode(self.query.text()))
+
+        self.worker = IdentifyWorker(self.log, self.abort, title,
+                authors, identifiers)
+
+        # self.worker.start()
+
+class FullFetch(QDialog): # {{{
+
+    def __init__(self, log, parent=None):
+        QDialog.__init__(self, parent)
+        self.log = log
+
+        self.setWindowTitle(_('Downloading metadata...'))
+        self.setWindowIcon(QIcon(I('metadata.png')))
+
+        self.stack = QStackedWidget()
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+        l.addWidget(self.stack)
+
+        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
+        l.addWidget(self.bb)
+        self.bb.rejected.connect(self.reject)
+
+        self.identify_widget = IdentifyWidget(log, self)
+        self.stack.addWidget(self.identify_widget)
+        self.resize(850, 500)
+
+    def accept(self):
+        # Prevent pressing Enter from closing the dialog
+        pass
+
+    def start(self, title=None, authors=None, identifiers={}):
+        self.identify_widget.start(title=title, authors=authors,
+                identifiers=identifiers)
+        self.exec_()
+# }}}
+
+if __name__ == '__main__':
+    app = QApplication([])
+    d = FullFetch(Log())
+    d.start(title='great gatsby', authors=['Fitzgerald'])
+

src/calibre/gui2/preferences/server.py

@@ -57,17 +57,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
         r('autolaunch_server', config)
 
-    def set_server_options(self):
-        c = self.proxy
-        c.set('port', self.opt_port.value())
-        c.set('username', unicode(self.opt_username.text()).strip())
-        p = unicode(self.opt_password.text()).strip()
-        if not p:
-            p = None
-        c.set('password', p)
-
     def start_server(self):
-        self.set_server_options()
+        ConfigWidgetBase.commit(self)
         self.gui.start_content_server(check_started=False)
         while not self.gui.content_server.is_running and self.gui.content_server.exception is None:
             time.sleep(1)

src/calibre/manual/conf.py

@@ -126,7 +126,7 @@ html_use_modindex = False
 html_use_index = False
 
 # If true, the reST sources are included in the HTML build as _sources/<name>.
-html_copy_source = False
+html_copy_source = True
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'calibredoc'

src/calibre/manual/faq.rst

@@ -99,7 +99,8 @@ We just need some information from you:
     device.
 
 Once you send us the output for a particular operating system, support for the device in that operating system
-will appear in the next release of |app|.
+will appear in the next release of |app|. To send us the output, open a bug report and attach the output to it.
+See `calibre bugs <http://calibre-ebook.com/bugs>`_.
 
 My device is not being detected by |app|?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/manual/gui.rst

@@ -71,7 +71,7 @@ Edit metadata
 
 |emii| The :guilabel:`Edit metadata` action has six variations, which can be accessed by clicking the down arrow on the right side of the button.
 
-    1. **Edit metadata individually**: This allows you to edit the metadata of books one-by-one, with the option of fetching metadata, including covers from the internet. It also allows you to add/remove particular ebook formats from a book. For more detail see :ref:`metadata`.
+    1. **Edit metadata individually**: This allows you to edit the metadata of books one-by-one, with the option of fetching metadata, including covers from the internet. It also allows you to add/remove particular ebook formats from a book. 
     2. **Edit metadata in bulk**: This allows you to edit common metadata fields for large numbers of books simulataneously. It operates on all the books you have selected in the :ref:`Library view <search_sort>`.
     3. **Download metadata and covers**: Downloads metadata and covers (if available), for the books that are selected in the book list.
     4. **Download only metadata**: Downloads only metadata (if available), for the books that are selected in the book list.
@@ -79,6 +79,7 @@ Edit metadata
     6. **Download only social metadata**: Downloads only social metadata such as tags and reviews (if available), for the books that are selected in the book list.
     7. **Merge Book Records**: Gives you the capability of merging the metadata and formats of two or more book records together. You can choose to either delete or keep the records that were not clicked first.
 
+For more details see :ref:`metadata`.
 
 .. _convert_ebooks:
 

src/calibre/manual/index.rst

@@ -70,7 +70,7 @@ Customizing |app|'s e-book conversion
 .. toctree::
    :maxdepth: 2
 
-   viewer
+   conversion
 
 Editing e-book metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -78,7 +78,7 @@ Editing e-book metadata
 .. toctree::
    :maxdepth: 2
 
-   viewer
+   metadata
 
 Frequently Asked Questions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^

src/calibre/manual/server.rst

@@ -16,7 +16,7 @@ Here, we will show you how to integrate the |app| content server into another se
 Using a reverse proxy
 -----------------------
 
-This is the simplest approach as it allows you to use the binary calibre install with no external dependencies/system integration requirements.
+A reverse proxy is when your normal server accepts incoming requests and passes them onto the calibre server. It then reads the response from the calibre server and forwards it to the client. This means that you can simply run the calibre server as normal without trying to integrate it closely with your main server, and you can take advantage of whatever authentication systems you main server has in place. This is the simplest approach as it allows you to use the binary calibre install with no external dependencies/system integration requirements. Below, is an example of how to achieve this with Apache as your main server, but it will work with any server that supports Reverse Proxies.
 
 First start the |app| content server as shown below::
 
@@ -33,7 +33,7 @@ The exact technique for enabling the proxy modules will vary depending on your A
     RewriteRule ^/calibre/(.*) http://localhost:8080/calibre/$1 [proxy]
     RewriteRule ^/calibre http://localhost:8080 [proxy]
 
-That's all, you will now be able to access the |app| Content Server under the /calibre URL in your apache server.
+That's all, you will now be able to access the |app| Content Server under the /calibre URL in your apache server. The above rules pass all requests under /calibre to the calibre server running on port 8080 and thanks to the --url-prefix option above, the calibre server handles them transparently.
 
 .. note:: If you are willing to devote an entire VirtualHost to the content server, then there is no need to use --url-prefix and RewriteRule, instead just use the ProxyPass directive.
 

src/calibre/utils/config.py

@@ -784,6 +784,7 @@ def write_tweaks(raw):
 
 
 tweaks = read_tweaks()
+test_eight_code = tweaks.get('test_eight_code', False)
 
 def migrate():
     if hasattr(os, 'geteuid') and os.geteuid() == 0:

src/calibre/utils/logging.py

@@ -14,7 +14,7 @@ import sys, traceback, cStringIO
 from functools import partial
 from threading import RLock
 
-
+from calibre import isbytestring, force_unicode, as_unicode
 
 class Stream(object):
 
@@ -63,15 +63,16 @@ class FileStream(Stream):
 
 class HTMLStream(Stream):
 
-    def __init__(self, stream=sys.stdout):
+    color = {
-        Stream.__init__(self, stream)
-        self.color = {
             DEBUG: '<span style="color:green">',
             INFO:'<span>',
             WARN: '<span style="color:yellow">',
             ERROR: '<span style="color:red">'
             }
-        self.normal = '</span>'
+    normal = '</span>'
+
+    def __init__(self, stream=sys.stdout):
+        Stream.__init__(self, stream)
 
     def prints(self, level, *args, **kwargs):
         self.stream.write(self.color[level])
@@ -82,6 +83,43 @@ class HTMLStream(Stream):
     def flush(self):
         self.stream.flush()
 
+class UnicodeHTMLStream(HTMLStream):
+
+    def __init__(self):
+        self.clear()
+
+    def flush(self):
+        pass
+
+    def prints(self, level, *args, **kwargs):
+        col = self.color[level]
+        if col != self.last_col:
+            if self.data:
+                self.data.append(self.normal)
+            self.data.append(col)
+            self.last_col = col
+
+        sep  = kwargs.get(u'sep', u' ')
+        end  = kwargs.get(u'end', u'\n')
+
+        for arg in args:
+            if isbytestring(arg):
+                arg = force_unicode(arg)
+            elif not isinstance(arg, unicode):
+                arg = as_unicode(arg)
+            self.data.append(arg+sep)
+        self.data.append(end)
+
+    def clear(self):
+        self.data = []
+        self.last_col = self.color[INFO]
+
+    @property
+    def html(self):
+        end = self.normal if self.data else u''
+        return u''.join(self.data) + end
+
+
 class Log(object):
 
     DEBUG = DEBUG