Sync with trunk.

Changelog.yaml

@@ -19,6 +19,124 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.7.44
+  date: 2011-02-04
+
+  new features:
+    - title: "Nook Color driver: Send downloaded news to the My Files/Magazines folder on the Nook Color. Also when getting the list of books on the device look at all folders in My Files, not just My Files/Books."
+
+    - title: "MOBI Output: Use the book uuid as the ASIN field and set cdetype to EBOK to allow Amazon furthest read tracking to work with calibre generated MOBI files."
+      tickets: [8721]
+
+    - title: "Comic input: Add an option to override the image size in the generated comic. Useful if you have a device whose screen size is not coverred by one of the available output profiles."
+      tickets: [7837]
+
+    - title: "Add a restore database option to the Library maintenance menu in the GUI"
+
+    - title: "TXT Output: Allow output in the textile markup language"
+
+    - title: "PML Output: Create multi-level Table of Contents"
+
+    - title: "Driver for the Archos 7O"
+
+    - title: "Search and Replace in the Bulk metadata dialog can now operate on the title_sort field as well"
+      tickets: [8732]
+
+    - title: "Allow changing the case of authors/tags/series etc. via the edit metadata dialog"
+
+    - title: "Connect/share menu: Re-organize to make it a little less easy to select email and delete instead of just email by mistake"
+
+    - title: "Heuristics: Improved Scene break detection and add option to control what scene breaks are replaced by."
+
+    - title: "SONY driver: Add option to not preserve aspect ratio of cover thumbnails."
+
+    - title: "BiBTeX catalog: Add on device column when available"
+
+    - title: "Add search to the plugin preferences dialog"
+  
+  bug fixes:
+    - title: "Fix a bug that could cause fiels to be lost when changing metadata on east asian windows installs if the title and/or author is very long."
+      tickets: [8620]
+
+    - title: "Tag browser: Fix searching with items in a user category not owrking if the main category is hidden"
+      tickets: [8741]
+
+    - title: "Make completion for author/series/tags/etc. fields less disruptive"
+
+    - title: "Fix regression that broke the content server when user categories/custom columns are present"
+
+    - title: "Catalog generation: Handle user supplied templates more robustly"
+
+    - title: "Move the Tags to apply to newly added books option into Preferences->Adding books"
+      tickets: [8730]
+
+    - title: "Workaround for bug in Qt on OS X that caused crashes when reading metedata from two or more EPUB files with HTML covers that used embedded fonts. Now the embedded fonts are ignored on OS X."
+      tickets: [8643]
+
+    - title: "Fix regression that broke the use of the group searched terms tweak"
+      tickets: [8739]
+
+    - title: "Fix template program regression triggered by recursively calling the processor"
+
+    - title: "Fix mimetype sent by content server for PDB files"
+
+    - title: "OPF: Write title_sort as a calibre custom metadata field rather than as a file-as attribute on the title. This conforms to the OPF spec"
+      tickets: [7883]
+
+    - title: "SONY driver: Fix thumbnails being sent to SD card are sent to the wrong location. Also use correct thumbnail size so that the SONY does not regenerate the thumbnail on disconnect"
+
+    - title: "Do not discard the result of a conversion if the user opens the edit metadata dialog while the conversion is running"
+      tickets: [8672]
+
+    - title: "CHM Input: When the chm file lacks a hhc, lookf for index.html instead"
+      tickets: [8688]
+
+    - title: "EPUB Input: Filter some invalid media types from the spine"
+
+    - title: "RTF Input: More encoding handlig fixes."
+      tickets: [8678]
+
+    - title: "Linux binary build: Restore functioning of CALIBRE_DEVELOP_FROM, which was accidentally removed a few versions ago"
+
+    - title: "RTF Output: Retain html headings as rtf headings when converting to rtf. Also fix output of italics."
+      tickets: [8641, 8640]
+
+    - title: "LIT Input: Fix regression that broke handling of LIT files that contain txt data instead of html"
+
+    - title: "MOBI Input: Handle more non printing ASCII codes"
+      tickets: [8646]
+
+    - title: "Handle empty cover files more gracefully"
+      tickets: [8656]
+
+    - title: "Catalog geenration: Fix error when Pocketbook is connected and trying to geenrate catalog"
+      tickets: [8651]
+
+    - title: "Heuristics: Italicize common cases, reduce false positives."
+
+    - title: "Fix regression that caused reporting of device connection errors to break"
+
+  improved recipes:
+    - MSN Japan
+    - The Onion
+    - La Tribuna de
+    - Wall Street Journal
+    - "20 Minutos"
+    - LA Times
+    - Endgadget Japan
+    - Ledevoir
+    - Vijesti
+
+  new recipes:
+    - title: "Cinco Dias and BBC Mundo"
+      author: Luis Hernandez
+
+    - title: "Explosm"
+      author: Andromeda Rabbit
+
+    - title: "Cinco Dias"
+      author: Luis Hernandez
+
 
 - version: 0.7.43
   date: 2011-01-28

resources/catalog/stylesheet.css

@@ -1,57 +1,10 @@
 body { background-color: white; }
 
-p.title  {
-	margin-top:0em;
-	margin-bottom:0em;
-	text-align:center;
-	font-style:italic;
-	font-size:xx-large;
-	}
-
-p.series_id {
-	margin-top:0em;
-	margin-bottom:0em;
-	text-align:center;
-	}
-
 a.series_id {
 	font-style:normal;
 	font-size:large;
 	}
 
-p.author {
-	font-size:large;
-	margin-top:0em;
-	margin-bottom:0em;
-	text-align: center;
-	text-indent: 0em;
-  	}
-
-p.author_index {
-	font-size:large;
-	font-weight:bold;
-	text-align:left;
-	margin-top:0px;
-	margin-bottom:-2px;
-	text-indent: 0em;
-	}
-
-p.genres {
-	font-style:normal;
-	margin-top:0.5em;
-	margin-bottom:0em;
-	text-align: left;
-	text-indent: 0.0in;
-	}
-
-p.formats {
-	font-size:90%;
-	margin-top:0em;
-	margin-bottom:0.5em;
-	text-align: left;
-	text-indent: 0.0in;
-	}
-
 /*
 * 	Minimize widows and orphans by logically grouping chunks
 *   Some reports of problems with Sony (ADE) ereaders
@@ -77,71 +30,6 @@ div.initial_letter {
 	page-break-before:always;
 	}
 
-p.author_title_letter_index {
-	font-size:x-large;
-	text-align:center;
-	font-weight:bold;
-	margin-top:0px;
-	margin-bottom:0px;
-	}
-
-p.date_index {
-	font-size:x-large;
-	text-align:center;
-	font-weight:bold;
-	margin-top:1em;
-	margin-bottom:0px;
-	}
-
-p.series {
-	font-style:italic;
-	margin-top:2px;
-	margin-bottom:0px;
-	margin-left:2em;
-	text-align:left;
-	text-indent:-2em;
-	}
-
-p.series_letter_index {
-	font-size:x-large;
-	text-align:center;
-	font-weight:bold;
-	margin-top:1em;
-	margin-bottom:0px;
-	}
-
-p.read_book {
-	text-align:left;
-	margin-top:0px;
-	margin-bottom:0px;
-	margin-left:2em;
-	text-indent:-2em;
-	}
-
-p.unread_book {
-	text-align:left;
-	margin-top:0px;
-	margin-bottom:0px;
-	margin-left:2em;
-	text-indent:-2em;
-	}
-
-p.wishlist_item {
-	text-align:left;
-	margin-top:0px;
-	margin-bottom:0px;
-	margin-left:2em;
-	text-indent:-2em;
-	}
-
-p.date_read {
-	text-align:left;
-	margin-top:0px;
-	margin-bottom:0px;
-	margin-left:6em;
-	text-indent:-6em;
-	}
-
 hr.annotations_divider {
 	width:50%;
 	margin-left:1em;
@@ -175,6 +63,102 @@ hr.merged_comments_divider {
     border-left: solid white 0px;
 	}
 
+p.date_read {
+	text-align:left;
+	margin-top:0px;
+	margin-bottom:0px;
+	margin-left:6em;
+	text-indent:-6em;
+	}
+
+p.author {
+	font-size:large;
+	margin-top:0em;
+	margin-bottom:0em;
+	text-align: center;
+	text-indent: 0em;
+  	}
+
+p.author_index {
+	font-size:large;
+	font-weight:bold;
+	text-align:left;
+	margin-top:0px;
+	margin-bottom:-2px;
+	text-indent: 0em;
+	}
+
+p.author_title_letter_index {
+	font-size:x-large;
+	text-align:center;
+	font-weight:bold;
+	margin-top:0px;
+	margin-bottom:0px;
+	}
+
+p.date_index {
+	font-size:x-large;
+	text-align:center;
+	font-weight:bold;
+	margin-top:1em;
+	margin-bottom:0px;
+	}
+
+p.formats {
+	font-size:90%;
+	margin-top:0em;
+	margin-bottom:0.5em;
+	text-align: left;
+	text-indent: 0.0in;
+	}
+
+p.genres {
+	font-style:normal;
+	margin-top:0.5em;
+	margin-bottom:0em;
+	text-align: left;
+	text-indent: 0.0in;
+	}
+
+p.series {
+	font-style:italic;
+	margin-top:0.25em;
+	margin-bottom:0em;
+	margin-left:2em;
+	text-align:left;
+	text-indent:-2em;
+	}
+
+p.series_id {
+	margin-top:0em;
+	margin-bottom:0em;
+	text-align:center;
+	}
+
+p.series_letter_index {
+	font-size:x-large;
+	text-align:center;
+	font-weight:bold;
+	margin-top:1em;
+	margin-bottom:0px;
+	}
+
+p.title  {
+	margin-top:0em;
+	margin-bottom:0em;
+	text-align:center;
+	font-style:italic;
+	font-size:xx-large;
+	}
+
+p.wishlist_item, p.unread_book, p.read_book {
+	text-align:left;
+	margin-top:0px;
+	margin-bottom:0px;
+	margin-left:2em;
+	text-indent:-2em;
+	}
+
 td.publisher, td.date {
 	font-weight:bold;
 	text-align:center;

resources/recipes/wsj.recipe

@@ -131,6 +131,7 @@ class WallStreetJournal(BasicNewsRecipe):
                 'description':desc, 'date':''})
 
             self.log('\tFound WN article:', title)
+            self.log('\t\t', desc)
 
         return articles
 
@@ -157,17 +158,23 @@ class WallStreetJournal(BasicNewsRecipe):
             meta = a.find(attrs={'class':'meta_sectionName'})
             if meta is not None:
                 meta.extract()
-            title = self.tag_to_string(a).strip() + ' [%s]'%self.tag_to_string(meta)
+                meta = self.tag_to_string(meta).strip()
+            if meta:
+                title = self.tag_to_string(a).strip() + ' [%s]'%meta
+            else:
+                title = self.tag_to_string(a).strip()
             url = 'http://online.wsj.com'+a['href']
             desc = ''
-            p = container.find('p')
+            for p in container.findAll('p'):
-            if p is not None:
                 desc = self.tag_to_string(p)
+                if not 'Subscriber Content' in desc:
+                    break
 
             articles.append({'title':title, 'url':url,
                 'description':desc, 'date':''})
 
             self.log('\tFound article:', title)
+            self.log('\t\t', desc)
 
         return articles
 

resources/recipes/wsj_free.recipe

@@ -140,12 +140,17 @@ class WallStreetJournal(BasicNewsRecipe):
             meta = a.find(attrs={'class':'meta_sectionName'})
             if meta is not None:
                 meta.extract()
-            title = self.tag_to_string(a).strip() + ' [%s]'%self.tag_to_string(meta)
+                meta = self.tag_to_string(meta).strip()
+            if meta:
+                title = self.tag_to_string(a).strip() + ' [%s]'%meta
+            else:
+                title = self.tag_to_string(a).strip()
             url = 'http://online.wsj.com'+a['href']
             desc = ''
-            p = container.find('p')
+            for p in container.findAll('p'):
-            if p is not None:
                 desc = self.tag_to_string(p)
+                if not 'Subscriber Content' in desc:
+                    break
 
             articles.append({'title':title, 'url':url,
                 'description':desc, 'date':''})

resources/template-functions.json

@@ -12,7 +12,7 @@
     "re": "def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):\n    return re.sub(pattern, replacement, val)\n", 
     "add": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x + y)\n", 
     "lookup": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if len(args) == 2: # here for backwards compatibility\n        if val:\n            return formatter.vformat('{'+args[0].strip()+'}', [], kwargs)\n        else:\n            return formatter.vformat('{'+args[1].strip()+'}', [], kwargs)\n    if (len(args) % 2) != 1:\n        raise ValueError(_('lookup requires either 2 or an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return formatter.vformat('{' + args[i].strip() + '}', [], kwargs)\n        if re.search(args[i], val):\n            return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)\n        i += 2\n", 
-    "template": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    template = template.replace('[[', '{').replace(']]', '}')\n    return formatter.safe_format(template, kwargs, 'TEMPLATE', mi)\n", 
+    "template": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    template = template.replace('[[', '{').replace(']]', '}')\n    return formatter.__class__().safe_format(template, kwargs, 'TEMPLATE', mi)\n", 
     "print": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    print args\n    return None\n", 
     "titlecase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return titlecase(val)\n", 
     "test": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):\n    if val:\n        return value_if_set\n    else:\n        return value_not_set\n", 

setup/publish.py

@@ -43,7 +43,7 @@ class Stage3(Command):
 
    description = 'Stage 3 of the publish process'
    sub_commands = ['upload_user_manual', 'upload_demo', 'sdist',
-            'upload_to_google_code', 'upload_to_sourceforge',
+            'upload_to_sourceforge', 'upload_to_google_code',
             'tag_release', 'upload_to_server',
             'upload_to_mobileread',
    ]

setup/upload.py

@@ -324,7 +324,7 @@ class UploadToServer(Command):
 
     def run(self, opts):
         check_call('ssh divok rm -f %s/calibre-\*.tar.gz'%DOWNLOADS, shell=True)
-        check_call('scp dist/calibre-*.tar.gz divok:%s/'%DOWNLOADS, shell=True)
+        #check_call('scp dist/calibre-*.tar.gz divok:%s/'%DOWNLOADS, shell=True)
         check_call('gpg --armor --detach-sign dist/calibre-*.tar.gz',
                 shell=True)
         check_call('scp dist/calibre-*.tar.gz.asc divok:%s/signatures/'%DOWNLOADS,

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.43'
+__version__   = '0.7.44'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/customize/builtins.py

@@ -474,7 +474,7 @@ from calibre.devices.binatone.driver import README
 from calibre.devices.hanvon.driver import N516, EB511, ALEX, AZBOOKA, THEBOOK
 from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \
-        SOVOS, PICO, SUNSTECH_EB700
+        SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O
 from calibre.devices.sne.driver import SNE
 from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, KOGAN, \
         GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, Q600, LUMIREAD, ALURATEK_COLOR, \
@@ -581,7 +581,7 @@ plugins += [
     ELONEX,
     TECLAST_K3,
     NEWSMY,
-    PICO, SUNSTECH_EB700,
+    PICO, SUNSTECH_EB700, ARCHOS7O,
     IPAPYRUS,
     SOVOS,
     EDGE,

src/calibre/devices/eb600/driver.py

@@ -183,9 +183,8 @@ class BOOQ(EB600):
 
     FORMATS = ['epub', 'mobi', 'prc', 'fb2', 'pdf', 'doc', 'rtf', 'txt', 'html']
 
-    VENDOR_NAME = 'NETRONIX'
+    VENDOR_NAME = ['NETRONIX', '36LBOOKS']
-    WINDOWS_MAIN_MEM = 'EB600'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['EB600', 'ELEQTOR']
-    WINDOWS_CARD_A_MEM = 'EB600'
 
 class MENTOR(EB600):
 

src/calibre/devices/teclast/driver.py

@@ -41,6 +41,16 @@ class NEWSMY(TECLAST_K3):
     WINDOWS_MAIN_MEM = 'NEWSMY'
     WINDOWS_CARD_A_MEM = 'USBDISK____SD'
 
+class ARCHOS7O(TECLAST_K3):
+    name = 'Archos 7O device interface'
+    gui_name = 'Archos'
+    description    = _('Communicate with the Archos reader.')
+
+    FORMATS = ['epub', 'mobi', 'fb2', 'rtf', 'ap', 'html', 'pdf', 'txt']
+
+    VENDOR_NAME      = 'ARCHOS'
+    WINDOWS_MAIN_MEM = 'USB-MSC'
+
 class PICO(NEWSMY):
     name = 'Pico device interface'
     gui_name = 'Pico'

src/calibre/ebooks/__init__.py

@@ -113,7 +113,7 @@ def render_html_svg_workaround(path_to_html, log, width=590, height=750):
 
 def render_html(path_to_html, width=590, height=750, as_xhtml=True):
     from PyQt4.QtWebKit import QWebPage
-    from PyQt4.Qt import QEventLoop, QPalette, Qt, SIGNAL, QUrl, QSize
+    from PyQt4.Qt import QEventLoop, QPalette, Qt, QUrl, QSize
     from calibre.gui2 import is_ok_to_use_qt
     if not is_ok_to_use_qt(): return None
     path_to_html = os.path.abspath(path_to_html)
@@ -127,8 +127,7 @@ def render_html(path_to_html, width=590, height=750, as_xhtml=True):
         page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
         loop = QEventLoop()
         renderer = HTMLRenderer(page, loop)
-        page.connect(page, SIGNAL('loadFinished(bool)'), renderer,
+        page.loadFinished.connect(renderer, type=Qt.QueuedConnection)
-                Qt.QueuedConnection)
         if as_xhtml:
             page.mainFrame().setContent(open(path_to_html, 'rb').read(),
                     'application/xhtml+xml', QUrl.fromLocalFile(path_to_html))
@@ -136,6 +135,7 @@ def render_html(path_to_html, width=590, height=750, as_xhtml=True):
             page.mainFrame().load(QUrl.fromLocalFile(path_to_html))
         loop.exec_()
     renderer.loop = renderer.page = None
+    page.loadFinished.disconnect()
     del page
     del loop
     if isinstance(renderer.exception, ParserError) and as_xhtml:

src/calibre/ebooks/conversion/utils.py

@@ -28,8 +28,8 @@ class HeuristicProcessor(object):
         self.linereg = re.compile('(?<=<p).*?(?=</p>)', re.IGNORECASE|re.DOTALL)
         self.blankreg = re.compile(r'\s*(?P<openline><p(?!\sclass=\"(softbreak|whitespace)\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
         self.anyblank = re.compile(r'\s*(?P<openline><p[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
-        self.multi_blank = re.compile(r'(\s*<p[^>]*>\s*</p>){2,}(?!\s*<h\d)', re.IGNORECASE)
+        self.multi_blank = re.compile(r'(\s*<p[^>]*>\s*</p>(\s*<div[^>]*>\s*</div>\s*)*){2,}(?!\s*<h\d)', re.IGNORECASE)
-        self.any_multi_blank = re.compile(r'(\s*<p[^>]*>\s*</p>){2,}', re.IGNORECASE)
+        self.any_multi_blank = re.compile(r'(\s*<p[^>]*>\s*</p>(\s*<div[^>]*>\s*</div>\s*)*){2,}', re.IGNORECASE)
         self.line_open = "<(?P<outer>p|div)[^>]*>\s*(<(?P<inner1>font|span|[ibu])[^>]*>)?\s*(<(?P<inner2>font|span|[ibu])[^>]*>)?\s*(<(?P<inner3>font|span|[ibu])[^>]*>)?\s*"
         self.line_close = "(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)>)?\s*</(?P=outer)>"
         self.single_blank = re.compile(r'(\s*<p[^>]*>\s*</p>)', re.IGNORECASE)
@@ -384,6 +384,8 @@ class HeuristicProcessor(object):
         html = re.sub(r"\s*<(font|[ibu]|em|strong)[^>]*>\s*(<(font|[ibu]|em|strong)[^>]*>\s*</(font|[ibu]|em|strong)>\s*){0,2}\s*</(font|[ibu]|em|strong)>", " ", html)
         html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]>\s*</span>){0,2}\s*</span>\s*", " ", html)
         html = re.sub(r"\s*<(font|[ibu]|em|strong)[^>]*>\s*(<(font|[ibu]|em|strong)[^>]*>\s*</(font|[ibu]|em|strong)>\s*){0,2}\s*</(font|[ibu]|em|strong)>", " ", html)
+        # delete surrounding divs from empty paragraphs
+        html = re.sub('<div[^>]*>\s*<p[^>]*>\s*</p>\s*</div>', '<p> </p>', html)
         # Empty heading tags
         html = re.sub(r'(?i)<h\d+>\s*</h\d+>', '', html)
         self.deleted_nbsps = True
@@ -561,7 +563,6 @@ class HeuristicProcessor(object):
         # Determine whether the document uses interleaved blank lines
         self.blanks_between_paragraphs = self.analyze_blanks(html)
 
-        #self.dump(html, 'before_chapter_markup')
         # detect chapters/sections to match xpath or splitting logic
 
         if getattr(self.extra_opts, 'markup_chapter_headings', False):

src/calibre/ebooks/metadata/epub.py

@@ -14,7 +14,8 @@ from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ptempfile import TemporaryDirectory, PersistentTemporaryFile
-from calibre import CurrentDir
+from calibre import CurrentDir, walk
+from calibre.constants import isosx
 
 class EPubException(Exception):
     pass
@@ -159,6 +160,13 @@ def get_cover(opf, opf_path, stream, reader=None):
     with TemporaryDirectory('_epub_meta') as tdir:
         with CurrentDir(tdir):
             zf.extractall()
+            if isosx:
+                # On OS X trying to render an HTML cover which uses embedded
+                # fonts more than once in the same process causes a crash in Qt
+                # so be safe and remove the fonts.
+                for f in walk('.'):
+                    if os.path.splitext(f)[1].lower() in ('.ttf', '.otf'):
+                        os.remove(f)
             opf_path = opf_path.replace('/', os.sep)
             cpage = os.path.join(tdir, os.path.dirname(opf_path), cpage)
             if not os.path.exists(cpage):

src/calibre/ebooks/pml/pmlml.py

@@ -16,7 +16,6 @@ from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace
 from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.ebooks.pdb.ereader import image_name
 from calibre.ebooks.pml import unipmlcode
-from calibre.utils.cleantext import clean_ascii_chars
 
 TAG_MAP = {
     'b'       : 'B',

src/calibre/gui2/filename_pattern.ui

@@ -17,13 +17,10 @@
    <item>
     <widget class="QLabel" name="label">
      <property name="text">
-      <string>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
+      <string>&lt;div style=&quot;font-size:10pt;&quot;&gt;
-&lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
+&lt;p&gt;Set a regular expression pattern to use when trying to guess ebook metadata from filenames. &lt;/p&gt;
-p, li { white-space: pre-wrap; }
+&lt;p&gt;A &lt;a href=&quot;http://calibre-ebook.com/user_manual/regexp.html&quot;&gt;tutorial&lt;/a&gt; on using regular expressions is available.&lt;/p&gt;
-&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Candara'; font-size:10pt; font-weight:400; font-style:normal;&quot;&gt;
+&lt;p&gt;Use the &lt;b&gt;Test&lt;/b&gt; functionality below to test your regular expression on a few sample filenames (remember to include the file extension). The group names for the various metadata entries are documented in tooltips.&lt;/p&gt;&lt;/div&gt;</string>
-&lt;p style=&quot; margin-top:12px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;Set a regular expression pattern to use when trying to guess ebook metadata from filenames. &lt;/p&gt;
-&lt;p style=&quot; margin-top:12px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;A &lt;a href=&quot;http://docs.python.org/lib/re-syntax.html&quot;&gt;&lt;span style=&quot; text-decoration: underline; color:#0000ff;&quot;&gt;reference&lt;/span&gt;&lt;/a&gt; on the syntax of regular expressions is available.&lt;/p&gt;
-&lt;p style=&quot; margin-top:12px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;Use the &lt;span style=&quot; font-weight:600;&quot;&gt;Test&lt;/span&gt; functionality below to test your regular expression on a few sample filenames. The group names for the various metadata entries are documented in tooltips.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
      </property>
      <property name="textFormat">
       <enum>Qt::RichText</enum>
@@ -104,8 +101,8 @@ p, li { white-space: pre-wrap; }
             <rect>
              <x>0</x>
              <y>0</y>
-             <width>277</width>
+             <width>305</width>
-             <height>276</height>
+             <height>263</height>
             </rect>
            </property>
            <layout class="QGridLayout" name="gridLayout_2">

src/calibre/gui2/preferences/adding.py

@@ -7,7 +7,8 @@ __docformat__ = 'restructuredtext en'
 
 
 
-from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
+    CommaSeparatedList
 from calibre.gui2.preferences.adding_ui import Ui_Form
 from calibre.utils.config import prefs
 from calibre.gui2.widgets import FilenamePattern
@@ -22,6 +23,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('read_file_metadata', prefs)
         r('swap_author_names', prefs)
         r('add_formats_to_existing', prefs)
+        r('new_book_tags', prefs, setting=CommaSeparatedList)
 
         self.filename_pattern = FilenamePattern(self)
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)

src/calibre/gui2/preferences/adding.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>1010</width>
+    <width>750</width>
     <height>339</height>
    </rect>
   </property>
@@ -27,10 +27,37 @@
    <item row="1" column="0">
     <widget class="QCheckBox" name="opt_read_file_metadata">
      <property name="text">
-      <string>Read metadata from &amp;file contents rather than file name</string>
+      <string>Read &amp;metadata from &amp;file contents rather than file name</string>
      </property>
     </widget>
    </item>
+   <item row="1" column="1">
+    <layout class="QHBoxLayout" name="horizontalLayout">
+     <item>
+      <spacer name="horizontalSpacer">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>40</width>
+         <height>20</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+     <item>
+      <widget class="QCheckBox" name="opt_swap_author_names">
+       <property name="toolTip">
+        <string>Swap the firstname and lastname of the author. This affects only metadata read from file names.</string>
+       </property>
+       <property name="text">
+        <string>&amp;Swap author firstname and lastname</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
    <item row="2" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_add_formats_to_existing">
      <property name="toolTip">
@@ -44,7 +71,24 @@ Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;,
      </property>
     </widget>
    </item>
-   <item row="3" column="0" colspan="2">
+   <item row="3" column="0">
+    <widget class="QLabel" name="label_230">
+     <property name="text">
+      <string>&amp;Tags to apply when adding a book:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_new_book_tags</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="1">
+    <widget class="QLineEdit" name="opt_new_book_tags">
+     <property name="toolTip">
+      <string>A comma-separated list of tags that will be applied to books added to the library</string>
+     </property>
+    </widget>
+   </item>
+   <item row="4" column="0" colspan="2">
     <widget class="QGroupBox" name="metadata_box">
      <property name="title">
       <string>&amp;Configure metadata from file name</string>
@@ -66,16 +110,6 @@ Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;,
      </layout>
     </widget>
    </item>
-   <item row="1" column="1">
-    <widget class="QCheckBox" name="opt_swap_author_names">
-     <property name="toolTip">
-      <string>Swap the firstname and lastname of the author. This affects only metadata read from file names.</string>
-     </property>
-     <property name="text">
-      <string>&amp;Swap author firstname and lastname</string>
-     </property>
-    </widget>
-   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/preferences/behavior.py

@@ -9,8 +9,7 @@ import re
 
 from PyQt4.Qt import Qt, QVariant, QListWidgetItem
 
-from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget
-    CommaSeparatedList
 from calibre.gui2.preferences.behavior_ui import Ui_Form
 from calibre.gui2 import config, info_dialog, dynamic
 from calibre.utils.config import prefs
@@ -49,7 +48,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         restrictions = sorted(saved_searches().names(), key=sort_key)
         choices = [('', '')] + [(x, x) for x in restrictions]
         r('gui_restriction', db.prefs, choices=choices)
-        r('new_book_tags', prefs, setting=CommaSeparatedList)
         self.reset_confirmation_button.clicked.connect(self.reset_confirmation_dialogs)
 
         self.input_up_button.clicked.connect(self.up_input)

src/calibre/gui2/preferences/behavior.ui

@@ -164,20 +164,6 @@
        </property>
       </widget>
      </item>
-     <item row="4" column="1">
-      <widget class="QLineEdit" name="opt_new_book_tags">
-       <property name="toolTip">
-        <string>A comma-separated list of tags that will be applied to books added to the library</string>
-       </property>
-      </widget>
-     </item>
-     <item row="4" column="0">
-      <widget class="QLabel" name="label_230">
-       <property name="text">
-        <string>Tags to apply when adding a book:</string>
-       </property>
-      </widget>
-     </item>
     </layout>
    </item>
    <item row="6" column="0" colspan="2">

src/calibre/gui2/tag_view.py

@@ -1041,23 +1041,28 @@ class TagsModel(QAbstractItemModel): # {{{
 
     def tokens(self):
         ans = []
+        # Tags can be in the news and the tags categories. However, because of
+        # the desire to use two different icons (tags and news), the nodes are
+        # not shared, which can lead to the possibility of searching twice for
+        # the same tag. The tags_seen set helps us prevent that
         tags_seen = set()
+        # Tag nodes are in their own category and possibly in user categories.
+        # They will be 'checked' in both places, but we want to put the node
+        # into the search string only once. The nodes_seen set helps us do that
+        nodes_seen = set()
         row_index = -1
 
         for i, key in enumerate(self.row_map):
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
             row_index += 1
-            if key.startswith('@'):
-                # User category, so skip it. The tag will be marked in its real category
-                continue
             category_item = self.root_item.children[row_index]
             for tag_item in category_item.child_tags():
                 tag = tag_item.tag
                 if tag.state != TAG_SEARCH_STATES['clear']:
                     prefix = ' not ' if tag.state == TAG_SEARCH_STATES['mark_minus'] \
                                      else ''
-                    category = key if key != 'news' else 'tag'
+                    category = tag.category if key != 'news' else 'tag'
                     if tag.name and tag.name[0] == u'\u2605': # char is a star. Assume rating
                         ans.append('%s%s:%s'%(prefix, category, len(tag.name)))
                     else:
@@ -1065,6 +1070,9 @@ class TagsModel(QAbstractItemModel): # {{{
                             if tag.name in tags_seen:
                                 continue
                             tags_seen.add(tag.name)
+                        if tag in nodes_seen:
+                            continue
+                        nodes_seen.add(tag)
                         ans.append('%s%s:"=%s"'%(prefix, category, tag.name))
         return ans
 

src/calibre/library/catalog.py

@@ -4456,20 +4456,32 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                                             self.generateAuthorAnchor(book['author']))
 
             if publisher == ' ':
-                publisherTag = body.find('td', attrs={'class':'publisher'})
+                try:
-                publisherTag.contents[0].replaceWith(' ')
+                    publisherTag = body.find('td', attrs={'class':'publisher'})
+                    publisherTag.contents[0].replaceWith(' ')
+                except:
+                    pass
 
             if not genres:
-                genresTag = body.find('p',attrs={'class':'genres'})
+                try:
-                genresTag.extract()
+                    genresTag = body.find('p',attrs={'class':'genres'})
+                    genresTag.extract()
+                except:
+                    pass
 
             if not formats:
-                formatsTag = body.find('p',attrs={'class':'formats'})
+                try:
-                formatsTag.extract()
+                    formatsTag = body.find('p',attrs={'class':'formats'})
+                    formatsTag.extract()
+                except:
+                    pass
 
             if note_content == '':
-                tdTag = body.find('td', attrs={'class':'notes'})
+                try:
-                tdTag.contents[0].replaceWith(' ')
+                    tdTag = body.find('td', attrs={'class':'notes'})
+                    tdTag.contents[0].replaceWith(' ')
+                except:
+                    pass
 
             emptyTags = body.findAll('td', attrs={'class':'empty'})
             for mt in emptyTags:

src/calibre/library/custom_columns.py

@@ -445,7 +445,7 @@ class CustomColumns(object):
         rv = self._set_custom(id, val, label=label, num=num, append=append,
                          notify=notify, extra=extra,
                          allow_case_change=allow_case_change)
-        self.dirtied([id], commit=False)
+        self.dirtied(set([id])|rv, commit=False)
         if commit:
             self.conn.commit()
         return rv

src/calibre/library/database2.py

@@ -1412,7 +1412,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 icon = icon_map['search']
         for srch in saved_searches().names():
             items.append(Tag(srch, tooltip=saved_searches().lookup(srch),
-                             icon=icon, category='search'))
+                             sort=srch, icon=icon, category='search'))
         if len(items):
             if icon_map is not None:
                 icon_map['search'] = icon_map['search']
@@ -1692,7 +1692,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         '''
         books_to_refresh = self._set_authors(id, authors,
                                              allow_case_change=allow_case_change)
-        self.dirtied([id], commit=False)
+        self.dirtied(set([id])|books_to_refresh, commit=False)
         if commit:
             self.conn.commit()
         self.set_path(id, index_is_id=True)
@@ -1768,7 +1768,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.conn.execute('''DELETE FROM publishers WHERE (SELECT COUNT(id)
                              FROM books_publishers_link
                              WHERE publisher=publishers.id) < 1''')
-        books_to_refresh = set()
+        books_to_refresh = set([])
         if publisher:
             case_change = False
             if not isinstance(publisher, unicode):
@@ -1793,7 +1793,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 bks = self.conn.get('''SELECT book FROM books_publishers_link
                                        WHERE publisher=?''', (aid,))
                 books_to_refresh |= set([bk[0] for bk in bks])
-        self.dirtied([id], commit=False)
+        self.dirtied(set([id])|books_to_refresh, commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['publisher'], publisher, row_is_id=True)
@@ -2206,7 +2206,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 bks = self.conn.get('SELECT book FROM books_tags_link WHERE tag=?',
                                         (tid,))
                 books_to_refresh |= set([bk[0] for bk in bks])
-        self.dirtied([id], commit=False)
+        self.dirtied(set([id])|books_to_refresh, commit=False)
         if commit:
             self.conn.commit()
         tags = u','.join(self.get_tags(id))