KG updates

resources/recipes/economist.recipe

@@ -9,7 +9,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.BeautifulSoup import Tag, NavigableString
 
-import mechanize, string, urllib, time, re
+import string, time, re
 
 class Economist(BasicNewsRecipe):
 
@@ -18,19 +18,19 @@ class Economist(BasicNewsRecipe):
 
     __author__ = "Kovid Goyal"
     INDEX = 'http://www.economist.com/printedition'
-    description = ('Global news and current affairs from a European perspective.'
-            ' Needs a subscription from ')+INDEX
+    description = 'Global news and current affairs from a European perspective.'
 
     oldest_article = 7.0
     cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg'
     remove_tags = [dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
             dict(attrs={'class':['dblClkTrk', 'ec-article-info']})]
     keep_only_tags = [dict(id='ec-article-body')]
-    needs_subscription = True
+    needs_subscription = False
     no_stylesheets = True
     preprocess_regexps = [(re.compile('</html>.*', re.DOTALL),
         lambda x:'</html>')]
 
+    '''
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
         br.open('http://www.economist.com')
@@ -50,6 +50,7 @@ class Economist(BasicNewsRecipe):
                     }))
         br.open(req).read()
         return br
+    '''
 
     def parse_index(self):
         try:

resources/recipes/economist_free.recipe

@@ -7,12 +7,12 @@ from lxml import html
 
 class Economist(BasicNewsRecipe):
 
-    title = 'The Economist (free)'
+    title = 'The Economist (RSS)'
     language = 'en'
 
     __author__ = "Kovid Goyal"
     description = ('Global news and current affairs from a European perspective.'
-            ' Much slower than the subscription based version.')
+            ' Much slower than the print edition based version.')
 
     oldest_article = 7.0
     cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg'

resources/recipes/freenature.recipe

@@ -1,4 +1,5 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Tag
 import re
 
 class NatureNews(BasicNewsRecipe):
@@ -10,17 +11,76 @@ class NatureNews(BasicNewsRecipe):
     max_articles_per_feed = 50
 
     no_stylesheets = True
-    remove_tags_before = dict(name='h1', attrs={'class':'heading entry-title'})
-    remove_tags_after  = dict(name='h2', attrs={'id':'comments'})
+    keep_only_tags = [dict(name='div', attrs={'id':'content'})]
+#    remove_tags_before = dict(name='h1', attrs={'class':'heading entry-title'})
+#    remove_tags_after  = dict(name='h2', attrs={'id':'comments'})
     remove_tags = [
        dict(name='h2', attrs={'id':'comments'}),
        dict(attrs={'alt':'Advertisement'}),
        dict(name='div', attrs={'class':'ad'}),
+       dict(attrs={'class':'Z3988'}),
+       dict(attrs={'class':['formatpublished','type-of-article','cleardiv','disclaimer','buttons','comments xoxo']}),
+       dict(name='a', attrs={'href':'#comments'}),
+       dict(name='h2',attrs={'class':'subheading plusicon icon-add-comment'})
     ]
 
     preprocess_regexps = [
         (re.compile(r'<p>ADVERTISEMENT</p>', re.DOTALL|re.IGNORECASE), lambda match: ''),
         ]
 
+    extra_css             = '''
+                            .author { text-align: right; font-size: small; line-height:1em; margin-top:0px; margin-left:0; margin-right:0; margin-bottom: 0; }
+                            .imagedescription { font-size: small; font-style:italic; line-height:1em; margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; }
+                            .imagecredit { font-size: x-small; font-style: normal; font-weight: bold}
+                            '''
+
     feeds = [('Nature News', 'http://feeds.nature.com/news/rss/most_recent')]
 
+    def preprocess_html(self,soup):
+        # The author name is slightly buried - dig it up
+        author = soup.find('p', {'class':'byline'})
+        if author:
+            # Find out the author's name
+            authornamediv = author.find('span',{'class':'author fn'})
+            authornamelink = authornamediv.find('a')
+            if authornamelink:
+                authorname = authornamelink.contents[0]
+            else:
+                authorname = authornamediv.contents[0]
+            # Stick the author's name in the byline tag
+            tag = Tag(soup,'div')
+            tag['class'] = 'author'
+            tag.insert(0,authorname.strip())
+            author.replaceWith(tag)
+
+        # Change the intro from a p to a div
+        intro = soup.find('p',{'class':'intro'})
+        if intro:
+            tag = Tag(soup,'div')
+            tag['class'] = 'intro'
+            tag.insert(0,intro.contents[0])
+            intro.replaceWith(tag)
+
+        # Change span class=imagedescription to div
+        descr = soup.find('span',{'class':'imagedescription'})
+        if descr:
+            tag = Tag(soup,'div')
+            tag['class'] = 'imagedescription'
+            tag.insert(0,descr.renderContents())
+            descr.replaceWith(tag)
+
+        # The references are in a list, let's make them simpler
+        reflistcont =  soup.find('ul',{'id':'article-refrences'})
+        if reflistcont:
+            reflist = reflistcont.li.renderContents()
+            tag = Tag(soup,'div')
+            tag['class'] = 'article-references'
+            tag.insert(0,reflist)
+            reflistcont.replaceWith(tag)
+
+        # Within the id=content div, we need to remove all the stuff after the end of the class=entry-content
+        entrycontent = soup.find('div',{'class':'entry-content'})
+        for nextSibling in entrycontent.findNextSiblings():
+            nextSibling.extract()
+
+        return soup

resources/recipes/ihned.recipe

@@ -0,0 +1,182 @@
+import re, time
+from calibre import strftime
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class IHNed(BasicNewsRecipe):
+
+
+    stahnout_vsechny = False
+        #True   = stahuje vsechny z homepage
+        #False  = stahuje pouze dnesni clanky (ze dne, kdy je skript spusten)
+
+    title       = 'iHNed'
+    __author__  = 'Karel Bílek'
+    language = 'cs'
+    description = 'Zprávy z iHNed.cz'
+    timefmt = ' [%a, %d %b, %Y]'
+    needs_subscription = False
+    remove_tags = [dict(attrs={'class':['borderbottom', 'web', 'foot', 'reklama', 'd-elm d-rellinks', 'd-elm']}),
+                 dict(style=['text-align: center;']),
+                 dict(id=['r-bfull']),
+                 dict(name=['script', 'noscript', 'style'])]
+    encoding = 'windows-1250'
+    no_stylesheets = True
+    remove_tags_before = dict(attrs={'class':'d-nadtit'})
+    remove_tags_after = dict(attrs={'class':'like'})
+
+    conversion_options = {
+      'linearize_tables' : True,
+    }
+
+
+
+    def preprocess_html(self, soup):
+
+        def makeurl(wat):
+            return "http://ihned.cz"+wat;
+
+        for h1 in soup.findAll('h1'):
+             a = h1.find('a')
+             if a:
+                 string = a.string
+                 if string:
+                     soup.a.replaceWith(string)
+        for a in soup.findAll('a',  href=True) :
+            cil = str(a['href'])
+            if cil.startswith("/") or  cil.startswith("index"):
+                a['href'] = makeurl(cil)
+        return soup
+
+
+    def parse_index(self):
+
+        def makeurl(wat):
+            if wat.startswith("/") or  wat.startswith("index"):
+                return "http://ihned.cz"+wat;
+            else:
+                return wat
+
+
+        articles = {} #vysledek, asi
+        key = None #soucasna sekce
+        ans = [] #vsechny sekce
+
+        articles["Hlavní"] = []
+        ans.append("Hlavní")
+
+        was = {}
+
+        def parse_subpage(url, name):
+            articles[name] = []
+            ans.append(name)
+
+
+            soup = self.index_to_soup(url)
+            otvirak = soup.find(True, attrs={'class':['otv']})
+            if otvirak:
+
+                #the code is copypasted here because I don't know python. simple as that.
+                a = otvirak.find('a', href=True)
+                title = self.tag_to_string(a, use_alt=True).strip()
+                txt = otvirak.find(True, attrs={'class':['txt']})
+                description = ''
+                if txt:
+                    match = re.match(r'<div class="txt">\s*([^<]*)\s*<a', str(txt), re.L)
+                    if match:
+                        description = match.group(1)
+
+                pubdate = strftime('%d. %m.')
+                if not title in was:
+                    articles[name].append(
+                          dict(title=title, url=makeurl(a['href']), date=pubdate,
+                                description=description,
+                                content=''))
+
+            otv234 = soup.find(True, attrs={'class':['otv234', 'col2a']})
+            if otv234:
+                for ow in otv234.findAll(True, attrs={'class':['ow']}):
+                    a = ow.find('a', href=True)
+                    title = self.tag_to_string(a, use_alt=True).strip()
+                    description=''
+                    prx = ow.find(True, attrs={'class':['prx']});
+                    if prx:
+                        description = str(prx.string)
+                    nfo = ow.find(True, attrs={'class':['nfo']});
+                    pubdate = ''
+                    if nfo:
+                        dtime = time.localtime();
+                        day = dtime[2]
+                        month = dtime[1]
+
+                        pubdate = strftime('%d. %m.')
+
+                        match = re.search(r'([0-9]*)\.([0-9]*)\.', str(nfo))
+
+                        if self.stahnout_vsechny or (int(day) == int(match.group(1)) and int(month) == int(match.group(2))):
+                            if not title in was:
+                                articles[name].append(
+                                      dict(title=title, url=makeurl(a['href']), date=pubdate,
+                                            description=description,
+                                            content=''))
+
+
+
+
+
+
+        soup = self.index_to_soup('http://ihned.cz/')
+        otvirak = soup.find(True, attrs={'class':['otv']})
+        if otvirak:
+            a = otvirak.find('a', href=True)
+            title = self.tag_to_string(a, use_alt=True).strip()
+            txt = otvirak.find(True, attrs={'class':['txt']})
+            description = ''
+            if txt:
+                match = re.match(r'<div class="txt">\s*([^<]*)\s*<a', str(txt), re.L)
+                if match:
+                    description = match.group(1)
+
+            pubdate = strftime('%d. %m.')
+            feed = "Hlavní"
+            articles[feed].append(
+                      dict(title=title, url=(a['href']), date=pubdate,
+                            description=description,
+                            content=''))
+            was[title]=1
+
+        otvirak2345 = soup.find(True, attrs={'class':['otv2345']})
+        if otvirak2345:
+            for otv2 in otvirak2345.findAll(True, attrs={'class':['otv2-5']}):
+                a = otv2.find('a', attrs={'class':['tit2']}, href=True)
+                title = self.tag_to_string(a, use_alt=True).strip()
+                description=''
+                span = otv2.find('span');
+                if span:
+                    match = re.match(r'<span>\s*([^<]*)\s*<a', str(span), re.L)
+                    if match:
+                        description = match.group(1)
+                feed = "Hlavní"
+                pubdate = strftime('%d. %m.')
+                articles[feed].append(
+                          dict(title=title, url=(a['href']), date=pubdate,
+                                description=description,
+                                content=''))
+                was[title]=1
+
+
+        parse_subpage("http://komentare.ihned.cz/", "Komentáře")
+        parse_subpage("http://domaci.ihned.cz", "Domácí")
+        parse_subpage("http://ekonomika.ihned.cz", "Ekonomika")
+        parse_subpage("http://zahranicni.ihned.cz/", "Zahraničí");
+        parse_subpage("http://finweb.ihned.cz/", "Finance");
+        parse_subpage("http://digiweb.ihned.cz/", "DigiWeb");
+        parse_subpage("http://kultura.ihned.cz/", "Kultura")
+        parse_subpage("http://sport.ihned.cz/", "Sport");
+
+        #seradi kategorie
+        ans = self.sort_index_by(ans, {'Hlavni':1, 'Domácí':2, 'Ekonomika':5, 'Zahraničí':3, 'Finance':6, 'DigiWeb':7, 'Kultura':8, 'Sport':9, 'Komentáře':4})
+
+        #vrati, ale pouze, kdyz je v kategoriich...
+        ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
+        return ans
+

resources/recipes/kath_net.recipe

@@ -0,0 +1,16 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1295262156(BasicNewsRecipe):
+    title          = u'kath.net'
+    __author__     = 'Bobus'
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    feeds          = [(u'kath.net', u'http://www.kath.net/2005/xml/index.xml')]
+
+
+    def print_version(self, url):
+        return url+"&print=yes"
+
+    extra_css = 'td.textb {font-size: medium;}'
+

resources/recipes/mail_and_guardian.recipe

@@ -0,0 +1,32 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1295081935(BasicNewsRecipe):
+     title          = u'Mail & Guardian ZA News'
+     __author__     = '77ja65'
+     language = 'en'
+     oldest_article = 7
+     max_articles_per_feed = 30
+     no_stylesheets = True
+     masthead_url          =  'http://c1608832.cdn.cloudfiles.rackspacecloud.com/mg_logo.gif'
+     remove_tags_after = [dict(id='content')]
+
+     feeds          = [
+         (u'National News', u'http://www.mg.co.za/rss/national'),
+         (u'Top Stories', u'http://www.mg.co.za/rss'),
+         (u'Africa News', u'http://www.mg.co.za/rss/africa'),
+         (u'Sport', u'http://www.mg.co.za/rss/sport'),
+         (u'Business', u'http://www.mg.co.za/rss/business'),
+         (u'And In Other News', u'http://www.mg.co.za/rss/and-in-other-news'),
+         (u'World News', u'http://www.mg.co.za/rss/world')
+         ]
+
+     def print_version(self, url):
+           return url.replace('http://www.mg.co.za/article/',
+ 'http://www.mg.co.za/printformat/single/')
+
+     extra_css = '''
+                     h1{font-family:Arial,Helvetica,sans-serif; font-
+ weight:bold;font-size:large;}
+                     h2{font-family:Arial,Helvetica,sans-serif; font-
+ weight:normal;font-size:small;}
+                 '''

resources/recipes/nikkei_sub_economy.recipe

@@ -27,6 +27,9 @@ class NikkeiNet_sub_economy(BasicNewsRecipe):
                        {'class':"JSID_basePageMove JSID_baseAsyncSubmit cmn-form_area JSID_optForm_utoken"},
                        {'class':"cmn-article_keyword cmn-clearfix"},
                        {'class':"cmn-print_headline cmn-clearfix"},
+                       {'class':"cmn-article_list"},
+                       dict(id="ABOUT-NIKKEI"),
+                       {'class':"cmn-sub_market"},
                          ]
     remove_tags_after = {'class':"cmn-pr_list"}
 

resources/recipes/sportsillustrated.recipe

@@ -1,5 +1,5 @@
 from calibre.web.feeds.recipes import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
+#from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from urllib import quote
 
 class SportsIllustratedRecipe(BasicNewsRecipe) :
@@ -91,7 +91,7 @@ class SportsIllustratedRecipe(BasicNewsRecipe) :
         #   expire : no idea what value to use
         # All this comes from the Javascript function that redirects to the print version. It's called PT() and is defined in the file 48.js
 
-    def preprocess_html(self, soup):
+    '''def preprocess_html(self, soup):
         header = soup.find('div', attrs = {'class' : 'siv_artheader'})
         homeMadeSoup = BeautifulSoup('<html><head></head><body></body></html>')
         body = homeMadeSoup.body
@@ -115,4 +115,5 @@ class SportsIllustratedRecipe(BasicNewsRecipe) :
             body.append(para)
 
         return homeMadeSoup
+        '''
 

resources/template-functions.json

@@ -0,0 +1,28 @@
+{
+    "contains": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, test, value_if_present, value_if_not):\n    if re.search(test, val):\n        return value_if_present\n    else:\n        return value_if_not\n", 
+    "divide": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x / y)\n", 
+    "uppercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.upper()\n", 
+    "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    res = ''\n    for i in range(0, len(args)):\n        res += args[i]\n    return res\n", 
+    "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n    return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", 
+    "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n    if val:\n        return val\n    else:\n        return value_if_empty\n", 
+    "field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return formatter.get_value(name, [], kwargs)\n", 
+    "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return capitalize(val)\n", 
+    "list_item": "def evaluate(self, formatter, kwargs, mi, locals, val, index, sep):\n    if not val:\n        return ''\n    index = int(index)\n    val = val.split(sep)\n    try:\n        return val[index]\n    except:\n        return ''\n", 
+    "shorten": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, leading, center_string, trailing):\n    l = max(0, int(leading))\n    t = max(0, int(trailing))\n    if len(val) > l + len(center_string) + t:\n        return val[0:l] + center_string + ('' if t == 0 else val[-t:])\n    else:\n        return val\n", 
+    "re": "def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):\n    return re.sub(pattern, replacement, val)\n", 
+    "add": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x + y)\n", 
+    "lookup": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if len(args) == 2: # here for backwards compatibility\n        if val:\n            return formatter.vformat('{'+args[0].strip()+'}', [], kwargs)\n        else:\n            return formatter.vformat('{'+args[1].strip()+'}', [], kwargs)\n    if (len(args) % 2) != 1:\n        raise ValueError(_('lookup requires either 2 or an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return formatter.vformat('{' + args[i].strip() + '}', [], kwargs)\n        if re.search(args[i], val):\n            return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)\n        i += 2\n", 
+    "template": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    template = template.replace('[[', '{').replace(']]', '}')\n    return formatter.safe_format(template, kwargs, 'TEMPLATE', mi)\n", 
+    "print": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    print args\n    return None\n", 
+    "titlecase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return titlecase(val)\n", 
+    "test": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):\n    if val:\n        return value_if_set\n    else:\n        return value_not_set\n", 
+    "eval": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    from formatter import eval_formatter\n    template = template.replace('[[', '{').replace(']]', '}')\n    return eval_formatter.safe_format(template, locals, 'EVAL', None)\n", 
+    "multiply": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x * y)\n", 
+    "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x - y)\n", 
+    "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n    return unicode(len(val.split(sep)))\n", 
+    "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.lower()\n", 
+    "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n    locals[target] = value\n    return value\n", 
+    "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if (len(args) % 2) != 1:\n        raise ValueError(_('switch requires an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return args[i]\n        if re.search(args[i], val):\n            return args[i+1]\n        i += 2\n", 
+    "strcmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    v = strcmp(x, y)\n    if v < 0:\n        return lt\n    if v == 0:\n        return eq\n    return gt\n", 
+    "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    if x < y:\n        return lt\n    if x == y:\n        return eq\n    return gt\n"
+}

setup/resources.py

@@ -84,6 +84,23 @@ class Resources(Command):
 
             cPickle.dump(complete, open(dest, 'wb'), -1)
 
+        self.info('\tCreating template-functions.json')
+        dest = self.j(self.RESOURCES, 'template-functions.json')
+        function_dict = {}
+        import inspect
+        from calibre.utils.formatter_functions import all_builtin_functions
+        for obj in all_builtin_functions:
+            eval_func = inspect.getmembers(obj,
+                    lambda x: inspect.ismethod(x) and x.__name__ == 'evaluate')
+            try:
+                lines = [l[4:] for l in inspect.getsourcelines(eval_func[0][1])[0]]
+            except:
+                continue
+            lines = ''.join(lines)
+            function_dict[obj.name] = lines
+        import json
+        json.dump(function_dict, open(dest, 'wb'), indent=4)
+
     def clean(self):
         for x in ('scripts', 'recipes', 'ebook-convert-complete'):
             x = self.j(self.RESOURCES, x+'.pickle')

src/calibre/ebooks/fb2/input.py

@@ -104,7 +104,11 @@ class FB2Input(InputFormatPlugin):
         entries = [(f, guess_type(f)[0]) for f in os.listdir('.')]
         opf.create_manifest(entries)
         opf.create_spine(['index.xhtml'])
-
+        if mi.cover_data and mi.cover_data[1]:
+            with open('fb2_cover_calibre_mi.jpg', 'wb') as f:
+                f.write(mi.cover_data[1])
+            opf.guide.set_cover(os.path.abspath('fb2_cover_calibre_mi.jpg'))
+        else:
             for img in doc.xpath('//f:coverpage/f:image', namespaces=NAMESPACES):
                 href = img.get('{%s}href'%XLINK_NS, img.get('href', None))
                 if href is not None:

src/calibre/ebooks/mobi/reader.py

@@ -541,6 +541,16 @@ class MobiReader(object):
                             pass
                         elif tag.tag == 'img':
                             tag.set('height', height)
+                        else:
+                            if tag.tag == 'div' and not tag.text and \
+                                    (not tag.tail or not tag.tail.strip()) and \
+                                    not len(list(tag.iterdescendants())):
+                                # Paragraph spacer
+                                # Insert nbsp so that the element is never
+                                # discarded by a renderer
+                                tag.text = u'\u00a0' # nbsp
+                                styles.append('height: %s' %
+                                        self.ensure_unit(height))
                             else:
                                 styles.append('margin-top: %s' % self.ensure_unit(height))
             if attrib.has_key('width'):

src/calibre/ebooks/oeb/iterator.py

@@ -227,7 +227,7 @@ class EbookIterator(object):
                 self.log.warn('Missing spine item:', repr(spath))
 
         cover = self.opf.cover
-        if self.ebook_ext in ('lit', 'mobi', 'prc', 'opf') and cover:
+        if self.ebook_ext in ('lit', 'mobi', 'prc', 'opf', 'fb2') and cover:
             cfile = os.path.join(self.base, 'calibre_iterator_cover.html')
             chtml = (TITLEPAGE%os.path.relpath(cover, self.base).replace(os.sep,
                 '/')).encode('utf-8')

src/calibre/ebooks/pml/pmlconverter.py

@@ -34,18 +34,15 @@ class PML_HTMLizer(object):
         'ra',
         'c',
         'r',
-        't',
         's',
         'l',
         'k',
-        'T',
         'FN',
         'SB',
     ]
 
     STATES_VALUE_REQ = [
         'a',
-        'T',
         'FN',
         'SB',
     ]
@@ -96,8 +93,6 @@ class PML_HTMLizer(object):
         'Sb': 'sb',
         'c': 'c',
         'r': 'r',
-        't': 't',
-        'T': 'T',
         'i': 'i',
         'I': 'i',
         'u': 'u',
@@ -133,8 +128,6 @@ class PML_HTMLizer(object):
     DIV_STATES = [
         'c',
         'r',
-        't',
-        'T',
         'FN',
         'SB',
     ]
@@ -255,8 +248,6 @@ class PML_HTMLizer(object):
 
         for key, val in self.state.items():
             if val[0]:
-                if key == 'T':
-                    self.state['T'][0] = False
                 if key in self.DIV_STATES:
                     div.append(key)
                 elif key in self.SPAN_STATES:
@@ -506,6 +497,9 @@ class PML_HTMLizer(object):
         self.toc = TOC()
         self.file_name = file_name
 
+        indent_state = {'t': False, 'T': False}
+        adv_indent_val = ''
+
         for s in self.STATES:
             self.state[s] = [False, ''];
 
@@ -515,6 +509,8 @@ class PML_HTMLizer(object):
 
             parsed = []
             empty = True
+            basic_indent = indent_state['t']
+            adv_indent = indent_state['T']
 
             # Must use StringIO, cStringIO does not support unicode
             line = StringIO.StringIO(line)
@@ -527,7 +523,7 @@ class PML_HTMLizer(object):
                 if c == '\\':
                     c = line.read(1)
 
-                    if c in 'qcrtTiIuobBlk':
+                    if c in 'qcriIuobBlk':
                         text = self.process_code(c, line)
                     elif c in 'FS':
                         l = line.read(1)
@@ -574,6 +570,15 @@ class PML_HTMLizer(object):
                     elif c == 'w':
                         empty = False
                         text = '<hr width="%s" />' % self.code_value(line)
+                    elif c == 't':
+                        indent_state[c] = not indent_state[c]
+                        if indent_state[c]:
+                            basic_indent = True
+                    elif c == 'T':
+                        indent_state[c] = not indent_state[c]
+                        if indent_state[c]:
+                            adv_indent = True
+                            adv_indent_val = self.code_value(line)
                     elif c == '-':
                         empty = False
                         text = '&shy;'
@@ -590,6 +595,16 @@ class PML_HTMLizer(object):
             if not empty:
                 text = self.end_line()
                 parsed.append(text)
+                
+                if basic_indent:
+                    parsed.insert(0, self.STATES_TAGS['t'][0])
+                    parsed.append(self.STATES_TAGS['t'][1])
+                elif adv_indent:
+                    parsed.insert(0, self.STATES_TAGS['T'][0] % adv_indent_val)
+                    parsed.append(self.STATES_TAGS['T'][1])
+                    indent_state['T'] = False
+                    adv_indent_val = ''
+                
                 output.append(u''.join(parsed))
             line.close()
 

src/calibre/ebooks/rtf/input.py

@@ -286,7 +286,6 @@ class RTFInput(InputFormatPlugin):
         try:
             xml = self.generate_xml(stream.name)
         except RtfInvalidCodeException, e:
-            raise
             raise ValueError(_('This RTF file has a feature calibre does not '
             'support. Convert it to HTML first and then try it.\n%s')%e)
 

src/calibre/ebooks/rtf2xml/ParseRtf.py

@@ -226,10 +226,6 @@ class ParseRtf:
         try:
             return_value = process_tokens_obj.process_tokens()
         except InvalidRtfException, msg:
-            try:
-                os.remove(self.__temp_file)
-            except OSError:
-                pass
             #Check to see if the file is correctly encoded
             encode_obj = default_encoding.DefaultEncoding(
             in_file = self.__temp_file,
@@ -241,13 +237,16 @@ class ParseRtf:
             check_encoding_obj = check_encoding.CheckEncoding(
                     bug_handler = RtfInvalidCodeException,
                         )
-            enc = encode_obj.get_codepage()
-            if enc != 'mac_roman':
-                enc = 'cp' + enc
+            enc = 'cp' + encode_obj.get_codepage()
+            msg = 'Exception in token processing'
             if check_encoding_obj.check_encoding(self.__file, enc):
                 file_name = self.__file if isinstance(self.__file, str) \
                                     else self.__file.encode('utf-8')
                 msg = 'File %s does not appear to be correctly encoded.\n' % file_name
+            try:
+                os.remove(self.__temp_file)
+            except OSError:
+                pass
             raise InvalidRtfException, msg
         delete_info_obj = delete_info.DeleteInfo(
             in_file = self.__temp_file,

src/calibre/ebooks/rtf2xml/default_encoding.py

@@ -74,9 +74,6 @@ class DefaultEncoding:
         if not self.__datafetched:
             self._encoding()
             self.__datafetched = True
-        if self.__platform == 'Macintosh':
-            code_page = self.__code_page
-        else:
             code_page = 'ansicpg' + self.__code_page
         return self.__platform, code_page, self.__default_num
 
@@ -94,49 +91,60 @@ class DefaultEncoding:
 
     def _encoding(self):
         with open(self.__file, 'r') as read_obj:
+            cpfound = False
             if not self.__fetchraw:
                 for line in read_obj:
                     self.__token_info = line[:16]
                     if self.__token_info == 'mi<mk<rtfhed-end':
                         break
-                    if self.__token_info == 'cw<ri<ansi-codpg':
-                        #cw<ri<ansi-codpg<nu<10000
-                        self.__code_page = line[20:-1] if int(line[20:-1]) \
-                                            else '1252'
                     if self.__token_info == 'cw<ri<macintosh_':
                         self.__platform = 'Macintosh'
-                        self.__code_page = 'mac_roman'
                     elif self.__token_info == 'cw<ri<pc________':
                         self.__platform = 'IBMPC'
-                        self.__code_page = '437'
                     elif self.__token_info == 'cw<ri<pca_______':
                         self.__platform = 'OS/2'
-                        self.__code_page = '850'
+                    if self.__token_info == 'cw<ri<ansi-codpg' \
+                        and int(line[20:-1]):
+                            self.__code_page = line[20:-1]
                     if self.__token_info == 'cw<ri<deflt-font':
                         self.__default_num = line[20:-1]
+                        cpfound = True
                         #cw<ri<deflt-font<nu<0
+                if self.__platform != 'Windows' and \
+                        not cpfound:
+                    if self.__platform == 'Macintosh':
+                       self.__code_page = '10000'
+                    elif self.__platform == 'IBMPC':
+                        self.__code_page = '437'
+                    elif self.__platform == 'OS/2':
+                        self.__code_page = '850'
             else:
                 fenc = re.compile(r'\\(mac|pc|ansi|pca)[\\ \{\}\t\n]+')
                 fenccp = re.compile(r'\\ansicpg(\d+)[\\ \{\}\t\n]+')
+
                 for line in read_obj:
+                    if fenc.search(line):
+                        enc = fenc.search(line).group(1)
                     if fenccp.search(line):
                         cp = fenccp.search(line).group(1)
                         if not int(cp):
                             self.__code_page = cp
+                        cpfound = True
                         break
-                    if fenc.search(line):
-                        enc = fenc.search(line).group(1)
+                if self.__platform != 'Windows' and \
+                        not cpfound:
                     if enc == 'mac':
-                            self.__code_page = 'mac_roman'
+                        self.__code_page = '10000'
                     elif enc == 'pc':
                         self.__code_page = '437'
                     elif enc == 'pca':
                         self.__code_page = '850'
 
-# if __name__ == '__main__':
-    # encode_obj = DefaultEncoding(
-            # in_file = sys.argv[1],
-            # bug_handler = Exception,
-            # check_raw = True,
-            # )
-    # print encode_obj.get_codepage()
+if __name__ == '__main__':
+    import sys
+    encode_obj = DefaultEncoding(
+            in_file = sys.argv[1],
+            bug_handler = Exception,
+            check_raw = True,
+            )
+    print encode_obj.get_codepage()

src/calibre/ebooks/rtf2xml/delete_info.py

@@ -20,7 +20,7 @@ import sys, os, tempfile
 from calibre.ebooks.rtf2xml import copy
 
 class DeleteInfo:
-    """Delelet unecessary destination groups"""
+    """Delete unecessary destination groups"""
     def __init__(self,
             in_file ,
             bug_handler,
@@ -31,17 +31,14 @@ class DeleteInfo:
         self.__bug_handler = bug_handler
         self.__copy = copy
         self.__write_to = tempfile.mktemp()
+        self.__run_level = run_level
+        self.__initiate_allow()
         self.__bracket_count= 0
         self.__ob_count = 0
         self.__cb_count = 0
-        # self.__after_asterisk = False
-        # self.__delete = 0
-        self.__initiate_allow()
         self.__ob = 0
         self.__write_cb = False
-        self.__run_level = run_level
         self.__found_delete = False
-        # self.__list = False
 
     def __initiate_allow(self):
         """
@@ -57,6 +54,8 @@ class DeleteInfo:
                             'cw<an<annotation',
                             'cw<cm<comment___',
                             'cw<it<lovr-table',
+                            # info table
+                            'cw<di<company___',
                             # 'cw<ls<list______',
                         )
         self.__not_allowable = (
@@ -116,7 +115,6 @@ class DeleteInfo:
         """
         # Test for {\*}, in which case don't enter
         # delete state
-        # self.__after_asterisk = False # only enter this function once
         self.__found_delete = True
         if self.__token_info == 'cb<nu<clos-brack':
             if self.__delete_count == self.__cb_count:
@@ -128,7 +126,7 @@ class DeleteInfo:
                 # not sure what happens here!
                 # believe I have a '{\*}
                 if self.__run_level > 3:
-                    msg = 'flag problem\n'
+                    msg = 'Flag problem\n'
                     raise self.__bug_handler, msg
                 return True
         elif self.__token_info in self.__allowable :
@@ -173,8 +171,8 @@ class DeleteInfo:
         Return True for all control words.
         Return False otherwise.
         """
-        if self.__delete_count == self.__cb_count and self.__token_info ==\
-            'cb<nu<clos-brack':
+        if self.__delete_count == self.__cb_count and \
+                self.__token_info == 'cb<nu<clos-brack':
             self.__state = 'default'
             if self.__write_cb:
                 self.__write_cb = False
@@ -186,32 +184,24 @@ class DeleteInfo:
             return False
 
     def delete_info(self):
-        """Main method for handling other methods. Read one line in at
+        """Main method for handling other methods. Read one line at
         a time, and determine whether to print the line based on the state."""
         with open(self.__file, 'r') as read_obj:
             with open(self.__write_to, 'w') as self.__write_obj:
                 for line in read_obj:
                     #ob<nu<open-brack<0001
-                    to_print = True
                     self.__token_info = line[:16]
                     if self.__token_info == 'ob<nu<open-brack':
                         self.__ob_count = line[-5:-1]
                     if self.__token_info == 'cb<nu<clos-brack':
                         self.__cb_count = line[-5:-1]
+                    # Get action to perform
                     action = self.__state_dict.get(self.__state)
                     if not action:
-                        sys.stderr.write(_('No action in dictionary state is "%s" \n')
+                        sys.stderr.write('No action in dictionary state is "%s" \n'
                                 % self.__state)
-                    to_print = action(line)
-                    # if self.__after_asterisk:
-                        # to_print = self.__asterisk_func(line)
-                    # elif self.__list:
-                        # self.__in_list_func(line)
-                    # elif self.__delete:
-                        # to_print = self.__delete_func(line)
-                    # else:
-                        # to_print = self.__default_func(line)
-                    if to_print:
+                    # Print if allowed by action
+                    if action(line):
                         self.__write_obj.write(line)
         copy_obj = copy.Copy(bug_handler = self.__bug_handler)
         if self.__copy:

src/calibre/ebooks/rtf2xml/info.py

@@ -15,8 +15,10 @@
 #                                                                       #
 #                                                                       #
 #########################################################################
-import sys, os, tempfile
+import sys, os, tempfile, re
+
 from calibre.ebooks.rtf2xml import copy
+
 class Info:
     """
     Make tags for document-information
@@ -42,12 +44,14 @@ class Info:
         self.__copy = copy
         self.__run_level = run_level
         self.__write_to = tempfile.mktemp()
+
     def __initiate_values(self):
         """
         Initiate all values.
         """
         self.__text_string = ''
         self.__state = 'before_info_table'
+        self.rmspace = re.compile(r'\s+')
         self.__state_dict = {
         'before_info_table': self.__before_info_table_func,
         'after_info_table': self.__after_info_table_func,
@@ -58,27 +62,49 @@ class Info:
         self.__info_table_dict = {
         'cw<di<title_____'  : (self.__found_tag_with_text_func, 'title'),
         'cw<di<author____'  : (self.__found_tag_with_text_func, 'author'),
+        'cw<di<operator__'  : (self.__found_tag_with_text_func, 'operator'),
+        'cw<di<manager___'  : (self.__found_tag_with_text_func, 'manager'),
+        'cw<di<company___'  : (self.__found_tag_with_text_func, 'company'),
         'cw<di<keywords__'  : (self.__found_tag_with_text_func, 'keywords'),
+        'cw<di<category__'  : (self.__found_tag_with_text_func, 'category'),
         'cw<di<doc-notes_'  : (self.__found_tag_with_text_func, 'doc-notes'),
         'cw<di<subject___'  : (self.__found_tag_with_text_func, 'subject'),
-        'cw<di<operator__'  : (self.__found_tag_with_text_func, 'operator'),
+        'cw<di<linkbase__'  : (self.__found_tag_with_text_func, 'hyperlink-base'),
+        
         'cw<di<create-tim'  : (self.__found_tag_with_tokens_func, 'creation-time'),
         'cw<di<revis-time'  : (self.__found_tag_with_tokens_func, 'revision-time'),
-        'cw<di<edit-time_'  : (self.__single_field_func, 'editing-time'),
+        'cw<di<edit-time_'  : (self.__found_tag_with_tokens_func, 'editing-time'),
+        'cw<di<print-time'  : (self.__found_tag_with_tokens_func, 'printing-time'),
+        'cw<di<backuptime'  : (self.__found_tag_with_tokens_func, 'backup-time'),
+        
         'cw<di<num-of-wor'  : (self.__single_field_func, 'number-of-words'),
         'cw<di<num-of-chr'  : (self.__single_field_func, 'number-of-characters'),
+        'cw<di<numofchrws'  : (self.__single_field_func, 'number-of-characters-without-space'),
         'cw<di<num-of-pag'  : (self.__single_field_func, 'number-of-pages'),
+        'cw<di<version___'  : (self.__single_field_func, 'version'),
+        'cw<di<intern-ver'  : (self.__single_field_func, 'internal-version-number'),
+        'cw<di<internalID'  : (self.__single_field_func, 'internal-id-number'),
         }
         self.__token_dict = {
         'year______'        : 'year',
         'month_____'        : 'month',
         'day_______'        : 'day',
         'minute____'        : 'minute',
+        'second____'        : 'second',
         'revis-time'        : 'revision-time',
+        'create-tim'        : 'creation-time',
+        'edit-time_'        : 'editing-time',
+        'print-time'        : 'printing-time',
+        'backuptime'        : 'backup-time',
         'num-of-wor'        : 'number-of-words',
         'num-of-chr'        : 'number-of-characters',
+        'numofchrws'        : 'number-of-characters-without-space',
         'num-of-pag'        : 'number-of-pages',
+        'version___'        : 'version',
+        'intern-ver'        : 'internal-version-number',
+        'internalID'        : 'internal-id-number',
         }
+
     def __before_info_table_func(self, line):
         """
         Required:
@@ -92,6 +118,7 @@ class Info:
         if self.__token_info == 'mi<mk<doc-in-beg':
             self.__state = 'in_info_table'
         self.__write_obj.write(line)
+
     def __in_info_table_func(self, line):
         """
         Requires:
@@ -112,6 +139,7 @@ class Info:
                 action(line, tag)
             else:
                 self.__write_obj.write(line)
+
     def __found_tag_with_text_func(self, line, tag):
         """
         Requires:
@@ -126,6 +154,7 @@ class Info:
         """
         self.__tag = tag
         self.__state = 'collect_text'
+
     def __collect_text_func(self, line):
         """
         Requires:
@@ -139,6 +168,8 @@ class Info:
         """
         if self.__token_info == 'mi<mk<docinf-end':
             self.__state = 'in_info_table'
+            #Don't print empty tags
+            if len(self.rmspace.sub('',self.__text_string)):
                 self.__write_obj.write(
                     'mi<tg<open______<%s\n'
                     'tx<nu<__________<%s\n'
@@ -147,6 +178,7 @@ class Info:
             self.__text_string = ''
         elif line[0:2] == 'tx':
             self.__text_string += line[17:-1]
+
     def __found_tag_with_tokens_func(self, line, tag):
         """
         Requires:
@@ -163,6 +195,7 @@ class Info:
         self.__state = 'collect_tokens'
         self.__text_string = 'mi<tg<empty-att_<%s' % tag
         #mi<tg<empty-att_<page-definition<margin>33\n
+
     def __collect_tokens_func(self, line):
         """
         Requires:
@@ -194,18 +227,19 @@ class Info:
             att = line[6:16]
             value = line[20:-1]
             att_changed = self.__token_dict.get(att)
-            if att_changed == None:
+            if att_changed is None:
                 if self.__run_level > 3:
-                    msg = 'no dictionary match for %s\n' % att
+                    msg = 'No dictionary match for %s\n' % att
                     raise self.__bug_handler, msg
             else:
                 self.__text_string += '<%s>%s' % (att_changed, value)
+
     def __single_field_func(self, line, tag):
         value = line[20:-1]
         self.__write_obj.write(
-        'mi<tg<empty-att_<%s'
-        '<%s>%s\n' % (tag, tag, value)
+        'mi<tg<empty-att_<%s<%s>%s\n' % (tag, tag, value)
         )
+
     def __after_info_table_func(self, line):
         """
         Requires:
@@ -217,6 +251,7 @@ class Info:
             the file.
         """
         self.__write_obj.write(line)
+
     def fix_info(self):
         """
         Requires:
@@ -234,20 +269,15 @@ class Info:
             information table, simply write the line to the output file.
         """
         self.__initiate_values()
-        read_obj = open(self.__file, 'r')
-        self.__write_obj = open(self.__write_to, 'w')
-        line_to_read = 1
-        while line_to_read:
-            line_to_read = read_obj.readline()
-            line = line_to_read
+        with open(self.__file, 'r') as read_obj:
+            with open(self.__write_to, 'wb') as self.__write_obj:
+                for line in read_obj:
                     self.__token_info = line[:16]
                     action = self.__state_dict.get(self.__state)
-            if action == None:
-                sys.stderr.write('no no matching state in module styles.py\n')
+                    if action is None:
+                        sys.stderr.write('No matching state in module styles.py\n')
                         sys.stderr.write(self.__state + '\n')
                     action(line)
-        read_obj.close()
-        self.__write_obj.close()
         copy_obj = copy.Copy(bug_handler = self.__bug_handler)
         if self.__copy:
             copy_obj.copy_file(self.__write_to, "info.data")

src/calibre/ebooks/rtf2xml/process_tokens.py

@@ -70,7 +70,7 @@ class ProcessTokens:
         ';'                  :	('mc', ';', self.ms_sub_func),
         # this must be wrong
         '-'                  :	('mc', '-', self.ms_sub_func),
-        'line'               :  ('mi', 'hardline-break', self.hardline_func), #calibre
+        'line'               :  ('mi', 'hardline-break', self.direct_conv_func), #calibre
         # misc => ml
         '*'                  :	('ml', 'asterisk__', self.default_func),
         ':'                  :	('ml', 'colon_____', self.default_func),
@@ -78,7 +78,6 @@ class ProcessTokens:
         'backslash'          :	('nu', '\\', self.text_func),
         'ob'                 :	('nu', '{', self.text_func),
         'cb'                 :	('nu', '}', self.text_func),
-        #'line'               :  ('nu', ' ', self.text_func), calibre
         # paragraph formatting => pf
         'page'               :  ('pf', 'page-break', self.default_func),
         'par'                :	('pf', 'par-end___', self.default_func),
@@ -231,11 +230,15 @@ class ProcessTokens:
         'trhdr'              :  ('tb', 'row-header', self.default_func),
         # preamble => pr
         # document information => di
+        # TODO integrate \userprops
         'info'               :	('di', 'doc-info__', self.default_func),
+        'title'              :	('di', 'title_____', self.default_func),
         'author'             :	('di', 'author____', self.default_func),
         'operator'           :	('di', 'operator__', self.default_func),
-        'title'              :	('di', 'title_____', self.default_func),
+        'manager'            :	('di', 'manager___', self.default_func),
+        'company'            :	('di', 'company___', self.default_func),
         'keywords'           :  ('di', 'keywords__', self.default_func),
+        'category'           :  ('di', 'category__', self.default_func),
         'doccomm'            :  ('di', 'doc-notes_', self.default_func),
         'comment'            :  ('di', 'doc-notes_', self.default_func),
         'subject'            :  ('di', 'subject___', self.default_func),
@@ -244,11 +247,19 @@ class ProcessTokens:
         'mo'                 :	('di', 'month_____', self.default_func),
         'dy'                 :	('di', 'day_______', self.default_func),
         'min'                :	('di', 'minute____', self.default_func),
+        'sec'                :	('di', 'second____', self.default_func),
         'revtim'             :	('di', 'revis-time', self.default_func),
+        'edmins'             :	('di', 'edit-time_', self.default_func),
+        'printim'            :	('di', 'print-time', self.default_func),
+        'buptim'             :	('di', 'backuptime', self.default_func),
         'nofwords'           :	('di', 'num-of-wor', self.default_func),
         'nofchars'           :	('di', 'num-of-chr', self.default_func),
+        'nofcharsws'         :	('di', 'numofchrws', self.default_func),
         'nofpages'           :	('di', 'num-of-pag', self.default_func),
-        'edmins'             :	('di', 'edit-time_', self.default_func),
+        'version'            :	('di', 'version___', self.default_func),
+        'vern'               :	('di', 'intern-ver', self.default_func),
+        'hlinkbase'          :	('di', 'linkbase__', self.default_func),
+        'id'                 :	('di', 'internalID', self.default_func),
         # headers and footers => hf
         'headerf'            :	('hf', 'head-first', self.default_func),
         'headerl'            :	('hf', 'head-left_', self.default_func),
@@ -605,7 +616,7 @@ class ProcessTokens:
     def ms_sub_func(self, pre, token, num):
         return 'tx<mc<__________<%s\n' % token
 
-    def hardline_func(self, pre, token, num):
+    def direct_conv_func(self, pre, token, num):
         return 'mi<tg<empty_____<%s\n' % token
 
     def default_func(self, pre, token, num):

src/calibre/ebooks/rtf2xml/tokenize.py

@@ -27,11 +27,13 @@ class Tokenize:
             bug_handler,
             copy = None,
             run_level = 1,
+            # out_file = None,
         ):
         self.__file = in_file
         self.__bug_handler = bug_handler
         self.__copy = copy
         self.__write_to = tempfile.mktemp()
+        # self.__out_file = out_file
         self.__compile_expressions()
         #variables
         self.__uc_char = 0
@@ -113,6 +115,8 @@ class Tokenize:
 
     def __sub_reg_split(self,input_file):
         input_file = self.__replace_spchar.mreplace(input_file)
+        # this is for older RTF
+        input_file = self.__par_exp.sub('\n\\par \n', input_file)
         input_file = self.__ms_hex_exp.sub("\\mshex0\g<1> ", input_file)
         input_file = self.__utf_ud.sub("\\{\\uc0 \g<1>\\}", input_file)
         #remove \n in bin data
@@ -153,8 +157,6 @@ class Tokenize:
             # put a backslash in front of to eliminate special cases and
             # make processing easier
             "}": "\\}",
-            # this is for older RTF
-            r'\\$': '\\par ',
             }
         self.__replace_spchar = MReplace(SIMPLE_RPL)
         #add ;? in case of char following \u
@@ -168,10 +170,12 @@ class Tokenize:
         #why keep backslash whereas \is replaced before?
         #remove \n from endline char
         self.__splitexp = re.compile(r"(\\[{}]|\n|\\[^\s\\{}&]+(?:[ \t\r\f\v])?)")
+        #this is for old RTF
+        self.__par_exp = re.compile(r'\\\n+')
+        # self.__par_exp = re.compile(r'\\$')
         #self.__bin_exp = re.compile(r"\\bin(-?\d{1,8}) {0,1}")
         #self.__utf_exp = re.compile(r"^\\u(-?\d{3,6})")
         #self.__splitexp = re.compile(r"(\\[\\{}]|{|}|\n|\\[^\s\\{}&]+(?:\s)?)")
-        #self.__par_exp = re.compile(r'\\$')
         #self.__remove_line = re.compile(r'\n+')
         #self.__mixed_exp = re.compile(r"(\\[a-zA-Z]+\d+)(\D+)")
         ##self.num_exp = re.compile(r"(\*|:|[a-zA-Z]+)(.*)")
@@ -199,7 +203,24 @@ class Tokenize:
         copy_obj = copy.Copy(bug_handler = self.__bug_handler)
         if self.__copy:
             copy_obj.copy_file(self.__write_to, "tokenize.data")
+        # if self.__out_file:
+            # self.__file = self.__out_file
         copy_obj.rename(self.__write_to, self.__file)
         os.remove(self.__write_to)
         
         #self.__special_tokens = [ '_', '~', "'", '{', '}' ]
+
+# import sys
+# def main(args=sys.argv):
+    # if len(args) < 1:
+        # print 'No file'
+        # return
+    # file = 'data_tokens.txt'
+    # if len(args) == 3:
+        # file = args[2]
+    # to = Tokenize(args[1], Exception, out_file = file)
+    # to.tokenize()
+
+
+# if __name__ == '__main__':
+    # sys.exit(main())

src/calibre/gui2/__init__.py

@@ -85,7 +85,7 @@ def _config():
     c.add_opt('LRF_ebook_viewer_options', default=None,
               help=_('Options for the LRF ebook viewer'))
     c.add_opt('internally_viewed_formats', default=['LRF', 'EPUB', 'LIT',
-        'MOBI', 'PRC', 'HTML', 'FB2', 'PDB', 'RB', 'SNB'],
+        'MOBI', 'PRC', 'AZW', 'HTML', 'FB2', 'PDB', 'RB', 'SNB'],
               help=_('Formats that are viewed using the internal viewer'))
     c.add_opt('column_map', default=ALL_COLUMNS,
               help=_('Columns to be displayed in the book list'))
@@ -505,7 +505,7 @@ class FileDialog(QObject):
         self.selected_files = []
         if mode == QFileDialog.AnyFile:
             f = unicode(QFileDialog.getSaveFileName(parent, title, initial_dir, ftext, ""))
-            if f and os.path.exists(f):
+            if f:
                 self.selected_files.append(f)
         elif mode == QFileDialog.ExistingFile:
             f = unicode(QFileDialog.getOpenFileName(parent, title, initial_dir, ftext, ""))

src/calibre/gui2/actions/catalog.py

@@ -9,7 +9,7 @@ import re, os, shutil
 
 from PyQt4.Qt import QModelIndex
 
-from calibre.gui2 import choose_dir, error_dialog, info_dialog, warning_dialog
+from calibre.gui2 import choose_dir, error_dialog, warning_dialog
 from calibre.gui2.tools import generate_catalog
 from calibre.utils.config import dynamic
 from calibre.gui2.actions import InterfaceAction

src/calibre/gui2/actions/edit_metadata.py

@@ -94,7 +94,7 @@ class EditMetadataAction(InterfaceAction):
             get_social_metadata = config['get_social_metadata']
         else:
             get_social_metadata = set_social_metadata
-        from calibre.gui2.metadata import DoDownload
+        from calibre.gui2.metadata.bulk_download import DoDownload
         if set_social_metadata is not None and set_social_metadata:
             x = _('social metadata')
         else:

src/calibre/gui2/convert/metadata.py

@@ -68,6 +68,9 @@ class MetadataWidget(Widget, Ui_Form):
     def initialize_metadata_options(self):
         self.initialize_combos()
         self.author.editTextChanged.connect(self.deduce_author_sort)
+        self.author.set_separator('&')
+        self.author.set_space_before_sep(True)
+        self.author.update_items_cache(self.db.all_author_names())
 
         mi = self.db.get_metadata(self.book_id, index_is_id=True)
         self.title.setText(mi.title)
@@ -75,7 +78,7 @@ class MetadataWidget(Widget, Ui_Form):
             self.publisher.setCurrentIndex(self.publisher.findText(mi.publisher))
         self.author_sort.setText(mi.author_sort if mi.author_sort else '')
         self.tags.setText(', '.join(mi.tags if mi.tags else []))
-        self.tags.update_tags_cache(self.db.all_tags())
+        self.tags.update_items_cache(self.db.all_tags())
         self.comment.setPlainText(mi.comments if mi.comments else '')
         if mi.series:
             self.series.setCurrentIndex(self.series.findText(mi.series))

src/calibre/gui2/convert/metadata.ui

@@ -190,7 +190,7 @@
         </widget>
        </item>
        <item row="4" column="1">
-        <widget class="TagsLineEdit" name="tags">
+        <widget class="CompleteLineEdit" name="tags">
          <property name="toolTip">
           <string>Tags categorize the book. This is particularly useful while searching. &lt;br&gt;&lt;br&gt;They can be any words or phrases, separated by commas.</string>
          </property>
@@ -255,7 +255,7 @@
         </widget>
        </item>
        <item row="1" column="1">
-        <widget class="EnComboBox" name="author">
+        <widget class="CompleteComboBox" name="author">
          <property name="editable">
           <bool>true</bool>
          </property>
@@ -310,7 +310,12 @@
    <header>widgets.h</header>
   </customwidget>
   <customwidget>
-   <class>TagsLineEdit</class>
+   <class>CompleteComboBox</class>
+   <extends>QComboBox</extends>
+   <header>widgets.h</header>
+  </customwidget>
+  <customwidget>
+   <class>CompleteLineEdit</class>
    <extends>QLineEdit</extends>
    <header>widgets.h</header>
   </customwidget>

src/calibre/gui2/custom_column_widgets.py

@@ -14,7 +14,7 @@ from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \
         QPushButton
 
 from calibre.utils.date import qt_to_dt, now
-from calibre.gui2.widgets import TagsLineEdit, EnComboBox
+from calibre.gui2.widgets import CompleteLineEdit, EnComboBox
 from calibre.gui2.comments_editor import Editor as CommentsEditor
 from calibre.gui2 import UNDEFINED_QDATE, error_dialog
 from calibre.utils.config import tweaks
@@ -212,7 +212,7 @@ class Text(Base):
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
         values.sort(key=sort_key)
         if self.col_metadata['is_multiple']:
-            w = TagsLineEdit(parent, values)
+            w = CompleteLineEdit(parent, values)
             w.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred)
         else:
             w = EnComboBox(parent)
@@ -226,7 +226,7 @@ class Text(Base):
         val = self.normalize_db_val(val)
         if self.col_metadata['is_multiple']:
             self.setter(val)
-            self.widgets[1].update_tags_cache(self.all_values)
+            self.widgets[1].update_items_cache(self.all_values)
         else:
             idx = None
             for i, c in enumerate(self.all_values):
@@ -656,7 +656,7 @@ class RemoveTags(QWidget):
         layout.setSpacing(5)
         layout.setContentsMargins(0, 0, 0, 0)
 
-        self.tags_box = TagsLineEdit(parent, values)
+        self.tags_box = CompleteLineEdit(parent, values)
         layout.addWidget(self.tags_box, stretch = 1)
         # self.tags_box.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred)
 
@@ -678,7 +678,7 @@ class BulkText(BulkBase):
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
         values.sort(key=sort_key)
         if self.col_metadata['is_multiple']:
-            w = TagsLineEdit(parent, values)
+            w = CompleteLineEdit(parent, values)
             w.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred)
             self.widgets = [QLabel('&'+self.col_metadata['name']+': ' +
                                    _('tags to add'), parent), w]
@@ -697,7 +697,7 @@ class BulkText(BulkBase):
 
     def initialize(self, book_ids):
         if self.col_metadata['is_multiple']:
-            self.widgets[1].update_tags_cache(self.all_values)
+            self.widgets[1].update_items_cache(self.all_values)
         else:
             val = self.get_initial_value(book_ids)
             self.initial_val = val = self.normalize_db_val(val)

src/calibre/gui2/device.py

@@ -1018,7 +1018,8 @@ class DeviceMixin(object): # {{{
         ids = [self.library_view.model().id(r) \
                for r in self.library_view.selectionModel().selectedRows()] \
                                 if send_ids is None else send_ids
-        if not self.device_manager or not ids or len(ids) == 0:
+        if not self.device_manager or not ids or len(ids) == 0 or \
+                not self.device_manager.is_device_connected:
             return
 
         settings = self.device_manager.device.settings()

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -15,15 +15,16 @@ from calibre.ebooks.metadata import string_to_authors, authors_to_string
 from calibre.ebooks.metadata.book.base import composite_formatter
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
-from calibre.gui2 import error_dialog, ResizableDialog
+from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.utils.config import dynamic
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import sort_key, capitalize
-from calibre.utils.config import prefs
+from calibre.utils.config import prefs, tweaks
 from calibre.utils.magick.draw import identify_data
+from calibre.utils.date import qt_to_dt
 
-def get_cover_data(path):
+def get_cover_data(path): # {{{
     old = prefs['read_file_metadata']
     if not old:
         prefs['read_file_metadata'] = True
@@ -46,7 +47,7 @@ def get_cover_data(path):
         prefs['read_file_metadata'] = old
 
     return cdata, area
-
+# }}}
 
 
 class MyBlockingBusy(QDialog): # {{{
@@ -132,7 +133,8 @@ class MyBlockingBusy(QDialog): # {{{
         remove_all, remove, add, au, aus, do_aus, rating, pub, do_series, \
             do_autonumber, do_remove_format, remove_format, do_swap_ta, \
             do_remove_conv, do_auto_author, series, do_series_restart, \
-            series_start_value, do_title_case, cover_action, clear_series = self.args
+            series_start_value, do_title_case, cover_action, clear_series, \
+            pubdate = self.args
 
 
         # first loop: do author and title. These will commit at the end of each
@@ -209,6 +211,9 @@ class MyBlockingBusy(QDialog): # {{{
             if clear_series:
                 self.db.set_series(id, '', notify=False, commit=False)
 
+            if pubdate is not None:
+                self.db.set_pubdate(id, pubdate, notify=False, commit=False)
+
             if do_series:
                 if do_series_restart:
                     if self.series_start_value is None:
@@ -274,8 +279,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.changed = False
 
         all_tags = self.db.all_tags()
-        self.tags.update_tags_cache(all_tags)
-        self.remove_tags.update_tags_cache(all_tags)
+        self.tags.update_items_cache(all_tags)
+        self.remove_tags.update_items_cache(all_tags)
 
         self.initialize_combos()
 
@@ -288,6 +293,13 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.series.editTextChanged.connect(self.series_changed)
         self.tag_editor_button.clicked.connect(self.tag_editor)
         self.autonumber_series.stateChanged[int].connect(self.auto_number_changed)
+        self.pubdate.setMinimumDate(UNDEFINED_QDATE)
+        pubdate_format = tweaks['gui_pubdate_display_format']
+        if pubdate_format is not None:
+            self.pubdate.setDisplayFormat(pubdate_format)
+        self.pubdate.setSpecialValueText(_('Undefined'))
+        self.clear_pubdate_button.clicked.connect(self.clear_pubdate)
+        self.pubdate.dateChanged.connect(self.do_apply_pubdate)
 
         if len(self.db.custom_field_keys(include_composites=False)) == 0:
             self.central_widget.removeTab(1)
@@ -304,6 +316,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.central_widget.setCurrentIndex(tab)
         self.exec_()
 
+    def do_apply_pubdate(self, *args):
+        self.apply_pubdate.setChecked(True)
+
+    def clear_pubdate(self, *args):
+        self.pubdate.setDate(UNDEFINED_QDATE)
+
     def button_clicked(self, which):
         if which == self.button_box.button(QDialogButtonBox.Apply):
             self.do_again = True
@@ -709,6 +727,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             self.authors.addItem(name)
         self.authors.setEditText('')
         
+        self.authors.set_separator('&')
+        self.authors.set_space_before_sep(True)
+        self.authors.update_items_cache(self.db.all_author_names())
+
     def initialize_series(self):
         all_series = self.db.all_series()
         all_series.sort(key=lambda x : sort_key(x[1]))
@@ -733,8 +755,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         if d.result() == QDialog.Accepted:
             tag_string = ', '.join(d.tags)
             self.tags.setText(tag_string)
-            self.tags.update_tags_cache(self.db.all_tags())
-            self.remove_tags.update_tags_cache(self.db.all_tags())
+            self.tags.update_items_cache(self.db.all_tags())
+            self.remove_tags.update_items_cache(self.db.all_tags())
 
     def auto_number_changed(self, state):
         if state:
@@ -783,6 +805,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         do_remove_conv = self.remove_conversion_settings.isChecked()
         do_auto_author = self.auto_author_sort.isChecked()
         do_title_case = self.change_title_to_title_case.isChecked()
+        pubdate = None
+        if self.apply_pubdate.isChecked():
+            pubdate = qt_to_dt(self.pubdate.date())
+
         cover_action = None
         if self.cover_remove.isChecked():
             cover_action = 'remove'
@@ -794,7 +820,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         args = (remove_all, remove, add, au, aus, do_aus, rating, pub, do_series,
                 do_autonumber, do_remove_format, remove_format, do_swap_ta,
                 do_remove_conv, do_auto_author, series, do_series_restart,
-                series_start_value, do_title_case, cover_action, clear_series)
+                series_start_value, do_title_case, cover_action, clear_series,
+                pubdate)
 
         bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.')
                 %len(self.ids), args, self.db, self.ids,

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -75,13 +75,31 @@
              </property>
             </widget>
            </item>
+           <item row="0" column="1">
+            <widget class="CompleteComboBox" name="authors">
+             <property name="editable">
+              <bool>true</bool>
+             </property>
+            </widget>
+           </item>
            <item row="1" column="1">
+            <layout class="QHBoxLayout" name="horizontalLayout_2">
+             <item>
               <widget class="QCheckBox" name="auto_author_sort">
                <property name="text">
                 <string>A&amp;utomatically set author sort</string>
                </property>
               </widget>
              </item>
+             <item>
+              <widget class="QCheckBox" name="swap_title_and_author">
+               <property name="text">
+                <string>&amp;Swap title and author</string>
+               </property>
+              </widget>
+             </item>
+            </layout>
+           </item>
            <item row="2" column="0">
             <widget class="QLabel" name="label_8">
              <property name="text">
@@ -95,7 +113,7 @@
              </property>
             </widget>
            </item>
-           <item row="2" column="1" colspan="2">
+           <item row="2" column="1">
             <widget class="EnLineEdit" name="author_sort">
              <property name="toolTip">
               <string>Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles.</string>
@@ -115,7 +133,7 @@
              </property>
             </widget>
            </item>
-           <item row="3" column="1" colspan="2">
+           <item row="3" column="1">
             <widget class="QSpinBox" name="rating">
              <property name="toolTip">
               <string>Rating of this book. 0-5 stars</string>
@@ -156,7 +174,7 @@
              </property>
             </widget>
            </item>
-           <item row="4" column="1" colspan="2">
+           <item row="4" column="1">
             <widget class="EnComboBox" name="publisher">
              <property name="editable">
               <bool>true</bool>
@@ -177,7 +195,7 @@
             </widget>
            </item>
            <item row="5" column="1">
-            <widget class="TagsLineEdit" name="tags">
+            <widget class="CompleteLineEdit" name="tags">
              <property name="toolTip">
               <string>Tags categorize the book. This is particularly useful while searching. &lt;br&gt;&lt;br&gt;They can be any words or phrases, separated by commas.</string>
              </property>
@@ -202,13 +220,16 @@
              <property name="text">
               <string>&amp;Remove tags:</string>
              </property>
+             <property name="alignment">
+              <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+             </property>
              <property name="buddy">
               <cstring>remove_tags</cstring>
              </property>
             </widget>
            </item>
            <item row="6" column="1">
-            <widget class="TagsLineEdit" name="remove_tags">
+            <widget class="CompleteLineEdit" name="remove_tags">
              <property name="toolTip">
               <string>Comma separated list of tags to remove from the books. </string>
              </property>
@@ -220,7 +241,7 @@
               <string>Check this box to remove all tags from the books.</string>
              </property>
              <property name="text">
-              <string>Remove all</string>
+              <string>Remove &amp;all</string>
              </property>
             </widget>
            </item>
@@ -241,9 +262,13 @@
             </widget>
            </item>
            <item row="7" column="1">
-            <layout class="QHBoxLayout" name="HLayout_34">
-             <item>
             <widget class="EnComboBox" name="series">
+             <property name="sizePolicy">
+              <sizepolicy hsizetype="Preferred" vsizetype="Fixed">
+               <horstretch>0</horstretch>
+               <verstretch>0</verstretch>
+              </sizepolicy>
+             </property>
              <property name="toolTip">
               <string>List of known series. You can add new series.</string>
              </property>
@@ -257,36 +282,24 @@
               <enum>QComboBox::InsertAlphabetically</enum>
              </property>
              <property name="sizeAdjustPolicy">
-                <enum>QComboBox::AdjustToContents</enum>
+              <enum>QComboBox::AdjustToMinimumContentsLengthWithIcon</enum>
+             </property>
+             <property name="minimumContentsLength">
+              <number>40</number>
              </property>
             </widget>
            </item>
-             <item>
+           <item row="7" column="2">
             <widget class="QCheckBox" name="clear_series">
              <property name="toolTip">
               <string>If checked, the series will be cleared</string>
              </property>
              <property name="text">
-                <string>Clear series</string>
+              <string>&amp;Clear series</string>
              </property>
             </widget>
            </item>
-             <item>
-              <spacer name="HSpacer_344">
-               <property name="orientation">
-                <enum>Qt::Horizontal</enum>
-               </property>
-               <property name="sizeHint" stdset="0">
-                <size>
-                 <width>20</width>
-                 <height>0</height>
-                </size>
-               </property>
-              </spacer>
-             </item>
-            </layout>
-           </item>
-           <item row="8" column="1" colspan="2">
+           <item row="8" column="1">
             <layout class="QHBoxLayout" name="HLayout_3">
              <item>
               <widget class="QCheckBox" name="autonumber_series">
@@ -297,7 +310,7 @@ you selected them. So if you selected Book A and then Book B,
 Book A will have series number 1 and Book B series number 2.</string>
                </property>
                <property name="text">
-                <string>Automatically number books in this series</string>
+                <string>&amp;Automatically number books in this series</string>
                </property>
               </widget>
              </item>
@@ -312,7 +325,7 @@ for that series. Checking this box will tell calibre to start numbering
 from the value in the box</string>
                </property>
                <property name="text">
-                <string>Force numbers to start with </string>
+                <string>&amp;Force numbers to start with:</string>
                </property>
               </widget>
              </item>
@@ -332,60 +345,83 @@ from the value in the box</string>
                </property>
               </widget>
              </item>
-             <item>
-              <spacer name="HSpacer_34">
-               <property name="orientation">
-                <enum>Qt::Horizontal</enum>
-               </property>
-               <property name="sizeHint" stdset="0">
-                <size>
-                 <width>20</width>
-                 <height>10</height>
-                </size>
-               </property>
-              </spacer>
-             </item>
             </layout>
            </item>
-           <item row="9" column="0">
-            <widget class="QLabel" name="label_5">
+           <item row="10" column="0">
+            <widget class="QLabel" name="label_9">
              <property name="text">
-              <string>Remove &amp;format:</string>
+              <string>&amp;Published:</string>
+             </property>
+             <property name="alignment">
+              <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
              </property>
              <property name="buddy">
-              <cstring>remove_format</cstring>
+              <cstring>pubdate</cstring>
              </property>
             </widget>
            </item>
-           <item row="9" column="1">
-            <widget class="QComboBox" name="remove_format"/>
-           </item>
-           <item row="0" column="1">
-            <widget class="EnComboBox" name="authors">
-             <property name="editable">
+           <item row="10" column="1">
+            <layout class="QHBoxLayout" name="horizontalLayout_4">
+             <item>
+              <widget class="QDateEdit" name="pubdate">
+               <property name="displayFormat">
+                <string>MMM yyyy</string>
+               </property>
+               <property name="calendarPopup">
                 <bool>true</bool>
                </property>
               </widget>
              </item>
-           <item row="11" column="0" colspan="2">
-            <widget class="QCheckBox" name="swap_title_and_author">
+             <item>
+              <widget class="QToolButton" name="clear_pubdate_button">
+               <property name="toolTip">
+                <string>Clear published date</string>
+               </property>
                <property name="text">
-              <string>&amp;Swap title and author</string>
+                <string>...</string>
+               </property>
+               <property name="icon">
+                <iconset resource="../../../../resources/images.qrc">
+                 <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
                </property>
               </widget>
              </item>
-           <item row="12" column="0" colspan="2">
+            </layout>
+           </item>
+           <item row="10" column="2">
+            <widget class="QCheckBox" name="apply_pubdate">
+             <property name="text">
+              <string>&amp;Apply date</string>
+             </property>
+            </widget>
+           </item>
+           <item row="13" column="0" colspan="3">
+            <layout class="QHBoxLayout" name="horizontalLayout_3">
+             <item>
               <widget class="QCheckBox" name="change_title_to_title_case">
                <property name="toolTip">
                 <string>Force the title to be in title case. If both this and swap authors are checked,
 title and author are swapped before the title case is set</string>
                </property>
                <property name="text">
-              <string>Change title to title case</string>
+                <string>Change title to title &amp;case</string>
                </property>
               </widget>
              </item>
-           <item row="10" column="0" colspan="2">
+             <item>
+              <spacer name="horizontalSpacer">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>40</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+             <item>
               <widget class="QCheckBox" name="remove_conversion_settings">
                <property name="toolTip">
                 <string>Remove stored conversion settings for the selected books.
@@ -397,20 +433,9 @@ Future conversion of these books will use the default settings.</string>
                </property>
               </widget>
              </item>
-           <item row="14" column="0" colspan="3">
-            <spacer name="verticalSpacer_2">
-             <property name="orientation">
-              <enum>Qt::Vertical</enum>
-             </property>
-             <property name="sizeHint" stdset="0">
-              <size>
-               <width>20</width>
-               <height>40</height>
-              </size>
-             </property>
-            </spacer>
+            </layout>
            </item>
-           <item row="13" column="0" colspan="3">
+           <item row="14" column="0" colspan="3">
             <widget class="QGroupBox" name="groupBox">
              <property name="title">
               <string>Change &amp;cover</string>
@@ -440,6 +465,55 @@ Future conversion of these books will use the default settings.</string>
              </layout>
             </widget>
            </item>
+           <item row="15" column="0">
+            <spacer name="verticalSpacer_2">
+             <property name="orientation">
+              <enum>Qt::Vertical</enum>
+             </property>
+             <property name="sizeHint" stdset="0">
+              <size>
+               <width>20</width>
+               <height>40</height>
+              </size>
+             </property>
+            </spacer>
+           </item>
+           <item row="12" column="0">
+            <spacer name="verticalSpacer">
+             <property name="orientation">
+              <enum>Qt::Vertical</enum>
+             </property>
+             <property name="sizeType">
+              <enum>QSizePolicy::Fixed</enum>
+             </property>
+             <property name="sizeHint" stdset="0">
+              <size>
+               <width>20</width>
+               <height>15</height>
+              </size>
+             </property>
+            </spacer>
+           </item>
+           <item row="11" column="0">
+            <widget class="QLabel" name="label_5">
+             <property name="text">
+              <string>Remove &amp;format:</string>
+             </property>
+             <property name="buddy">
+              <cstring>remove_format</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="11" column="1">
+            <widget class="QComboBox" name="remove_format">
+             <property name="maximumSize">
+              <size>
+               <width>120</width>
+               <height>16777215</height>
+              </size>
+             </property>
+            </widget>
+           </item>
           </layout>
          </widget>
          <widget class="QWidget" name="tab">
@@ -881,7 +955,12 @@ not multiple and the destination field is multiple</string>
    <header>widgets.h</header>
   </customwidget>
   <customwidget>
-   <class>TagsLineEdit</class>
+   <class>CompleteComboBox</class>
+   <extends>QComboBox</extends>
+   <header>widgets.h</header>
+  </customwidget>
+  <customwidget>
+   <class>CompleteLineEdit</class>
    <extends>QLineEdit</extends>
    <header>widgets.h</header>
   </customwidget>
@@ -902,14 +981,9 @@ not multiple and the destination field is multiple</string>
   <tabstop>remove_tags</tabstop>
   <tabstop>remove_all_tags</tabstop>
   <tabstop>series</tabstop>
-  <tabstop>clear_series</tabstop>
   <tabstop>autonumber_series</tabstop>
   <tabstop>series_numbering_restarts</tabstop>
   <tabstop>series_start_number</tabstop>
-  <tabstop>remove_format</tabstop>
-  <tabstop>remove_conversion_settings</tabstop>
-  <tabstop>swap_title_and_author</tabstop>
-  <tabstop>change_title_to_title_case</tabstop>
   <tabstop>button_box</tabstop>
   <tabstop>search_field</tabstop>
   <tabstop>search_mode</tabstop>

src/calibre/gui2/dialogs/metadata_single.py

@@ -16,7 +16,7 @@ from PyQt4.Qt import SIGNAL, QObject, Qt, QTimer, QDate, \
 
 from calibre.gui2 import error_dialog, file_icon_provider, dynamic, \
                            choose_files, choose_images, ResizableDialog, \
-                           warning_dialog, question_dialog
+                           warning_dialog, question_dialog, UNDEFINED_QDATE
 from calibre.gui2.dialogs.metadata_single_ui import Ui_MetadataSingleDialog
 from calibre.gui2.dialogs.fetch_metadata import FetchMetadata
 from calibre.gui2.dialogs.tag_editor import TagEditor
@@ -491,11 +491,15 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         self.formats.setAcceptDrops(True)
         self.cover_changed = False
         self.cpixmap = None
-        self.pubdate.setMinimumDate(QDate(100,1,1))
+        self.pubdate.setMinimumDate(UNDEFINED_QDATE)
         pubdate_format = tweaks['gui_pubdate_display_format']
         if pubdate_format is not None:
             self.pubdate.setDisplayFormat(pubdate_format)
-        self.date.setMinimumDate(QDate(100,1,1))
+        self.date.setMinimumDate(UNDEFINED_QDATE)
+        self.pubdate.setSpecialValueText(_('Undefined'))
+        self.date.setSpecialValueText(_('Undefined'))
+        self.clear_pubdate_button.clicked.connect(self.clear_pubdate)
+
 
         self.connect(self.cover, SIGNAL('cover_changed(PyQt_PyObject)'), self.cover_dropped)
         QObject.connect(self.cover_button, SIGNAL("clicked(bool)"), \
@@ -552,7 +556,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         tags = self.db.tags(row)
         self.original_tags = ', '.join(tags.split(',')) if tags else ''
         self.tags.setText(self.original_tags)
-        self.tags.update_tags_cache(self.db.all_tags())
+        self.tags.update_items_cache(self.db.all_tags())
         rating = self.db.rating(row)
         if rating > 0:
             self.rating.setValue(int(rating/2.))
@@ -615,6 +619,9 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
 
         self.show()
 
+    def clear_pubdate(self, *args):
+        self.pubdate.setDate(UNDEFINED_QDATE)
+
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
         layout = w.layout()
@@ -718,6 +725,10 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         au = ' & '.join([a.strip().replace('|', ',') for a in au.split(',')])
         self.authors.setEditText(au)
         
+        self.authors.set_separator('&')
+        self.authors.set_space_before_sep(True)
+        self.authors.update_items_cache(self.db.all_author_names())
+
     def initialize_series(self):
         self.series.setSizeAdjustPolicy(self.series.AdjustToContentsOnFirstShow)
         all_series = self.db.all_series()
@@ -769,7 +780,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         if d.result() == QDialog.Accepted:
             tag_string = ', '.join(d.tags)
             self.tags.setText(tag_string)
-            self.tags.update_tags_cache(self.db.all_tags())
+            self.tags.update_items_cache(self.db.all_tags())
 
 
     def fetch_metadata(self):

src/calibre/gui2/dialogs/metadata_single.ui

@@ -100,246 +100,6 @@
                     </property>
                    </widget>
                   </item>
-                  <item row="1" column="0">
-                   <widget class="QLabel" name="label">
-                    <property name="text">
-                     <string>Title &amp;sort: </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>title_sort</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="1" column="1">
-                   <widget class="EnLineEdit" name="title_sort">
-                    <property name="toolTip">
-                     <string>Specify how this book should be sorted when by title. For example, The Exorcist might be sorted as Exorcist, The.</string>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="2" column="0">
-                   <widget class="QLabel" name="label_2">
-                    <property name="text">
-                     <string>&amp;Author(s): </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>authors</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="2" column="1">
-                   <widget class="EnComboBox" name="authors">
-                    <property name="editable">
-                     <bool>true</bool>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="3" column="0">
-                   <widget class="QLabel" name="label_8">
-                    <property name="text">
-                     <string>Author S&amp;ort: </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>author_sort</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="3" column="1">
-                   <widget class="EnLineEdit" name="author_sort">
-                    <property name="toolTip">
-                     <string>Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles.
-If the box is colored green, then text matches the individual author's sort strings. If it is colored red, then the authors and this text do not match.</string>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="4" column="0">
-                   <widget class="QLabel" name="label_6">
-                    <property name="text">
-                     <string>&amp;Rating:</string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>rating</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="4" column="1" colspan="2">
-                   <widget class="QSpinBox" name="rating">
-                    <property name="toolTip">
-                     <string>Rating of this book. 0-5 stars</string>
-                    </property>
-                    <property name="whatsThis">
-                     <string>Rating of this book. 0-5 stars</string>
-                    </property>
-                    <property name="buttonSymbols">
-                     <enum>QAbstractSpinBox::PlusMinus</enum>
-                    </property>
-                    <property name="suffix">
-                     <string> stars</string>
-                    </property>
-                    <property name="maximum">
-                     <number>5</number>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="5" column="0">
-                   <widget class="QLabel" name="label_3">
-                    <property name="text">
-                     <string>&amp;Publisher: </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>publisher</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="5" column="1" colspan="2">
-                   <widget class="EnComboBox" name="publisher">
-                    <property name="editable">
-                     <bool>true</bool>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="6" column="0">
-                   <widget class="QLabel" name="label_4">
-                    <property name="text">
-                     <string>Ta&amp;gs: </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>tags</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="7" column="0">
-                   <widget class="QLabel" name="label_7">
-                    <property name="text">
-                     <string>&amp;Series:</string>
-                    </property>
-                    <property name="textFormat">
-                     <enum>Qt::PlainText</enum>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>series</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="7" column="1">
-                   <layout class="QHBoxLayout" name="_3">
-                    <property name="spacing">
-                     <number>5</number>
-                    </property>
-                    <item>
-                     <widget class="EnComboBox" name="series">
-                      <property name="toolTip">
-                       <string>List of known series. You can add new series.</string>
-                      </property>
-                      <property name="whatsThis">
-                       <string>List of known series. You can add new series.</string>
-                      </property>
-                      <property name="editable">
-                       <bool>true</bool>
-                      </property>
-                      <property name="insertPolicy">
-                       <enum>QComboBox::InsertAlphabetically</enum>
-                      </property>
-                     </widget>
-                    </item>
-                   </layout>
-                  </item>
-                  <item row="8" column="1" colspan="2">
-                   <widget class="QDoubleSpinBox" name="series_index">
-                    <property name="enabled">
-                     <bool>false</bool>
-                    </property>
-                    <property name="prefix">
-                     <string>Book </string>
-                    </property>
-                    <property name="maximum">
-                     <double>9999.989999999999782</double>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="9" column="0">
-                   <widget class="QLabel" name="label_9">
-                    <property name="text">
-                     <string>IS&amp;BN:</string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>isbn</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="9" column="1" colspan="2">
-                   <widget class="QLineEdit" name="isbn"/>
-                  </item>
-                  <item row="10" column="0">
-                   <widget class="QLabel" name="label_11">
-                    <property name="text">
-                     <string>&amp;Date:</string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>date</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="10" column="1" colspan="2">
-                   <widget class="QDateEdit" name="date">
-                    <property name="displayFormat">
-                     <string>dd MMM yyyy</string>
-                    </property>
-                    <property name="calendarPopup">
-                     <bool>true</bool>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="11" column="0">
-                   <widget class="QLabel" name="label_10">
-                    <property name="text">
-                     <string>Publishe&amp;d:</string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>pubdate</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="11" column="1" colspan="2">
-                   <widget class="QDateEdit" name="pubdate">
-                    <property name="displayFormat">
-                     <string>MMM yyyy</string>
-                    </property>
-                    <property name="calendarPopup">
-                     <bool>true</bool>
-                    </property>
-                   </widget>
-                  </item>
                   <item row="0" column="2" rowspan="4">
                    <layout class="QVBoxLayout" name="verticalLayout_7">
                     <item>
@@ -446,10 +206,136 @@ Using this button to create author sort will change author sort from red to gree
                     </item>
                    </layout>
                   </item>
+                  <item row="1" column="0">
+                   <widget class="QLabel" name="label">
+                    <property name="text">
+                     <string>Title &amp;sort: </string>
+                    </property>
+                    <property name="alignment">
+                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                    </property>
+                    <property name="buddy">
+                     <cstring>title_sort</cstring>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="1" column="1">
+                   <widget class="EnLineEdit" name="title_sort">
+                    <property name="toolTip">
+                     <string>Specify how this book should be sorted when by title. For example, The Exorcist might be sorted as Exorcist, The.</string>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="2" column="0">
+                   <widget class="QLabel" name="label_2">
+                    <property name="text">
+                     <string>&amp;Author(s): </string>
+                    </property>
+                    <property name="alignment">
+                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                    </property>
+                    <property name="buddy">
+                     <cstring>authors</cstring>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="2" column="1">
+                   <widget class="CompleteComboBox" name="authors">
+                    <property name="editable">
+                     <bool>true</bool>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="3" column="0">
+                   <widget class="QLabel" name="label_8">
+                    <property name="text">
+                     <string>Author S&amp;ort: </string>
+                    </property>
+                    <property name="alignment">
+                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                    </property>
+                    <property name="buddy">
+                     <cstring>author_sort</cstring>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="3" column="1">
+                   <widget class="EnLineEdit" name="author_sort">
+                    <property name="toolTip">
+                     <string>Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles.
+If the box is colored green, then text matches the individual author's sort strings. If it is colored red, then the authors and this text do not match.</string>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="4" column="0">
+                   <widget class="QLabel" name="label_6">
+                    <property name="text">
+                     <string>&amp;Rating:</string>
+                    </property>
+                    <property name="alignment">
+                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                    </property>
+                    <property name="buddy">
+                     <cstring>rating</cstring>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="4" column="1" colspan="2">
+                   <widget class="QSpinBox" name="rating">
+                    <property name="toolTip">
+                     <string>Rating of this book. 0-5 stars</string>
+                    </property>
+                    <property name="whatsThis">
+                     <string>Rating of this book. 0-5 stars</string>
+                    </property>
+                    <property name="buttonSymbols">
+                     <enum>QAbstractSpinBox::PlusMinus</enum>
+                    </property>
+                    <property name="suffix">
+                     <string> stars</string>
+                    </property>
+                    <property name="maximum">
+                     <number>5</number>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="5" column="0">
+                   <widget class="QLabel" name="label_3">
+                    <property name="text">
+                     <string>&amp;Publisher: </string>
+                    </property>
+                    <property name="alignment">
+                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                    </property>
+                    <property name="buddy">
+                     <cstring>publisher</cstring>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="5" column="1" colspan="2">
+                   <widget class="EnComboBox" name="publisher">
+                    <property name="editable">
+                     <bool>true</bool>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="6" column="0">
+                   <widget class="QLabel" name="label_4">
+                    <property name="text">
+                     <string>Ta&amp;gs: </string>
+                    </property>
+                    <property name="alignment">
+                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                    </property>
+                    <property name="buddy">
+                     <cstring>tags</cstring>
+                    </property>
+                   </widget>
+                  </item>
                   <item row="6" column="1">
                    <layout class="QHBoxLayout" name="_2">
                     <item>
-                     <widget class="TagsLineEdit" name="tags">
+                     <widget class="CompleteLineEdit" name="tags">
                       <property name="toolTip">
                        <string>Tags categorize the book. This is particularly useful while searching. &lt;br&gt;&lt;br&gt;They can be any words or phrases, separated by commas.</string>
                       </property>
@@ -471,6 +357,45 @@ Using this button to create author sort will change author sort from red to gree
                     </property>
                    </widget>
                   </item>
+                  <item row="7" column="0">
+                   <widget class="QLabel" name="label_7">
+                    <property name="text">
+                     <string>&amp;Series:</string>
+                    </property>
+                    <property name="textFormat">
+                     <enum>Qt::PlainText</enum>
+                    </property>
+                    <property name="alignment">
+                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                    </property>
+                    <property name="buddy">
+                     <cstring>series</cstring>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="7" column="1">
+                   <layout class="QHBoxLayout" name="_3">
+                    <property name="spacing">
+                     <number>5</number>
+                    </property>
+                    <item>
+                     <widget class="EnComboBox" name="series">
+                      <property name="toolTip">
+                       <string>List of known series. You can add new series.</string>
+                      </property>
+                      <property name="whatsThis">
+                       <string>List of known series. You can add new series.</string>
+                      </property>
+                      <property name="editable">
+                       <bool>true</bool>
+                      </property>
+                      <property name="insertPolicy">
+                       <enum>QComboBox::InsertAlphabetically</enum>
+                      </property>
+                     </widget>
+                    </item>
+                   </layout>
+                  </item>
                   <item row="7" column="2">
                    <widget class="QToolButton" name="remove_series_button">
                     <property name="toolTip">
@@ -485,6 +410,92 @@ Using this button to create author sort will change author sort from red to gree
                     </property>
                    </widget>
                   </item>
+                  <item row="8" column="1" colspan="2">
+                   <widget class="QDoubleSpinBox" name="series_index">
+                    <property name="enabled">
+                     <bool>false</bool>
+                    </property>
+                    <property name="prefix">
+                     <string>Book </string>
+                    </property>
+                    <property name="maximum">
+                     <double>9999.989999999999782</double>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="9" column="0">
+                   <widget class="QLabel" name="label_9">
+                    <property name="text">
+                     <string>IS&amp;BN:</string>
+                    </property>
+                    <property name="alignment">
+                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                    </property>
+                    <property name="buddy">
+                     <cstring>isbn</cstring>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="9" column="1" colspan="2">
+                   <widget class="QLineEdit" name="isbn"/>
+                  </item>
+                  <item row="10" column="0">
+                   <widget class="QLabel" name="label_11">
+                    <property name="text">
+                     <string>&amp;Date:</string>
+                    </property>
+                    <property name="alignment">
+                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                    </property>
+                    <property name="buddy">
+                     <cstring>date</cstring>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="10" column="1" colspan="2">
+                   <widget class="QDateEdit" name="date">
+                    <property name="displayFormat">
+                     <string>dd MMM yyyy</string>
+                    </property>
+                    <property name="calendarPopup">
+                     <bool>true</bool>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="11" column="0">
+                   <widget class="QLabel" name="label_10">
+                    <property name="text">
+                     <string>Publishe&amp;d:</string>
+                    </property>
+                    <property name="alignment">
+                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                    </property>
+                    <property name="buddy">
+                     <cstring>pubdate</cstring>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="11" column="1">
+                   <widget class="QDateEdit" name="pubdate">
+                    <property name="displayFormat">
+                     <string>MMM yyyy</string>
+                    </property>
+                    <property name="calendarPopup">
+                     <bool>true</bool>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="11" column="2">
+                   <widget class="QToolButton" name="clear_pubdate_button">
+                    <property name="toolTip">
+                     <string>Clear published date</string>
+                    </property>
+                    <property name="icon">
+                     <iconset resource="../../../../resources/images.qrc">
+                      <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
+                    </property>
+                   </widget>
+                  </item>
                  </layout>
                 </widget>
                </item>
@@ -831,10 +842,15 @@ Using this button to create author sort will change author sort from red to gree
    <header>widgets.h</header>
   </customwidget>
   <customwidget>
-   <class>TagsLineEdit</class>
+   <class>CompleteLineEdit</class>
    <extends>QLineEdit</extends>
    <header>widgets.h</header>
   </customwidget>
+  <customwidget>
+   <class>CompleteComboBox</class>
+   <extends>QComboBox</extends>
+   <header>widgets.h</header>
+  </customwidget>
   <customwidget>
    <class>FormatList</class>
    <extends>QListWidget</extends>

src/calibre/gui2/dialogs/search.py

@@ -31,6 +31,9 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.authors_box.setEditText('')
         self.authors_box.completer().setCompletionMode(QCompleter.PopupCompletion)
         self.authors_box.setAutoCompletionCaseSensitivity(Qt.CaseInsensitive)
+        self.authors_box.set_separator('&')
+        self.authors_box.set_space_before_sep(True)
+        self.authors_box.update_items_cache(db.all_author_names())
 
         all_series = db.all_series()
         all_series.sort(key=lambda x : sort_key(x[1]))
@@ -42,7 +45,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.series_box.setAutoCompletionCaseSensitivity(Qt.CaseInsensitive)
 
         all_tags = db.all_tags()
-        self.tags_box.update_tags_cache(all_tags)
+        self.tags_box.update_items_cache(all_tags)
 
         self.box_last_values = copy.deepcopy(box_values)
         if self.box_last_values:

src/calibre/gui2/dialogs/search.ui

@@ -265,7 +265,7 @@
         </widget>
        </item>
        <item row="2" column="1">
-        <widget class="EnComboBox" name="authors_box">
+        <widget class="CompleteComboBox" name="authors_box">
          <property name="toolTip">
           <string>Enter an author's name. Only one author can be used.</string>
          </property>
@@ -279,7 +279,7 @@
         </widget>
        </item>
        <item row="4" column="1">
-        <widget class="TagsLineEdit" name="tags_box">
+        <widget class="CompleteLineEdit" name="tags_box">
          <property name="toolTip">
           <string>Enter tags separated by spaces</string>
          </property>
@@ -360,10 +360,15 @@
    <header>widgets.h</header>
   </customwidget>
   <customwidget>
-   <class>TagsLineEdit</class>
+   <class>CompleteLineEdit</class>
    <extends>QLineEdit</extends>
    <header>widgets.h</header>
   </customwidget>
+  <customwidget>
+   <class>CompleteComboBox</class>
+   <extends>QComboBox</extends>
+   <header>widgets.h</header>
+  </customwidget>
  </customwidgets>
  <tabstops>
   <tabstop>all</tabstop>

src/calibre/gui2/dialogs/template_dialog.py

@@ -3,8 +3,11 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __license__   = 'GPL v3'
 
+import json
+
 from PyQt4.Qt import Qt, QDialog, QDialogButtonBox
 from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog
+from calibre.utils.formatter_functions import formatter_functions
 
 class TemplateDialog(QDialog, Ui_TemplateDialog):
 
@@ -17,9 +20,41 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
         self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint))
         self.setWindowIcon(icon)
 
+        self.textbox.setTabStopWidth(10)
+        self.source_code.setTabStopWidth(10)
+        self.documentation.setReadOnly(True)
+        self.source_code.setReadOnly(True)
+
         if text is not None:
             self.textbox.setPlainText(text)
-        self.textbox.setTabStopWidth(50)
         self.buttonBox.button(QDialogButtonBox.Ok).setText(_('&OK'))
         self.buttonBox.button(QDialogButtonBox.Cancel).setText(_('&Cancel'))
 
+        try:
+            with open(P('template-functions.json'), 'rb') as f:
+                self.builtin_source_dict = json.load(f, encoding='utf-8')
+        except:
+            self.builtin_source_dict = {}
+
+        self.funcs = formatter_functions.get_functions()
+        self.builtins = formatter_functions.get_builtins()
+
+        func_names = sorted(self.funcs)
+        self.function.clear()
+        self.function.addItem('')
+        self.function.addItems(func_names)
+        self.function.setCurrentIndex(0)
+        self.function.currentIndexChanged[str].connect(self.function_changed)
+
+    def function_changed(self, toWhat):
+        name = unicode(toWhat)
+        self.source_code.clear()
+        self.documentation.clear()
+        if name in self.funcs:
+            self.documentation.setPlainText(self.funcs[name].doc)
+            if name in self.builtins:
+                if name in self.builtin_source_dict:
+                    self.source_code.setPlainText(self.builtin_source_dict[name])
+            else:
+                self.source_code.setPlainText(self.funcs[name].program_text)
+

src/calibre/gui2/dialogs/template_dialog.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>500</width>
-    <height>235</height>
+    <width>588</width>
+    <height>546</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -33,6 +33,62 @@
      </property>
     </widget>
    </item>
+   <item>
+    <layout class="QGridLayout" name="gridLayout">
+     <item row="0" column="0">
+      <widget class="QLabel" name="label">
+       <property name="text">
+        <string>Function &amp;name:</string>
+       </property>
+       <property name="buddy">
+        <cstring>function</cstring>
+       </property>
+      </widget>
+     </item>
+     <item row="0" column="1">
+      <widget class="QComboBox" name="function"/>
+     </item>
+     <item row="1" column="0">
+      <widget class="QLabel" name="label_2">
+       <property name="text">
+        <string>&amp;Documentation:</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
+       </property>
+       <property name="buddy">
+        <cstring>documentation</cstring>
+       </property>
+      </widget>
+     </item>
+     <item row="2" column="0">
+      <widget class="QLabel" name="label_3">
+       <property name="text">
+        <string>Python &amp;code:</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
+       </property>
+       <property name="buddy">
+        <cstring>source_code</cstring>
+       </property>
+      </widget>
+     </item>
+     <item row="1" column="1">
+      <widget class="QPlainTextEdit" name="documentation">
+       <property name="maximumSize">
+        <size>
+         <width>16777215</width>
+         <height>75</height>
+        </size>
+       </property>
+      </widget>
+     </item>
+     <item row="2" column="1">
+      <widget class="QPlainTextEdit" name="source_code"/>
+     </item>
+    </layout>
+   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/init.py

@@ -148,7 +148,6 @@ class StatusBar(QStatusBar): # {{{
                 self.get_version() + ' ' + _('created by Kovid Goyal')
         self.device_string = ''
         self.update_label = QLabel('')
-        self.update_label.setOpenExternalLinks(True)
         self.addPermanentWidget(self.update_label)
         self.update_label.setVisible(False)
         self._font = QFont()
@@ -174,8 +173,9 @@ class StatusBar(QStatusBar): # {{{
         self.clearMessage()
 
     def new_version_available(self, ver, url):
-        msg = (u'<span style="color:red; font-weight: bold">%s: <a href="%s">%s<a></span>') % (
-                _('Update found'), url, ver)
+        msg = (u'<span style="color:red; font-weight: bold">%s: <a'
+               ' href="update:%s">%s<a></span>') % (
+                _('Update found'), ver, ver)
         self.update_label.setText(msg)
         self.update_label.setCursor(Qt.PointingHandCursor)
         self.update_label.setVisible(True)
@@ -240,6 +240,13 @@ class LayoutMixin(object): # {{{
             self.status_bar.addPermanentWidget(button)
         self.status_bar.addPermanentWidget(self.jobs_button)
         self.setStatusBar(self.status_bar)
+        self.status_bar.update_label.linkActivated.connect(self.update_link_clicked)
+
+    def update_link_clicked(self, url):
+        url = unicode(url)
+        if url.startswith('update:'):
+            version = url.partition(':')[-1]
+            self.update_found(version, force=True)
 
     def finalize_layout(self):
         self.status_bar.initialize(self.system_tray_icon)

src/calibre/gui2/library/delegates.py

@@ -16,7 +16,7 @@ from PyQt4.Qt import QColor, Qt, QModelIndex, QSize, \
                      QComboBox, QTextDocument
 
 from calibre.gui2 import UNDEFINED_QDATE, error_dialog
-from calibre.gui2.widgets import EnLineEdit, TagsLineEdit
+from calibre.gui2.widgets import EnLineEdit, CompleteLineEdit
 from calibre.utils.date import now, format_date
 from calibre.utils.config import tweaks
 from calibre.utils.formatter import validation_formatter
@@ -173,9 +173,9 @@ class TagsDelegate(QStyledItemDelegate): # {{{
         if self.db:
             col = index.model().column_map[index.column()]
             if not index.model().is_custom_column(col):
-                editor = TagsLineEdit(parent, self.db.all_tags())
+                editor = CompleteLineEdit(parent, self.db.all_tags())
             else:
-                editor = TagsLineEdit(parent,
+                editor = CompleteLineEdit(parent,
                         sorted(list(self.db.all_custom(label=self.db.field_metadata.key_to_label(col))),
                                key=sort_key))
                 return editor
@@ -184,6 +184,31 @@ class TagsDelegate(QStyledItemDelegate): # {{{
         return editor
 # }}}
 
+class CompleteDelegate(QStyledItemDelegate): # {{{
+    def __init__(self, parent, sep, items_func_name, space_before_sep=False):
+        QStyledItemDelegate.__init__(self, parent)
+        self.sep = sep
+        self.items_func_name = items_func_name
+        self.space_before_sep = space_before_sep
+
+    def set_database(self, db):
+        self.db = db
+
+    def createEditor(self, parent, option, index):
+        if self.db and hasattr(self.db, self.items_func_name):
+            col = index.model().column_map[index.column()]
+            if not index.model().is_custom_column(col):
+                editor = CompleteLineEdit(parent, getattr(self.db, self.items_func_name)(),
+                    self.sep, self.space_before_sep)
+            else:
+                editor = CompleteLineEdit(parent,
+                    sorted(list(self.db.all_custom(label=self.db.field_metadata.key_to_label(col))),
+                    key=sort_key), self.sep, self.space_before_sep)
+        else:
+            editor = EnLineEdit(parent)
+        return editor
+# }}}
+
 class CcDateDelegate(QStyledItemDelegate): # {{{
     '''
     Delegate for custom columns dates. Because this delegate stores the

src/calibre/gui2/library/views.py

@@ -13,7 +13,7 @@ from PyQt4.Qt import QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, \
     QPoint, QPixmap, QUrl, QImage, QPainter, QColor, QRect
 
 from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \
-    TextDelegate, DateDelegate, TagsDelegate, CcTextDelegate, \
+    TextDelegate, DateDelegate, CompleteDelegate, CcTextDelegate, \
     CcBoolDelegate, CcCommentsDelegate, CcDateDelegate, CcTemplateDelegate, \
     CcEnumDelegate
 from calibre.gui2.library.models import BooksModel, DeviceBooksModel
@@ -76,8 +76,8 @@ class BooksView(QTableView): # {{{
         self.rating_delegate = RatingDelegate(self)
         self.timestamp_delegate = DateDelegate(self)
         self.pubdate_delegate = PubDateDelegate(self)
-        self.tags_delegate = TagsDelegate(self)
-        self.authors_delegate = TextDelegate(self)
+        self.tags_delegate = CompleteDelegate(self, ',', 'all_tags')
+        self.authors_delegate = CompleteDelegate(self, '&', 'all_author_names', True)
         self.series_delegate = TextDelegate(self)
         self.publisher_delegate = TextDelegate(self)
         self.text_delegate = TextDelegate(self)
@@ -410,8 +410,7 @@ class BooksView(QTableView): # {{{
         self.save_state()
         self._model.set_database(db)
         self.tags_delegate.set_database(db)
-        self.authors_delegate.set_auto_complete_function(
-                lambda: [(x, y.replace('|', ',')) for (x, y) in db.all_authors()])
+        self.authors_delegate.set_database(db)
         self.series_delegate.set_auto_complete_function(db.all_series)
         self.publisher_delegate.set_auto_complete_function(db.all_publishers)
 

src/calibre/gui2/metadata/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/gui2/preferences/template_functions.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import traceback
+import json, traceback
 
 from calibre.gui2 import error_dialog
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget
@@ -73,6 +73,12 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.textBrowser.setHtml(help_text)
 
     def initialize(self):
+        try:
+            with open(P('template-functions.json'), 'rb') as f:
+                self.builtin_source_dict = json.load(f, encoding='utf-8')
+        except:
+            self.builtin_source_dict = {}
+
         self.funcs = formatter_functions.get_functions()
         self.builtins = formatter_functions.get_builtins()
 
@@ -179,8 +185,13 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         func = self.funcs[txt]
         self.argument_count.setValue(func.arg_count)
         self.documentation.setText(func.doc)
-        self.program.setPlainText(func.program_text)
         if txt in self.builtins:
+            if hasattr(func, 'program_text'):
+                self.program.setPlainText(func.program_text)
+            elif txt in self.builtin_source_dict:
+                self.program.setPlainText(self.builtin_source_dict[txt])
+            else:
+                self.program.setPlainText(_('function source code not available'))
             self.documentation.setReadOnly(True)
             self.argument_count.setReadOnly(True)
             self.program.setReadOnly(True)

src/calibre/gui2/tag_view.py

@@ -730,7 +730,7 @@ class TagsModel(QAbstractItemModel): # {{{
             else:
                 collapse_model = 'partition'
                 collapse_template = tweaks['categories_collapsed_popularity_template']
-        collapse_letter = None
+        collapse_letter = collapse_letter_sk = None
 
         for i, r in enumerate(self.row_map):
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
@@ -782,8 +782,17 @@ class TagsModel(QAbstractItemModel): # {{{
                         ts = tag.sort
                         if not ts:
                             ts = ' '
-                        if upper(ts[0]) != collapse_letter:
+                        try:
+                            sk = sort_key(ts)[0]
+                        except:
+                            sk = ts[0]
+
+                        if sk != collapse_letter_sk:
                             collapse_letter = upper(ts[0])
+                            try:
+                                collapse_letter_sk = sort_key(collapse_letter)[0]
+                            except:
+                                collapse_letter_sk = collapse_letter
                             sub_cat = TagTreeItem(parent=category,
                                      data = collapse_letter,
                                      category_icon = category_node.icon,

src/calibre/gui2/update.py

@@ -52,8 +52,7 @@ class UpdateNotification(QDialog):
         self.label = QLabel('<p>'+
             _('%s has been updated to version <b>%s</b>. '
             'See the <a href="http://calibre-ebook.com/whats-new'
-            '">new features</a>. Visit the download pa'
-            'ge?')%(__appname__, version))
+            '">new features</a>.')%(__appname__, version))
         self.label.setOpenExternalLinks(True)
         self.label.setWordWrap(True)
         self.setWindowTitle(_('Update available!'))
@@ -94,13 +93,13 @@ class UpdateMixin(object):
                     type=Qt.QueuedConnection)
             self.update_checker.start()
 
-    def update_found(self, version):
+    def update_found(self, version, force=False):
         os = 'windows' if iswindows else 'osx' if isosx else 'linux'
         url = 'http://calibre-ebook.com/download_%s'%os
         self.status_bar.new_version_available(version, url)
 
-        if config.get('new_version_notification') and \
-                dynamic.get('update to version %s'%version, True):
+        if force or (config.get('new_version_notification') and \
+                dynamic.get('update to version %s'%version, True)):
             self._update_notification__ = UpdateNotification(version,
                     parent=self)
             self._update_notification__.show()

src/calibre/gui2/widgets.py

@@ -386,11 +386,13 @@ class LineEditECM(object):
         action_lower_case = case_menu.addAction(_('Lower Case'))
         action_swap_case = case_menu.addAction(_('Swap Case'))
         action_title_case = case_menu.addAction(_('Title Case'))
+        action_capitalize = case_menu.addAction(_('Capitalize'))
 
         self.connect(action_upper_case, SIGNAL('triggered()'), self.upper_case)
         self.connect(action_lower_case, SIGNAL('triggered()'), self.lower_case)
         self.connect(action_swap_case, SIGNAL('triggered()'), self.swap_case)
         self.connect(action_title_case, SIGNAL('triggered()'), self.title_case)
+        self.connect(action_capitalize, SIGNAL('triggered()'), self.capitalize)
 
         menu.addMenu(case_menu)
         menu.exec_(event.globalPos())
@@ -408,6 +410,10 @@ class LineEditECM(object):
         from calibre.utils.titlecase import titlecase
         self.setText(titlecase(unicode(self.text())))
 
+    def capitalize(self):
+        from calibre.utils.icu import capitalize
+        self.setText(capitalize(unicode(self.text())))
+
 
 class EnLineEdit(LineEditECM, QLineEdit):
 
@@ -420,46 +426,47 @@ class EnLineEdit(LineEditECM, QLineEdit):
     pass
 
 
-class TagsCompleter(QCompleter):
+class ItemsCompleter(QCompleter):
 
     '''
     A completer object that completes a list of tags. It is used in conjunction
     with a CompleterLineEdit.
     '''
 
-    def __init__(self, parent, all_tags):
-        QCompleter.__init__(self, all_tags, parent)
-        self.all_tags = set(all_tags)
+    def __init__(self, parent, all_items):
+        QCompleter.__init__(self, all_items, parent)
+        self.all_items = set(all_items)
 
-    def update(self, text_tags, completion_prefix):
-        tags = list(self.all_tags.difference(text_tags))
-        model = QStringListModel(tags, self)
+    def update(self, text_items, completion_prefix):
+        items = list(self.all_items.difference(text_items))
+        model = QStringListModel(items, self)
         self.setModel(model)
 
         self.setCompletionPrefix(completion_prefix)
         if completion_prefix.strip() != '':
             self.complete()
 
-    def update_tags_cache(self, tags):
-        self.all_tags = set(tags)
-        model = QStringListModel(tags, self)
+    def update_items_cache(self, items):
+        self.all_items = set(items)
+        model = QStringListModel(items, self)
         self.setModel(model)
 
 
-class TagsLineEdit(EnLineEdit):
+class CompleteLineEdit(EnLineEdit):
 
     '''
     A QLineEdit that can complete parts of text separated by separator.
     '''
 
-    def __init__(self, parent=0, tags=[]):
+    def __init__(self, parent=0, complete_items=[], sep=',', space_before_sep=False):
         EnLineEdit.__init__(self, parent)
 
-        self.separator = ','
+        self.separator = sep
+        self.space_before_sep = space_before_sep
 
         self.connect(self, SIGNAL('textChanged(QString)'), self.text_changed)
 
-        self.completer = TagsCompleter(self, tags)
+        self.completer = ItemsCompleter(self, complete_items)
         self.completer.setCaseSensitivity(Qt.CaseInsensitive)
 
         self.connect(self,
@@ -470,32 +477,43 @@ class TagsLineEdit(EnLineEdit):
 
         self.completer.setWidget(self)
 
-    def update_tags_cache(self, tags):
-        self.completer.update_tags_cache(tags)
+    def update_items_cache(self, complete_items):
+        self.completer.update_items_cache(complete_items)
+        
+    def set_separator(self, sep):
+        self.separator = sep
+        
+    def set_space_before_sep(self, space_before):
+        self.space_before_sep = space_before
 
     def text_changed(self, text):
         all_text = unicode(text)
         text = all_text[:self.cursorPosition()]
-        prefix = text.split(',')[-1].strip()
+        prefix = text.split(self.separator)[-1].strip()
 
-        text_tags = []
+        text_items = []
         for t in all_text.split(self.separator):
             t1 = unicode(t).strip()
             if t1 != '':
-                text_tags.append(t)
-        text_tags = list(set(text_tags))
+                text_items.append(t)
+        text_items = list(set(text_items))
 
         self.emit(SIGNAL('text_changed(PyQt_PyObject, PyQt_PyObject)'),
-            text_tags, prefix)
+            text_items, prefix)
 
     def complete_text(self, text):
         cursor_pos = self.cursorPosition()
         before_text = unicode(self.text())[:cursor_pos]
         after_text = unicode(self.text())[cursor_pos:]
-        prefix_len = len(before_text.split(',')[-1].strip())
-        self.setText('%s%s%s %s' % (before_text[:cursor_pos - prefix_len],
-            text, self.separator, after_text))
-        self.setCursorPosition(cursor_pos - prefix_len + len(text) + 2)
+        prefix_len = len(before_text.split(self.separator)[-1].strip())
+        if self.space_before_sep:
+            complete_text_pat = '%s%s %s %s'
+            len_extra = 3
+        else:
+            complete_text_pat = '%s%s%s %s'
+            len_extra = 2
+        self.setText(complete_text_pat % (before_text[:cursor_pos - prefix_len], text, self.separator, after_text))
+        self.setCursorPosition(cursor_pos - prefix_len + len(text) + len_extra)
 
 
 class EnComboBox(QComboBox):
@@ -522,6 +540,22 @@ class EnComboBox(QComboBox):
             idx = 0
         self.setCurrentIndex(idx)
 
+class CompleteComboBox(EnComboBox):
+    
+    def __init__(self, *args):
+        EnComboBox.__init__(self, *args)
+        self.setLineEdit(CompleteLineEdit(self))
+
+    def update_items_cache(self, complete_items):
+        self.lineEdit().update_items_cache(complete_items)
+        
+    def set_separator(self, sep):
+        self.lineEdit().set_separator(sep)
+        
+    def set_space_before_sep(self, space_before):
+        self.lineEdit().set_space_before_sep(space_before)
+
+
 class HistoryLineEdit(QComboBox):
 
     lost_focus = pyqtSignal()

src/calibre/library/catalog.py

@@ -1037,7 +1037,7 @@ class EPUB_MOBI(CatalogPlugin):
                          'by_recently_added_series_title_template',
                          'by_month_added_normal_title_template',
                          'by_month_added_series_title_template']
-            execfile(P(os.path.join('catalog','section_list_templates.py')),locals())
+            execfile(P('catalog/section_list_templates.py'), locals())
             for t in templates:
                 setattr(self,t,eval(t))
 
@@ -4252,13 +4252,13 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                             xmlns=XHTML_NS,
                             )
 
-                generated_html = P(os.path.join('catalog','template.xhtml'),
+                generated_html = P('catalog/template.xhtml',
                         data=True).decode('utf-8').format(**args)
                 generated_html = substitute_entites(generated_html)
                 return BeautifulSoup(generated_html)
 
             # Generate the template arguments
-            css = P(os.path.join('catalog','stylesheet.css'), data=True).decode('utf-8')
+            css = P('catalog/stylesheet.css', data=True).decode('utf-8')
             title_str = title = escape(book['title'])
             series = ''
             series_index = ''
@@ -4443,7 +4443,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
         def generateMastheadImage(self, out_path):
             from calibre.ebooks.conversion.config import load_defaults
             from calibre.utils.fonts import fontconfig
-            font_path = default_font = P(os.path.join('fonts','liberation','LiberationSerif-Bold.ttf'))
+            font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf')
             recs = load_defaults('mobi_output')
             masthead_font_family = recs.get('masthead_font', 'Default')
 
@@ -4482,12 +4482,16 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
 
         def generateRatingString(self, book):
             rating = ''
+            try:
                 if 'rating' in book:
                     stars = int(book['rating']) / 2
                     if stars:
                         star_string = self.FULL_RATING_SYMBOL * stars
                         empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars)
                         rating = '%s%s' % (star_string,empty_stars)
+            except:
+                # Rating could be None
+                pass
             return rating
 
         def generateShortDescription(self, description, dest=None):
@@ -4829,13 +4833,6 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
             self.progressInt = coarse_progress + fine_progress
             self.reporter(self.progressInt, self.progressString)
 
-        class NotImplementedError:
-            def __init__(self, error):
-                self.error.append(error)
-
-            def logerror(self):
-                self.opts.log.info('%s not implemented' % error)
-
     def run(self, path_to_output, opts, db, notification=DummyReporter()):
         opts.log = log
         opts.fmt = self.fmt = path_to_output.rpartition('.')[2]

src/calibre/library/cli.py

@@ -693,10 +693,7 @@ def command_catalog(args, dbpath):
                             }
 
     with plugin:
-        plugin.run(args[1], opts, get_db(dbpath, opts))
-    return 0
-
-# end of GR additions
+        return int(bool(plugin.run(args[1], opts, get_db(dbpath, opts))))
 
 def parse_series_string(db, label, value):
     val = unicode(value).strip()

src/calibre/library/database.py

@@ -1060,6 +1060,10 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
         return [ (i[0], i[1]) for i in \
                 self.conn.get('SELECT id, name FROM authors')]
 
+    def all_author_names(self):
+        return filter(None, [i[0].strip().replace('|', ',') for i in self.conn.get(
+            'SELECT name FROM authors')])
+
     def all_publishers(self):
         return [ (i[0], i[1]) for i in \
                 self.conn.get('SELECT id, name FROM publishers')]

src/calibre/library/database2.py

@@ -690,11 +690,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi = Metadata(None)
 
         aut_list = row[fm['au_map']]
-        aut_list = [p.split(':::') for p in aut_list.split(':#:')]
+        if aut_list:
+            aut_list = [p.split(':::') for p in aut_list.split(':#:') if p]
+        else:
+            aut_list = []
         aum = []
         aus = {}
         for (author, author_sort) in aut_list:
-            aum.append(author)
+            aum.append(author.replace('|', ','))
             aus[author] = author_sort.replace('|', ',')
         mi.title       = row[fm['title']]
         mi.authors     = aum

src/calibre/library/sqlite.py

@@ -100,7 +100,7 @@ class AumSortedConcatenate(object):
         keys = self.ans.keys()
         l = len(keys)
         if l == 0:
-            return 'Unknown:::Unknown'
+            return None
         if l == 1:
             return self.ans[keys[0]]
         return ':#:'.join([self.ans[v] for v in sorted(keys)])

src/calibre/manual/conversion.rst

@@ -547,6 +547,7 @@ Some limitations of PDF input are:
     * Extraction of vector images and tables from within the document is also not supported.
     * Some PDFs use special glyphs to represent ll or ff or fi, etc. Conversion of these may or may not work depending on just how they are represented internally in the PDF.
     * Some PDFs store their images upside down with a rotation instruction, |app| currently doesn't support that instruction, so the images will be rotated in the output as well. 
+    * Links and Tables of Contents are not supported
 
 To re-iterate **PDF is a really, really bad** format to use as input. If you absolutely must use PDF, then be prepared for an
 output ranging anywhere from decent to unusable, depending on the input PDF.

src/calibre/manual/faq.rst

@@ -101,6 +101,17 @@ We just need some information from you:
 Once you send us the output for a particular operating system, support for the device in that operating system
 will appear in the next release of |app|.
 
+My device is not being detected by |app|?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Follow these steps to find the problem:
+
+    * Make sure that you are connecting only a single device to your computer at a time. Do not have another |app| supported device like an iPhone/iPad etc. at the same time.
+    * Make sure you are running the latest version of |app|. The latest version can always be downloaded from `http://calibre-ebook.com/download`_.
+    * Ensure your operating system is seeing the device. That is, the device should be mounted as a disk that you can access using Windows explorer or whatever the file management program on your computer is
+    * In calibre, go to Preferences->Plugins->Device Interface plugin and make sure the plugin for your device is enabled.
+    * If all the above steps fail, go to Preferences->Miscellaneous and click debug device detection with your device attached and post the output as a ticket on `http://bugs.calibre-ebook.com`_.
+
 How does |app| manage collections on my SONY reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -437,10 +448,24 @@ My antivirus program claims |app| is a virus/trojan?
 
 Your antivirus program is wrong. |app| is a completely open source product. You can actually browse the source code yourself (or hire someone to do it for you) to verify that it is not a virus. Please report the false identification to whatever company you buy your antivirus software from. If the antivirus program is preventing you from downloading/installing |app|, disable it temporarily, install |app| and then re-enable it.
 
+How do I backup |app|?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The most important thing to backup is the |app| library folder, that contains all your books and metadata. This is the folder you chose for your |app| library when you ran |app| for the first time. You can get the path to the library folder by clicking the |app| icon on the main toolbar. You must backup this complete folder with all its files and sub-folders. 
+
+You can switch |app| to using a backed up library folder by simply clicking the |app| icon on the toolbar and choosing your backup library folder. 
+
+If you want to backup the |app| configuration/plugins, you have to backup the config directory. You can find this config directory via :guilabel:`Preferences->Miscellaneous`. Note that restoring configuration directories is not officially supported, but should work in most cases. Just copy the contents of the backup directory into the current configuration directory to restore.
+
 How do I use purchased EPUB books with |app|?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Most purchased EPUB books have `DRM <http://wiki.mobileread.com/wiki/DRM>`_. This prevents |app| from opening them. You can still use |app| to store and transfer them to your e-book reader. First, you must authorize your reader on a windows machine with Adobe Digital Editions. Once this is done, EPUB books transferred with |app| will work fine on your reader. When you purchase an epub book from a website, you will get an ".acsm" file. This file should be opened with Adobe Digital Editions, which will then download the actual ".epub" e-book. The e-book file will be stored in the folder "My Digital Editions", from where you can add it to |app|.
 
+I am getting a "Permission Denied" error?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A permission denied error can occur because of many possible reasons, none of them having anything to do with |app|. You can get permission denied errors if you are using an SD card with write protect enabled. Or if you, or some program you used changed the file permissions of the files in question to read only. Or if there is a filesystem error on the device which caused your operating system to mount the filesystem in read only mode or mark a particular file as read only pending recovery. Or if the files have their owner set to a user other than you. You will need to fix the underlying cause of the permissions error before resuming to use |app|. Read the error message carefully, see what file it points to and fix the permissions on that file.
+
 Can I have the comment metadata show up on my reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

src/calibre/utils/date.py

@@ -148,6 +148,9 @@ def format_date(dt, format, assume_utc=False, as_utc=False):
         if len(mo.group(0)) == 2: return '%02d'%(dt.year % 100)
         return '%04d'%dt.year
 
+    if dt == UNDEFINED_DATE:
+        return ''
+
     format = re.sub('d{1,4}', format_day, format)
     format = re.sub('M{1,4}', format_month, format)
     return re.sub('yyyy|yy', format_year, format)

src/calibre/utils/formatter.py

@@ -274,9 +274,9 @@ class TemplateFormatter(string.Formatter):
                     colon += 1
 
                 funcs = formatter_functions.get_functions()
-                if fmt[colon:p] in funcs:
-                    field = fmt[colon:p]
-                    func = funcs[field]
+                fname = fmt[colon:p]
+                if fname in funcs:
+                    func = funcs[fname]
                     if func.arg_count == 2:
                         # only one arg expected. Don't bother to scan. Avoids need
                         # for escaping characters
@@ -292,6 +292,8 @@ class TemplateFormatter(string.Formatter):
                     else:
                         val = func.eval_(self, self.kwargs, self.book, self.locals,
                                         val, *args).strip()
+                else:
+                    return _('%s: unknown function')%fname
         if val:
             val = self._do_format(val, dispfmt)
         if not val:

src/calibre/utils/formatter_functions.py

@@ -77,7 +77,7 @@ class FormatterFunction(object):
                                         exc_traceback)[-2:]).replace('\n', '')
             return _('Exception ' + info)
 
-
+all_builtin_functions = []
 class BuiltinFormatterFunction(FormatterFunction):
     def __init__(self):
         formatter_functions.register_builtin(self)
@@ -88,6 +88,7 @@ class BuiltinFormatterFunction(FormatterFunction):
         except:
             lines = []
         self.program_text = ''.join(lines)
+        all_builtin_functions.append(self)
 
 class BuiltinStrcmp(BuiltinFormatterFunction):
     name = 'strcmp'

src/calibre/utils/icu.py

@@ -80,7 +80,7 @@ def icu_case_sensitive_strcmp(collator, a, b):
 
 def icu_capitalize(s):
     s = lower(s)
-    return s.replace(s[0], upper(s[0]), 1)
+    return s.replace(s[0], upper(s[0]), 1) if s else s
 
 load_icu()
 load_collator()

src/calibre/web/feeds/news.py

@@ -839,7 +839,13 @@ class BasicNewsRecipe(Recipe):
         fetcher.image_url_processor = self.image_url_processor
         res, path, failures = fetcher.start_fetch(url), fetcher.downloaded_paths, fetcher.failed_links
         if not res or not os.path.exists(res):
-            raise Exception(_('Could not fetch article. Run with -vv to see the reason'))
+            msg = _('Could not fetch article.') + ' '
+            if self.debug:
+                msg += _('The debug traceback is available earlier in this log')
+            else:
+                msg += _('Run with -vv to see the reason')
+            raise Exception(msg)
+
         return res, path, failures
 
     def fetch_article(self, url, dir, f, a, num_of_feeds):
@@ -902,9 +908,6 @@ class BasicNewsRecipe(Recipe):
             feeds = feeds[:2]
         self.has_single_feed = len(feeds) == 1
 
-        if self.use_embedded_content is None:
-            self.use_embedded_content = feeds[0].has_embedded_content()
-
         index = os.path.join(self.output_dir, 'index.html')
 
         html = self.feeds2index(feeds)
@@ -939,7 +942,9 @@ class BasicNewsRecipe(Recipe):
                     url = None
                 if not url:
                     continue
-                func, arg = (self.fetch_embedded_article, article) if self.use_embedded_content else \
+                func, arg = (self.fetch_embedded_article, article) \
+                            if self.use_embedded_content or (self.use_embedded_content == None and feed.has_embedded_content()) \
+                            else \
                             ((self.fetch_obfuscated_article if self.articles_are_obfuscated \
                               else self.fetch_article), url)
                 req = WorkRequest(func, (arg, art_dir, f, a, len(feed)),