Fix #1246 (Authors import incomplete...) and various other minor regressions

src/calibre/ebooks/html.py

@@ -313,11 +313,11 @@ def opf_traverse(opf_reader, verbose=0, encoding=None):
     return ans
             
 
-
+convert_entities = functools.partial(entity_to_unicode, exceptions=['quot', 'apos', 'lt', 'gt', 'amp'])
 class PreProcessor(object):
     PREPROCESS = [
                   # Convert all entities, since lxml doesn't handle them well
-                  (re.compile(r'&(\S+?);'), entity_to_unicode),
+                  (re.compile(r'&(\S+?);'), convert_entities),
                   ]
                      
     # Fix pdftohtml markup
@@ -379,7 +379,6 @@ class PreProcessor(object):
             rules = []
         for rule in self.PREPROCESS + rules:
             html = rule[0].sub(rule[1], html)
-        
         return html
     
 class Parser(PreProcessor, LoggingInterface):
@@ -801,14 +800,14 @@ class Processor(Parser):
         for rule in sheet:
             self.stylesheet.add(rule)
         css = ''
-        if self.opts.override_css:
-            css += '\n\n' + self.opts.override_css
         css += '\n\n' + 'body {margin-top: 0pt; margin-bottom: 0pt; margin-left: 0pt; margin-right: 0pt;}'
         css += '\n\n@page {margin-top: %fpt; margin-bottom: %fpt; margin-left: %fpt; margin-right: %fpt}'%(self.opts.margin_top, self.opts.margin_bottom, self.opts.margin_left, self.opts.margin_right)
         # Workaround for anchor rendering bug in ADE
         css += '\n\na { color: inherit; text-decoration: inherit; cursor: default; }\na[href] { color: blue; text-decoration: underline; cursor:pointer; }'
         if self.opts.remove_paragraph_spacing:
             css += '\n\np {text-indent: 2em; margin-top:1pt; margin-bottom:1pt; padding:0pt; border:0pt;}'
+        if self.opts.override_css:
+            css += '\n\n' + self.opts.override_css
         self.override_css = self.css_parser.parseString(self.preprocess_css(css))
         for rule in reversed(self.specified_override_css):
             self.override_css.insertRule(rule, index=0)

src/calibre/ebooks/metadata/opf2.py

@@ -330,7 +330,7 @@ class Guide(ResourceCollection):
         @staticmethod
         def from_opf_resource_item(ref, basedir):
             title, href, type = ref.get('title', ''), ref.get('href'), ref.get('type')
-            res = Guide.Reference(href, basedir, is_path=False)
+            res = Guide.Reference(href, basedir, is_path=True)
             res.title = title
             res.type = type
             return res
@@ -866,7 +866,7 @@ class OPFCreator(MetaInformation):
                 self.manifest.append(ManifestItem(ncx_manifest_entry, self.base_path))
                 self.manifest[-1].id = 'ncx'
                 self.manifest[-1].mime_type = 'application/x-dtbncx+xml'
-        if not self.guide:
+        if self.guide is None:
             self.guide = Guide()
         if self.cover:
             cover = self.cover
@@ -886,7 +886,6 @@ class OPFCreator(MetaInformation):
 class OPFTest(unittest.TestCase):
     
     def setUp(self):
-        import cStringIO
         self.stream = cStringIO.StringIO(
 '''\
 <?xml version="1.0"  encoding="UTF-8"?>

src/calibre/gui2/main.py

@@ -771,7 +771,7 @@ class Main(MainWindow, Ui_MainWindow):
             d.exec_()
             return
 
-        dir = choose_dir(self, 'save to disk dialog', ('Choose destination directory'))
+        dir = choose_dir(self, 'save to disk dialog', _('Choose destination directory'))
         if not dir:
             return
         if self.current_view() == self.library_view:

src/calibre/library/cli.py

@@ -22,7 +22,7 @@ from calibre.library.database2 import LibraryDatabase2
 from calibre.ebooks.metadata.opf import OPFCreator, OPFReader
 from calibre.utils.genshi.template import MarkupTemplate
 
-FIELDS = set(['title', 'authors', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'formats', 'isbn', 'cover'])
+FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'formats', 'isbn', 'cover'])
 
 XML_TEMPLATE = '''\
 <?xml version="1.0"  encoding="UTF-8"?>
@@ -31,7 +31,7 @@ XML_TEMPLATE = '''\
     <record>
         <id>${record['id']}</id>
         <title>${record['title']}</title>
-        <authors>
+        <authors sort="${record['author_sort']}">
         <py:for each="author in record['authors']">
             <author>$author</author>
         </py:for>

src/calibre/library/database.py

@@ -9,7 +9,7 @@ from zlib import compress, decompress
 
 from calibre import sanitize_file_name
 from calibre.ebooks.metadata.meta import set_metadata, metadata_from_formats
-from calibre.ebooks.metadata.opf import OPFCreator
+from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.web.feeds.recipes import migrate_automatic_profile_to_automatic_recipe
@@ -1283,6 +1283,9 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
             self.set_series(id, mi.series)
         if mi.cover_data[1] is not None:
             self.set_cover(id, mi.cover_data[1])
+    
+        
+            
 
     def add_books(self, paths, formats, metadata, uris=[], add_duplicates=True):
         '''
@@ -1399,7 +1402,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
             if not au:
                 au = self.authors(index, index_is_id=index_is_id)
                 if not au:
-                    au = 'Unknown'
+                    au = _('Unknown')
                 au = au.split(',')[0]
             else:
                 au = au.replace(',', ';')
@@ -1421,7 +1424,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
                 name = au + ' - ' + title if byauthor else title + ' - ' + au
                 name += '_'+id
                 base  = dir if single_dir else tpath
-                mi = OPFCreator(base, self.get_metadata(idx, index_is_id=index_is_id))
+                mi = self.get_metadata(idx, index_is_id=index_is_id)
                 cover = self.cover(idx, index_is_id=index_is_id)
                 if cover is not None:
                     cname = sanitize_file_name(name) + '.jpg'
@@ -1431,7 +1434,8 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
                 f = open(os.path.join(base, sanitize_file_name(name)+'.opf'), 'wb')
                 if not mi.authors:
                     mi.authors = [_('Unknown')]
-                mi.render(f)
+                opf = OPFCreator(base, mi)
+                opf.render(f)
                 f.close()
                 
                 fmts = self.formats(idx, index_is_id=index_is_id)
@@ -1458,7 +1462,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
     def import_book(self, mi, formats):
         series_index = 1 if mi.series_index is None else mi.series_index
         if not mi.authors:
-            mi.authors = ['Unknown']
+            mi.authors = [_('Unknown')]
         aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors)
         obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)',
                           (mi.title, None, series_index, aus))
@@ -1554,6 +1558,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
         return duplicates
 
     def export_single_format_to_dir(self, dir, indices, format, index_is_id=False):
+        dir = os.path.abspath(dir)
         if not index_is_id:
             indices = map(self.id, indices)
         failures = []
@@ -1572,6 +1577,8 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
                 au = _('Unknown')
             fname = '%s - %s.%s'%(title, au, format.lower())
             fname = sanitize_file_name(fname)
+            if not os.path.exists(dir):
+                os.makedirs(dir)
             f = open(os.path.join(dir, fname), 'w+b')
             f.write(data)
             f.seek(0)

src/calibre/library/database2.py

@@ -284,7 +284,7 @@ class ResultCache(SearchQueryParser):
             field += 's'
         if   field == 'date': field = 'timestamp'
         elif field == 'title': field = 'sort'
-        elif field == 'author': field = 'author_sort'
+        elif field == 'authors': field = 'author_sort'
         fcmp = self.seriescmp if field == 'series' else \
             functools.partial(self.cmp, FIELD_MAP[field], 
                               str=field not in ('size', 'rating', 'timestamp'))
@@ -792,6 +792,8 @@ class LibraryDatabase2(LibraryDatabase):
             self.set_series(id, mi.series, notify=False)
         if mi.cover_data[1] is not None:
             self.set_cover(id, mi.cover_data[1])
+        elif mi.cover is not None and os.access(mi.cover,os.R_OK):
+            self.set_cover(id, open(mi.cover, 'rb').read())
         if mi.tags:
             self.set_tags(id, mi.tags, notify=False)
         if mi.comments:
@@ -1105,7 +1107,7 @@ class LibraryDatabase2(LibraryDatabase):
         '''
         if prefix is None:
             prefix = self.library_path
-        FIELDS = set(['title', 'authors', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'isbn'])
+        FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'isbn'])
         data = []
         for record in self.data:
             if record is None: continue