GwR revisions 2-10 EXTH fix, thumb tweaks, catalog.svg

Changelog.yaml

@@ -4,6 +4,26 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.
 
+- version: 0.6.39
+  date: 2010-02-11
+
+  new features:
+    - title: "Add ability to control how author sort strings are automatically generated from author strings, via the config file tweaks.py"
+
+    - title: "Handle broken EPUB files from Project Gutenberg that have invalid OCF containers"
+      tickets: [4832]
+
+  bug fixes:
+    - title: "Fix regression in 0.6.38 that broke setting bookmarks in the viewer"
+
+    - title: "HTML Input: Ignore filenames that are encoded incorerctly."
+
+  new recipes:
+
+    - title: Radikal
+      author: Darko Miletic
+
+
 - version: 0.6.38
   date: 2010-02-09
 

resources/default_tweaks.py

@@ -16,3 +16,12 @@ defaults.
 # next - Next available number
 # const - Assign the number 1 always
 series_index_auto_increment = 'next'
+
+
+
+# The algorithm used to copy author to author_sort
+# Possible values are:
+#  invert: use "fn ln" -> "ln, fn" (the original algorithm)
+#  copy  : copy author to author_sort without modification
+#  comma : use 'copy' if there is a ',' in the name, otherwise use 'invert'
+author_sort_copy_method = 'invert'

resources/recipes/radikal_tr.recipe

@@ -0,0 +1,45 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+radikal.com.tr
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Radikal_tr(BasicNewsRecipe):
+    title                 = 'Radikal - Turkey'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Turkey'
+    publisher             = 'radikal'
+    category              = 'news, politics, Turkey'
+    oldest_article        = 2
+    max_articles_per_feed = 150
+    no_stylesheets        = True
+    encoding              = 'cp1254'
+    use_embedded_content  = False
+    masthead_url          = 'http://www.radikal.com.tr/D/i/1/V2/radikal_logo.jpg'
+    language              = 'tr'
+    extra_css             = ' @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} .article_description,body{font-family: Arial,Verdana,Helvetica,sans1,sans-serif } '
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [dict(name=['embed','iframe','object','link','base'])]
+    remove_tags_before = dict(name='h1')
+    remove_tags_after = dict(attrs={'id':'haberDetayYazi'})
+
+
+    feeds = [(u'Yazarlar', u'http://www.radikal.com.tr/d/rss/RssYazarlar.xml')]
+
+    def print_version(self, url):
+        articleid = url.rpartition('ArticleID=')[2]
+        return 'http://www.radikal.com.tr/Default.aspx?aType=HaberYazdir&ArticleID=' + articleid
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)
+

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.6.38'
+__version__   = '0.6.39'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/ebooks/metadata/__init__.py

@@ -10,9 +10,10 @@ import os, mimetypes, sys, re
 from urllib import unquote, quote
 from urlparse import urlparse
 
-
 from calibre import relpath
 
+from calibre.utils.config import tweaks
+
 _author_pat = re.compile(',?\s+(and|with)\s+', re.IGNORECASE)
 def string_to_authors(raw):
     raw = raw.replace('&&', u'\uffff')
@@ -27,6 +28,9 @@ def authors_to_string(authors):
         return ''
 
 def author_to_author_sort(author):
+    method = tweaks['author_sort_copy_method']
+    if method == 'copy' or (method == 'comma' and author.count(',') > 0):
+        return author
     tokens = author.split()
     tokens = tokens[-1:] + tokens[:-1]
     if len(tokens) > 1:

src/calibre/ebooks/metadata/epub.py

@@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 '''Read meta information from epub files'''
 
-import os
+import os, re
 from cStringIO import StringIO
 from contextlib import closing
 
@@ -29,15 +29,15 @@ class Container(dict):
     def __init__(self, stream=None):
         if not stream: return
         soup = BeautifulStoneSoup(stream.read())
-        container = soup.find('container')
+        container = soup.find(name=re.compile(r'container$', re.I))
         if not container:
-            raise OCFException("<container/> element missing")
+            raise OCFException("<container> element missing")
         if container.get('version', None) != '1.0':
             raise EPubException("unsupported version of OCF")
-        rootfiles = container.find('rootfiles')
+        rootfiles = container.find(re.compile(r'rootfiles$', re.I))
         if not rootfiles:
             raise EPubException("<rootfiles/> element missing")
-        for rootfile in rootfiles.findAll('rootfile'):
+        for rootfile in rootfiles.findAll(re.compile(r'rootfile$', re.I)):
             try:
                 self[rootfile['media-type']] = rootfile['full-path']
             except KeyError:

src/calibre/ebooks/metadata/mobi.py

@@ -134,11 +134,13 @@ class MetadataUpdater(object):
             if id == 106:
                 self.timestamp = content
             elif id == 201:
-                rindex, = self.cover_rindex, = unpack('>I', content)
-                self.cover_record = self.record(rindex + image_base)
+                rindex, = self.cover_rindex, = unpack('>i', content)
+                if rindex > 0 :
+                    self.cover_record = self.record(rindex + image_base)
             elif id == 202:
-                rindex, = self.thumbnail_rindex, = unpack('>I', content)
-                self.thumbnail_record = self.record(rindex + image_base)
+                rindex, = self.thumbnail_rindex, = unpack('>i', content)
+                if rindex > 0 :
+                    self.thumbnail_record = self.record(rindex + image_base)
 
     def patch(self, off, new_record0):
         # Save the current size of each record

src/calibre/ebooks/oeb/base.py

@@ -16,9 +16,10 @@ from urllib import unquote as urlunquote
 from urlparse import urljoin
 
 from lxml import etree, html
+from cssutils import CSSParser
 
 import calibre
-from cssutils import CSSParser
+from calibre.constants import filesystem_encoding
 from calibre.translations.dynamic import translate
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
@@ -434,10 +435,18 @@ class DirContainer(object):
 
     def namelist(self):
         names = []
-        for root, dirs, files in os.walk(self.rootdir):
+        base = self.rootdir
+        if isinstance(base, unicode):
+            base = base.encode(filesystem_encoding)
+        for root, dirs, files in os.walk(base):
             for fname in files:
                 fname = os.path.join(root, fname)
                 fname = fname.replace('\\', '/')
+                if not isinstance(fname, unicode):
+                    try:
+                        fname = fname.decode(filesystem_encoding)
+                    except:
+                        continue
                 names.append(fname)
         return names
 

src/calibre/ebooks/pdf/reflow.py

@@ -303,6 +303,22 @@ class Region(object):
         for x in self.columns:
             yield x
 
+    def absorb_regions(self, regions, at):
+        for region in regions:
+            self.absorb_region(region, at)
+
+    def absorb_region(self, region, at):
+        src_iter = lambda x:x if at == 'bottom' else reversed
+        if len(region.columns) == len(self.columns):
+            for src, dest in zip(region.columns, self.columns):
+                for elem in src_iter(src):
+                    if at == 'bottom':
+                        dest.append(elem)
+                    else:
+                        dest.insert(0, elem)
+        else:
+            pass
+
     def linearize(self):
         self.elements = []
         for x in self.columns:
@@ -444,7 +460,7 @@ class Page(object):
             for i, region in enumerate(self.regions):
                 if region.is_small:
                     found = True
-                    regions = []
+                    regions = [region]
                     for j in range(i+1, len(self.regions)):
                         if self.regions[j].is_small:
                             regions.append(self.regions[j])
@@ -452,8 +468,10 @@ class Page(object):
                             break
                     prev_region = None if i == 0 else i-1
                     next_region = j if self.regions[j] not in regions else None
+                    absorb_at = 'bottom'
                     if prev_region is None and next_region is not None:
                         absorb_into = next_region
+                        absorb_at = 'top'
                     elif next_region is None and prev_region is not None:
                         absorb_into = prev_region
                     elif prev_region is None and next_region is None:
@@ -471,8 +489,9 @@ class Page(object):
                                or abs(avg_column_count - len(prev_region.columns)) \
                                > abs(avg_column_count - len(next_region.columns)):
                                    absorb_into = next_region
+                                   absorb_at = 'top'
                     if absorb_into is not None:
-                        absorb_into.absorb_region(regions)
+                        absorb_into.absorb_regions(regions, absorb_at)
                         absorbed.update(regions)
                     i = j
         for region in absorbed:

src/calibre/ebooks/pml/pmlconverter.py

@@ -182,10 +182,10 @@ class PML_HTMLizer(object):
         return pml
 
     def strip_pml(self, pml):
-        pml = re.sub(r'\\C\d=".+*"', '', pml)
-        pml = re.sub(r'\\Fn=".+*"', '', pml)
-        pml = re.sub(r'\\Sd=".+*"', '', pml)
-        pml = re.sub(r'\\.=".+*"', '', pml)
+        pml = re.sub(r'\\C\d=".*"', '', pml)
+        pml = re.sub(r'\\Fn=".*"', '', pml)
+        pml = re.sub(r'\\Sd=".*"', '', pml)
+        pml = re.sub(r'\\.=".*"', '', pml)
         pml = re.sub(r'\\X\d', '', pml)
         pml = re.sub(r'\\S[pbd]', '', pml)
         pml = re.sub(r'\\Fn', '', pml)

src/calibre/library/catalog.py

@@ -945,7 +945,7 @@ class EPUB_MOBI(CatalogPlugin):
                     this_title['author'] = " & ".join(record['authors'])
                 else:
                     this_title['author'] = 'Unknown'
-                this_title['author_sort'] = record['author_sort'].capitalize() if len(record['author_sort']) \
+                this_title['author_sort'] = record['author_sort'].title() if len(record['author_sort'].strip()) \
                      else self.author_to_author_sort(this_title['author'])
                 this_title['id'] = record['id']
                 if record['publisher']:
@@ -3178,7 +3178,7 @@ class EPUB_MOBI(CatalogPlugin):
 
         if catalog_source_built:
             recommendations = []
-            recommendations.append(('cover', I('catalog.svg'), OptionRecommendation.HIGH))
+            # recommendations.append(('cover', I('catalog.svg'), OptionRecommendation.HIGH))
 
             dp = getattr(opts, 'debug_pipeline', None)
             if dp is not None:

src/calibre/translations/calibre.pot

@@ -4,9 +4,9 @@
 #
 msgid ""
 msgstr ""
-"Project-Id-Version: calibre 0.6.38\n"
-"POT-Creation-Date: 2010-02-08 22:00+MST\n"
-"PO-Revision-Date: 2010-02-08 22:00+MST\n"
+"Project-Id-Version: calibre 0.6.39\n"
+"POT-Creation-Date: 2010-02-09 20:45+MST\n"
+"PO-Revision-Date: 2010-02-09 20:45+MST\n"
 "Last-Translator: Automatically generated\n"
 "Language-Team: LANGUAGE\n"
 "MIME-Version: 1.0\n"
@@ -36,10 +36,10 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/html/convert_from.py:1894
 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/html/convert_from.py:1896
 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/output.py:24
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:225
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:256
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:259
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:359
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:229
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:260
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:263
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:363
 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/ereader.py:35
 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/ereader.py:60
 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/fb2.py:46
@@ -67,9 +67,9 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader.py:799
 #: /home/kovid/work/calibre/src/calibre/ebooks/odt/input.py:49
 #: /home/kovid/work/calibre/src/calibre/ebooks/odt/input.py:51
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:898
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:903
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:969
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:907
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:912
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:978
 #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/reader.py:137
 #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/reader.py:139
 #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/transforms/jacket.py:105
@@ -1317,7 +1317,7 @@ msgstr ""
 msgid "Comic"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:358
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:362
 #: /home/kovid/work/calibre/src/calibre/ebooks/pdf/manipulate/info.py:45
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/book_info.py:98
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/book_info.py:99
@@ -1328,7 +1328,7 @@ msgstr ""
 msgid "Title"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:359
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:363
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/fetch_metadata.py:57
 #: /home/kovid/work/calibre/src/calibre/gui2/library.py:168
 #: /home/kovid/work/calibre/src/calibre/gui2/library.py:408
@@ -1336,18 +1336,18 @@ msgstr ""
 msgid "Author(s)"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:360
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:364
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/fetch_metadata.py:59
 #: /home/kovid/work/calibre/src/calibre/gui2/library.py:173
 msgid "Publisher"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:361
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:365
 #: /home/kovid/work/calibre/src/calibre/ebooks/pdf/manipulate/info.py:49
 msgid "Producer"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:362
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:366
 #: /home/kovid/work/calibre/src/calibre/gui2/convert/metadata_ui.py:183
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/book_info.py:100
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/book_info_ui.py:67
@@ -1356,7 +1356,7 @@ msgstr ""
 msgid "Comments"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:370
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:374
 #: /home/kovid/work/calibre/src/calibre/gui2/library.py:174
 #: /home/kovid/work/calibre/src/calibre/gui2/library.py:353
 #: /home/kovid/work/calibre/src/calibre/gui2/library.py:1064
@@ -1366,7 +1366,7 @@ msgstr ""
 msgid "Tags"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:372
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:376
 #: /home/kovid/work/calibre/src/calibre/gui2/library.py:175
 #: /home/kovid/work/calibre/src/calibre/gui2/library.py:369
 #: /home/kovid/work/calibre/src/calibre/gui2/status.py:95
@@ -1374,22 +1374,22 @@ msgstr ""
 msgid "Series"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:373
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:377
 msgid "Language"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:375
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:379
 #: /home/kovid/work/calibre/src/calibre/gui2/library.py:1063
 msgid "Timestamp"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:377
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:381
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/fetch_metadata.py:61
 #: /home/kovid/work/calibre/src/calibre/gui2/library.py:171
 msgid "Published"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:379
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/__init__.py:383
 msgid "Rights"
 msgstr ""
 
@@ -1561,7 +1561,7 @@ msgid ""
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/opf2.py:1083
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1336
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1345
 msgid "Cover"
 msgstr ""
 
@@ -1590,70 +1590,70 @@ msgstr ""
 msgid "All articles"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1337
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1346
 msgid "Title Page"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1338
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1347
 #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/transforms/htmltoc.py:15
 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main.py:53
 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:188
 msgid "Table of Contents"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1339
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1348
 msgid "Index"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1340
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1349
 msgid "Glossary"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1341
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1350
 msgid "Acknowledgements"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1342
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1351
 msgid "Bibliography"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1343
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1352
 msgid "Colophon"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1344
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1353
 msgid "Copyright"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1345
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1354
 msgid "Dedication"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1346
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1355
 msgid "Epigraph"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1347
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1356
 msgid "Foreword"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1348
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1357
 msgid "List of Illustrations"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1349
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1358
 msgid "List of Tables"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1350
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1359
 msgid "Notes"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1351
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1360
 msgid "Preface"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1352
+#: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1361
 msgid "Main Text"
 msgstr ""