Sync to trunk.

recipes/den_of_geek.recipe

@@ -0,0 +1,21 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1316944753(BasicNewsRecipe):
+    title          = u'Den of Geek'
+    __author__            = 'Jaded'
+    language = 'en'
+    description = 'From science fiction enthusiasts through to gaming fanatics, Den of Geek has become the one-stop UK website for people genuinely passionate about their entertainment media. Den of Geek covers popular culture but always with an edgy, UK centric slant that sets it apart from the crowd.'
+    category               = 'Movies, TV, Games, Comics, Cult, News, Reviews'
+    language              = 'en'
+
+    oldest_article = 7
+    max_articles_per_feed = 100
+    auto_cleanup = True
+
+    no_stylesheets        = True
+    use_embedded_content  = True
+    publication_type      = 'newsportal'
+    masthead_url ='http://www.denofgeek.com/siteimage/scale/0/0/logo.gif'
+    cover_url ='http://a5.sphotos.ak.fbcdn.net/hphotos-ak-snc6/166479_180131695357862_139191826118516_354818_4993703_n.jpg'
+
+    feeds          = [(u'Movies', u'http://www.denofgeek.com/movies/rss/'), (u'TV', u'http://www.denofgeek.com/television/rss/'), (u'Comics & Books', u'http://www.denofgeek.com/comics/rss/'), (u'Games', u'http://www.denofgeek.com/games/rss/'), (u'DVD/Blu-ray', u'http://www.denofgeek.com/Reviews/rss/')]

src/calibre/db/__init__.py

@@ -7,6 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+SPOOL_SIZE = 30*1024*1024
 
 '''
 Rewrite of the calibre database backend.

src/calibre/db/backend.py

@@ -8,7 +8,7 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 # Imports {{{
-import os, shutil, uuid, json, glob, time, tempfile
+import os, shutil, uuid, json, glob, time
 from functools import partial
 
 import apsw
@@ -16,7 +16,8 @@ import apsw
 from calibre import isbytestring, force_unicode, prints
 from calibre.constants import (iswindows, filesystem_encoding,
         preferred_encoding)
-from calibre.ptempfile import PersistentTemporaryFile
+from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile
+from calibre.db import SPOOL_SIZE
 from calibre.db.schema_upgrades import SchemaUpgrade
 from calibre.library.field_metadata import FieldMetadata
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
@@ -38,7 +39,7 @@ Differences in semantics from pysqlite:
 
 '''
 
-SPOOL_SIZE = 30*1024*1024
+
 
 class DynamicFilter(object): # {{{
 
@@ -805,7 +806,7 @@ class DB(object):
                         shutil.copyfileobj(f, pt)
                     return pt.name
                 if as_file:
-                    ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE)
+                    ret = SpooledTemporaryFile(SPOOL_SIZE)
                     shutil.copyfileobj(f, ret)
                     ret.seek(0)
                 else:

src/calibre/devices/kobo/driver.py

@@ -51,11 +51,11 @@ class KOBO(USBMS):
 
     EXTRA_CUSTOMIZATION_MESSAGE = [
             _('The Kobo supports several collections including ')+\
-                    'Read, Closed, Im_Reading ' +\
+                    'Read, Closed, Im_Reading. ' +\
             _('Create tags for automatic management'),
     ]
 
-    EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['tags'])
+    EXTRA_CUSTOMIZATION_DEFAULT = [', '.join(['tags'])]
 
     OPT_COLLECTIONS = 0
 

src/calibre/ebooks/metadata/sources/amazon.py

@@ -73,6 +73,20 @@ class Worker(Thread): # Get details {{{
             8: ['août'],
             9: ['sept'],
             12: ['déc'],
+            },
+            'es': {
+                1: ['enero'],
+                2: ['febrero'],
+                3: ['marzo'],
+                4: ['abril'],
+                5: ['mayo'],
+                6: ['junio'],
+                7: ['julio'],
+                8: ['agosto'],
+                9: ['septiembre', 'setiembre'],
+                10: ['octubre'],
+                11: ['noviembre'],
+                12: ['diciembre'],
             },
                 'jp': {
             1: [u'1月'],
@@ -101,13 +115,16 @@ class Worker(Thread): # Get details {{{
                  text()="Dettagli prodotto" or \
                  text()="Product details" or \
                  text()="Détails sur le produit" or \
+                 text()="Detalles del producto" or \
                  text()="登録情報"]/../div[@class="content"]
             '''
+        # Editor: is for Spanish
         self.publisher_xpath = '''
             descendant::*[starts-with(text(), "Publisher:") or \
                     starts-with(text(), "Verlag:") or \
                     starts-with(text(), "Editore:") or \
                     starts-with(text(), "Editeur") or \
+                    starts-with(text(), "Editor:") or \
                     starts-with(text(), "出版社:")]
             '''
         self.language_xpath =    '''
@@ -116,12 +133,14 @@ class Worker(Thread): # Get details {{{
                 or text() = "Language" \
                 or text() = "Sprache:" \
                 or text() = "Lingua:" \
+                or text() = "Idioma:" \
                 or starts-with(text(), "Langue") \
                 or starts-with(text(), "言語") \
                 ]
             '''
+
         self.ratings_pat = re.compile(
-            r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち) ([\d\.]+)( (stars|Sternen|stelle)){0,1}')
+            r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち|de un máximo de) ([\d\.]+)( (stars|Sternen|stelle|estrellas)){0,1}')
 
         lm = {
                 'eng': ('English', 'Englisch'),
@@ -143,6 +162,7 @@ class Worker(Thread): # Get details {{{
         for i, vals in self.months.iteritems():
             for x in vals:
                 ans = ans.replace(x, self.english_months[i])
+        ans = ans.replace(' de ', ' ')
         return ans
 
     def run(self):
@@ -422,6 +442,7 @@ class Amazon(Source):
             'uk' : _('UK'),
             'it' : _('Italy'),
             'jp' : _('Japan'),
+            'es' : _('Spain'),
     }
 
     options = (
@@ -789,6 +810,16 @@ if __name__ == '__main__': # tests {{{
             ),
     ] # }}}
 
+    es_tests = [ # {{{
+            (
+                {'identifiers':{'isbn': '8483460831'}},
+                [title_test('Tiempos Interesantes',
+                    exact=True), authors_test(['Terry Pratchett'])
+                 ]
+
+            ),
+    ] # }}}
+
     jp_tests = [ # {{{
             ( # isbn -> title, authors
                 {'identifiers':{'isbn': '9784101302720' }},
@@ -804,6 +835,6 @@ if __name__ == '__main__': # tests {{{
     ] # }}}
 
     test_identify_plugin(Amazon.name, com_tests)
-    #test_identify_plugin(Amazon.name, jp_tests)
+    #test_identify_plugin(Amazon.name, es_tests)
 # }}}
 

src/calibre/ebooks/metadata/worker.py

@@ -249,6 +249,7 @@ class SaveWorker(Thread):
             recs[pref.name] = getattr(self.opts, pref.name)
 
         plugboards = self.db.prefs.get('plugboards', {})
+        template_functions = self.db.prefs.get('user_template_functions', [])
 
         for i, task in enumerate(tasks):
             tids = [x[-1] for x in task]
@@ -260,7 +261,7 @@ class SaveWorker(Thread):
             job = ParallelJob('save_book',
                     'Save books (%d of %d)'%(i, len(tasks)),
                     lambda x,y:x,
-                    args=[tids, dpath, plugboards, self.path, recs])
+                    args=[tids, dpath, plugboards, template_functions, self.path, recs])
             jobs.add(job)
             server.add_job(job)
 
@@ -312,9 +313,12 @@ class SaveWorker(Thread):
                 break
 
 
-def save_book(ids, dpath, plugboards, path, recs, notification=lambda x,y:x):
+def save_book(ids, dpath, plugboards, template_functions, path, recs,
+              notification=lambda x,y:x):
     from calibre.library.save_to_disk import config, save_serialized_to_disk
     from calibre.customize.ui import apply_null_metadata
+    from calibre.utils.formatter_functions import load_user_template_functions
+    load_user_template_functions(template_functions)
     opts = config().parse()
     for name in recs:
         setattr(opts, name, recs[name])

src/calibre/ebooks/pdf/images.cpp

@@ -126,7 +126,7 @@ void XMLImages::add(GfxState *state, Object *ref, Stream *str,
 
     if (img->type == jpeg) {
         int c;
-        str = ((DCTStream *)str)->getRawStream();
+        str = str->getNextStream();
         str->reset();
 
         // copy the stream

src/calibre/ebooks/pdf/reflow.cpp

@@ -625,7 +625,12 @@ static string get_link_dest(LinkAction *link, PDFDoc *doc) {
   return oss.str();
 }
 
+#if (POPPLER_MAJOR_VERSION == 0) && (POPPLER_MINOR_VERSION < 17)
 void XMLOutputDev::process_link(Link* link){
+#else
+void XMLOutputDev::process_link(AnnotLink* link){
+#endif
+
   double _x1, _y1, _x2, _y2;
   int x1, y1, x2, y2;
   

src/calibre/ebooks/pdf/reflow.h

@@ -244,6 +244,11 @@ class XMLOutputDev : public OutputDev {
     XMLImages *images;
     PDFDoc *doc;
 
+#if (POPPLER_MAJOR_VERSION == 0) && (POPPLER_MINOR_VERSION < 17)
     void process_link(Link* link);
+#else
+    void process_link(AnnotLink* link);
+#endif
+
 };
 }

src/calibre/gui2/metadata/basic_widgets.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import textwrap, re, os, errno
+import textwrap, re, os, errno, shutil
 
 from PyQt4.Qt import (Qt, QDateEdit, QDate, pyqtSignal, QMessageBox,
     QIcon, QToolButton, QWidget, QLabel, QGridLayout, QApplication,
@@ -33,8 +33,9 @@ from calibre.gui2.comments_editor import Editor
 from calibre.library.comments import comments_to_html
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.utils.icu import strcmp
-from calibre.ptempfile import PersistentTemporaryFile
+from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile
 from calibre.gui2.languages import LanguagesEdit as LE
+from calibre.db import SPOOL_SIZE
 
 def save_dialog(parent, title, msg, det_msg=''):
     d = QMessageBox(parent)
@@ -43,8 +44,6 @@ def save_dialog(parent, title, msg, det_msg=''):
     d.setStandardButtons(QMessageBox.Yes | QMessageBox.No | QMessageBox.Cancel)
     return d.exec_()
 
-
-
 '''
 The interface common to all widgets used to set basic metadata
 class BasicMetadataWidget(object):
@@ -731,8 +730,12 @@ class FormatsManager(QWidget):
             else:
                 old_extensions.add(ext)
         for ext in new_extensions:
-            db.add_format(id_, ext, open(paths[ext], 'rb'), notify=False,
+            with SpooledTemporaryFile(SPOOL_SIZE) as spool:
-                    index_is_id=True)
+                with open(paths[ext], 'rb') as f:
+                    shutil.copyfileobj(f, spool)
+                spool.seek(0)
+                db.add_format(id_, ext, spool, notify=False,
+                        index_is_id=True)
         dbfmts = db.formats(id_, index_is_id=True)
         db_extensions = set([f.lower() for f in (dbfmts.split(',') if dbfmts
             else [])])

src/calibre/library/database2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 The database used to store ebook metadata
 '''
 import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
-        json, uuid, tempfile, hashlib, copy
+        json, uuid, hashlib, copy
 from collections import defaultdict
 import threading, random
 from itertools import repeat
@@ -26,7 +26,8 @@ from calibre.library.sqlite import connect, IntegrityError
 from calibre.library.prefs import DBPrefs
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.constants import preferred_encoding, iswindows, filesystem_encoding
-from calibre.ptempfile import PersistentTemporaryFile, base_dir
+from calibre.ptempfile import (PersistentTemporaryFile,
+        base_dir, SpooledTemporaryFile)
 from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
@@ -610,7 +611,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f:
                     f.write(cdata)
             for format in formats:
-                with tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) as stream:
+                with SpooledTemporaryFile(SPOOL_SIZE) as stream:
                     try:
                         self.copy_format_to(id, format, stream, index_is_id=True)
                         stream.seek(0)
@@ -694,7 +695,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                         shutil.copyfileobj(f, pt)
                     return pt.name
                 if as_file:
-                    ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE)
+                    ret = SpooledTemporaryFile(SPOOL_SIZE)
                     shutil.copyfileobj(f, ret)
                     ret.seek(0)
                 else:
@@ -1282,7 +1283,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                             shutil.copyfileobj(f, pt)
                             ret = pt.name
                 elif as_file:
-                    ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE)
+                    ret = SpooledTemporaryFile(SPOOL_SIZE)
                     shutil.copyfileobj(f, ret)
                     ret.seek(0)
                     # Various bits of code try to use the name as the default

src/calibre/ptempfile.py

@@ -181,4 +181,17 @@ class TemporaryFile(object):
 
 
 
+class SpooledTemporaryFile(tempfile.SpooledTemporaryFile):
+
+    def __init__(self, max_size=0, suffix="", prefix="", dir=None, mode='w+b',
+            bufsize=-1):
+        if prefix == None:
+            prefix = ''
+        if suffix is None:
+            suffix = ''
+        if dir is None:
+            dir = base_dir()
+        tempfile.SpooledTemporaryFile.__init__(self, max_size=max_size, suffix=suffix,
+                prefix=prefix, dir=dir, mode=mode, bufsize=bufsize)
+
 

src/calibre/utils/date.py

@@ -170,32 +170,40 @@ def format_date(dt, format, assume_utc=False, as_utc=False):
     if format == 'iso':
         return isoformat(dt, assume_utc=assume_utc, as_utc=as_utc)
 
+    if dt == UNDEFINED_DATE:
+        return ''
+
     strf = partial(strftime, t=dt.timetuple())
 
-    def format_day(mo):
+    def format_day(dy):
-        l = len(mo.group(0))
+        l = len(dy)
         if l == 1: return '%d'%dt.day
         if l == 2: return '%02d'%dt.day
         if l == 3: return strf('%a')
         return strf('%A')
 
     def format_month(mo):
-        l = len(mo.group(0))
+        l = len(mo)
         if l == 1: return '%d'%dt.month
         if l == 2: return '%02d'%dt.month
         if l == 3: return strf('%b')
         return strf('%B')
 
-    def format_year(mo):
+    def format_year(yr):
-        if len(mo.group(0)) == 2: return '%02d'%(dt.year % 100)
+        if len(yr) == 2: return '%02d'%(dt.year % 100)
         return '%04d'%dt.year
 
-    if dt == UNDEFINED_DATE:
+    def repl_func(mo):
-        return ''
+        s = mo.group(0)
+        if s is None:
+            return ''
+        if s[0] == 'd':
+            return format_day(s)
+        if s[0] == 'M':
+            return format_month(s)
+        return format_year(s)
 
-    format = re.sub('d{1,4}', format_day, format)
+    return re.sub('(d{1,4}|M{1,4}|(?:yyyy|yy))', repl_func, format)
-    format = re.sub('M{1,4}', format_month, format)
-    return re.sub('yyyy|yy', format_year, format)
 
 def replace_months(datestr, clang):
     # Replace months by english equivalent for parse_date

src/calibre/web/feeds/news.py

@@ -13,7 +13,7 @@ from functools import partial
 from contextlib import nested, closing
 
 
-from calibre import (browser, __appname__, iswindows,
+from calibre import (browser, __appname__, iswindows, force_unicode,
                     strftime, preferred_encoding, as_unicode)
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
 from calibre.ebooks.metadata.opf2 import OPFCreator
@@ -49,7 +49,7 @@ class BasicNewsRecipe(Recipe):
 
     #: A couple of lines that describe the content this recipe downloads.
     #: This will be used primarily in a GUI that presents a list of recipes.
-    description = ''
+    description = u''
 
     #: The author of this recipe
     __author__             = __appname__
@@ -112,8 +112,6 @@ class BasicNewsRecipe(Recipe):
     #: If set to "optional" the use of a username and password becomes optional
     needs_subscription     = False
 
-    #:
-
     #: If True the navigation bar is center aligned, otherwise it is left aligned
     center_navbar = True
 
@@ -1205,12 +1203,22 @@ class BasicNewsRecipe(Recipe):
         mi.author_sort = __appname__
         mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title()
         mi.timestamp = nowf()
+        article_titles, aseen = [], set()
+        for f in feeds:
+            for a in f:
+                if a.title and a.title not in aseen:
+                    aseen.add(a.title)
+                    article_titles.append(force_unicode(a.title, 'utf-8'))
+
         mi.comments = self.description
+        if not isinstance(mi.comments, unicode):
+            mi.comments = mi.comments.decode('utf-8', 'replace')
+        mi.comments += ('\n\n' + _('Articles in this issue: ') + '\n' +
+                '\n\n'.join(article_titles))
+
         language = canonicalize_lang(self.language)
         if language is not None:
             mi.language = language
-        if not isinstance(mi.comments, unicode):
-            mi.comments = mi.comments.decode('utf-8', 'replace')
         mi.pubdate = nowf()
         opf_path = os.path.join(dir, 'index.opf')
         ncx_path = os.path.join(dir, 'index.ncx')
@@ -1256,6 +1264,7 @@ class BasicNewsRecipe(Recipe):
         self.play_order_counter = 0
         self.play_order_map = {}
 
+
         def feed_index(num, parent):
             f = feeds[num]
             for j, a in enumerate(f):