From 90bb4382e009585f94d700fa92b4a7db38cdb30c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 24 Sep 2011 15:53:08 -0600 Subject: [PATCH 1/9] Amazon metadata download plugin: Add option to donload metadata from amazon.es --- src/calibre/ebooks/metadata/sources/amazon.py | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index fa72766a0a..122e3ac19b 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -73,6 +73,20 @@ class Worker(Thread): # Get details {{{ 8: ['août'], 9: ['sept'], 12: ['déc'], + }, + 'es': { + 1: ['enero'], + 2: ['febrero'], + 3: ['marzo'], + 4: ['abril'], + 5: ['mayo'], + 6: ['junio'], + 7: ['julio'], + 8: ['agosto'], + 9: ['septiembre', 'setiembre'], + 10: ['octubre'], + 11: ['noviembre'], + 12: ['diciembre'], }, 'jp': { 1: [u'1月'], @@ -101,13 +115,16 @@ class Worker(Thread): # Get details {{{ text()="Dettagli prodotto" or \ text()="Product details" or \ text()="Détails sur le produit" or \ + text()="Detalles del producto" or \ text()="登録情報"]/../div[@class="content"] ''' + # Editor: is for Spanish self.publisher_xpath = ''' descendant::*[starts-with(text(), "Publisher:") or \ starts-with(text(), "Verlag:") or \ starts-with(text(), "Editore:") or \ starts-with(text(), "Editeur") or \ + starts-with(text(), "Editor:") or \ starts-with(text(), "出版社:")] ''' self.language_xpath = ''' @@ -116,12 +133,14 @@ class Worker(Thread): # Get details {{{ or text() = "Language" \ or text() = "Sprache:" \ or text() = "Lingua:" \ + or text() = "Idioma:" \ or starts-with(text(), "Langue") \ or starts-with(text(), "言語") \ ] ''' + self.ratings_pat = re.compile( - r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち) ([\d\.]+)( (stars|Sternen|stelle)){0,1}') + r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち|de un máximo de) ([\d\.]+)( (stars|Sternen|stelle|estrellas)){0,1}') lm = { 'eng': ('English', 'Englisch'), @@ -143,6 +162,7 @@ class Worker(Thread): # Get details {{{ for i, vals in self.months.iteritems(): for x in vals: ans = ans.replace(x, self.english_months[i]) + ans = ans.replace(' de ', ' ') return ans def run(self): @@ -422,6 +442,7 @@ class Amazon(Source): 'uk' : _('UK'), 'it' : _('Italy'), 'jp' : _('Japan'), + 'es' : _('Spain'), } options = ( @@ -789,6 +810,16 @@ if __name__ == '__main__': # tests {{{ ), ] # }}} + es_tests = [ # {{{ + ( + {'identifiers':{'isbn': '8483460831'}}, + [title_test('Tiempos Interesantes', + exact=True), authors_test(['Terry Pratchett']) + ] + + ), + ] # }}} + jp_tests = [ # {{{ ( # isbn -> title, authors {'identifiers':{'isbn': '9784101302720' }}, @@ -804,6 +835,6 @@ if __name__ == '__main__': # tests {{{ ] # }}} test_identify_plugin(Amazon.name, com_tests) - #test_identify_plugin(Amazon.name, jp_tests) + #test_identify_plugin(Amazon.name, es_tests) # }}} From 4ece2a59e274d5d5be733c54c2dd3be4aeac38dd Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 25 Sep 2011 09:00:27 +0200 Subject: [PATCH 2/9] Allow user-defined functions to be directly used in save-to-disk templates --- src/calibre/ebooks/metadata/worker.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index cab582a264..660240571b 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -249,6 +249,7 @@ class SaveWorker(Thread): recs[pref.name] = getattr(self.opts, pref.name) plugboards = self.db.prefs.get('plugboards', {}) + template_functions = self.db.prefs.get('user_template_functions', []) for i, task in enumerate(tasks): tids = [x[-1] for x in task] @@ -260,7 +261,7 @@ class SaveWorker(Thread): job = ParallelJob('save_book', 'Save books (%d of %d)'%(i, len(tasks)), lambda x,y:x, - args=[tids, dpath, plugboards, self.path, recs]) + args=[tids, dpath, plugboards, template_functions, self.path, recs]) jobs.add(job) server.add_job(job) @@ -312,9 +313,12 @@ class SaveWorker(Thread): break -def save_book(ids, dpath, plugboards, path, recs, notification=lambda x,y:x): +def save_book(ids, dpath, plugboards, template_functions, path, recs, + notification=lambda x,y:x): from calibre.library.save_to_disk import config, save_serialized_to_disk from calibre.customize.ui import apply_null_metadata + from calibre.utils.formatter_functions import load_user_template_functions + load_user_template_functions(template_functions) opts = config().parse() for name in recs: setattr(opts, name, recs[name]) From 30f29c61174bdef6d10b6375a9430004b6f7a3ed Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 25 Sep 2011 09:40:35 -0600 Subject: [PATCH 3/9] Fix regression that broke customization of Kobo device plugin --- src/calibre/devices/kobo/driver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index fa4796a5a9..393b96442b 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -51,11 +51,11 @@ class KOBO(USBMS): EXTRA_CUSTOMIZATION_MESSAGE = [ _('The Kobo supports several collections including ')+\ - 'Read, Closed, Im_Reading ' +\ + 'Read, Closed, Im_Reading. ' +\ _('Create tags for automatic management'), ] - EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['tags']) + EXTRA_CUSTOMIZATION_DEFAULT = [', '.join(['tags'])] OPT_COLLECTIONS = 0 @@ -659,7 +659,7 @@ class KOBO(USBMS): "Read":2, "Closed":3, "Shortlist":4, - # "Preview":99, # Unsupported as we don't want to change it + # "Preview":99, # Unsupported as we don't want to change it } # Define lists for the ReadStatus From 8cb2e172540cb981416efbf8cb16a230c9d4f162 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 25 Sep 2011 11:57:31 -0600 Subject: [PATCH 4/9] Fix #856158 (Truncating files when adding from withing library directory) --- src/calibre/db/backend.py | 6 +++--- src/calibre/gui2/metadata/basic_widgets.py | 15 +++++++++------ src/calibre/ptempfile.py | 13 +++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index c29e0d5c12..7dd6e052ea 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -8,7 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' # Imports {{{ -import os, shutil, uuid, json, glob, time, tempfile +import os, shutil, uuid, json, glob, time from functools import partial import apsw @@ -16,7 +16,7 @@ import apsw from calibre import isbytestring, force_unicode, prints from calibre.constants import (iswindows, filesystem_encoding, preferred_encoding) -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile from calibre.db.schema_upgrades import SchemaUpgrade from calibre.library.field_metadata import FieldMetadata from calibre.ebooks.metadata import title_sort, author_to_author_sort @@ -805,7 +805,7 @@ class DB(object): shutil.copyfileobj(f, pt) return pt.name if as_file: - ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE) + ret = SpooledTemporaryFile(SPOOL_SIZE) shutil.copyfileobj(f, ret) ret.seek(0) else: diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index fe20be765f..fbde89b7b7 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import textwrap, re, os, errno +import textwrap, re, os, errno, shutil from PyQt4.Qt import (Qt, QDateEdit, QDate, pyqtSignal, QMessageBox, QIcon, QToolButton, QWidget, QLabel, QGridLayout, QApplication, @@ -33,8 +33,9 @@ from calibre.gui2.comments_editor import Editor from calibre.library.comments import comments_to_html from calibre.gui2.dialogs.tag_editor import TagEditor from calibre.utils.icu import strcmp -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile from calibre.gui2.languages import LanguagesEdit as LE +from calibre.db.backend import SPOOL_SIZE def save_dialog(parent, title, msg, det_msg=''): d = QMessageBox(parent) @@ -43,8 +44,6 @@ def save_dialog(parent, title, msg, det_msg=''): d.setStandardButtons(QMessageBox.Yes | QMessageBox.No | QMessageBox.Cancel) return d.exec_() - - ''' The interface common to all widgets used to set basic metadata class BasicMetadataWidget(object): @@ -731,8 +730,12 @@ class FormatsManager(QWidget): else: old_extensions.add(ext) for ext in new_extensions: - db.add_format(id_, ext, open(paths[ext], 'rb'), notify=False, - index_is_id=True) + with SpooledTemporaryFile(SPOOL_SIZE) as spool: + with open(paths[ext], 'rb') as f: + shutil.copyfileobj(f, spool) + spool.seek(0) + db.add_format(id_, ext, spool, notify=False, + index_is_id=True) dbfmts = db.formats(id_, index_is_id=True) db_extensions = set([f.lower() for f in (dbfmts.split(',') if dbfmts else [])]) diff --git a/src/calibre/ptempfile.py b/src/calibre/ptempfile.py index 34df148f15..aca8397f53 100644 --- a/src/calibre/ptempfile.py +++ b/src/calibre/ptempfile.py @@ -181,4 +181,17 @@ class TemporaryFile(object): +class SpooledTemporaryFile(tempfile.SpooledTemporaryFile): + + def __init__(self, max_size=0, suffix="", prefix="", dir=None, mode='w+b', + bufsize=-1): + if prefix == None: + prefix = '' + if suffix is None: + suffix = '' + if dir is None: + dir = base_dir() + tempfile.SpooledTemporaryFile.__init__(self, max_size=max_size, suffix=suffix, + prefix=prefix, dir=dir, mode=mode, bufsize=bufsize) + From 31ad166e9928a7fe809079f154eb54554527fa32 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 25 Sep 2011 12:00:25 -0600 Subject: [PATCH 5/9] Ensure use of SpooledTempFile does not temporarily create files outside the calibre temp dir --- src/calibre/library/database2.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index d6c2ddd659..f491b16291 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' The database used to store ebook metadata ''' import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \ - json, uuid, tempfile, hashlib, copy + json, uuid, hashlib, copy from collections import defaultdict import threading, random from itertools import repeat @@ -26,7 +26,8 @@ from calibre.library.sqlite import connect, IntegrityError from calibre.library.prefs import DBPrefs from calibre.ebooks.metadata.book.base import Metadata from calibre.constants import preferred_encoding, iswindows, filesystem_encoding -from calibre.ptempfile import PersistentTemporaryFile, base_dir +from calibre.ptempfile import (PersistentTemporaryFile, + base_dir, SpooledTemporaryFile) from calibre.customize.ui import run_plugins_on_import from calibre import isbytestring from calibre.utils.filenames import ascii_filename @@ -610,7 +611,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f: f.write(cdata) for format in formats: - with tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) as stream: + with SpooledTemporaryFile(SPOOL_SIZE) as stream: try: self.copy_format_to(id, format, stream, index_is_id=True) stream.seek(0) @@ -694,7 +695,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): shutil.copyfileobj(f, pt) return pt.name if as_file: - ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE) + ret = SpooledTemporaryFile(SPOOL_SIZE) shutil.copyfileobj(f, ret) ret.seek(0) else: @@ -1282,7 +1283,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): shutil.copyfileobj(f, pt) ret = pt.name elif as_file: - ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) + ret = SpooledTemporaryFile(SPOOL_SIZE) shutil.copyfileobj(f, ret) ret.seek(0) # Various bits of code try to use the name as the default From 64dce130f94df42f2e23f71d0a08ad958672cac3 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 25 Sep 2011 20:22:39 +0200 Subject: [PATCH 6/9] Fix #858737 --- src/calibre/utils/date.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index 99be4af47c..00190ac23d 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -170,32 +170,40 @@ def format_date(dt, format, assume_utc=False, as_utc=False): if format == 'iso': return isoformat(dt, assume_utc=assume_utc, as_utc=as_utc) + if dt == UNDEFINED_DATE: + return '' + strf = partial(strftime, t=dt.timetuple()) - def format_day(mo): - l = len(mo.group(0)) + def format_day(dy): + l = len(dy) if l == 1: return '%d'%dt.day if l == 2: return '%02d'%dt.day if l == 3: return strf('%a') return strf('%A') def format_month(mo): - l = len(mo.group(0)) + l = len(mo) if l == 1: return '%d'%dt.month if l == 2: return '%02d'%dt.month if l == 3: return strf('%b') return strf('%B') - def format_year(mo): - if len(mo.group(0)) == 2: return '%02d'%(dt.year % 100) + def format_year(yr): + if len(yr) == 2: return '%02d'%(dt.year % 100) return '%04d'%dt.year - if dt == UNDEFINED_DATE: - return '' + def repl_func(mo): + s = mo.group(0) + if s is None: + return '' + if s[0] == 'd': + return format_day(s) + if s[0] == 'M': + return format_month(s) + return format_year(s) - format = re.sub('d{1,4}', format_day, format) - format = re.sub('M{1,4}', format_month, format) - return re.sub('yyyy|yy', format_year, format) + return re.sub('(d{1,4}|M{1,4}|(?:yyyy|yy))', repl_func, format) def replace_months(datestr, clang): # Replace months by english equivalent for parse_date From bebe2467cc95162f745ef2b94202b63e5a54a484 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 25 Sep 2011 13:17:40 -0600 Subject: [PATCH 7/9] News download: Add list of articles in the downloaded issue to the comments metadata of the generated ebook. Makes it possible to search for a particular article in the calibre library. Fixes #851717 ([enhancement] News downloading options for search) --- src/calibre/web/feeds/news.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index da037ca43b..d50cd632f0 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -13,7 +13,7 @@ from functools import partial from contextlib import nested, closing -from calibre import (browser, __appname__, iswindows, +from calibre import (browser, __appname__, iswindows, force_unicode, strftime, preferred_encoding, as_unicode) from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag from calibre.ebooks.metadata.opf2 import OPFCreator @@ -49,7 +49,7 @@ class BasicNewsRecipe(Recipe): #: A couple of lines that describe the content this recipe downloads. #: This will be used primarily in a GUI that presents a list of recipes. - description = '' + description = u'' #: The author of this recipe __author__ = __appname__ @@ -112,8 +112,6 @@ class BasicNewsRecipe(Recipe): #: If set to "optional" the use of a username and password becomes optional needs_subscription = False - #: - #: If True the navigation bar is center aligned, otherwise it is left aligned center_navbar = True @@ -1205,12 +1203,22 @@ class BasicNewsRecipe(Recipe): mi.author_sort = __appname__ mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() mi.timestamp = nowf() + article_titles, aseen = [], set() + for f in feeds: + for a in f: + if a.title and a.title not in aseen: + aseen.add(a.title) + article_titles.append(force_unicode(a.title, 'utf-8')) + mi.comments = self.description + if not isinstance(mi.comments, unicode): + mi.comments = mi.comments.decode('utf-8', 'replace') + mi.comments += ('\n\n' + _('Articles in this issue: ') + '\n' + + '\n\n'.join(article_titles)) + language = canonicalize_lang(self.language) if language is not None: mi.language = language - if not isinstance(mi.comments, unicode): - mi.comments = mi.comments.decode('utf-8', 'replace') mi.pubdate = nowf() opf_path = os.path.join(dir, 'index.opf') ncx_path = os.path.join(dir, 'index.ncx') @@ -1256,6 +1264,7 @@ class BasicNewsRecipe(Recipe): self.play_order_counter = 0 self.play_order_map = {} + def feed_index(num, parent): f = feeds[num] for j, a in enumerate(f): From dc9b181dae236b86e28bb30b9242c2faca1d1786 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 26 Sep 2011 12:07:20 -0600 Subject: [PATCH 8/9] Fix #855587 (Doesn't build with poppler 0.17/0.18) --- src/calibre/ebooks/pdf/images.cpp | 2 +- src/calibre/ebooks/pdf/reflow.cpp | 5 +++++ src/calibre/ebooks/pdf/reflow.h | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/pdf/images.cpp b/src/calibre/ebooks/pdf/images.cpp index 8ca7448001..466c69af4b 100644 --- a/src/calibre/ebooks/pdf/images.cpp +++ b/src/calibre/ebooks/pdf/images.cpp @@ -126,7 +126,7 @@ void XMLImages::add(GfxState *state, Object *ref, Stream *str, if (img->type == jpeg) { int c; - str = ((DCTStream *)str)->getRawStream(); + str = str->getNextStream(); str->reset(); // copy the stream diff --git a/src/calibre/ebooks/pdf/reflow.cpp b/src/calibre/ebooks/pdf/reflow.cpp index 65b5de6ae0..8103e1a03d 100644 --- a/src/calibre/ebooks/pdf/reflow.cpp +++ b/src/calibre/ebooks/pdf/reflow.cpp @@ -625,7 +625,12 @@ static string get_link_dest(LinkAction *link, PDFDoc *doc) { return oss.str(); } +#if (POPPLER_MAJOR_VERSION == 0) && (POPPLER_MINOR_VERSION < 17) void XMLOutputDev::process_link(Link* link){ +#else +void XMLOutputDev::process_link(AnnotLink* link){ +#endif + double _x1, _y1, _x2, _y2; int x1, y1, x2, y2; diff --git a/src/calibre/ebooks/pdf/reflow.h b/src/calibre/ebooks/pdf/reflow.h index 768799f004..d99137d376 100644 --- a/src/calibre/ebooks/pdf/reflow.h +++ b/src/calibre/ebooks/pdf/reflow.h @@ -244,6 +244,11 @@ class XMLOutputDev : public OutputDev { XMLImages *images; PDFDoc *doc; +#if (POPPLER_MAJOR_VERSION == 0) && (POPPLER_MINOR_VERSION < 17) void process_link(Link* link); +#else + void process_link(AnnotLink* link); +#endif + }; } From 589ca8b572786050cb993d5c386f8ec8c4bb96cf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 26 Sep 2011 17:04:26 -0600 Subject: [PATCH 9/9] ... --- src/calibre/db/__init__.py | 1 + src/calibre/db/backend.py | 3 ++- src/calibre/gui2/metadata/basic_widgets.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 3c7c86b932..826b7a99fd 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -7,6 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +SPOOL_SIZE = 30*1024*1024 ''' Rewrite of the calibre database backend. diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 7dd6e052ea..d7bb251506 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -17,6 +17,7 @@ from calibre import isbytestring, force_unicode, prints from calibre.constants import (iswindows, filesystem_encoding, preferred_encoding) from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile +from calibre.db import SPOOL_SIZE from calibre.db.schema_upgrades import SchemaUpgrade from calibre.library.field_metadata import FieldMetadata from calibre.ebooks.metadata import title_sort, author_to_author_sort @@ -38,7 +39,7 @@ Differences in semantics from pysqlite: ''' -SPOOL_SIZE = 30*1024*1024 + class DynamicFilter(object): # {{{ diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index fbde89b7b7..267462bd5e 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -35,7 +35,7 @@ from calibre.gui2.dialogs.tag_editor import TagEditor from calibre.utils.icu import strcmp from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile from calibre.gui2.languages import LanguagesEdit as LE -from calibre.db.backend import SPOOL_SIZE +from calibre.db import SPOOL_SIZE def save_dialog(parent, title, msg, det_msg=''): d = QMessageBox(parent)