diff --git a/recipes/den_of_geek.recipe b/recipes/den_of_geek.recipe new file mode 100644 index 0000000000..7f44bcb259 --- /dev/null +++ b/recipes/den_of_geek.recipe @@ -0,0 +1,21 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1316944753(BasicNewsRecipe): + title = u'Den of Geek' + __author__ = 'Jaded' + language = 'en' + description = 'From science fiction enthusiasts through to gaming fanatics, Den of Geek has become the one-stop UK website for people genuinely passionate about their entertainment media. Den of Geek covers popular culture but always with an edgy, UK centric slant that sets it apart from the crowd.' + category = 'Movies, TV, Games, Comics, Cult, News, Reviews' + language = 'en' + + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + + no_stylesheets = True + use_embedded_content = True + publication_type = 'newsportal' + masthead_url ='http://www.denofgeek.com/siteimage/scale/0/0/logo.gif' + cover_url ='http://a5.sphotos.ak.fbcdn.net/hphotos-ak-snc6/166479_180131695357862_139191826118516_354818_4993703_n.jpg' + + feeds = [(u'Movies', u'http://www.denofgeek.com/movies/rss/'), (u'TV', u'http://www.denofgeek.com/television/rss/'), (u'Comics & Books', u'http://www.denofgeek.com/comics/rss/'), (u'Games', u'http://www.denofgeek.com/games/rss/'), (u'DVD/Blu-ray', u'http://www.denofgeek.com/Reviews/rss/')] diff --git a/recipes/icons/den_of_geek.png b/recipes/icons/den_of_geek.png new file mode 100644 index 0000000000..a3e285b540 Binary files /dev/null and b/recipes/icons/den_of_geek.png differ diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 3c7c86b932..826b7a99fd 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -7,6 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +SPOOL_SIZE = 30*1024*1024 ''' Rewrite of the calibre database backend. diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index c29e0d5c12..d7bb251506 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -8,7 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' # Imports {{{ -import os, shutil, uuid, json, glob, time, tempfile +import os, shutil, uuid, json, glob, time from functools import partial import apsw @@ -16,7 +16,8 @@ import apsw from calibre import isbytestring, force_unicode, prints from calibre.constants import (iswindows, filesystem_encoding, preferred_encoding) -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile +from calibre.db import SPOOL_SIZE from calibre.db.schema_upgrades import SchemaUpgrade from calibre.library.field_metadata import FieldMetadata from calibre.ebooks.metadata import title_sort, author_to_author_sort @@ -38,7 +39,7 @@ Differences in semantics from pysqlite: ''' -SPOOL_SIZE = 30*1024*1024 + class DynamicFilter(object): # {{{ @@ -805,7 +806,7 @@ class DB(object): shutil.copyfileobj(f, pt) return pt.name if as_file: - ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE) + ret = SpooledTemporaryFile(SPOOL_SIZE) shutil.copyfileobj(f, ret) ret.seek(0) else: diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index fa4796a5a9..393b96442b 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -51,11 +51,11 @@ class KOBO(USBMS): EXTRA_CUSTOMIZATION_MESSAGE = [ _('The Kobo supports several collections including ')+\ - 'Read, Closed, Im_Reading ' +\ + 'Read, Closed, Im_Reading. ' +\ _('Create tags for automatic management'), ] - EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['tags']) + EXTRA_CUSTOMIZATION_DEFAULT = [', '.join(['tags'])] OPT_COLLECTIONS = 0 @@ -659,7 +659,7 @@ class KOBO(USBMS): "Read":2, "Closed":3, "Shortlist":4, - # "Preview":99, # Unsupported as we don't want to change it + # "Preview":99, # Unsupported as we don't want to change it } # Define lists for the ReadStatus diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index fa72766a0a..122e3ac19b 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -73,6 +73,20 @@ class Worker(Thread): # Get details {{{ 8: ['août'], 9: ['sept'], 12: ['déc'], + }, + 'es': { + 1: ['enero'], + 2: ['febrero'], + 3: ['marzo'], + 4: ['abril'], + 5: ['mayo'], + 6: ['junio'], + 7: ['julio'], + 8: ['agosto'], + 9: ['septiembre', 'setiembre'], + 10: ['octubre'], + 11: ['noviembre'], + 12: ['diciembre'], }, 'jp': { 1: [u'1月'], @@ -101,13 +115,16 @@ class Worker(Thread): # Get details {{{ text()="Dettagli prodotto" or \ text()="Product details" or \ text()="Détails sur le produit" or \ + text()="Detalles del producto" or \ text()="登録情報"]/../div[@class="content"] ''' + # Editor: is for Spanish self.publisher_xpath = ''' descendant::*[starts-with(text(), "Publisher:") or \ starts-with(text(), "Verlag:") or \ starts-with(text(), "Editore:") or \ starts-with(text(), "Editeur") or \ + starts-with(text(), "Editor:") or \ starts-with(text(), "出版社:")] ''' self.language_xpath = ''' @@ -116,12 +133,14 @@ class Worker(Thread): # Get details {{{ or text() = "Language" \ or text() = "Sprache:" \ or text() = "Lingua:" \ + or text() = "Idioma:" \ or starts-with(text(), "Langue") \ or starts-with(text(), "言語") \ ] ''' + self.ratings_pat = re.compile( - r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち) ([\d\.]+)( (stars|Sternen|stelle)){0,1}') + r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち|de un máximo de) ([\d\.]+)( (stars|Sternen|stelle|estrellas)){0,1}') lm = { 'eng': ('English', 'Englisch'), @@ -143,6 +162,7 @@ class Worker(Thread): # Get details {{{ for i, vals in self.months.iteritems(): for x in vals: ans = ans.replace(x, self.english_months[i]) + ans = ans.replace(' de ', ' ') return ans def run(self): @@ -422,6 +442,7 @@ class Amazon(Source): 'uk' : _('UK'), 'it' : _('Italy'), 'jp' : _('Japan'), + 'es' : _('Spain'), } options = ( @@ -789,6 +810,16 @@ if __name__ == '__main__': # tests {{{ ), ] # }}} + es_tests = [ # {{{ + ( + {'identifiers':{'isbn': '8483460831'}}, + [title_test('Tiempos Interesantes', + exact=True), authors_test(['Terry Pratchett']) + ] + + ), + ] # }}} + jp_tests = [ # {{{ ( # isbn -> title, authors {'identifiers':{'isbn': '9784101302720' }}, @@ -804,6 +835,6 @@ if __name__ == '__main__': # tests {{{ ] # }}} test_identify_plugin(Amazon.name, com_tests) - #test_identify_plugin(Amazon.name, jp_tests) + #test_identify_plugin(Amazon.name, es_tests) # }}} diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index cab582a264..660240571b 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -249,6 +249,7 @@ class SaveWorker(Thread): recs[pref.name] = getattr(self.opts, pref.name) plugboards = self.db.prefs.get('plugboards', {}) + template_functions = self.db.prefs.get('user_template_functions', []) for i, task in enumerate(tasks): tids = [x[-1] for x in task] @@ -260,7 +261,7 @@ class SaveWorker(Thread): job = ParallelJob('save_book', 'Save books (%d of %d)'%(i, len(tasks)), lambda x,y:x, - args=[tids, dpath, plugboards, self.path, recs]) + args=[tids, dpath, plugboards, template_functions, self.path, recs]) jobs.add(job) server.add_job(job) @@ -312,9 +313,12 @@ class SaveWorker(Thread): break -def save_book(ids, dpath, plugboards, path, recs, notification=lambda x,y:x): +def save_book(ids, dpath, plugboards, template_functions, path, recs, + notification=lambda x,y:x): from calibre.library.save_to_disk import config, save_serialized_to_disk from calibre.customize.ui import apply_null_metadata + from calibre.utils.formatter_functions import load_user_template_functions + load_user_template_functions(template_functions) opts = config().parse() for name in recs: setattr(opts, name, recs[name]) diff --git a/src/calibre/ebooks/pdf/images.cpp b/src/calibre/ebooks/pdf/images.cpp index 8ca7448001..466c69af4b 100644 --- a/src/calibre/ebooks/pdf/images.cpp +++ b/src/calibre/ebooks/pdf/images.cpp @@ -126,7 +126,7 @@ void XMLImages::add(GfxState *state, Object *ref, Stream *str, if (img->type == jpeg) { int c; - str = ((DCTStream *)str)->getRawStream(); + str = str->getNextStream(); str->reset(); // copy the stream diff --git a/src/calibre/ebooks/pdf/reflow.cpp b/src/calibre/ebooks/pdf/reflow.cpp index 65b5de6ae0..8103e1a03d 100644 --- a/src/calibre/ebooks/pdf/reflow.cpp +++ b/src/calibre/ebooks/pdf/reflow.cpp @@ -625,7 +625,12 @@ static string get_link_dest(LinkAction *link, PDFDoc *doc) { return oss.str(); } +#if (POPPLER_MAJOR_VERSION == 0) && (POPPLER_MINOR_VERSION < 17) void XMLOutputDev::process_link(Link* link){ +#else +void XMLOutputDev::process_link(AnnotLink* link){ +#endif + double _x1, _y1, _x2, _y2; int x1, y1, x2, y2; diff --git a/src/calibre/ebooks/pdf/reflow.h b/src/calibre/ebooks/pdf/reflow.h index 768799f004..d99137d376 100644 --- a/src/calibre/ebooks/pdf/reflow.h +++ b/src/calibre/ebooks/pdf/reflow.h @@ -244,6 +244,11 @@ class XMLOutputDev : public OutputDev { XMLImages *images; PDFDoc *doc; +#if (POPPLER_MAJOR_VERSION == 0) && (POPPLER_MINOR_VERSION < 17) void process_link(Link* link); +#else + void process_link(AnnotLink* link); +#endif + }; } diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index fe20be765f..267462bd5e 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import textwrap, re, os, errno +import textwrap, re, os, errno, shutil from PyQt4.Qt import (Qt, QDateEdit, QDate, pyqtSignal, QMessageBox, QIcon, QToolButton, QWidget, QLabel, QGridLayout, QApplication, @@ -33,8 +33,9 @@ from calibre.gui2.comments_editor import Editor from calibre.library.comments import comments_to_html from calibre.gui2.dialogs.tag_editor import TagEditor from calibre.utils.icu import strcmp -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile from calibre.gui2.languages import LanguagesEdit as LE +from calibre.db import SPOOL_SIZE def save_dialog(parent, title, msg, det_msg=''): d = QMessageBox(parent) @@ -43,8 +44,6 @@ def save_dialog(parent, title, msg, det_msg=''): d.setStandardButtons(QMessageBox.Yes | QMessageBox.No | QMessageBox.Cancel) return d.exec_() - - ''' The interface common to all widgets used to set basic metadata class BasicMetadataWidget(object): @@ -731,8 +730,12 @@ class FormatsManager(QWidget): else: old_extensions.add(ext) for ext in new_extensions: - db.add_format(id_, ext, open(paths[ext], 'rb'), notify=False, - index_is_id=True) + with SpooledTemporaryFile(SPOOL_SIZE) as spool: + with open(paths[ext], 'rb') as f: + shutil.copyfileobj(f, spool) + spool.seek(0) + db.add_format(id_, ext, spool, notify=False, + index_is_id=True) dbfmts = db.formats(id_, index_is_id=True) db_extensions = set([f.lower() for f in (dbfmts.split(',') if dbfmts else [])]) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index d6c2ddd659..f491b16291 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' The database used to store ebook metadata ''' import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \ - json, uuid, tempfile, hashlib, copy + json, uuid, hashlib, copy from collections import defaultdict import threading, random from itertools import repeat @@ -26,7 +26,8 @@ from calibre.library.sqlite import connect, IntegrityError from calibre.library.prefs import DBPrefs from calibre.ebooks.metadata.book.base import Metadata from calibre.constants import preferred_encoding, iswindows, filesystem_encoding -from calibre.ptempfile import PersistentTemporaryFile, base_dir +from calibre.ptempfile import (PersistentTemporaryFile, + base_dir, SpooledTemporaryFile) from calibre.customize.ui import run_plugins_on_import from calibre import isbytestring from calibre.utils.filenames import ascii_filename @@ -610,7 +611,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f: f.write(cdata) for format in formats: - with tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) as stream: + with SpooledTemporaryFile(SPOOL_SIZE) as stream: try: self.copy_format_to(id, format, stream, index_is_id=True) stream.seek(0) @@ -694,7 +695,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): shutil.copyfileobj(f, pt) return pt.name if as_file: - ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE) + ret = SpooledTemporaryFile(SPOOL_SIZE) shutil.copyfileobj(f, ret) ret.seek(0) else: @@ -1282,7 +1283,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): shutil.copyfileobj(f, pt) ret = pt.name elif as_file: - ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) + ret = SpooledTemporaryFile(SPOOL_SIZE) shutil.copyfileobj(f, ret) ret.seek(0) # Various bits of code try to use the name as the default diff --git a/src/calibre/ptempfile.py b/src/calibre/ptempfile.py index 34df148f15..aca8397f53 100644 --- a/src/calibre/ptempfile.py +++ b/src/calibre/ptempfile.py @@ -181,4 +181,17 @@ class TemporaryFile(object): +class SpooledTemporaryFile(tempfile.SpooledTemporaryFile): + + def __init__(self, max_size=0, suffix="", prefix="", dir=None, mode='w+b', + bufsize=-1): + if prefix == None: + prefix = '' + if suffix is None: + suffix = '' + if dir is None: + dir = base_dir() + tempfile.SpooledTemporaryFile.__init__(self, max_size=max_size, suffix=suffix, + prefix=prefix, dir=dir, mode=mode, bufsize=bufsize) + diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index 99be4af47c..00190ac23d 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -170,32 +170,40 @@ def format_date(dt, format, assume_utc=False, as_utc=False): if format == 'iso': return isoformat(dt, assume_utc=assume_utc, as_utc=as_utc) + if dt == UNDEFINED_DATE: + return '' + strf = partial(strftime, t=dt.timetuple()) - def format_day(mo): - l = len(mo.group(0)) + def format_day(dy): + l = len(dy) if l == 1: return '%d'%dt.day if l == 2: return '%02d'%dt.day if l == 3: return strf('%a') return strf('%A') def format_month(mo): - l = len(mo.group(0)) + l = len(mo) if l == 1: return '%d'%dt.month if l == 2: return '%02d'%dt.month if l == 3: return strf('%b') return strf('%B') - def format_year(mo): - if len(mo.group(0)) == 2: return '%02d'%(dt.year % 100) + def format_year(yr): + if len(yr) == 2: return '%02d'%(dt.year % 100) return '%04d'%dt.year - if dt == UNDEFINED_DATE: - return '' + def repl_func(mo): + s = mo.group(0) + if s is None: + return '' + if s[0] == 'd': + return format_day(s) + if s[0] == 'M': + return format_month(s) + return format_year(s) - format = re.sub('d{1,4}', format_day, format) - format = re.sub('M{1,4}', format_month, format) - return re.sub('yyyy|yy', format_year, format) + return re.sub('(d{1,4}|M{1,4}|(?:yyyy|yy))', repl_func, format) def replace_months(datestr, clang): # Replace months by english equivalent for parse_date diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index da037ca43b..d50cd632f0 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -13,7 +13,7 @@ from functools import partial from contextlib import nested, closing -from calibre import (browser, __appname__, iswindows, +from calibre import (browser, __appname__, iswindows, force_unicode, strftime, preferred_encoding, as_unicode) from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag from calibre.ebooks.metadata.opf2 import OPFCreator @@ -49,7 +49,7 @@ class BasicNewsRecipe(Recipe): #: A couple of lines that describe the content this recipe downloads. #: This will be used primarily in a GUI that presents a list of recipes. - description = '' + description = u'' #: The author of this recipe __author__ = __appname__ @@ -112,8 +112,6 @@ class BasicNewsRecipe(Recipe): #: If set to "optional" the use of a username and password becomes optional needs_subscription = False - #: - #: If True the navigation bar is center aligned, otherwise it is left aligned center_navbar = True @@ -1205,12 +1203,22 @@ class BasicNewsRecipe(Recipe): mi.author_sort = __appname__ mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() mi.timestamp = nowf() + article_titles, aseen = [], set() + for f in feeds: + for a in f: + if a.title and a.title not in aseen: + aseen.add(a.title) + article_titles.append(force_unicode(a.title, 'utf-8')) + mi.comments = self.description + if not isinstance(mi.comments, unicode): + mi.comments = mi.comments.decode('utf-8', 'replace') + mi.comments += ('\n\n' + _('Articles in this issue: ') + '\n' + + '\n\n'.join(article_titles)) + language = canonicalize_lang(self.language) if language is not None: mi.language = language - if not isinstance(mi.comments, unicode): - mi.comments = mi.comments.decode('utf-8', 'replace') mi.pubdate = nowf() opf_path = os.path.join(dir, 'index.opf') ncx_path = os.path.join(dir, 'index.ncx') @@ -1256,6 +1264,7 @@ class BasicNewsRecipe(Recipe): self.play_order_counter = 0 self.play_order_map = {} + def feed_index(num, parent): f = feeds[num] for j, a in enumerate(f):