Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Create a unicode friendly logging mechanism that should help remove unicode errors on non-english OSes.

Browse Source
This commit is contained in:
Kovid Goyal 2008-05-04 05:50:45 -07:00
parent 5b99ec33c3
commit 12676bceff
8 changed files with 157 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/calibre/__init__.py
View File

@ -7,7 +7,7 @@ __author__ = "Kovid Goyal <kovid at kovidgoyal.net>"
__appname__ = 'calibre' __appname__ = 'calibre'
import sys, os, logging, mechanize, locale, copy, cStringIO, re, subprocess, \ import sys, os, logging, mechanize, locale, copy, cStringIO, re, subprocess, \
textwrap, atexit, cPickle textwrap, atexit, cPickle, codecs
from gettext import GNUTranslations from gettext import GNUTranslations
from math import floor from math import floor
from optparse import OptionParser as _OptionParser from optparse import OptionParser as _OptionParser
@ -31,6 +31,13 @@ try:
except: except:
pass pass
try:
preferred_encoding = locale.getpreferredencoding()
codecs.lookup(preferred_encoding)
except:
preferred_encoding = 'utf-8'
def osx_version(): def osx_version():
if isosx: if isosx:
import platform import platform
@ -486,3 +493,37 @@ def english_sort(x, y):
Comapare two english phrases ignoring starting prepositions. Comapare two english phrases ignoring starting prepositions.
''' '''
return cmp(_spat.sub('', x), _spat.sub('', y)) return cmp(_spat.sub('', x), _spat.sub('', y))
class LoggingInterface:
def __init__(self, logger):
self.__logger = logger
def ___log(self, func, msg, args, kwargs):
args = [msg] + list(args)
for i in range(len(args)):
if isinstance(args[i], unicode):
args[i] = args[i].encode(preferred_encoding, 'replace')
func(*args, **kwargs)
def log_debug(self, msg, *args, **kwargs):
self.___log(self.__logger.debug, msg, args, kwargs)
def log_info(self, msg, *args, **kwargs):
self.___log(self.__logger.info, msg, args, kwargs)
def log_warning(self, msg, *args, **kwargs):
self.___log(self.__logger.warning, msg, args, kwargs)
def log_warn(self, msg, *args, **kwargs):
self.___log(self.__logger.warning, msg, args, kwargs)
def log_error(self, msg, *args, **kwargs):
self.___log(self.__logger.error, msg, args, kwargs)
def log_critical(self, msg, *args, **kwargs):
self.___log(self.__logger.critical, msg, args, kwargs)
def log_exception(self, msg, *args):
self.___log(self.__logger.exception, msg, args, {})

55
src/calibre/ebooks/lrf/html/convert_from.py
View File

@ -29,7 +29,8 @@ from calibre.ebooks.lrf import Book, entity_to_unicode
from calibre.ebooks.lrf import option_parser as lrf_option_parser from calibre.ebooks.lrf import option_parser as lrf_option_parser
from calibre.ebooks import ConversionError from calibre.ebooks import ConversionError
from calibre.ebooks.lrf.html.table import Table from calibre.ebooks.lrf.html.table import Table
from calibre import filename_to_utf8, setup_cli_handlers, __appname__, fit_image from calibre import filename_to_utf8, setup_cli_handlers, __appname__, \
fit_image, LoggingInterface
from calibre.ptempfile import PersistentTemporaryFile from calibre.ptempfile import PersistentTemporaryFile
from calibre.ebooks.metadata.opf import OPFReader from calibre.ebooks.metadata.opf import OPFReader
from calibre.devices.interface import Device from calibre.devices.interface import Device
@ -77,7 +78,7 @@ def tag_regex(tagname):
return dict(open=r'(?:<\s*%(t)s\s+[^<>]*?>|<\s*%(t)s\s*>)'%dict(t=tagname), \ return dict(open=r'(?:<\s*%(t)s\s+[^<>]*?>|<\s*%(t)s\s*>)'%dict(t=tagname), \
close=r'</\s*%(t)s\s*>'%dict(t=tagname)) close=r'</\s*%(t)s\s*>'%dict(t=tagname))
class HTMLConverter(object): class HTMLConverter(object, LoggingInterface):
SELECTOR_PAT = re.compile(r"([A-Za-z0-9\-\_\:\.]+[A-Za-z0-9\-\_\:\.\s\,]*)\s*\{([^\}]*)\}") SELECTOR_PAT = re.compile(r"([A-Za-z0-9\-\_\:\.]+[A-Za-z0-9\-\_\:\.\s\,]*)\s*\{([^\}]*)\}")
PAGE_BREAK_PAT = re.compile(r'page-break-(?:after|before)\s*:\s*(\w+)', re.IGNORECASE) PAGE_BREAK_PAT = re.compile(r'page-break-(?:after|before)\s*:\s*(\w+)', re.IGNORECASE)
IGNORED_TAGS = (Comment, Declaration, ProcessingInstruction) IGNORED_TAGS = (Comment, Declaration, ProcessingInstruction)
@ -159,7 +160,7 @@ class HTMLConverter(object):
def __getattr__(self, attr): def __getattr__(self, attr):
if hasattr(self.options, attr): if hasattr(self.options, attr):
return getattr(self.options, attr) return getattr(self.options, attr)
return object.__getattr__(self, attr) return object.__getattribute__(self, attr)
def __setattr__(self, attr, val): def __setattr__(self, attr, val):
if hasattr(self.options, attr): if hasattr(self.options, attr):
@ -202,7 +203,7 @@ class HTMLConverter(object):
''' '''
# Defaults for various formatting tags # Defaults for various formatting tags
object.__setattr__(self, 'options', options) object.__setattr__(self, 'options', options)
self.logger = logger LoggingInterface.__init__(self, logger)
self.fonts = fonts #: dict specifying font families to use self.fonts = fonts #: dict specifying font families to use
# Memory # Memory
self.scaled_images = {} #: Temporary files with scaled version of images self.scaled_images = {} #: Temporary files with scaled version of images
@ -273,9 +274,9 @@ class HTMLConverter(object):
if link['path'] == path: if link['path'] == path:
self.links.remove(link) self.links.remove(link)
break break
self.logger.warn('Could not process '+path) self.log_warn('Could not process '+path)
if self.verbose: if self.verbose:
self.logger.exception(' ') self.log_exception(' ')
self.links = self.process_links() self.links = self.process_links()
self.link_level += 1 self.link_level += 1
paths = [link['path'] for link in self.links] paths = [link['path'] for link in self.links]
@ -288,7 +289,7 @@ class HTMLConverter(object):
self.book.addTocEntry(ascii_text, tb) self.book.addTocEntry(ascii_text, tb)
if self.base_font_size > 0: if self.base_font_size > 0:
self.logger.info('\tRationalizing font sizes...') self.log_info('\tRationalizing font sizes...')
self.book.rationalize_font_sizes(self.base_font_size) self.book.rationalize_font_sizes(self.base_font_size)
def is_baen(self, soup): def is_baen(self, soup):
@ -304,9 +305,9 @@ class HTMLConverter(object):
if not self.book_designer and self.is_book_designer(raw): if not self.book_designer and self.is_book_designer(raw):
self.book_designer = True self.book_designer = True
self.logger.info(_('\tBook Designer file detected.')) self.log_info(_('\tBook Designer file detected.'))
self.logger.info(_('\tParsing HTML...')) self.log_info(_('\tParsing HTML...'))
if self.baen: if self.baen:
nmassage.extend(HTMLConverter.BAEN) nmassage.extend(HTMLConverter.BAEN)
@ -328,7 +329,7 @@ class HTMLConverter(object):
if not self.baen and self.is_baen(soup): if not self.baen and self.is_baen(soup):
self.baen = True self.baen = True
self.logger.info(_('\tBaen file detected. Re-parsing...')) self.log_info(_('\tBaen file detected. Re-parsing...'))
return self.preprocess(raw) return self.preprocess(raw)
if self.book_designer: if self.book_designer:
t = soup.find(id='BookTitle') t = soup.find(id='BookTitle')
@ -344,7 +345,7 @@ class HTMLConverter(object):
try: try:
dump = open(os.path.join(tdir, 'html2lrf-verbose.html'), 'wb') dump = open(os.path.join(tdir, 'html2lrf-verbose.html'), 'wb')
dump.write(unicode(soup).encode('utf-8')) dump.write(unicode(soup).encode('utf-8'))
self.logger.info(_('Written preprocessed HTML to ')+dump.name) self.log_info(_('Written preprocessed HTML to ')+dump.name)
dump.close() dump.close()
except: except:
pass pass
@ -357,7 +358,7 @@ class HTMLConverter(object):
self.css.update(self.override_css) self.css.update(self.override_css)
self.file_name = os.path.basename(path) self.file_name = os.path.basename(path)
self.logger.info(_('Processing %s'), path if self.verbose else self.file_name) self.log_info(_('Processing %s'), path if self.verbose else self.file_name)
upath = path.encode('utf-8') if isinstance(path, unicode) else path upath = path.encode('utf-8') if isinstance(path, unicode) else path
if not os.path.exists(upath): if not os.path.exists(upath):
upath = upath.replace('&', '%26') #convertlit replaces & with %26 in file names upath = upath.replace('&', '%26') #convertlit replaces & with %26 in file names
@ -371,7 +372,7 @@ class HTMLConverter(object):
raw = xml_to_unicode(raw, self.verbose)[0] raw = xml_to_unicode(raw, self.verbose)[0]
f.close() f.close()
soup = self.preprocess(raw) soup = self.preprocess(raw)
self.logger.info(_('\tConverting to BBeB...')) self.log_info(_('\tConverting to BBeB...'))
self.current_style = {} self.current_style = {}
self.page_break_found = False self.page_break_found = False
self.target_prefix = path self.target_prefix = path
@ -571,7 +572,7 @@ class HTMLConverter(object):
hasattr(target.parent, 'objId'): hasattr(target.parent, 'objId'):
self.book.addTocEntry(ascii_text, tb) self.book.addTocEntry(ascii_text, tb)
else: else:
self.logger.debug(_("Cannot add link %s to TOC"), ascii_text) self.log_debug(_("Cannot add link %s to TOC"), ascii_text)
def get_target_block(fragment, targets): def get_target_block(fragment, targets):
@ -891,7 +892,7 @@ class HTMLConverter(object):
try: try:
im = PILImage.open(path) im = PILImage.open(path)
except IOError, err: except IOError, err:
self.logger.warning('Unable to process image: %s\n%s', original_path, err) self.log_warning('Unable to process image: %s\n%s', original_path, err)
return return
encoding = detect_encoding(im) encoding = detect_encoding(im)
@ -913,7 +914,7 @@ class HTMLConverter(object):
self.scaled_images[path] = pt self.scaled_images[path] = pt
return pt.name return pt.name
except (IOError, SystemError), err: # PIL chokes on interlaced PNG images as well a some GIF images except (IOError, SystemError), err: # PIL chokes on interlaced PNG images as well a some GIF images
self.logger.warning(_('Unable to process image %s. Error: %s')%(path, err)) self.log_warning(_('Unable to process image %s. Error: %s')%(path, err))
return None return None
pheight = int(self.current_page.pageStyle.attrs['textheight']) pheight = int(self.current_page.pageStyle.attrs['textheight'])
@ -951,7 +952,7 @@ class HTMLConverter(object):
self.rotated_images[path] = pt self.rotated_images[path] = pt
width, height = im.size width, height = im.size
except IOError: # PIL chokes on interlaced PNG files and since auto-rotation is not critical we ignore the error except IOError: # PIL chokes on interlaced PNG files and since auto-rotation is not critical we ignore the error
self.logger.debug(_('Unable to process interlaced PNG %s'), original_path) self.log_debug(_('Unable to process interlaced PNG %s'), original_path)
finally: finally:
pt.close() pt.close()
@ -966,7 +967,7 @@ class HTMLConverter(object):
try: try:
self.images[path] = ImageStream(path, encoding=encoding) self.images[path] = ImageStream(path, encoding=encoding)
except LrsError, err: except LrsError, err:
self.logger.warning(_('Could not process image: %s\n%s'), original_path, err) self.log_warning(_('Could not process image: %s\n%s'), original_path, err)
return return
im = Image(self.images[path], x0=0, y0=0, x1=width, y1=height,\ im = Image(self.images[path], x0=0, y0=0, x1=width, y1=height,\
@ -1026,7 +1027,7 @@ class HTMLConverter(object):
if number_of_paragraphs > 2: if number_of_paragraphs > 2:
self.end_page() self.end_page()
self.logger.debug('Forcing page break at %s', tagname) self.log_debug('Forcing page break at %s', tagname)
return end_page return end_page
def block_properties(self, tag_css): def block_properties(self, tag_css):
@ -1434,7 +1435,7 @@ class HTMLConverter(object):
self.targets[self.target_prefix+tag[key]] = self.current_block self.targets[self.target_prefix+tag[key]] = self.current_block
self.current_block.must_append = True self.current_block.must_append = True
else: else:
self.logger.debug('Could not follow link to '+tag['href']) self.log_debug('Could not follow link to '+tag['href'])
self.process_children(tag, tag_css, tag_pseudo_css) self.process_children(tag, tag_css, tag_pseudo_css)
elif tag.has_key('name') or tag.has_key('id'): elif tag.has_key('name') or tag.has_key('id'):
self.process_anchor(tag, tag_css, tag_pseudo_css) self.process_anchor(tag, tag_css, tag_pseudo_css)
@ -1453,9 +1454,9 @@ class HTMLConverter(object):
dropcaps = tag.has_key('class') and tag['class'] == 'libprs500_dropcaps' dropcaps = tag.has_key('class') and tag['class'] == 'libprs500_dropcaps'
self.process_image(path, tag_css, width, height, dropcaps=dropcaps) self.process_image(path, tag_css, width, height, dropcaps=dropcaps)
elif not urlparse(tag['src'])[0]: elif not urlparse(tag['src'])[0]:
self.logger.warn('Could not find image: '+tag['src']) self.log_warn('Could not find image: '+tag['src'])
else: else:
self.logger.debug("Failed to process: %s", str(tag)) self.log_debug("Failed to process: %s", str(tag))
elif tagname in ['style', 'link']: elif tagname in ['style', 'link']:
ncss, npcss = {}, {} ncss, npcss = {}, {}
if tagname == 'style': if tagname == 'style':
@ -1475,7 +1476,7 @@ class HTMLConverter(object):
self.page_break_found = True self.page_break_found = True
ncss, npcss = self.parse_css(src) ncss, npcss = self.parse_css(src)
except IOError: except IOError:
self.logger.warn('Could not read stylesheet: '+tag['href']) self.log_warn('Could not read stylesheet: '+tag['href'])
if ncss: if ncss:
update_css(ncss, self.css) update_css(ncss, self.css)
self.css.update(self.override_css) self.css.update(self.override_css)
@ -1605,7 +1606,7 @@ class HTMLConverter(object):
if not self.disable_chapter_detection and tagname.startswith('h'): if not self.disable_chapter_detection and tagname.startswith('h'):
if self.chapter_regex.search(src): if self.chapter_regex.search(src):
self.logger.debug('Detected chapter %s', src) self.log_debug('Detected chapter %s', src)
self.end_page() self.end_page()
self.page_break_found = True self.page_break_found = True
@ -1656,9 +1657,9 @@ class HTMLConverter(object):
try: try:
self.process_table(tag, tag_css) self.process_table(tag, tag_css)
except Exception, err: except Exception, err:
self.logger.warning(_('An error occurred while processing a table: %s. Ignoring table markup.'), str(err)) self.log_warning(_('An error occurred while processing a table: %s. Ignoring table markup.'), str(err))
self.logger.debug('', exc_info=True) self.log_debug('', exc_info=True)
self.logger.debug(_('Bad table:\n%s'), str(tag)[:300]) self.log_debug(_('Bad table:\n%s'), str(tag)[:300])
self.in_table = False self.in_table = False
self.process_children(tag, tag_css, tag_pseudo_css) self.process_children(tag, tag_css, tag_pseudo_css)
finally: finally:

10
src/calibre/gui2/library.py
View File

@ -11,17 +11,11 @@ from PyQt4.QtGui import QTableView, QProgressDialog, QAbstractItemView, QColor,
from PyQt4.QtCore import QAbstractTableModel, QVariant, Qt, QString, \ from PyQt4.QtCore import QAbstractTableModel, QVariant, Qt, QString, \
QCoreApplication, SIGNAL, QObject, QSize, QModelIndex QCoreApplication, SIGNAL, QObject, QSize, QModelIndex
from calibre import Settings from calibre import Settings, preferred_encoding
from calibre.ptempfile import PersistentTemporaryFile from calibre.ptempfile import PersistentTemporaryFile
from calibre.library.database import LibraryDatabase, SearchToken from calibre.library.database import LibraryDatabase, SearchToken
from calibre.gui2 import NONE, TableView, qstring_to_unicode from calibre.gui2 import NONE, TableView, qstring_to_unicode
try:
pe = locale.getpreferredencoding()
codecs.lookup(pe)
except:
pe = 'utf-8'
class LibraryDelegate(QItemDelegate): class LibraryDelegate(QItemDelegate):
COLOR = QColor("blue") COLOR = QColor("blue")
SIZE = 16 SIZE = 16
@ -304,7 +298,7 @@ class BooksModel(QAbstractTableModel):
dt = self.db.timestamp(row) dt = self.db.timestamp(row)
if dt: if dt:
dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight) dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight)
return QVariant(dt.strftime(BooksView.TIME_FMT).decode(pe, 'replace')) return QVariant(dt.strftime(BooksView.TIME_FMT).decode(preferred_encoding, 'replace'))
elif col == 4: elif col == 4:
r = self.db.rating(row) r = self.db.rating(row)
r = r/2 if r else 0 r = r/2 if r else 0

8
src/calibre/translations/__init__.py
View File

@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
Manage translation of user visible strings. Manage translation of user visible strings.
''' '''
import sys, os, cStringIO, tempfile, subprocess, functools import sys, os, cStringIO, tempfile, subprocess, functools, tarfile
check_call = functools.partial(subprocess.check_call, shell=True) check_call = functools.partial(subprocess.check_call, shell=True)
from calibre.translations.pygettext import main as pygettext from calibre.translations.pygettext import main as pygettext
@ -35,6 +35,8 @@ def main(args=sys.argv):
print 'Creating translations template' print 'Creating translations template'
pygettext(buf, ['-p', tdir]+files) pygettext(buf, ['-p', tdir]+files)
src = buf.getvalue() src = buf.getvalue()
tempdir = tempfile.mkdtemp()
tf = tarfile.open(os.path.join(tempdir, 'translations.tar.bz2'), 'w:bz2')
fd, fname = tempfile.mkstemp(suffix='.pot') fd, fname = tempfile.mkstemp(suffix='.pot')
os.write(fd,src) os.write(fd,src)
@ -46,13 +48,17 @@ def main(args=sys.argv):
else: else:
print 'Merging', os.path.basename(po) print 'Merging', os.path.basename(po)
check_call('msgmerge -v -U -N --backup=none '+po + ' ' + fname) check_call('msgmerge -v -U -N --backup=none '+po + ' ' + fname)
tf.add(po, os.path.basename(po))
buf = cStringIO.StringIO() buf = cStringIO.StringIO()
print 'Compiling translations' print 'Compiling translations'
msgfmt(buf, [po]) msgfmt(buf, [po])
translations[tr] = buf.getvalue() translations[tr] = buf.getvalue()
open(os.path.join(tdir, 'data.py'), 'wb').write('translations = '+repr(translations)) open(os.path.join(tdir, 'data.py'), 'wb').write('translations = '+repr(translations))
os.close(fd) os.close(fd)
tf.add(fname, 'strings.pot')
tf.close()
os.unlink(fname) os.unlink(fname)
print 'Translations tarball is in', os.path.join(tempdir, 'translations.tar.bz2')
return 0 return 0
if __name__ == '__main__': if __name__ == '__main__':

2
src/calibre/translations/data.py
View File

File diff suppressed because one or more lines are too long

97
src/calibre/translations/it.po
View File

@ -8,7 +8,7 @@ msgid ""
msgstr "" msgstr ""
"Project-Id-Version: it\n" "Project-Id-Version: it\n"
"POT-Creation-Date: 2008-05-02 17:09+PDT\n" "POT-Creation-Date: 2008-05-02 17:09+PDT\n"
"PO-Revision-Date: 2008-04-05 14:25+0200\n" "PO-Revision-Date: 2008-05-03 22:51+0200\n"
"Last-Translator: Iacopo Benesperi <iacchi@iacchi.org>\n" "Last-Translator: Iacopo Benesperi <iacchi@iacchi.org>\n"
"Language-Team: italiano\n" "Language-Team: italiano\n"
"MIME-Version: 1.0\n" "MIME-Version: 1.0\n"
@ -79,8 +79,7 @@ msgstr ""
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:108 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:108
msgid "Output file name. Default is derived from input filename" msgid "Output file name. Default is derived from input filename"
msgstr "" msgstr "Nome del file in uscita. Il nome predefinito è preso dal file in ingresso"
"Nome del file in uscita. Il nome predefinito è preso dal file in ingresso"
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:110 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:110
msgid "" msgid ""
@ -110,8 +109,7 @@ msgstr ""
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:118 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:118
msgid "Set the space between words in pts. Default is %default" msgid "Set the space between words in pts. Default is %default"
msgstr "" msgstr "Imposta lo spazio tra le parole in punti. Il valore predefinito è %default"
"Imposta lo spazio tra le parole in punti. Il valore predefinito è %default"
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:120 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:120
msgid "Separate paragraphs by blank lines." msgid "Separate paragraphs by blank lines."
@ -119,8 +117,7 @@ msgstr "Separa i paragrafi con linee bianche"
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:122 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:122
msgid "Add a header to all the pages with title and author." msgid "Add a header to all the pages with title and author."
msgstr "" msgstr "Aggiunge a tutte le pagine un'intestazione contenente il titolo e l'autore"
"Aggiunge a tutte le pagine un'intestazione contenente il titolo e l'autore"
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:124 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:124
msgid "" msgid ""
@ -219,8 +216,7 @@ msgstr ""
"ignorati. Predefinita: %default" "ignorati. Predefinita: %default"
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:169 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:169
msgid "" msgid "Prevent the automatic insertion of page breaks before detected chapters."
"Prevent the automatic insertion of page breaks before detected chapters."
msgstr "" msgstr ""
"Previene l'inserimento automatico di interruzioni di pagina prima dei " "Previene l'inserimento automatico di interruzioni di pagina prima dei "
"capitoli individuati" "capitoli individuati"
@ -253,8 +249,7 @@ msgstr ""
"perciò ignorata se la pagina corrente ha solo pochi elementi" "perciò ignorata se la pagina corrente ha solo pochi elementi"
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:186 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/__init__.py:186
msgid "" msgid "Force a page break before tags whoose names match this regular expression."
"Force a page break before tags whoose names match this regular expression."
msgstr "" msgstr ""
"Forza un'interruzione di pagina prima dei tag i cui nomi corrispondono a " "Forza un'interruzione di pagina prima dei tag i cui nomi corrispondono a "
"questa espressione regolare" "questa espressione regolare"
@ -384,6 +379,10 @@ msgid ""
"\n" "\n"
"%prog converts mybook.fb2 to mybook.lrf" "%prog converts mybook.fb2 to mybook.lrf"
msgstr "" msgstr ""
"%prog [opzioni] miolibro.fb2\n"
"\n"
"\n"
"%prog converte miolibro.fb2 in miolibro.lrf"
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/fb2/convert_from.py:23 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/fb2/convert_from.py:23
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/txt/convert_from.py:22 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/txt/convert_from.py:22
@ -582,13 +581,11 @@ msgstr "La categoria a cui questo libro appartiene. Es: Storia"
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/meta.py:559 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/meta.py:559
msgid "Path to a graphic that will be set as this files' thumbnail" msgid "Path to a graphic that will be set as this files' thumbnail"
msgstr "" msgstr "Percorso a un'immagine che verrà impostata come miniatura di questo file"
"Percorso a un'immagine che verrà impostata come miniatura di questo file"
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/meta.py:562 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/meta.py:562
msgid "Path to a txt file containing the comment to be stored in the lrf file." msgid "Path to a txt file containing the comment to be stored in the lrf file."
msgstr "" msgstr "Percorso a un file TXT contenente il commento che verrà incluso nel file LRF"
"Percorso a un file TXT contenente il commento che verrà incluso nel file LRF"
#: /home/kovid/work/calibre/src/calibre/ebooks/lrf/meta.py:566 #: /home/kovid/work/calibre/src/calibre/ebooks/lrf/meta.py:566
msgid "Extract thumbnail from LRF file" msgid "Extract thumbnail from LRF file"
@ -1004,8 +1001,7 @@ msgstr "Nessun formato disponibile"
#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/lrf_single.py:82 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/lrf_single.py:82
msgid "Cannot convert %s as this book has no supported formats" msgid "Cannot convert %s as this book has no supported formats"
msgstr "" msgstr "Impossibile convertire %s perché questo libro non ha formati supportati"
"Impossibile convertire %s perché questo libro non ha formati supportati"
#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/lrf_single.py:86 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/lrf_single.py:86
msgid "Choose the format to convert into LRF" msgid "Choose the format to convert into LRF"
@ -1149,8 +1145,7 @@ msgstr "Ca&mbia l'immagine di copertina:"
#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/lrf_single_ui.py:609 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/lrf_single_ui.py:609
#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/metadata_single_ui.py:334 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/metadata_single_ui.py:334
msgid "Browse for an image to use as the cover of this book." msgid "Browse for an image to use as the cover of this book."
msgstr "" msgstr "Sfoglia per trovare un'immagine da usare come copertina per questo libro"
"Sfoglia per trovare un'immagine da usare come copertina per questo libro"
#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/lrf_single_ui.py:611 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/lrf_single_ui.py:611
msgid "Use cover from &source file" msgid "Use cover from &source file"
@ -1517,10 +1512,8 @@ msgid "Fetch cover image from server"
msgstr "Scarica immagine di copertina dal server" msgstr "Scarica immagine di copertina dal server"
#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/metadata_single_ui.py:337 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/metadata_single_ui.py:337
msgid "" msgid "Change the username and/or password for your account at LibraryThing.com"
"Change the username and/or password for your account at LibraryThing.com" msgstr "Cambia il nome utente e/o password del proprio account su LibraryThing.com"
msgstr ""
"Cambia il nome utente e/o password del proprio account su LibraryThing.com"
#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/metadata_single_ui.py:338 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/metadata_single_ui.py:338
msgid "Change password" msgid "Change password"
@ -1811,6 +1804,8 @@ msgid ""
"For help with writing advanced news recipes, please visit <a href=\"http://" "For help with writing advanced news recipes, please visit <a href=\"http://"
"calibre.kovidgoyal.net/user_manual/news.html\">User Recipes</a>" "calibre.kovidgoyal.net/user_manual/news.html\">User Recipes</a>"
msgstr "" msgstr ""
"Per un aiuto su come scrivere formule di notizie avanzate, visitare <a href=\"http://"
"calibre.kovidgoyal.net/user_manual/news.html\">Formule utente</a>"
#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles_ui.py:242 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles_ui.py:242
msgid "Recipe source code (python)" msgid "Recipe source code (python)"
@ -2287,7 +2282,7 @@ msgstr ""
#: /home/kovid/work/calibre/src/calibre/gui2/main_ui.py:266 #: /home/kovid/work/calibre/src/calibre/gui2/main_ui.py:266
msgid "calibre" msgid "calibre"
msgstr "" msgstr "calibre"
#: /home/kovid/work/calibre/src/calibre/gui2/main_ui.py:267 #: /home/kovid/work/calibre/src/calibre/gui2/main_ui.py:267
msgid "" msgid ""
@ -2303,6 +2298,17 @@ msgid ""
"a><br /><br /><span style=\" font-weight:600;\">calibre</span>: %1 by <span " "a><br /><br /><span style=\" font-weight:600;\">calibre</span>: %1 by <span "
"style=\" font-weight:600;\">Kovid Goyal</span> %2<br />%3</p></body></html>" "style=\" font-weight:600;\">Kovid Goyal</span> %2<br />%3</p></body></html>"
msgstr "" msgstr ""
"<html><head><meta name=\"qrichtext\" content=\"1\" /><style type=\"text/css"
"\">\n"
"p, li { white-space: pre-wrap; }\n"
"</style></head><body style=\" font-family:'Sans Serif'; font-size:9pt; font-"
"weight:400; font-style:normal;\">\n"
"<p style=\" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-"
"right:0px; -qt-block-indent:0; text-indent:0px;\">Visitare <a href=\"http://calibre.kovidgoyal.net/user_manual\"><span style=\" text-decoration: "
"underline; color:#0000ff;\">calibre.kovidgoyal.net</span></a> per la "
"guida<br /><br /><span style=\" font-weight:600;\">calibre</span>: %1 di "
"<span style=\" font-weight:600;\">Kovid Goyal</span> %2<br />%3</p></body></"
"html>"
#: /home/kovid/work/calibre/src/calibre/gui2/main_ui.py:271 #: /home/kovid/work/calibre/src/calibre/gui2/main_ui.py:271
msgid "Advanced search" msgid "Advanced search"
@ -2535,8 +2541,7 @@ msgstr ""
msgid "" msgid ""
"The directory in which to store the downloaded feeds. Defaults to the " "The directory in which to store the downloaded feeds. Defaults to the "
"current directory." "current directory."
msgstr "" msgstr "La cartella in cui salvare i feed scaricati. Perdefinita: cartella corrente"
"La cartella in cui salvare i feed scaricati. Perdefinita: cartella corrente"
#: /home/kovid/work/calibre/src/calibre/web/feeds/main.py:55 #: /home/kovid/work/calibre/src/calibre/web/feeds/main.py:55
msgid "Dont show the progress bar" msgid "Dont show the progress bar"
@ -2587,8 +2592,7 @@ msgstr "\tLink falliti:"
#: /home/kovid/work/calibre/src/calibre/web/feeds/news.py:549 #: /home/kovid/work/calibre/src/calibre/web/feeds/news.py:549
msgid "Could not fetch article. Run with --debug to see the reason" msgid "Could not fetch article. Run with --debug to see the reason"
msgstr "" msgstr "Impossibile scaricare l'articolo. Eseguire con --debug per vedere la ragione"
"Impossibile scaricare l'articolo. Eseguire con --debug per vedere la ragione"
#: /home/kovid/work/calibre/src/calibre/web/feeds/news.py:571 #: /home/kovid/work/calibre/src/calibre/web/feeds/news.py:571
msgid "Got feeds from index page" msgid "Got feeds from index page"
@ -2732,40 +2736,3 @@ msgstr "Non scaricare i fogli di stile CSS"
msgid "Show detailed output information. Useful for debugging" msgid "Show detailed output information. Useful for debugging"
msgstr "Mostra un output dettagliato. Utile per il debugging" msgstr "Mostra un output dettagliato. Utile per il debugging"
#~ msgid ""
#~ "For help with writing advanced news recipes, please visit <a href="
#~ "\"http://libprs500.kovidgoyal.net/user_manual/news.html\">User Recipes</a>"
#~ msgstr ""
#~ "Per un aiuto su come scrivere formule di notizie avanzate, visitare <a "
#~ "href=\"http://libprs500.kovidgoyal.net/user_manual/news.html\">Formule "
#~ "utente</a>"
#~ msgid "libprs500"
#~ msgstr "libprs500"
#~ msgid ""
#~ "<html><head><meta name=\"qrichtext\" content=\"1\" /><style type=\"text/"
#~ "css\">\n"
#~ "p, li { white-space: pre-wrap; }\n"
#~ "</style></head><body style=\" font-family:'Sans Serif'; font-size:9pt; "
#~ "font-weight:400; font-style:normal;\">\n"
#~ "<p style=\" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-"
#~ "right:0px; -qt-block-indent:0; text-indent:0px;\">For help visit <a href="
#~ "\"http://libprs500.kovidgoyal.net/user_manual\"><span style=\" text-"
#~ "decoration: underline; color:#0000ff;\">libprs500.kovidgoyal.net</span></"
#~ "a><br /><br /><span style=\" font-weight:600;\">libprs500</span>: %1 by "
#~ "<span style=\" font-weight:600;\">Kovid Goyal</span> %2<br />%3</p></"
#~ "body></html>"
#~ msgstr ""
#~ "<html><head><meta name=\"qrichtext\" content=\"1\" /><style type=\"text/"
#~ "css\">\n"
#~ "p, li { white-space: pre-wrap; }\n"
#~ "</style></head><body style=\" font-family:'Sans Serif'; font-size:9pt; "
#~ "font-weight:400; font-style:normal;\">\n"
#~ "<p style=\" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-"
#~ "right:0px; -qt-block-indent:0; text-indent:0px;\">Visitare <a href="
#~ "\"http://libprs500.kovidgoyal.net/user_manual\"><span style=\" text-"
#~ "decoration: underline; color:#0000ff;\">libprs500.kovidgoyal.net</span></"
#~ "a> per la guida<br /><br /><span style=\" font-weight:600;\">libprs500</"
#~ "span>: %1 di <span style=\" font-weight:600;\">Kovid Goyal</span> %2<br />"
#~ "%3</p></body></html>"

45
src/calibre/web/feeds/news.py
View File

@ -10,7 +10,7 @@ __docformat__ = "restructuredtext en"
import logging, os, cStringIO, time, traceback, re, urlparse import logging, os, cStringIO, time, traceback, re, urlparse
from collections import defaultdict from collections import defaultdict
from calibre import browser, __appname__, iswindows from calibre import browser, __appname__, iswindows, LoggingInterface
from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
from calibre.ebooks.metadata.opf import OPFCreator from calibre.ebooks.metadata.opf import OPFCreator
from calibre.ebooks.lrf import entity_to_unicode from calibre.ebooks.lrf import entity_to_unicode
@ -24,7 +24,7 @@ from calibre.ebooks.lrf.web.profiles import FullContentProfile
from calibre.ptempfile import PersistentTemporaryFile from calibre.ptempfile import PersistentTemporaryFile
class BasicNewsRecipe(object): class BasicNewsRecipe(object, LoggingInterface):
''' '''
Abstract base class that contains logic needed in all feed fetchers. Abstract base class that contains logic needed in all feed fetchers.
''' '''
@ -363,17 +363,18 @@ class BasicNewsRecipe(object):
@param parser: Command line option parser. Used to intelligently merge options. @param parser: Command line option parser. Used to intelligently merge options.
@param progress_reporter: A Callable that takes two arguments: progress (a number between 0 and 1) and a string message. The message should be optional. @param progress_reporter: A Callable that takes two arguments: progress (a number between 0 and 1) and a string message. The message should be optional.
''' '''
LoggingInterface.__init__(self, logging.getLogger('feeds2disk'))
for attr in ('username', 'password', 'lrf', 'output_dir', 'verbose', 'debug', 'test'): for attr in ('username', 'password', 'lrf', 'output_dir', 'verbose', 'debug', 'test'):
setattr(self, attr, getattr(options, attr)) setattr(self, attr, getattr(options, attr))
self.output_dir = os.path.abspath(self.output_dir) self.output_dir = os.path.abspath(self.output_dir)
if options.test: if options.test:
self.max_articles_per_feed = 2 self.max_articles_per_feed = 2
self.simultaneous_downloads = min(4, self.simultaneous_downloads) self.simultaneous_downloads = min(4, self.simultaneous_downloads)
self.logger = logging.getLogger('feeds2disk')
if self.debug: if self.debug:
self.logger.setLevel(logging.DEBUG) logging.getLogger('feeds2disk').setLevel(logging.DEBUG)
self.verbose = True self.verbose = True
self.report_progress = progress_reporter self.report_progress = progress_reporter
@ -467,20 +468,20 @@ class BasicNewsRecipe(object):
self.cleanup() self.cleanup()
self.report_progress(1, _('Download finished')) self.report_progress(1, _('Download finished'))
if self.failed_downloads: if self.failed_downloads:
self.logger.warning(_('Failed to download the following articles:')) self.log_warning(_('Failed to download the following articles:'))
for feed, article, debug in self.failed_downloads: for feed, article, debug in self.failed_downloads:
self.logger.warning(article.title+_(' from ')+feed.title) self.log_warning(article.title+_(' from ')+feed.title)
self.logger.debug(article.url) self.log_debug(article.url)
self.logger.debug(debug) self.log_debug(debug)
if self.partial_failures: if self.partial_failures:
self.logger.warning(_('Failed to download parts of the following articles:')) self.log_warning(_('Failed to download parts of the following articles:'))
for feed, atitle, aurl, debug in self.partial_failures: for feed, atitle, aurl, debug in self.partial_failures:
self.logger.warning(atitle + _(' from ') + feed) self.log_warning(atitle + _(' from ') + feed)
self.logger.debug(aurl) self.log_debug(aurl)
self.logger.warning(_('\tFailed links:')) self.log_warning(_('\tFailed links:'))
for l, tb in debug: for l, tb in debug:
self.logger.warning(l) self.log_warning(l)
self.logger.debug(tb) self.log_debug(tb)
return res return res
def feeds2index(self, feeds): def feeds2index(self, feeds):
@ -645,8 +646,8 @@ class BasicNewsRecipe(object):
cu = self.get_cover_url() cu = self.get_cover_url()
except Exception, err: except Exception, err:
cu = None cu = None
self.logger.error(_('Could not download cover: %s')%str(err)) self.log_error(_('Could not download cover: %s')%str(err))
self.logger.debug(traceback.format_exc()) self.log_debug(traceback.format_exc())
if cu is not None: if cu is not None:
ext = cu.rpartition('.')[-1] ext = cu.rpartition('.')[-1]
ext = ext.lower() if ext else 'jpg' ext = ext.lower() if ext else 'jpg'
@ -726,7 +727,7 @@ class BasicNewsRecipe(object):
a = request.requestID[1] a = request.requestID[1]
article = request.article article = request.article
self.logger.debug(_('\nDownloaded article %s from %s\n%s')%(article.title, article.url, request.stream.getvalue().decode('utf-8', 'ignore'))) self.log_debug(_('\nDownloaded article %s from %s\n%s')%(article.title, article.url, request.stream.getvalue().decode('utf-8', 'ignore')))
article.orig_url = article.url article.orig_url = article.url
article.url = 'article_%d/index.html'%a article.url = 'article_%d/index.html'%a
article.downloaded = True article.downloaded = True
@ -738,11 +739,11 @@ class BasicNewsRecipe(object):
def error_in_article_download(self, request, traceback): def error_in_article_download(self, request, traceback):
self.jobs_done += 1 self.jobs_done += 1
self.logger.error(_('Failed to download article: %s from %s\n')%(request.article.title, request.article.url)) self.log_error(_('Failed to download article: %s from %s\n')%(request.article.title, request.article.url))
debug = request.stream.getvalue().decode('utf-8', 'ignore') debug = request.stream.getvalue().decode('utf-8', 'ignore')
self.logger.debug(debug) self.log_debug(debug)
self.logger.debug(traceback) self.log_debug(traceback)
self.logger.debug('\n') self.log_debug('\n')
self.report_progress(float(self.jobs_done)/len(self.jobs), _('Article download failed: %s')%request.article.title) self.report_progress(float(self.jobs_done)/len(self.jobs), _('Article download failed: %s')%request.article.title)
self.failed_downloads.append((request.feed, request.article, debug)) self.failed_downloads.append((request.feed, request.article, debug))

43
src/calibre/web/fetch/simple.py
View File

@ -11,7 +11,8 @@ import sys, socket, os, urlparse, codecs, logging, re, time, copy, urllib2, thre
from urllib import url2pathname from urllib import url2pathname
from httplib import responses from httplib import responses
from calibre import setup_cli_handlers, browser, sanitize_file_name, OptionParser, relpath from calibre import setup_cli_handlers, browser, sanitize_file_name, \
OptionParser, relpath, LoggingInterface
from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.chardet import xml_to_unicode
@ -47,7 +48,7 @@ def save_soup(soup, target):
f.close() f.close()
class RecursiveFetcher(object): class RecursiveFetcher(object, LoggingInterface):
LINK_FILTER = tuple(re.compile(i, re.IGNORECASE) for i in LINK_FILTER = tuple(re.compile(i, re.IGNORECASE) for i in
('.exe\s*$', '.mp3\s*$', '.ogg\s*$', '^\s*mailto:', '^\s*$')) ('.exe\s*$', '.mp3\s*$', '.ogg\s*$', '^\s*mailto:', '^\s*$'))
#ADBLOCK_FILTER = tuple(re.compile(i, re.IGNORECASE) for it in #ADBLOCK_FILTER = tuple(re.compile(i, re.IGNORECASE) for it in
@ -58,7 +59,7 @@ class RecursiveFetcher(object):
CSS_IMPORT_PATTERN = re.compile(r'\@import\s+url\((.*?)\)', re.IGNORECASE) CSS_IMPORT_PATTERN = re.compile(r'\@import\s+url\((.*?)\)', re.IGNORECASE)
def __init__(self, options, logger, image_map={}, css_map={}, job_info=None): def __init__(self, options, logger, image_map={}, css_map={}, job_info=None):
self.logger = logger LoggingInterface.__init__(self, logger)
self.base_dir = os.path.abspath(os.path.expanduser(options.dir)) self.base_dir = os.path.abspath(os.path.expanduser(options.dir))
if not os.path.exists(self.base_dir): if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir) os.makedirs(self.base_dir)
@ -130,7 +131,7 @@ class RecursiveFetcher(object):
def fetch_url(self, url): def fetch_url(self, url):
f = None f = None
self.logger.debug('Fetching %s', url) self.log_debug('Fetching %s', url)
delta = time.time() - self.last_fetch_at delta = time.time() - self.last_fetch_at
if delta < self.delay: if delta < self.delay:
time.sleep(delta) time.sleep(delta)
@ -140,7 +141,7 @@ class RecursiveFetcher(object):
if hasattr(err, 'code') and responses.has_key(err.code): if hasattr(err, 'code') and responses.has_key(err.code):
raise FetchError, responses[err.code] raise FetchError, responses[err.code]
if err.reason[0] == 104: # Connection reset by peer if err.reason[0] == 104: # Connection reset by peer
self.logger.debug('Connection reset by peer retrying in 1 second.') self.log_debug('Connection reset by peer retrying in 1 second.')
time.sleep(1) time.sleep(1)
f = self.browser.open(url) f = self.browser.open(url)
else: else:
@ -152,9 +153,9 @@ class RecursiveFetcher(object):
def start_fetch(self, url): def start_fetch(self, url):
soup = BeautifulSoup(u'<a href="'+url+'" />') soup = BeautifulSoup(u'<a href="'+url+'" />')
self.logger.info('Downloading') self.log_info('Downloading')
res = self.process_links(soup, url, 0, into_dir='') res = self.process_links(soup, url, 0, into_dir='')
self.logger.info('%s saved to %s', url, res) self.log_info('%s saved to %s', url, res)
return res return res
def is_link_ok(self, url): def is_link_ok(self, url):
@ -191,8 +192,8 @@ class RecursiveFetcher(object):
try: try:
f = self.fetch_url(iurl) f = self.fetch_url(iurl)
except Exception, err: except Exception, err:
self.logger.warning('Could not fetch stylesheet %s', iurl) self.log_warning('Could not fetch stylesheet %s', iurl)
self.logger.debug('Error: %s', str(err), exc_info=True) self.log_debug('Error: %s', str(err), exc_info=True)
continue continue
stylepath = os.path.join(diskpath, 'style'+str(c)+'.css') stylepath = os.path.join(diskpath, 'style'+str(c)+'.css')
with self.stylemap_lock: with self.stylemap_lock:
@ -214,8 +215,8 @@ class RecursiveFetcher(object):
try: try:
f = self.fetch_url(iurl) f = self.fetch_url(iurl)
except Exception, err: except Exception, err:
self.logger.warning('Could not fetch stylesheet %s', iurl) self.log_warning('Could not fetch stylesheet %s', iurl)
self.logger.debug('Error: %s', str(err), exc_info=True) self.log_debug('Error: %s', str(err), exc_info=True)
continue continue
c += 1 c += 1
stylepath = os.path.join(diskpath, 'style'+str(c)+'.css') stylepath = os.path.join(diskpath, 'style'+str(c)+'.css')
@ -234,7 +235,7 @@ class RecursiveFetcher(object):
for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')): for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')):
iurl, ext = tag['src'], os.path.splitext(tag['src'])[1] iurl, ext = tag['src'], os.path.splitext(tag['src'])[1]
#if not ext: #if not ext:
# self.logger.debug('Skipping extensionless image %s', iurl) # self.log_debug('Skipping extensionless image %s', iurl)
# continue # continue
if not urlparse.urlsplit(iurl).scheme: if not urlparse.urlsplit(iurl).scheme:
iurl = urlparse.urljoin(baseurl, iurl, False) iurl = urlparse.urljoin(baseurl, iurl, False)
@ -245,8 +246,8 @@ class RecursiveFetcher(object):
try: try:
f = self.fetch_url(iurl) f = self.fetch_url(iurl)
except Exception, err: except Exception, err:
self.logger.warning('Could not fetch image %s', iurl) self.log_warning('Could not fetch image %s', iurl)
self.logger.debug('Error: %s', str(err), exc_info=True) self.log_debug('Error: %s', str(err), exc_info=True)
continue continue
c += 1 c += 1
imgpath = os.path.join(diskpath, sanitize_file_name('img'+str(c)+ext)) imgpath = os.path.join(diskpath, sanitize_file_name('img'+str(c)+ext))
@ -263,10 +264,10 @@ class RecursiveFetcher(object):
if not parts.scheme: if not parts.scheme:
iurl = urlparse.urljoin(baseurl, iurl, False) iurl = urlparse.urljoin(baseurl, iurl, False)
if not self.is_link_ok(iurl): if not self.is_link_ok(iurl):
self.logger.debug('Skipping invalid link: %s', iurl) self.log_debug('Skipping invalid link: %s', iurl)
return None return None
if filter and not self.is_link_wanted(iurl): if filter and not self.is_link_wanted(iurl):
self.logger.debug('Filtered link: '+iurl) self.log_debug('Filtered link: '+iurl)
return None return None
return iurl return iurl
@ -330,7 +331,7 @@ class RecursiveFetcher(object):
dsrc = xml_to_unicode(dsrc, self.verbose)[0] dsrc = xml_to_unicode(dsrc, self.verbose)[0]
soup = self.get_soup(dsrc) soup = self.get_soup(dsrc)
self.logger.debug('Processing images...') self.log_debug('Processing images...')
self.process_images(soup, f.geturl()) self.process_images(soup, f.geturl())
if self.download_stylesheets: if self.download_stylesheets:
self.process_stylesheets(soup, f.geturl()) self.process_stylesheets(soup, f.geturl())
@ -339,11 +340,11 @@ class RecursiveFetcher(object):
self.downloaded_paths.append(res) self.downloaded_paths.append(res)
self.filemap[nurl] = res self.filemap[nurl] = res
if recursion_level < self.max_recursions: if recursion_level < self.max_recursions:
self.logger.debug('Processing links...') self.log_debug('Processing links...')
self.process_links(soup, iurl, recursion_level+1) self.process_links(soup, iurl, recursion_level+1)
else: else:
self.process_return_links(soup, iurl) self.process_return_links(soup, iurl)
self.logger.debug('Recursion limit reached. Skipping links in %s', iurl) self.log_debug('Recursion limit reached. Skipping links in %s', iurl)
if callable(self.postprocess_html_ext): if callable(self.postprocess_html_ext):
soup = self.postprocess_html_ext(soup, soup = self.postprocess_html_ext(soup,
@ -356,8 +357,8 @@ class RecursiveFetcher(object):
self.localize_link(tag, 'href', res) self.localize_link(tag, 'href', res)
except Exception, err: except Exception, err:
self.failed_links.append((iurl, traceback.format_exc())) self.failed_links.append((iurl, traceback.format_exc()))
self.logger.warning('Could not fetch link %s', iurl) self.log_warning('Could not fetch link %s', iurl)
self.logger.debug('Error: %s', str(err), exc_info=True) self.log_debug('Error: %s', str(err), exc_info=True)
finally: finally:
self.current_dir = diskpath self.current_dir = diskpath
self.files += 1 self.files += 1