Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-07 18:24:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

Sync to pluginize

Browse Source
This commit is contained in:
John Schember 2009-04-11 07:39:12 -04:00
commit 1edbd88e73
76 changed files with 24321 additions and 11312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
installer/windows/freeze.py
View File

@ -14,7 +14,20 @@ IMAGEMAGICK_DIR = 'C:\\ImageMagick'
FONTCONFIG_DIR = 'C:\\fontconfig'
VC90 = r'C:\VC90.CRT'
import sys, os, py2exe, shutil, zipfile, glob, re
# ModuleFinder can't handle runtime changes to __path__, but win32com uses them
import sys
import py2exe.mf as modulefinder
import win32com
for p in win32com.__path__[1:]:
modulefinder.AddPackagePath("win32com", p)
for extra in ["win32com.shell"]: #,"win32com.mapi"
__import__(extra)
m = sys.modules[extra]
for p in m.__path__[1:]:
modulefinder.AddPackagePath(extra, p)
import os, py2exe, shutil, zipfile, glob, re
from distutils.core import setup
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
sys.path.insert(0, BASE_DIR)

2
src/calibre/constants.py
View File

@ -2,7 +2,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
__appname__ = 'calibre'
__version__ = '0.5.5'
__version__ = '0.5.6'
__author__ = "Kovid Goyal <kovid@kovidgoyal.net>"
'''
Various run time constants.

5
src/calibre/customize/builtins.py
View File

@ -189,6 +189,7 @@ class ComicMetadataReader(MetadataReaderPlugin):
def get_metadata(self, stream, ftype):
if ftype == 'cbr':
from calibre.libunrar import extract_member as extract_first
extract_first
else:
from calibre.libunzip import extract_member as extract_first
from calibre.ebooks.metadata import MetaInformation
@ -267,12 +268,14 @@ from calibre.ebooks.epub.input import EPUBInput
from calibre.ebooks.mobi.input import MOBIInput
from calibre.ebooks.pdf.input import PDFInput
from calibre.ebooks.txt.input import TXTInput
from calibre.ebooks.html.input import HTMLInput
from calibre.ebooks.oeb.output import OEBOutput
from calibre.ebooks.txt.output import TXTOutput
from calibre.ebooks.pdf.output import PDFOutput
from calibre.customize.profiles import input_profiles, output_profiles
plugins = [HTML2ZIP, EPUBInput, MOBIInput, PDFInput, TXTInput, OEBOutput, TXTOutput, PDFOutput]
plugins = [HTML2ZIP, EPUBInput, MOBIInput, PDFInput, HTMLInput,
TXTInput, OEBOutput, TXTOutput, PDFOutput]
plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
x.__name__.endswith('MetadataReader')]
plugins += [x for x in list(locals().values()) if isinstance(x, type) and \

2
src/calibre/customize/conversion.py
View File

@ -163,9 +163,9 @@ class InputFormatPlugin(Plugin):
for x in os.listdir('.'):
shutil.rmtree(x) if os.path.isdir(x) else os.remove(x)
ret = self.convert(stream, options, file_ext,
log, accelerators)
if options.debug_input is not None:
options.debug_input = os.path.abspath(options.debug_input)
if not os.path.exists(options.debug_input):

4
src/calibre/devices/__init__.py
View File

@ -13,7 +13,9 @@ def devices():
from calibre.devices.kindle.driver import KINDLE
from calibre.devices.kindle.driver import KINDLE2
from calibre.devices.blackberry.driver import BLACKBERRY
return (PRS500, PRS505, PRS700, CYBOOKG3, KINDLE, KINDLE2, BLACKBERRY)
from calibre.devices.eb600.driver import EB600
return (PRS500, PRS505, PRS700, CYBOOKG3, KINDLE, KINDLE2,
BLACKBERRY, EB600)
import time

3
src/calibre/devices/cybookg3/driver.py
View File

@ -11,7 +11,6 @@ from calibre.ebooks.metadata import authors_to_string
from calibre.devices.errors import FreeSpaceError
from calibre.devices.usbms.driver import USBMS
import calibre.devices.cybookg3.t2b as t2b
from calibre.devices.errors import FreeSpaceError
class CYBOOKG3(USBMS):
# Ordered list of supported formats
@ -123,7 +122,7 @@ class CYBOOKG3(USBMS):
filepath, ext = os.path.splitext(path)
# Delete the ebook auxiliary files
# Delete the ebook auxiliary file
if os.path.exists(filepath + '.mbp'):
os.unlink(filepath + '.mbp')
if os.path.exists(filepath + '.dat'):

2
src/calibre/devices/eb600/__init__.py Executable file
View File

@ -0,0 +1,2 @@
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'

41
src/calibre/devices/eb600/driver.py Executable file
View File

@ -0,0 +1,41 @@
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
'''
Device driver for the Netronix EB600
'''
from calibre.devices.usbms.driver import USBMS
class EB600(USBMS):
# Ordered list of supported formats
FORMATS = ['epub', 'prc', 'chm', 'djvu', 'html', 'rtf', 'txt', 'pdf']
DRM_FORMATS = ['prc', 'mobi', 'html', 'pdf', 'txt']
VENDOR_ID = [0x1f85]
PRODUCT_ID = [0x1688]
BCD = [0x110]
VENDOR_NAME = 'NETRONIX'
WINDOWS_MAIN_MEM = 'EBOOK'
WINDOWS_CARD_MEM = 'EBOOK'
OSX_MAIN_MEM = 'EB600 Internal Storage Media'
OSX_CARD_MEM = 'EB600 Card Storage Media'
MAIN_MEMORY_VOLUME_LABEL = 'EB600 Main Memory'
STORAGE_CARD_VOLUME_LABEL = 'EB600 Storage Card'
EBOOK_DIR_MAIN = ''
EBOOK_DIR_CARD = ''
SUPPORTS_SUB_DIRS = True
def windows_sort_drives(self, drives):
main = drives['main']
card = drives['card']
if card and main and card < main:
drives['main'] = card
drives['card'] = main
return drives

13
src/calibre/devices/usbms/device.py
View File

@ -174,6 +174,14 @@ class Device(_Device):
return prefix
def windows_sort_drives(self, drives):
'''
Called to disambiguate main memory and storage card for devices that
do not distinguish between them on the basis of `WINDOWS_CARD_NAME`.
For e.g.: The EB600
'''
return drives
def open_windows(self):
time.sleep(6)
drives = {}
@ -188,11 +196,14 @@ class Device(_Device):
if 'main' in drives.keys() and 'card' in drives.keys():
break
drives = self.windows_sort_drives(drives)
self._main_prefix = drives.get('main')
self._card_prefix = drives.get('card')
if not self._main_prefix:
raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.') % self.__class__.__name__)
raise DeviceError(
_('Unable to detect the %s disk drive. Try rebooting.') %
self.__class__.__name__)
def get_osx_mountpoints(self, raw=None):
if raw is None:

33
src/calibre/ebooks/epub/from_html.py
View File

@ -36,7 +36,7 @@ import os, sys, cStringIO, logging, re, functools, shutil
from lxml.etree import XPath
from lxml import html, etree
from PyQt4.Qt import QApplication, QPixmap
from PyQt4.Qt import QApplication, QPixmap, Qt
from calibre.ebooks.html_old import Processor, merge_metadata, get_filelist,\
opf_traverse, create_metadata, rebase_toc, Link, parser
@ -50,7 +50,7 @@ from calibre.ebooks.epub.pages import add_page_map
from calibre.ebooks.epub.fonts import Rationalizer
from calibre.constants import preferred_encoding
from calibre.customize.ui import run_plugins_on_postprocess
from calibre import walk, CurrentDir, to_unicode
from calibre import walk, CurrentDir, to_unicode, fit_image
content = functools.partial(os.path.join, u'content')
@ -112,6 +112,31 @@ def find_html_index(files):
return f, os.path.splitext(f)[1].lower()[1:]
return html_files[-1], os.path.splitext(html_files[-1])[1].lower()[1:]
def rescale_images(imgdir, screen_size, log):
pwidth, pheight = screen_size
if QApplication.instance() is None:
QApplication([])
for f in os.listdir(imgdir):
path = os.path.join(imgdir, f)
if os.path.splitext(f)[1] in ('.css', '.js'):
continue
p = QPixmap()
p.load(path)
if p.isNull():
continue
width, height = p.width(), p.height()
scaled, new_width, new_height = fit_image(width, height, pwidth,
pheight)
if scaled:
log.info('Rescaling image: '+f)
p.scaled(new_width, new_height, Qt.IgnoreAspectRatio,
Qt.SmoothTransformation).save(path, 'JPEG')
class HTMLProcessor(Processor, Rationalizer):
def __init__(self, htmlfile, opts, tdir, resource_map, htmlfiles, stylesheets):
@ -482,6 +507,10 @@ def convert(htmlfile, opts, notification=None, create_epub=True,
if os.stat(ncx_path).st_size > opts.profile.flow_size:
logger.warn('NCX still larger than allowed size at %d bytes. Menu based Table of Contents may not work on device.'%os.stat(ncx_path).st_size)
if opts.profile.screen_size is not None:
rescale_images(os.path.join(tdir, 'content', 'resources'),
opts.profile.screen_size, logger)
if create_epub:
epub = initialize_container(opts.output)
epub.add_dir(tdir)

2
src/calibre/ebooks/html/__init__.py
View File

@ -17,7 +17,7 @@ def tostring(root, strip_comments=False, pretty_print=False):
root.set('xmlns', 'http://www.w3.org/1999/xhtml')
root.set('{http://www.w3.org/1999/xhtml}xlink', 'http://www.w3.org/1999/xlink')
for x in root.iter():
if x.tag.rpartition('}')[-1].lower() == 'svg':
if hasattr(x.tag, 'rpartition') and x.tag.rpartition('}')[-1].lower() == 'svg':
x.set('xmlns', 'http://www.w3.org/2000/svg')
ans = _tostring(root, encoding='utf-8', pretty_print=pretty_print)

91
src/calibre/ebooks/html/input.py
View File

@ -11,14 +11,12 @@ __docformat__ = 'restructuredtext en'
Input plugin for HTML or OPF ebooks.
'''
import os, re, sys, cStringIO
import os, re, sys
from urlparse import urlparse, urlunparse
from urllib import unquote
from calibre.customize.conversion import InputFormatPlugin
from calibre.ebooks.metadata.meta import get_metadata
from calibre.ebooks.metadata.opf2 import OPF, OPFCreator
from calibre.ebooks.metadata import MetaInformation
from calibre.ebooks.metadata.opf2 import OPFCreator
from calibre.ebooks.chardet import xml_to_unicode
from calibre.customize.conversion import OptionRecommendation
from calibre import unicode_path
@ -213,63 +211,12 @@ def traverse(path_to_html_file, max_levels=sys.maxint, verbose=0, encoding=None)
sys.setrecursionlimit(orec)
def opf_traverse(opf_reader, verbose=0, encoding=None):
'''
Return a list of :class:`HTMLFile` objects in the order specified by the
`<spine>` element of the OPF.
:param opf_reader: An :class:`calibre.ebooks.metadata.opf2.OPF` instance.
:param encoding: Specify character encoding of HTML files. If `None` it is
auto-detected.
'''
if not opf_reader.spine:
raise ValueError('OPF does not have a spine')
flat = []
for path in opf_reader.spine.items():
path = os.path.abspath(path)
if path not in flat:
flat.append(os.path.abspath(path))
for item in opf_reader.manifest:
if 'html' in item.mime_type:
path = os.path.abspath(item.path)
if path not in flat:
flat.append(path)
for i, path in enumerate(flat):
if not os.path.exists(path):
path = path.replace('&', '%26')
if os.path.exists(path):
flat[i] = path
for item in opf_reader.itermanifest():
item.set('href', item.get('href').replace('&', '%26'))
ans = []
for path in flat:
if os.path.exists(path):
ans.append(HTMLFile(path, 0, encoding, verbose))
else:
print 'WARNING: OPF spine item %s does not exist'%path
ans = [f for f in ans if not f.is_binary]
return ans
def search_for_opf(dir):
for f in os.listdir(dir):
if f.lower().endswith('.opf'):
return OPF(open(os.path.join(dir, f), 'rb'), dir)
def get_filelist(htmlfile, dir, opts, log):
'''
Build list of files referenced by html file or try to detect and use an
OPF file instead.
'''
print 'Building file list...'
opf = search_for_opf(dir)
filelist = None
if opf is not None:
try:
filelist = opf_traverse(opf, verbose=opts.verbose,
encoding=opts.input_encoding)
except:
pass
if not filelist:
log.info('Building file list...')
filelist = traverse(htmlfile, max_levels=int(opts.max_levels),
verbose=opts.verbose,
encoding=opts.input_encoding)\
@ -278,7 +225,7 @@ def get_filelist(htmlfile, dir, opts, log):
log.debug('\tFound files...')
for f in filelist:
log.debug('\t\t', f)
return opf, filelist
return filelist
class HTMLInput(InputFormatPlugin):
@ -309,34 +256,32 @@ class HTMLInput(InputFormatPlugin):
def convert(self, stream, opts, file_ext, log,
accelerators):
from calibre.ebooks.metadata.meta import get_metadata
basedir = os.getcwd()
if hasattr(stream, 'name'):
basedir = os.path.dirname(stream.name)
if file_ext == 'opf':
opf = OPF(stream, basedir)
filelist = opf_traverse(opf, verbose=opts.verbose,
encoding=opts.input_encoding)
mi = MetaInformation(opf)
opfpath = stream.name
else:
opf, filelist = get_filelist(stream.name, basedir, opts, log)
mi = MetaInformation(opf)
mi.smart_update(get_metadata(stream, 'html'))
filelist = get_filelist(stream.name, basedir, opts, log)
mi = get_metadata(stream, 'html')
mi = OPFCreator(os.getcwdu(), mi)
mi.guide = None
entries = [(f.path, 'application/xhtml+xml') for f in filelist]
mi.create_manifest(entries)
mi.create_spine([f.path for f in filelist])
tocbuf = cStringIO.StringIO()
mi.render(open('metadata.opf', 'wb'), tocbuf, 'toc.ncx')
toc = tocbuf.getvalue()
if toc:
open('toc.ncx', 'wb').write(toc)
mi.render(open('metadata.opf', 'wb'))
opfpath = os.path.abspath('metadata.opf')
from calibre.ebooks.conversion.plumber import create_oebbook
return create_oebbook(log, os.path.abspath('metadata.opf'))
oeb = create_oebbook(log, opfpath)
from calibre.ebooks.oeb.transforms.package import Package
Package(os.getcwdu())(oeb, opts)
return oeb

31
src/calibre/ebooks/metadata/__init__.py
View File

@ -11,9 +11,7 @@ from urllib import unquote, quote
from urlparse import urlparse
from calibre.constants import __version__ as VERSION
from calibre import relpath
from calibre.utils.config import OptionParser
def string_to_authors(raw):
raw = raw.replace('&&', u'\uffff')
@ -189,11 +187,11 @@ class MetaInformation(object):
'publisher', 'series', 'series_index', 'rating',
'isbn', 'tags', 'cover_data', 'application_id', 'guide',
'manifest', 'spine', 'toc', 'cover', 'language',
'book_producer', 'timestamp'):
'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc'):
if hasattr(mi, attr):
setattr(ans, attr, getattr(mi, attr))
def __init__(self, title, authors=[_('Unknown')]):
def __init__(self, title, authors=(_('Unknown'),)):
'''
@param title: title or ``_('Unknown')`` or a MetaInformation object
@param authors: List of strings or []
@ -204,9 +202,9 @@ class MetaInformation(object):
title = mi.title
authors = mi.authors
self.title = title
self.author = authors # Needed for backward compatibility
self.author = list(authors) if authors else []# Needed for backward compatibility
#: List of strings or []
self.authors = authors
self.authors = list(authors) if authors else []
self.tags = getattr(mi, 'tags', [])
#: mi.cover_data = (ext, data)
self.cover_data = getattr(mi, 'cover_data', (None, None))
@ -214,7 +212,7 @@ class MetaInformation(object):
for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher',
'series', 'series_index', 'rating', 'isbn', 'language',
'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
'book_producer', 'timestamp'
'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc'
):
setattr(self, x, getattr(mi, x, None))
@ -229,13 +227,13 @@ class MetaInformation(object):
if mi.authors and mi.authors[0] != _('Unknown'):
self.authors = mi.authors
for attr in ('author_sort', 'title_sort', 'category',
'publisher', 'series', 'series_index', 'rating',
'isbn', 'application_id', 'manifest', 'spine', 'toc',
'cover', 'language', 'guide', 'book_producer',
'timestamp'):
val = getattr(mi, attr, None)
'timestamp', 'lccn', 'lcc', 'ddc'):
if hasattr(mi, attr):
val = getattr(mi, attr)
if val is not None:
setattr(self, attr, val)
@ -293,6 +291,13 @@ class MetaInformation(object):
fmt('Rating', self.rating)
if self.timestamp is not None:
fmt('Timestamp', self.timestamp.isoformat(' '))
if self.lccn:
fmt('LCCN', unicode(self.lccn))
if self.lcc:
fmt('LCC', unicode(self.lcc))
if self.ddc:
fmt('DDC', unicode(self.ddc))
return u'\n'.join(ans)
def to_html(self):
@ -302,6 +307,12 @@ class MetaInformation(object):
ans += [(_('Producer'), unicode(self.book_producer))]
ans += [(_('Comments'), unicode(self.comments))]
ans += [('ISBN', unicode(self.isbn))]
if self.lccn:
ans += [('LCCN', unicode(self.lccn))]
if self.lcc:
ans += [('LCC', unicode(self.lcc))]
if self.ddc:
ans += [('DDC', unicode(self.ddc))]
ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
if self.series:
ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]

8
src/calibre/ebooks/mobi/reader.py
View File

@ -59,7 +59,8 @@ class EXTHHeader(object):
elif id == 502:
# last update time
pass
elif id == 503 and (not title or title == _('Unknown')):
elif id == 503: # Long title
if not title or title == _('Unknown'):
title = content
#else:
# print 'unknown record', id, repr(content)
@ -87,6 +88,8 @@ class EXTHHeader(object):
content, '%Y-%m-%d',).date()
except:
pass
elif id == 108:
pass # Producer
#else:
# print 'unhandled metadata record', id, repr(content)
@ -522,7 +525,8 @@ class MobiReader(object):
else:
raise MobiError('Unknown compression algorithm: %s'%repr(self.book_header.compression_type))
if self.book_header.ancient and '<html' not in self.mobi_html[:300].lower():
self.mobi_html = self.mobi_html.replace('\r ', '\n\n').replace('\0', '')
self.mobi_html = self.mobi_html.replace('\r ', '\n\n ')
self.mobi_html = self.mobi_html.replace('\0', '')
return processed_records

2
src/calibre/ebooks/oeb/reader.py
View File

@ -573,7 +573,7 @@ class OEBReader(object):
item = self._find_ncx(opf)
self._toc_from_opf(opf, item)
self._pages_from_opf(opf, item)
self._ensure_cover_image()
#self._ensure_cover_image()
def main(argv=sys.argv):

68
src/calibre/ebooks/oeb/transforms/package.py
View File

@ -6,13 +6,14 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os
import os, re
from urllib import unquote as urlunquote
from functools import partial
from lxml import etree
import cssutils
from calibre import sanitize_file_name
from calibre.constants import islinux
from calibre.ebooks.oeb.base import OEB_DOCS, urlnormalize, urldefrag, \
rewrite_links
@ -36,15 +37,21 @@ class Package(object):
self.new_base_path = os.path.abspath(base)
def rewrite_links_in(self, item):
base = os.path.join(self.new_base_path, *item.href.split('/'))
old_href = item.old_href.split('#')[0]
new_href = item.href.split('#')[0]
base = os.path.join(self.old_base_path, *old_href.split('/'))
base = os.path.dirname(base)
self.log.debug('\tRewriting links in', base+'/'+
item.href.rpartition('/')[-1])
new_base = os.path.join(self.new_base_path, *new_href.split('/'))
new_base = os.path.dirname(new_base)
if etree.iselement(item.data):
self.rewrite_links_in_xml(item.data, base)
self.rewrite_links_in_xml(item.data, base, new_base)
elif hasattr(item.data, 'cssText'):
self.rewrite_links_in_css(item.data, base)
self.rewrite_links_in_css(item.data, base, new_base)
def link_replacer(self, link_, base=''):
def link_replacer(self, link_, base='', new_base=''):
link = urlnormalize(link_)
link, frag = urldefrag(link)
link = urlunquote(link).replace('/', os.sep)
@ -55,20 +62,33 @@ class Package(object):
link = link.lower()
if link not in self.map:
return link_
nlink = os.path.relpath(self.map[link], base)
nlink = os.path.relpath(self.map[link], new_base)
if frag:
nlink = '#'.join(nlink, frag)
nlink = '#'.join((nlink, frag))
return nlink.replace(os.sep, '/')
def rewrite_links_in_css(self, sheet, base):
repl = partial(self.link_replacer, base=base)
def rewrite_links_in_css(self, sheet, base, new_base):
repl = partial(self.link_replacer, base=base, new_base=new_base)
cssutils.replaceUrls(sheet, repl)
def rewrite_links_in_xml(self, root, base):
repl = partial(self.link_replacer, base=base)
def rewrite_links_in_xml(self, root, base, new_base):
repl = partial(self.link_replacer, base=base, new_base=new_base)
rewrite_links(root, repl)
def move_manifest_item(self, item):
def uniqify_name(self, new_href, hrefs):
c = 0
while new_href in hrefs:
c += 1
parts = new_href.split('/')
name, ext = os.path.splitext(parts[-1])
name = re.sub(r'_\d+$', '', name)
name += '_%d'%c
parts[-1] = name + ext
new_href = '/'.join(parts)
return new_href
def move_manifest_item(self, item, hrefs):
item.data # Make sure the data has been loaded and cached
old_abspath = os.path.join(self.old_base_path,
*(urldefrag(item.href)[0].split('/')))
@ -79,11 +99,17 @@ class Package(object):
new_href = 'content/'
elif item.href.lower().endswith('.ncx'):
new_href = ''
new_href += bname
new_href += sanitize_file_name(bname)
if new_href in hrefs:
new_href = self.uniqify_name(new_href, hrefs)
hrefs.add(new_href)
new_abspath = os.path.join(self.new_base_path, *new_href.split('/'))
new_abspath = os.path.abspath(new_abspath)
item.old_href = self.oeb.manifest.hrefs.pop(item.href).href
item.href = new_href
self.oeb.manifest.hrefs[item.href] = item
if not islinux:
old_abspath, new_abspath = old_abspath.lower(), new_abspath.lower()
if old_abspath != new_abspath:
@ -91,25 +117,33 @@ class Package(object):
def rewrite_links_in_toc(self, toc):
if toc.href:
toc.href = self.link_replacer(toc.href, base=self.new_base_path)
toc.href = self.link_replacer(toc.href, base=self.old_base_path,
new_base=self.new_base_path)
for x in toc:
self.rewrite_links_in_toc(x)
def __call__(self, oeb, context):
self.map = {}
self.log = self.oeb.log
self.log = oeb.log
self.oeb = oeb
self.old_base_path = os.path.abspath(oeb.container.rootdir)
hrefs = set([])
for item in self.oeb.manifest:
self.move_manifest_item(item)
self.move_manifest_item(item, hrefs)
self.log.debug('Rewriting links in OEB documents...')
for item in self.oeb.manifest:
self.rewrite_links_in(item)
if getattr(oeb.toc, 'nodes', False):
self.log.debug('Rewriting links in TOC...')
self.rewrite_links_in_toc(oeb.toc)
if hasattr(oeb, 'guide'):
self.log.debug('Rewriting links in guide...')
for ref in oeb.guide.values():
ref.href = self.link_replacer(ref.href, base=self.new_base_path)
ref.href = self.link_replacer(ref.href,
base=self.old_base_path,
new_base=self.new_base_path)

3
src/calibre/ebooks/oeb/writer.py
View File

@ -48,7 +48,8 @@ class OEBWriter(object):
pretty_print=pretty_print)
def __call__(self, oeb, path):
"""Read the book in the :class:`OEBBook` object :param:`oeb` to a file
"""
Read the book in the :class:`OEBBook` object :param:`oeb` to a file
at :param:`path`.
"""
version = int(self.version[0])

2
src/calibre/gui2/__init__.py
View File

@ -466,5 +466,3 @@ class Application(QApplication):
self.translator.loadFromData(data)
self.installTranslator(self.translator)

5
src/calibre/gui2/dialogs/config.py
View File

@ -199,7 +199,7 @@ class EmailAccounts(QAbstractTableModel):
return (account, self.accounts[account])
if role == Qt.ToolTipRole:
return self.tooltips[col]
if role == Qt.DisplayRole:
if role in [Qt.DisplayRole, Qt.EditRole]:
if col == 0:
return QVariant(account)
if col == 1:
@ -397,6 +397,9 @@ class ConfigDialog(QDialog, Ui_Dialog):
self.separate_cover_flow.setChecked(config['separate_cover_flow'])
self.setup_email_page()
self.category_view.setCurrentIndex(self.category_view.model().index(0))
self.delete_news.setEnabled(bool(self.sync_news.isChecked()))
self.connect(self.sync_news, SIGNAL('toggled(bool)'),
self.delete_news.setEnabled)
def setup_email_page(self):
opts = smtp_prefs().parse()

2
src/calibre/gui2/dialogs/config.ui
View File

@ -371,7 +371,7 @@
<item>
<widget class="QCheckBox" name="delete_news">
<property name="text">
<string>&amp;Delete news from library when it is sent to reader</string>
<string>&amp;Delete news from library when it is automatically sent to reader</string>
</property>
</widget>
</item>

2
src/calibre/gui2/dialogs/metadata_single.ui
View File

@ -324,7 +324,7 @@
<string>Book </string>
</property>
<property name="minimum">
<number>1</number>
<number>0</number>
</property>
<property name="maximum">
<number>10000</number>

3
src/calibre/gui2/dialogs/scheduler.py
View File

@ -82,7 +82,8 @@ def load_recipes():
recipes = []
for r in config.get('scheduled_recipes', []):
r = Recipe().unpickle(r)
if r.builtin and not str(r.id).startswith('recipe_'):
if r.builtin and \
(not str(r.id).startswith('recipe_') or not str(r.id) in recipe_modules):
continue
recipes.append(r)
return recipes

BIN
src/calibre/gui2/images/news/azstarnet.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 360 B

BIN
src/calibre/gui2/images/news/corriere_della_sera_en.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 524 B

BIN
src/calibre/gui2/images/news/corriere_della_sera_it.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 524 B

BIN
src/calibre/gui2/images/news/msdnmag_en.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 694 B

31
src/calibre/gui2/lrf_renderer/config.ui
View File

@ -1,3 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>ViewerConfig</class>
<widget class="QDialog" name="ViewerConfig">
@ -5,8 +6,8 @@
<rect>
<x>0</x>
<y>0</y>
<width>281</width>
<height>214</height>
<width>373</width>
<height>264</height>
</rect>
</property>
<property name="windowTitle">
@ -16,8 +17,8 @@
<iconset resource="../images.qrc">
<normaloff>:/images/config.svg</normaloff>:/images/config.svg</iconset>
</property>
<layout class="QGridLayout" >
<item row="0" column="0" colspan="2" >
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="0">
<widget class="QCheckBox" name="white_background">
<property name="text">
<string>Use white background</string>
@ -34,17 +35,7 @@
</property>
</widget>
</item>
<item row="3" column="1" >
<widget class="QDialogButtonBox" name="buttonBox" >
<property name="orientation" >
<enum>Qt::Horizontal</enum>
</property>
<property name="standardButtons" >
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
<item row="2" column="0" colspan="2" >
<item row="2" column="0">
<widget class="QLabel" name="label">
<property name="frameShape">
<enum>QFrame::Box</enum>
@ -63,6 +54,16 @@
</property>
</widget>
</item>
<item row="3" column="0">
<widget class="QDialogButtonBox" name="buttonBox">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
</layout>
</widget>
<resources>

30
src/calibre/gui2/main.py
View File

@ -1110,27 +1110,30 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
return
self._view_file(job.result)
def _view_file(self, name):
def _launch_viewer(self, name=None, viewer='ebook-viewer', internal=True):
self.setCursor(Qt.BusyCursor)
try:
ext = os.path.splitext(name)[1].upper().replace('.', '')
if ext in config['internally_viewed_formats']:
if ext == 'LRF':
args = ['lrfviewer', name]
self.job_manager.server.run_free_job('lrfviewer',
kwdargs=dict(args=args))
else:
args = ['ebook-viewer', name]
if isosx:
if internal:
args = [viewer]
if isosx and 'ebook' in viewer:
args.append('--raise-window')
self.job_manager.server.run_free_job('ebook-viewer',
if name is not None:
args.append(name)
self.job_manager.server.run_free_job(viewer,
kwdargs=dict(args=args))
else:
QDesktopServices.openUrl(QUrl('file:'+name))#launch(name)
QDesktopServices.openUrl(QUrl.fromLocalFile(name))#launch(name)
time.sleep(5) # User feedback
finally:
self.unsetCursor()
def _view_file(self, name):
ext = os.path.splitext(name)[1].upper().replace('.', '')
viewer = 'lrfviewer' if ext == 'LRF' else 'ebook-viewer'
internal = ext in config['internally_viewed_formats']
self._launch_viewer(name, viewer, internal)
def view_specific_format(self, triggered):
rows = self.library_view.selectionModel().selectedRows()
if not rows or len(rows) == 0:
@ -1165,8 +1168,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
rows = self.current_view().selectionModel().selectedRows()
if self.current_view() is self.library_view:
if not rows or len(rows) == 0:
d = error_dialog(self, _('Cannot view'), _('No book selected'))
d.exec_()
self._launch_viewer()
return
row = rows[0].row()

1
src/calibre/library/cli.py
View File

@ -15,6 +15,7 @@ from calibre import terminal_controller, preferred_encoding
from calibre.utils.config import OptionParser, prefs
try:
from calibre.utils.single_qt_application import send_message
send_message
except:
send_message = None
from calibre.ebooks.metadata.meta import get_metadata

22
src/calibre/library/database2.py
View File

@ -31,6 +31,21 @@ from calibre.customize.ui import run_plugins_on_import
from calibre import sanitize_file_name
from calibre.ebooks import BOOK_EXTENSIONS
if iswindows:
import calibre.utils.winshell as winshell
def delete_file(path):
try:
winshell.delete_file(path, silent=True, no_confirm=True)
except:
os.remove(path)
def delete_tree(path):
try:
winshell.delete_file(path, silent=True, no_confirm=True)
except:
shutil.rmtree(path)
copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'publisher':3, 'rating':4, 'timestamp':5,
@ -499,7 +514,7 @@ class LibraryDatabase2(LibraryDatabase):
def rmtree(self, path):
if not self.normpath(self.library_path).startswith(self.normpath(path)):
shutil.rmtree(path)
delete_tree(path)
def normpath(self, path):
path = os.path.abspath(os.path.realpath(path))
@ -745,6 +760,9 @@ class LibraryDatabase2(LibraryDatabase):
path = os.path.join(self.library_path, self.path(id, index_is_id=True))
self.data.remove(id)
if os.path.exists(path):
if iswindows:
winshell.delete_file(path, no_confirm=True, silent=True)
else:
self.rmtree(path)
parent = os.path.dirname(path)
if len(os.listdir(parent)) == 0:
@ -764,7 +782,7 @@ class LibraryDatabase2(LibraryDatabase):
ext = ('.' + format.lower()) if format else ''
path = os.path.join(path, name+ext)
try:
os.remove(path)
delete_file(path)
except:
traceback.print_exc()
self.conn.execute('DELETE FROM data WHERE book=? AND format=?', (id, format.upper()))

2
src/calibre/manual/faq.rst
View File

@ -105,7 +105,7 @@ Device Integration
What devices does |app| support?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
At the moment |app| has full support for the SONY PRS 500/505/700, Cybook Gen 3, Amazon Kindle 1 and 2 as well as the iPhone. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk.
At the moment |app| has full support for the SONY PRS 500/505/700, Cybook Gen 3, Amazon Kindle 1/2, Netronix EB600 and the iPhone. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk.
I used |app| to transfer some books to my reader, and now the SONY software hangs every time I connect the reader?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

3
src/calibre/startup.py
View File

@ -32,7 +32,8 @@ if not _run_once:
lang = prefs['language']
if lang is not None:
return lang
lang = locale.getdefaultlocale()[0]
lang = locale.getdefaultlocale(['LANGUAGE', 'LC_ALL', 'LC_CTYPE',
'LC_MESSAGES', 'LANG'])[0]
if lang is None and os.environ.has_key('LANG'): # Needed for OS X
try:
lang = os.environ['LANG']

1
src/calibre/trac/plugins/download.py
View File

@ -38,6 +38,7 @@ def get_linux_data(version='1.0.0'):
('exherbo', 'Exherbo'),
('foresight', 'Foresight 2.1'),
('ubuntu', 'Ubuntu Jaunty Jackalope'),
('linux_mint', 'Linux Mint Gloria'),
]:
data['supported'].append(CoolDistro(name, title,
prefix='http://calibre.kovidgoyal.net'))

BIN
src/calibre/trac/plugins/htdocs/images/linux_mint_logo.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 8.9 KiB

1240
src/calibre/translations/ar.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/bg.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/ca.po
View File

File diff suppressed because it is too large Load Diff

1222
src/calibre/translations/cs.po
View File

File diff suppressed because it is too large Load Diff

1465
src/calibre/translations/da.po
View File

File diff suppressed because it is too large Load Diff

1260
src/calibre/translations/de.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/el.po
View File

File diff suppressed because it is too large Load Diff

1258
src/calibre/translations/es.po
View File

File diff suppressed because it is too large Load Diff

1258
src/calibre/translations/fr.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/gl.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/he.po
View File

File diff suppressed because it is too large Load Diff

2695
src/calibre/translations/hr.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/hu.po
View File

File diff suppressed because it is too large Load Diff

1281
src/calibre/translations/it.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/ja.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/nb.po
View File

File diff suppressed because it is too large Load Diff

1260
src/calibre/translations/nds.po
View File

File diff suppressed because it is too large Load Diff

1222
src/calibre/translations/nl.po
View File

File diff suppressed because it is too large Load Diff

1198
src/calibre/translations/pl.po
View File

File diff suppressed because it is too large Load Diff

1278
src/calibre/translations/pt.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/ro.po
View File

File diff suppressed because it is too large Load Diff

1411
src/calibre/translations/ru.po
View File

File diff suppressed because it is too large Load Diff

1231
src/calibre/translations/sk.po
View File

File diff suppressed because it is too large Load Diff

1232
src/calibre/translations/sl.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/sv.po
View File

File diff suppressed because it is too large Load Diff

1189
src/calibre/translations/te.po
View File

File diff suppressed because it is too large Load Diff

400
src/calibre/utils/winshell.py Normal file
View File

@ -0,0 +1,400 @@
"""winshell - convenience functions to access Windows shell functionality
Certain aspects of the Windows user interface are grouped by
Microsoft as Shell functions. These include the Desktop, shortcut
icons, special folders (such as My Documents) and a few other things.
These are mostly available via the shell module of the win32all
extensions, but whenever I need to use them, I've forgotten the
various constants and so on.
Several of the shell items have two variants: personal and common,
or User and All Users. These refer to systems with profiles in use:
anything from NT upwards, and 9x with Profiles turned on. Where
relevant, the Personal/User version refers to that owned by the
logged-on user and visible only to that user; the Common/All Users
version refers to that maintained by an Administrator and visible
to all users of the system.
(c) Tim Golden <winshell@timgolden.me.uk> 25th November 2003
Licensed under the (GPL-compatible) MIT License:
http://www.opensource.org/licenses/mit-license.php
9th Nov 2005 0.2 . License changed to MIT
. Added functionality using SHFileOperation
25th Nov 2003 0.1 . Initial release by Tim Golden
"""
__VERSION__ = "0.2"
import os
from win32com import storagecon
from win32com.shell import shell, shellcon
import pythoncom
class x_winshell (Exception):
pass
#
# Although this can be done in one call, Win9x didn't
# support it, so I added this workaround.
#
def get_path (folder_id):
return shell.SHGetPathFromIDList (shell.SHGetSpecialFolderLocation (0, folder_id))
def desktop (common=0):
"What folder is equivalent to the current desktop?"
return get_path ((shellcon.CSIDL_DESKTOP, shellcon.CSIDL_COMMON_DESKTOPDIRECTORY)[common])
def common_desktop ():
#
# Only here because already used in code
#
return desktop (common=1)
def application_data (common=0):
"What folder holds application configuration files?"
return get_path ((shellcon.CSIDL_APPDATA, shellcon.CSIDL_COMMON_APPDATA)[common])
def favourites (common=0):
"What folder holds the Explorer favourites shortcuts?"
return get_path ((shellcon.CSIDL_FAVORITES, shellcon.CSIDL_COMMON_FAVORITES)[common])
bookmarks = favourites
def start_menu (common=0):
"What folder holds the Start Menu shortcuts?"
return get_path ((shellcon.CSIDL_STARTMENU, shellcon.CSIDL_COMMON_STARTMENU)[common])
def programs (common=0):
"What folder holds the Programs shortcuts (from the Start Menu)?"
return get_path ((shellcon.CSIDL_PROGRAMS, shellcon.CSIDL_COMMON_PROGRAMS)[common])
def startup (common=0):
"What folder holds the Startup shortcuts (from the Start Menu)?"
return get_path ((shellcon.CSIDL_STARTUP, shellcon.CSIDL_COMMON_STARTUP)[common])
def personal_folder ():
"What folder holds the My Documents files?"
return get_path (shellcon.CSIDL_PERSONAL)
my_documents = personal_folder
def recent ():
"What folder holds the Documents shortcuts (from the Start Menu)?"
return get_path (shellcon.CSIDL_RECENT)
def sendto ():
"What folder holds the SendTo shortcuts (from the Context Menu)?"
return get_path (shellcon.CSIDL_SENDTO)
#
# Internally abstracted function to handle one
# of several shell-based file manipulation
# routines. Not all the possible parameters
# are covered which might be passed to the
# underlying SHFileOperation API call, but
# only those which seemed useful to me at
# the time.
#
def _file_operation (
operation,
source_path,
target_path=None,
allow_undo=True,
no_confirm=False,
rename_on_collision=True,
silent=False,
hWnd=None
):
#
# At present the Python wrapper around SHFileOperation doesn't
# allow lists of files. Hopefully it will at some point, so
# take account of it here.
# If you pass this shell function a "/"-separated path with
# a wildcard, eg c:/temp/*.tmp, it gets confused. It's ok
# with a backslash, so convert here.
#
source_path = source_path or ""
if isinstance (source_path, basestring):
source_path = os.path.abspath (source_path)
else:
source_path = [os.path.abspath (i) for i in source_path]
target_path = target_path or ""
if isinstance (target_path, basestring):
target_path = os.path.abspath (target_path)
else:
target_path = [os.path.abspath (i) for i in target_path]
flags = 0
if allow_undo: flags |= shellcon.FOF_ALLOWUNDO
if no_confirm: flags |= shellcon.FOF_NOCONFIRMATION
if rename_on_collision: flags |= shellcon.FOF_RENAMEONCOLLISION
if silent: flags |= shellcon.FOF_SILENT
result, n_aborted = shell.SHFileOperation (
(hWnd or 0, operation, source_path, target_path, flags, None, None)
)
if result <> 0:
raise x_winshell, result
elif n_aborted:
raise x_winshell, "%d operations were aborted by the user" % n_aborted
def copy_file (
source_path,
target_path,
allow_undo=True,
no_confirm=False,
rename_on_collision=True,
silent=False,
hWnd=None
):
"""Perform a shell-based file copy. Copying in
this way allows the possibility of undo, auto-renaming,
and showing the "flying file" animation during the copy.
The default options allow for undo, don't automatically
clobber on a name clash, automatically rename on collision
and display the animation.
"""
_file_operation (
shellcon.FO_COPY,
source_path,
target_path,
allow_undo,
no_confirm,
rename_on_collision,
silent,
hWnd
)
def move_file (
source_path,
target_path,
allow_undo=True,
no_confirm=False,
rename_on_collision=True,
silent=False,
hWnd=None
):
"""Perform a shell-based file move. Moving in
this way allows the possibility of undo, auto-renaming,
and showing the "flying file" animation during the copy.
The default options allow for undo, don't automatically
clobber on a name clash, automatically rename on collision
and display the animation.
"""
_file_operation (
shellcon.FO_MOVE,
source_path,
target_path,
allow_undo,
no_confirm,
rename_on_collision,
silent,
hWnd
)
def rename_file (
source_path,
target_path,
allow_undo=True,
no_confirm=False,
rename_on_collision=True,
silent=False,
hWnd=None
):
"""Perform a shell-based file rename. Renaming in
this way allows the possibility of undo, auto-renaming,
and showing the "flying file" animation during the copy.
The default options allow for undo, don't automatically
clobber on a name clash, automatically rename on collision
and display the animation.
"""
_file_operation (
shellcon.FO_RENAME,
source_path,
target_path,
allow_undo,
no_confirm,
rename_on_collision,
silent,
hWnd
)
def delete_file (
source_path,
allow_undo=True,
no_confirm=False,
rename_on_collision=True,
silent=False,
hWnd=None
):
"""Perform a shell-based file delete. Deleting in
this way uses the system recycle bin, allows the
possibility of undo, and showing the "flying file"
animation during the delete.
The default options allow for undo, don't automatically
clobber on a name clash, automatically rename on collision
and display the animation.
"""
_file_operation (
shellcon.FO_DELETE,
source_path,
None,
allow_undo,
no_confirm,
rename_on_collision,
silent,
hWnd
)
def CreateShortcut (Path, Target, Arguments = "", StartIn = "", Icon = ("",0), Description = ""):
"""Create a Windows shortcut:
Path - As what file should the shortcut be created?
Target - What command should the desktop use?
Arguments - What arguments should be supplied to the command?
StartIn - What folder should the command start in?
Icon - (filename, index) What icon should be used for the shortcut?
Description - What description should the shortcut be given?
eg
CreateShortcut (
Path=os.path.join (desktop (), "PythonI.lnk"),
Target=r"c:\python\python.exe",
Icon=(r"c:\python\python.exe", 0),
Description="Python Interpreter"
)
"""
sh = pythoncom.CoCreateInstance (
shell.CLSID_ShellLink,
None,
pythoncom.CLSCTX_INPROC_SERVER,
shell.IID_IShellLink
)
sh.SetPath (Target)
sh.SetDescription (Description)
sh.SetArguments (Arguments)
sh.SetWorkingDirectory (StartIn)
sh.SetIconLocation (Icon[0], Icon[1])
persist = sh.QueryInterface (pythoncom.IID_IPersistFile)
persist.Save (Path, 1)
#
# Constants for structured storage
#
# These come from ObjIdl.h
FMTID_USER_DEFINED_PROPERTIES = "{F29F85E0-4FF9-1068-AB91-08002B27B3D9}"
FMTID_CUSTOM_DEFINED_PROPERTIES = "{D5CDD505-2E9C-101B-9397-08002B2CF9AE}"
PIDSI_TITLE = 0x00000002
PIDSI_SUBJECT = 0x00000003
PIDSI_AUTHOR = 0x00000004
PIDSI_CREATE_DTM = 0x0000000c
PIDSI_KEYWORDS = 0x00000005
PIDSI_COMMENTS = 0x00000006
PIDSI_TEMPLATE = 0x00000007
PIDSI_LASTAUTHOR = 0x00000008
PIDSI_REVNUMBER = 0x00000009
PIDSI_EDITTIME = 0x0000000a
PIDSI_LASTPRINTED = 0x0000000b
PIDSI_LASTSAVE_DTM = 0x0000000d
PIDSI_PAGECOUNT = 0x0000000e
PIDSI_WORDCOUNT = 0x0000000f
PIDSI_CHARCOUNT = 0x00000010
PIDSI_THUMBNAIL = 0x00000011
PIDSI_APPNAME = 0x00000012
PROPERTIES = (
PIDSI_TITLE,
PIDSI_SUBJECT,
PIDSI_AUTHOR,
PIDSI_CREATE_DTM,
PIDSI_KEYWORDS,
PIDSI_COMMENTS,
PIDSI_TEMPLATE,
PIDSI_LASTAUTHOR,
PIDSI_EDITTIME,
PIDSI_LASTPRINTED,
PIDSI_LASTSAVE_DTM,
PIDSI_PAGECOUNT,
PIDSI_WORDCOUNT,
PIDSI_CHARCOUNT,
PIDSI_APPNAME
)
#
# This was taken from someone else's example,
# but I can't find where. If you know, please
# tell me so I can give due credit.
#
def structured_storage (filename):
"""Pick out info from MS documents with embedded
structured storage (typically MS Word docs etc.)
Returns a dictionary of information found
"""
if not pythoncom.StgIsStorageFile (filename):
return {}
flags = storagecon.STGM_READ | storagecon.STGM_SHARE_EXCLUSIVE
storage = pythoncom.StgOpenStorage (filename, None, flags)
try:
properties_storage = storage.QueryInterface (pythoncom.IID_IPropertySetStorage)
except pythoncom.com_error:
return {}
property_sheet = properties_storage.Open (FMTID_USER_DEFINED_PROPERTIES)
try:
data = property_sheet.ReadMultiple (PROPERTIES)
finally:
property_sheet = None
title, subject, author, created_on, keywords, comments, template_used, \
updated_by, edited_on, printed_on, saved_on, \
n_pages, n_words, n_characters, \
application = data
result = {}
if title: result['title'] = title
if subject: result['subject'] = subject
if author: result['author'] = author
if created_on: result['created_on'] = created_on
if keywords: result['keywords'] = keywords
if comments: result['comments'] = comments
if template_used: result['template_used'] = template_used
if updated_by: result['updated_by'] = updated_by
if edited_on: result['edited_on'] = edited_on
if printed_on: result['printed_on'] = printed_on
if saved_on: result['saved_on'] = saved_on
if n_pages: result['n_pages'] = n_pages
if n_words: result['n_words'] = n_words
if n_characters: result['n_characters'] = n_characters
if application: result['application'] = application
return result
if __name__ == '__main__':
try:
print 'Desktop =>', desktop ()
print 'Common Desktop =>', desktop (1)
print 'Application Data =>', application_data ()
print 'Common Application Data =>', application_data (1)
print 'Bookmarks =>', bookmarks ()
print 'Common Bookmarks =>', bookmarks (1)
print 'Start Menu =>', start_menu ()
print 'Common Start Menu =>', start_menu (1)
print 'Programs =>', programs ()
print 'Common Programs =>', programs (1)
print 'Startup =>', startup ()
print 'Common Startup =>', startup (1)
print 'My Documents =>', my_documents ()
print 'Recent =>', recent ()
print 'SendTo =>', sendto ()
finally:
raw_input ("Press enter...")

16
src/calibre/web/feeds/main.py
View File

@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
CLI for downloading feeds.
'''
import sys, os, logging
import sys, os
from calibre.web.feeds.recipes import get_builtin_recipe, compile_recipe, titles
from calibre.web.fetch.simple import option_parser as _option_parser
from calibre.web.feeds.news import BasicNewsRecipe
@ -113,7 +113,7 @@ If you specify this option, any argument to %prog is ignored and a default recip
class RecipeError(Exception):
pass
def run_recipe(opts, recipe_arg, parser, notification=None, handler=None):
def run_recipe(opts, recipe_arg, parser, notification=None):
if notification is None:
from calibre.utils.terminfo import TerminalController, ProgressBar
term = TerminalController(sys.stdout)
@ -137,14 +137,6 @@ def run_recipe(opts, recipe_arg, parser, notification=None, handler=None):
if recipe is None:
raise RecipeError(recipe_arg+ ' is an invalid recipe')
if handler is None:
from calibre import ColoredFormatter
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG if opts.debug else logging.INFO if opts.verbose else logging.WARN)
handler.setFormatter(ColoredFormatter('%(levelname)s: %(message)s\n')) # The trailing newline is need because of the progress bar
logging.getLogger('feeds2disk').addHandler(handler)
recipe = recipe(opts, parser, notification)
if not os.path.exists(recipe.output_dir):
@ -153,7 +145,7 @@ def run_recipe(opts, recipe_arg, parser, notification=None, handler=None):
return recipe
def main(args=sys.argv, notification=None, handler=None):
def main(args=sys.argv, notification=None):
p = option_parser()
opts, args = p.parse_args(args=args[1:])
@ -161,7 +153,7 @@ def main(args=sys.argv, notification=None, handler=None):
p.print_help()
return 1
recipe_arg = args[0] if len(args) > 0 else None
run_recipe(opts, recipe_arg, p, notification=notification, handler=handler)
run_recipe(opts, recipe_arg, p, notification=notification)
return 0

112
src/calibre/web/feeds/news.py
View File

@ -7,7 +7,7 @@ Defines various abstract base classes that can be subclassed to create powerful
__docformat__ = "restructuredtext en"
import logging, os, cStringIO, time, traceback, re, urlparse, sys
import os, time, traceback, re, urlparse, sys
from collections import defaultdict
from functools import partial
from contextlib import nested, closing
@ -27,6 +27,7 @@ from calibre.web.feeds import feed_from_xml, templates, feeds_from_index, Feed
from calibre.web.fetch.simple import option_parser as web2disk_option_parser
from calibre.web.fetch.simple import RecursiveFetcher
from calibre.utils.threadpool import WorkRequest, ThreadPool, NoResultsPending
from calibre.utils.logging import Log
from calibre.ptempfile import PersistentTemporaryFile, \
PersistentTemporaryDirectory
@ -423,7 +424,7 @@ class BasicNewsRecipe(object):
'''
raise NotImplementedError
def get_obfuscated_article(self, url, logger):
def get_obfuscated_article(self, url):
'''
If you set :member:`articles_are_obfuscated` this method is called with
every article URL. It should return the path to a file on the filesystem
@ -443,6 +444,7 @@ class BasicNewsRecipe(object):
:param parser: Command line option parser. Used to intelligently merge options.
:param progress_reporter: A Callable that takes two arguments: progress (a number between 0 and 1) and a string message. The message should be optional.
'''
self.log = Log()
if not isinstance(self.title, unicode):
self.title = unicode(self.title, 'utf-8', 'replace')
@ -455,7 +457,6 @@ class BasicNewsRecipe(object):
if self.debug:
logging.getLogger('feeds2disk').setLevel(logging.DEBUG)
self.verbose = True
self.report_progress = progress_reporter
@ -560,20 +561,20 @@ class BasicNewsRecipe(object):
res = self.build_index()
self.report_progress(1, _('Download finished'))
if self.failed_downloads:
self.log_warning(_('Failed to download the following articles:'))
self.log.warning(_('Failed to download the following articles:'))
for feed, article, debug in self.failed_downloads:
self.log_warning(article.title+_(' from ')+feed.title)
self.log_debug(article.url)
self.log_debug(debug)
self.log.warning(article.title+_(' from ')+feed.title)
self.log.debug(article.url)
self.log.debug(debug)
if self.partial_failures:
self.log_warning(_('Failed to download parts of the following articles:'))
self.log.warning(_('Failed to download parts of the following articles:'))
for feed, atitle, aurl, debug in self.partial_failures:
self.log_warning(atitle + _(' from ') + feed)
self.log_debug(aurl)
self.log_warning(_('\tFailed links:'))
self.log.warning(atitle + _(' from ') + feed)
self.log.debug(aurl)
self.log.warning(_('\tFailed links:'))
for l, tb in debug:
self.log_warning(l)
self.log_debug(tb)
self.log.warning(l)
self.log.debug(tb)
return res
finally:
self.cleanup()
@ -636,20 +637,11 @@ class BasicNewsRecipe(object):
extra_css=self.extra_css).render(doctype='xhtml')
def create_logger(self, feed_number, article_number):
logger = logging.getLogger('feeds2disk.article_%d_%d'%(feed_number, article_number))
out = cStringIO.StringIO()
handler = logging.StreamHandler(out)
handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
handler.setLevel(logging.INFO if self.verbose else logging.WARNING)
if self.debug:
handler.setLevel(logging.DEBUG)
logger.addHandler(handler)
return logger, out
def _fetch_article(self, url, dir, logger, f, a, num_of_feeds):
def _fetch_article(self, url, dir, f, a, num_of_feeds):
self.web2disk_options.browser = self.get_browser() if self.multithreaded_fetch else self.browser
fetcher = RecursiveFetcher(self.web2disk_options, logger, self.image_map, self.css_map, (url, f, a, num_of_feeds))
fetcher = RecursiveFetcher(self.web2disk_options, self.log,
self.image_map, self.css_map,
(url, f, a, num_of_feeds))
fetcher.base_dir = dir
fetcher.current_dir = dir
fetcher.show_progress = False
@ -661,21 +653,21 @@ class BasicNewsRecipe(object):
raise Exception(_('Could not fetch article. Run with --debug to see the reason'))
return res, path, failures
def fetch_article(self, url, dir, logger, f, a, num_of_feeds):
return self._fetch_article(url, dir, logger, f, a, num_of_feeds)
def fetch_article(self, url, dir, f, a, num_of_feeds):
return self._fetch_article(url, dir, f, a, num_of_feeds)
def fetch_obfuscated_article(self, url, dir, logger, f, a, num_of_feeds):
path = os.path.abspath(self.get_obfuscated_article(url, logger))
def fetch_obfuscated_article(self, url, dir, f, a, num_of_feeds):
path = os.path.abspath(self.get_obfuscated_article(url))
url = ('file:'+path) if iswindows else ('file://'+path)
return self._fetch_article(url, dir, logger, f, a, num_of_feeds)
return self._fetch_article(url, dir, f, a, num_of_feeds)
def fetch_embedded_article(self, article, dir, logger, f, a, num_of_feeds):
def fetch_embedded_article(self, article, dir, f, a, num_of_feeds):
templ = templates.EmbeddedContent()
raw = templ.generate(article).render('html')
with PersistentTemporaryFile('_feeds2disk.html') as pt:
pt.write(raw)
url = ('file:'+pt.name) if iswindows else ('file://'+pt.name)
return self._fetch_article(url, dir, logger, f, a, num_of_feeds)
return self._fetch_article(url, dir, f, a, num_of_feeds)
def build_index(self):
@ -716,7 +708,6 @@ class BasicNewsRecipe(object):
art_dir = os.path.join(feed_dir, 'article_%d'%a)
if not os.path.isdir(art_dir):
os.makedirs(art_dir)
logger, stream = self.create_logger(f, a)
try:
url = self.print_version(article.url)
except NotImplementedError:
@ -726,10 +717,9 @@ class BasicNewsRecipe(object):
func, arg = (self.fetch_embedded_article, article) if self.use_embedded_content else \
((self.fetch_obfuscated_article if self.articles_are_obfuscated \
else self.fetch_article), url)
req = WorkRequest(func, (arg, art_dir, logger, f, a, len(feed)),
req = WorkRequest(func, (arg, art_dir, f, a, len(feed)),
{}, (f, a), self.article_downloaded,
self.error_in_article_download)
req.stream = stream
req.feed = feed
req.article = article
req.feed_dir = feed_dir
@ -768,8 +758,8 @@ class BasicNewsRecipe(object):
cu = self.get_cover_url()
except Exception, err:
cu = None
self.log_error(_('Could not download cover: %s')%str(err))
self.log_debug(traceback.format_exc())
self.log.error(_('Could not download cover: %s')%str(err))
self.log.debug(traceback.format_exc())
if cu is not None:
ext = cu.rpartition('.')[-1]
if '?' in ext:
@ -841,8 +831,8 @@ class BasicNewsRecipe(object):
f.write(html.encode('utf-8'))
renderer = render_html(hf)
if renderer.tb is not None:
self.logger.warning('Failed to render default cover')
self.logger.debug(renderer.tb)
self.log.warning('Failed to render default cover')
self.log.debug(renderer.tb)
else:
cover_file.write(renderer.data)
cover_file.flush()
@ -863,7 +853,7 @@ class BasicNewsRecipe(object):
manifest.append(os.path.join(dir, 'index.ncx'))
cpath = getattr(self, 'cover_path', None)
if cpath is None:
pf = PersistentTemporaryFile('_recipe_cover.jpg')
pf = open(os.path.join(dir, 'cover.jpg'), 'wb')
self.default_cover(pf)
cpath = pf.name
if cpath is not None and os.access(cpath, os.R_OK):
@ -944,7 +934,7 @@ class BasicNewsRecipe(object):
a = request.requestID[1]
article = request.article
self.log_debug(_('\nDownloaded article %s from %s\n%s')%(article.title, article.url, request.stream.getvalue().decode('utf-8', 'ignore')))
self.log.debug(_('\nDownloaded article %s from %s')%(article.title, article.url))
article.orig_url = article.url
article.url = 'article_%d/index.html'%a
article.downloaded = True
@ -956,11 +946,11 @@ class BasicNewsRecipe(object):
def error_in_article_download(self, request, traceback):
self.jobs_done += 1
self.log_error(_('Failed to download article: %s from %s\n')%(request.article.title, request.article.url))
self.log.error(_('Failed to download article: %s from %s\n')%(request.article.title, request.article.url))
debug = request.stream.getvalue().decode('utf-8', 'ignore')
self.log_debug(debug)
self.log_debug(traceback)
self.log_debug('\n')
self.log.debug(debug)
self.log.debug(traceback)
self.log.debug('\n')
self.report_progress(float(self.jobs_done)/len(self.jobs), _('Article download failed: %s')%request.article.title)
self.failed_downloads.append((request.feed, request.article, debug))
@ -990,7 +980,7 @@ class BasicNewsRecipe(object):
feed.populate_from_preparsed_feed(msg, [])
feed.description = unicode(err)
parsed_feeds.append(feed)
self.log_exception(msg)
self.log.exception(msg)
return parsed_feeds
@ -1033,6 +1023,28 @@ class BasicNewsRecipe(object):
nmassage.extend(entity_replace)
return BeautifulSoup(raw, markupMassage=nmassage)
@classmethod
def adeify_images(cls, soup):
'''
If your recipe when converted to EPUB has problems with images when
viewed in Adobe Digital Editions, call this method from within
:method:`postprocess_html`.
'''
for item in soup.findAll('img'):
for attrib in ['height','width','border','align','style']:
if item.has_key(attrib):
del item[attrib]
oldParent = item.parent
myIndex = oldParent.contents.index(item)
item.extract()
divtag = Tag(soup,'div')
brtag = Tag(soup,'br')
oldParent.insert(myIndex,divtag)
divtag.append(item)
divtag.append(brtag)
return soup
class CustomIndexRecipe(BasicNewsRecipe):
def custom_index(self):
@ -1057,7 +1069,7 @@ class CustomIndexRecipe(BasicNewsRecipe):
index = os.path.abspath(self.custom_index())
url = 'file:'+index if iswindows else 'file://'+index
self.web2disk_options.browser = self.browser
fetcher = RecursiveFetcher(self.web2disk_options, self.logger)
fetcher = RecursiveFetcher(self.web2disk_options, self.log)
fetcher.base_dir = self.output_dir
fetcher.current_dir = self.output_dir
fetcher.show_progress = False
@ -1069,7 +1081,7 @@ class AutomaticNewsRecipe(BasicNewsRecipe):
keep_only_tags = [dict(name=['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'])]
def fetch_embedded_article(self, article, dir, logger, f, a, num_of_feeds):
def fetch_embedded_article(self, article, dir, f, a, num_of_feeds):
if self.use_embedded_content:
self.web2disk_options.keep_only_tags = []
return BasicNewsRecipe.fetch_embedded_article(self, article, dir, logger, f, a, num_of_feeds)
return BasicNewsRecipe.fetch_embedded_article(self, article, dir, f, a, num_of_feeds)

5
src/calibre/web/feeds/recipes/__init__.py
View File

@ -8,7 +8,7 @@ recipe_modules = ['recipe_' + r for r in (
'newsweek', 'atlantic', 'economist', 'portfolio', 'the_register',
'usatoday', 'outlook_india', 'bbc', 'greader', 'wsj',
'wired', 'globe_and_mail', 'smh', 'espn', 'business_week', 'miami_herald',
'ars_technica', 'upi', 'new_yorker', 'irish_times', 'iht', 'lanacion',
'ars_technica', 'upi', 'new_yorker', 'irish_times', 'lanacion',
'discover_magazine', 'scientific_american', 'new_york_review_of_books',
'daily_telegraph', 'guardian', 'el_pais', 'new_scientist', 'b92',
'politika', 'moscow_times', 'latimes', 'japan_times', 'san_fran_chronicle',
@ -37,7 +37,8 @@ recipe_modules = ['recipe_' + r for r in (
'new_york_review_of_books_no_sub', 'politico', 'adventuregamers',
'mondedurable', 'instapaper', 'dnevnik_cro', 'vecernji_list',
'nacional_cro', '24sata', 'dnevni_avaz', 'glas_srpske', '24sata_rs',
'krstarica', 'krstarica_en', 'tanjug', 'laprensa_ni',
'krstarica', 'krstarica_en', 'tanjug', 'laprensa_ni', 'azstarnet',
'corriere_della_sera_it', 'corriere_della_sera_en', 'msdnmag_en',
)]
import re, imp, inspect, time, os

21
src/calibre/web/feeds/recipes/recipe_24sata_rs.py
View File

@ -9,6 +9,7 @@ __copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
import re
from calibre.web.feeds.recipes import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
class Ser24Sata(BasicNewsRecipe):
title = '24 Sata - Sr'
@ -39,14 +40,30 @@ class Ser24Sata(BasicNewsRecipe):
feeds = [(u'Vesti Dana', u'http://www.24sata.rs/rss.php')]
def cleanup_image_tags(self,soup):
for item in soup.findAll('img'):
for attrib in ['height','width','border','align']:
if item.has_key(attrib):
del item[attrib]
oldParent = item.parent
myIndex = oldParent.contents.index(item)
item.extract()
divtag = Tag(soup,'div')
brtag = Tag(soup,'br')
oldParent.insert(myIndex,divtag)
divtag.append(item)
divtag.append(brtag)
return soup
def preprocess_html(self, soup):
soup.html['xml:lang'] = 'sr-Latn-RS'
soup.html['lang'] = 'sr-Latn-RS'
mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
soup.head.insert(0,mtag)
return soup
return self.cleanup_image_tags(soup)
def print_version(self, url):
article, sep, rest = url.partition('#')
return article.replace('/show.php','/_print.php')
article_base, sep2, article_id = article.partition('id=')
return 'http://www.24sata.co.rs/_print.php?id=' + article_id

63
src/calibre/web/feeds/recipes/recipe_azstarnet.py Normal file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
'''
www.azstarnet.com
'''
from calibre.web.feeds.news import BasicNewsRecipe
class Azstarnet(BasicNewsRecipe):
title = 'Arizona Daily Star'
__author__ = 'Darko Miletic'
description = 'news from Arizona'
publisher = 'azstarnet.com'
category = 'news, politics, Arizona, USA'
delay = 1
oldest_article = 1
max_articles_per_feed = 100
no_stylesheets = True
use_embedded_content = False
encoding = 'utf-8'
needs_subscription = True
remove_javascript = True
html2lrf_options = [
'--comment', description
, '--category', category
, '--publisher', publisher
]
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
def get_browser(self):
br = BasicNewsRecipe.get_browser()
if self.username is not None and self.password is not None:
br.open('http://azstarnet.com/registration/retro.php')
br.select_form(nr=1)
br['email'] = self.username
br['pass' ] = self.password
br.submit()
return br
keep_only_tags = [dict(name='div', attrs={'id':'storycontent'})]
remove_tags = [
dict(name=['object','link','iframe','base','img'])
,dict(name='div',attrs={'class':'bannerinstory'})
]
feeds = [(u'Tucson Region', u'http://rss.azstarnet.com/index.php?site=metro')]
def preprocess_html(self, soup):
soup.html['dir' ] = 'ltr'
soup.html['lang'] = 'en-US'
mtag = '\n<meta http-equiv="Content-Language" content="en-US"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n'
soup.head.insert(0,mtag)
for item in soup.findAll(style=True):
del item['style']
return soup

27
src/calibre/web/feeds/recipes/recipe_blic.py
View File

@ -8,11 +8,12 @@ blic.rs
import re
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
class Blic(BasicNewsRecipe):
title = u'Blic'
__author__ = u'Darko Miletic'
description = u'Blic.co.yu online verzija najtiraznije novine u Srbiji donosi najnovije vesti iz Srbije i sveta, komentare, politicke analize, poslovne i ekonomske vesti, vesti iz regiona, intervjue, informacije iz kulture, reportaze, pokriva sve sportske dogadjaje, detaljan tv program, nagradne igre, zabavu, fenomenalni Blic strip, dnevni horoskop, arhivu svih dogadjaja'
title = 'Blic'
__author__ = 'Darko Miletic'
description = 'Blic.co.yu online verzija najtiraznije novine u Srbiji donosi najnovije vesti iz Srbije i sveta, komentare, politicke analize, poslovne i ekonomske vesti, vesti iz regiona, intervjue, informacije iz kulture, reportaze, pokriva sve sportske dogadjaje, detaljan tv program, nagradne igre, zabavu, fenomenalni Blic strip, dnevni horoskop, arhivu svih dogadjaja'
publisher = 'RINGIER d.o.o.'
category = 'news, politics, Serbia'
oldest_article = 2
@ -30,7 +31,7 @@ class Blic(BasicNewsRecipe):
, '--ignore-tables'
]
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True'
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} "'
preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
@ -44,10 +45,26 @@ class Blic(BasicNewsRecipe):
start_url, question, rest_url = url.partition('?')
return u'http://www.blic.rs/_print.php?' + rest_url
def cleanup_image_tags(self,soup):
for item in soup.findAll('img'):
for attrib in ['height','width','border','align']:
if item.has_key(attrib):
del item[attrib]
oldParent = item.parent
myIndex = oldParent.contents.index(item)
item.extract()
divtag = Tag(soup,'div')
brtag = Tag(soup,'br')
oldParent.insert(myIndex,divtag)
divtag.append(item)
divtag.append(brtag)
return soup
def preprocess_html(self, soup):
mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
soup.head.insert(0,mtag)
for item in soup.findAll(style=True):
del item['style']
return soup
return self.cleanup_image_tags(soup)

45
src/calibre/web/feeds/recipes/recipe_corriere_della_sera_en.py Normal file
View File

@ -0,0 +1,45 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
'''
www.corriere.it/english
'''
from calibre.web.feeds.news import BasicNewsRecipe
class Corriere_en(BasicNewsRecipe):
title = 'Corriere della Sera in English'
__author__ = 'Darko Miletic'
description = 'News from Milan and Italy'
oldest_article = 15
publisher = 'Corriere della Sera'
category = 'news, politics, Italy'
max_articles_per_feed = 100
no_stylesheets = True
use_embedded_content = False
encoding = 'cp1252'
remove_javascript = True
language = _('English')
html2lrf_options = [
'--comment', description
, '--category', category
, '--publisher', publisher
, '--ignore-tables'
]
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True'
keep_only_tags = [dict(name='div', attrs={'class':['news-dettaglio article','article']})]
remove_tags = [
dict(name=['base','object','link','embed','img'])
,dict(name='div', attrs={'class':'news-goback'})
,dict(name='ul', attrs={'class':'toolbar'})
]
remove_tags_after = dict(name='p', attrs={'class':'footnotes'})
feeds = [(u'Italian Life', u'http://www.corriere.it/rss/english.xml')]

55
src/calibre/web/feeds/recipes/recipe_corriere_della_sera_it.py Normal file
View File

@ -0,0 +1,55 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
'''
www.corriere.it
'''
from calibre.web.feeds.news import BasicNewsRecipe
class Corriere_it(BasicNewsRecipe):
title = 'Corriere della Sera'
__author__ = 'Darko Miletic'
description = 'News from Milan and Italy'
oldest_article = 7
publisher = 'Corriere della Sera'
category = 'news, politics, Italy'
max_articles_per_feed = 100
no_stylesheets = True
use_embedded_content = False
encoding = 'cp1252'
remove_javascript = True
language = _('Italian')
html2lrf_options = [
'--comment', description
, '--category', category
, '--publisher', publisher
, '--ignore-tables'
]
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True'
keep_only_tags = [dict(name='div', attrs={'class':['news-dettaglio article','article']})]
remove_tags = [
dict(name=['base','object','link','embed','img'])
,dict(name='div', attrs={'class':'news-goback'})
,dict(name='ul', attrs={'class':'toolbar'})
]
remove_tags_after = dict(name='p', attrs={'class':'footnotes'})
feeds = [
(u'Ultimora' , u'http://www.corriere.it/rss/ultimora.xml' )
,(u'Cronache' , u'http://www.corriere.it/rss/cronache.xml' )
,(u'Economia' , u'http://www.corriere.it/rss/economia.xml' )
,(u'Editoriali', u'http://www.corriere.it/rss/editoriali.xml')
,(u'Esteri' , u'http://www.corriere.it/rss/esteri.xml' )
,(u'Politica' , u'http://www.corriere.it/rss/politica.xml' )
,(u'Salute' , u'http://www.corriere.it/rss/salute.xml' )
,(u'Scienze' , u'http://www.corriere.it/rss/scienze.xml' )
,(u'Spettacolo', u'http://www.corriere.it/rss/spettacoli.xml')
,(u'Sport' , u'http://www.corriere.it/rss/sport.xml' )
]

61
src/calibre/web/feeds/recipes/recipe_msdnmag_en.py Normal file
View File

@ -0,0 +1,61 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
'''
msdn.microsoft.com/en-us/magazine
'''
from calibre.web.feeds.news import BasicNewsRecipe
class MSDNMagazine_en(BasicNewsRecipe):
title = 'MSDN Magazine'
__author__ = 'Darko Miletic'
description = 'The Microsoft Journal for Developers'
publisher = 'Microsoft Press'
category = 'news, IT, Microsoft, programming, windows'
oldest_article = 31
max_articles_per_feed = 100
no_stylesheets = True
use_embedded_content = False
encoding = 'utf-8'
remove_javascript = True
current_issue = 'http://msdn.microsoft.com/en-us/magazine/default.aspx'
language = _('English')
html2lrf_options = [
'--comment', description
, '--category', category
, '--publisher', publisher
]
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
feeds = [(u'Articles', u'http://msdn.microsoft.com/en-us/magazine/rss/default.aspx?z=z&iss=1')]
keep_only_tags = [dict(name='div', attrs={'class':'topic'})]
remove_tags = [
dict(name=['object','link','base','table'])
,dict(name='div', attrs={'class':'MTPS_CollapsibleRegion'})
]
def get_cover_url(self):
cover_url = None
soup = self.index_to_soup(self.current_issue)
link_item = soup.find('span',attrs={'class':'ContentsImageSpacer'})
if link_item:
imgt = link_item.find('img')
if imgt:
cover_url = imgt['src']
return cover_url
def preprocess_html(self, soup):
for item in soup.findAll('div',attrs={'class':['FeatureSmallHead','ColumnTypeSubTitle']}):
item.name="h2"
for item in soup.findAll('div',attrs={'class':['FeatureHeadline','ColumnTypeTitle']}):
item.name="h1"
for item in soup.findAll('div',attrs={'class':'ArticleTypeTitle'}):
item.name="h3"
return soup

10
src/calibre/web/feeds/recipes/recipe_new_yorker.py
View File

@ -9,6 +9,7 @@ newyorker.com
from calibre.web.feeds.news import BasicNewsRecipe
class NewYorker(BasicNewsRecipe):
title = u'The New Yorker'
__author__ = 'Darko Miletic'
description = 'The best of US journalism'
@ -41,3 +42,12 @@ class NewYorker(BasicNewsRecipe):
def print_version(self, url):
return url + '?printable=true'
def postprocess_html(self, soup, x):
body = soup.find('body')
if body:
html = soup.find('html')
if html:
body.extract()
html.insert(-1, body)
return soup

22
src/calibre/web/feeds/recipes/recipe_nspm.py
View File

@ -8,9 +8,10 @@ nspm.rs
import re
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
class Nspm(BasicNewsRecipe):
title = u'Nova srpska politicka misao'
title = 'Nova srpska politicka misao'
__author__ = 'Darko Miletic'
description = 'Casopis za politicku teoriju i drustvena istrazivanja'
publisher = 'NSPM'
@ -36,7 +37,7 @@ class Nspm(BasicNewsRecipe):
preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
remove_tags = [
dict(name=['a','img','link','object','embed'])
dict(name=['link','object','embed'])
,dict(name='td', attrs={'class':'buttonheading'})
]
@ -50,6 +51,21 @@ class Nspm(BasicNewsRecipe):
def print_version(self, url):
return url.replace('.html','/stampa.html')
def cleanup_image_tags(self,soup):
for item in soup.findAll('img'):
for attrib in ['height','width','border','align']:
if item.has_key(attrib):
del item[attrib]
oldParent = item.parent
myIndex = oldParent.contents.index(item)
item.extract()
divtag = Tag(soup,'div')
brtag = Tag(soup,'br')
oldParent.insert(myIndex,divtag)
divtag.append(item)
divtag.append(brtag)
return soup
def preprocess_html(self, soup):
lng = 'sr-Latn-RS'
soup.html['xml:lang'] = lng
@ -59,4 +75,4 @@ class Nspm(BasicNewsRecipe):
ftag['content'] = lng
for item in soup.findAll(style=True):
del item['style']
return soup
return self.cleanup_image_tags(soup)

53
src/calibre/web/feeds/recipes/recipe_tomshardware.py
View File

@ -1,38 +1,47 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
'''
tomshardware.com
tomshardware.com/us
'''
import urllib
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.web.feeds.recipes import BasicNewsRecipe
class Tomshardware(BasicNewsRecipe):
title = "Tom's Hardware US"
__author__ = 'Darko Miletic'
description = 'Hardware reviews and News'
publisher = "Tom's Hardware"
category = 'news, IT, hardware, USA'
no_stylesheets = True
needs_subscription = True
language = _('English')
INDEX = 'http://www.tomshardware.com'
LOGIN = 'http://www.tomshardware.com/membres/?r=%2Fus%2F#loginForm'
cover_url = 'http://img.bestofmedia.com/img/tomshardware/design/tomshardware.jpg'
LOGIN = INDEX + '/membres/'
remove_javascript = True
use_embedded_content= False
html2lrf_options = [ '--comment' , description
, '--category' , 'hardware,news'
, '--base-font-size', '10'
html2lrf_options = [
'--comment', description
, '--category', category
, '--publisher', publisher
]
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
def get_browser(self):
br = BasicNewsRecipe.get_browser()
br.open(self.INDEX+'/us/')
if self.username is not None and self.password is not None:
br.open(self.LOGIN)
br.select_form(name='connexion')
br['login'] = self.username
br['mdp' ] = self.password
br.submit()
data = urllib.urlencode({ 'action':'login_action'
,'r':self.INDEX+'/us/'
,'login':self.username
,'mdp':self.password
})
br.open(self.LOGIN,data)
return br
remove_tags = [
@ -49,10 +58,22 @@ class Tomshardware(BasicNewsRecipe):
main, sep, rest = url.rpartition('.html')
rmain, rsep, article_id = main.rpartition(',')
tmain, tsep, trest = rmain.rpartition('/reviews/')
rind = 'http://www.tomshardware.com/news_print.php?p1='
if tsep:
return 'http://www.tomshardware.com/review_print.php?p1=' + article_id
return 'http://www.tomshardware.com/news_print.php?p1=' + article_id
rind = 'http://www.tomshardware.com/review_print.php?p1='
return rind + article_id
def cleanup_image_tags(self,soup):
for item in soup.findAll('img'):
for attrib in ['height','width','border','align']:
if item.has_key(attrib):
del item[attrib]
return soup
def preprocess_html(self, soup):
del(soup.body['onload'])
return soup
for item in soup.findAll(style=True):
del item['style']
for it in soup.findAll('span'):
it.name="div"
return self.cleanup_image_tags(soup)

2
src/calibre/web/feeds/recipes/recipe_wash_post.py
View File

@ -17,7 +17,7 @@ class WashingtonPost(BasicNewsRecipe):
feeds = [ ('Today\'s Highlights', 'http://www.washingtonpost.com/wp-dyn/rss/linkset/2005/03/24/LI2005032400102.xml'),
('Politics', 'http://www.washingtonpost.com/wp-dyn/rss/politics/index.xml'),
('Nation', 'http://www.www.washingtonpost.com/wp-dyn/rss/nation/index.xml'),
('Nation', 'http://www.washingtonpost.com/wp-dyn/rss/nation/index.xml'),
('World', 'http://www.washingtonpost.com/wp-dyn/rss/world/index.xml'),
('Business', 'http://www.washingtonpost.com/wp-dyn/rss/business/index.xml'),
('Technology', 'http://www.washingtonpost.com/wp-dyn/rss/technology/index.xml'),

58
src/calibre/web/fetch/simple.py
View File

@ -7,18 +7,19 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
Fetch a webpage and its links recursively. The webpages are saved to disk in
UTF-8 encoding with any charset declarations removed.
'''
import sys, socket, os, urlparse, logging, re, time, copy, urllib2, threading, traceback
import sys, socket, os, urlparse, re, time, copy, urllib2, threading, traceback
from urllib import url2pathname, quote
from threading import RLock
from httplib import responses
from PIL import Image
from cStringIO import StringIO
from calibre import setup_cli_handlers, browser, sanitize_file_name, \
from calibre import browser, sanitize_file_name, \
relpath, unicode_path
from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
from calibre.ebooks.chardet import xml_to_unicode
from calibre.utils.config import OptionParser
from calibre.utils.logging import Log
class FetchError(Exception):
pass
@ -92,10 +93,11 @@ class RecursiveFetcher(object):
default_timeout = socket.getdefaulttimeout() # Needed here as it is used in __del__
DUMMY_LOCK = DummyLock()
def __init__(self, options, logger, image_map={}, css_map={}, job_info=None):
def __init__(self, options, log, image_map={}, css_map={}, job_info=None):
self.base_dir = os.path.abspath(os.path.expanduser(options.dir))
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
self.log = log
self.default_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(options.timeout)
self.verbose = options.verbose
@ -174,12 +176,15 @@ class RecursiveFetcher(object):
def fetch_url(self, url):
data = None
self.log_debug('Fetching %s', url)
self.log.debug('Fetching', url)
delta = time.time() - self.last_fetch_at
if delta < self.delay:
time.sleep(delta)
if re.search(r'\s+', url) is not None:
url = quote(url)
if re.search(r'\s+|,', url) is not None:
purl = list(urlparse.urlparse(url))
for i in range(2, 6):
purl[i] = quote(purl[i])
url = urlparse.urlunparse(purl)
with self.browser_lock:
try:
with closing(self.browser.open(url)) as f:
@ -190,7 +195,7 @@ class RecursiveFetcher(object):
raise FetchError, responses[err.code]
if getattr(err, 'reason', [0])[0] == 104 or \
getattr(getattr(err, 'args', [None])[0], 'errno', None) == -2: # Connection reset by peer or Name or service not know
self.log_debug('Temporary error, retrying in 1 second')
self.log.debug('Temporary error, retrying in 1 second')
time.sleep(1)
with closing(self.browser.open(url)) as f:
data = response(f.read()+f.read())
@ -204,9 +209,9 @@ class RecursiveFetcher(object):
def start_fetch(self, url):
soup = BeautifulSoup(u'<a href="'+url+'" />')
self.log_info('Downloading')
self.log.debug('Downloading')
res = self.process_links(soup, url, 0, into_dir='')
self.log_info('%s saved to %s', url, res)
self.log.debug('%s saved to %s'%( url, res))
return res
def is_link_ok(self, url):
@ -243,8 +248,7 @@ class RecursiveFetcher(object):
try:
data = self.fetch_url(iurl)
except Exception, err:
self.log_debug('Could not fetch stylesheet %s', iurl)
self.log_debug('Error: %s', str(err), exc_info=True)
self.log.exception('Could not fetch stylesheet %s'% iurl)
continue
stylepath = os.path.join(diskpath, 'style'+str(c)+'.css')
with self.stylemap_lock:
@ -267,8 +271,7 @@ class RecursiveFetcher(object):
try:
data = self.fetch_url(iurl)
except Exception, err:
self.log_warning('Could not fetch stylesheet %s', iurl)
self.log_debug('Error: %s', str(err), exc_info=True)
self.log.exception('Could not fetch stylesheet %s'% iurl)
continue
c += 1
stylepath = os.path.join(diskpath, 'style'+str(c)+'.css')
@ -291,9 +294,6 @@ class RecursiveFetcher(object):
iurl = self.image_url_processor(baseurl, iurl)
ext = os.path.splitext(iurl)[1]
ext = ext[:5]
#if not ext:
# self.log_debug('Skipping extensionless image %s', iurl)
# continue
if not urlparse.urlsplit(iurl).scheme:
iurl = urlparse.urljoin(baseurl, iurl, False)
with self.imagemap_lock:
@ -303,8 +303,7 @@ class RecursiveFetcher(object):
try:
data = self.fetch_url(iurl)
except Exception, err:
self.log_warning('Could not fetch image %s', iurl)
self.log_debug('Error: %s', str(err), exc_info=True)
self.log.exception('Could not fetch image %s'% iurl)
continue
c += 1
fname = sanitize_file_name('img'+str(c)+ext)
@ -330,10 +329,10 @@ class RecursiveFetcher(object):
if not parts.scheme:
iurl = urlparse.urljoin(baseurl, iurl, False)
if not self.is_link_ok(iurl):
self.log_debug('Skipping invalid link: %s', iurl)
self.log.debug('Skipping invalid link:', iurl)
return None
if filter and not self.is_link_wanted(iurl):
self.log_debug('Filtered link: '+iurl)
self.log.debug('Filtered link: '+iurl)
return None
return iurl
@ -401,7 +400,7 @@ class RecursiveFetcher(object):
base = soup.find('base', href=True)
if base is not None:
newbaseurl = base['href']
self.log_debug('Processing images...')
self.log.debug('Processing images...')
self.process_images(soup, newbaseurl)
if self.download_stylesheets:
self.process_stylesheets(soup, newbaseurl)
@ -416,11 +415,11 @@ class RecursiveFetcher(object):
self.downloaded_paths.append(res)
self.filemap[nurl] = res
if recursion_level < self.max_recursions:
self.log_debug('Processing links...')
self.log.debug('Processing links...')
self.process_links(soup, newbaseurl, recursion_level+1)
else:
self.process_return_links(soup, newbaseurl)
self.log_debug('Recursion limit reached. Skipping links in %s', iurl)
self.log.debug('Recursion limit reached. Skipping links in', iurl)
if callable(self.postprocess_html_ext):
soup = self.postprocess_html_ext(soup,
@ -434,8 +433,7 @@ class RecursiveFetcher(object):
self.localize_link(tag, 'href', res)
except Exception, err:
self.failed_links.append((iurl, traceback.format_exc()))
self.log_warning('Could not fetch link %s', iurl)
self.log_debug('Error: %s', str(err), exc_info=True)
self.log.exception('Could not fetch link', iurl)
finally:
self.current_dir = diskpath
self.files += 1
@ -478,12 +476,10 @@ def option_parser(usage=_('%prog URL\n\nWhere URL is for example http://google.c
return parser
def create_fetcher(options, logger=None, image_map={}):
if logger is None:
level = logging.DEBUG if options.verbose else logging.INFO
logger = logging.getLogger('web2disk')
setup_cli_handlers(logger, level)
return RecursiveFetcher(options, logger, image_map={})
def create_fetcher(options, image_map={}, log=None):
if log is None:
log = Log()
return RecursiveFetcher(options, log, image_map={})
def main(args=sys.argv):
parser = option_parser()