mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-06-23 15:30:45 -04:00
299 lines
14 KiB
Python
299 lines
14 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 mode: python -*-
|
|
from __future__ import absolute_import, division, print_function, unicode_literals
|
|
|
|
__license__ = 'GPL v3'
|
|
__copyright__ = '2010, Steffen Siebert <calibre at steffensiebert.de>'
|
|
__docformat__ = 'restructuredtext de'
|
|
__version__ = '1.5'
|
|
|
|
"""
|
|
Die Zeit EPUB
|
|
"""
|
|
|
|
import os
|
|
import zipfile
|
|
import re
|
|
import io
|
|
from calibre.web.feeds.news import BasicNewsRecipe
|
|
from calibre.ptempfile import PersistentTemporaryFile
|
|
from calibre import walk
|
|
try:
|
|
from urllib.parse import urlparse
|
|
except ImportError:
|
|
from urlparse import urlparse
|
|
from contextlib import closing
|
|
from calibre.utils.magick.draw import save_cover_data_to
|
|
|
|
|
|
class ZeitEPUBAbo(BasicNewsRecipe):
|
|
|
|
title = u'Die Zeit'
|
|
description = u'Das EPUB Abo der Zeit (needs subscription)'
|
|
language = 'de'
|
|
lang = 'de-DE'
|
|
|
|
__author__ = 'Steffen Siebert, revised by Tobias Isenberg (with some code by Kovid Goyal), updated by Henning Losert'
|
|
needs_subscription = True
|
|
|
|
conversion_options = {
|
|
'no_default_epub_cover': True,
|
|
# fixing the wrong left margin
|
|
'mobi_ignore_margins': True,
|
|
'keep_ligatures': True,
|
|
}
|
|
|
|
preprocess_regexps = [
|
|
# filtering for correct dashes ("Gedankenstrich" and "bis")
|
|
(re.compile(u' (-|\u2212)(?=[ ,])'), lambda match: u' \u2013'),
|
|
(re.compile(r'(?<=\d)-(?=\d)'), lambda match: u'\u2013'), # number-number
|
|
(re.compile(r'(?<=\d,)-(?= ?\u20AC)'), lambda match: u'\u2013'), # ,- Euro
|
|
# fix the number dash number dash for the title image that was broken
|
|
# by the previous line
|
|
(re.compile(r'(?<=\d\d\d\d)\u2013(?=\d?\d\.png)'), lambda match: '-'),
|
|
# filtering for certain dash cases
|
|
(re.compile(r'Bild - Zeitung'), lambda match: 'Bild-Zeitung'), # the obvious
|
|
(re.compile(r'EMail'), lambda match: 'E-Mail'), # the obvious
|
|
(re.compile(r'SBahn'), lambda match: 'S-Bahn'), # the obvious
|
|
(re.compile(r'UBoot'), lambda match: 'U-Boot'), # the obvious
|
|
(re.compile(r'T Shirt'), lambda match: 'T-Shirt'), # the obvious
|
|
(re.compile(r'TShirt'), lambda match: 'T-Shirt'), # the obvious
|
|
# the next two lines not only fix errors but also create new ones. this is due to additional errors in
|
|
# the typesetting such as missing commas or wrongly placed dashes. but
|
|
# more is fixed than broken.
|
|
# space too much before a connecting dash
|
|
(re.compile(r'(?<!und|der|\w\w,) -(?=\w)'), lambda match: '-'),
|
|
(re.compile(r'(?<=\w)- (?!und\b|oder\b|wie\b|aber\b|auch\b|sondern\b|bis\b|&|&\s|bzw\.|auf\b|eher\b)'),
|
|
lambda match: '-'), # space too much after a connecting dash
|
|
# filtering for missing spaces before the month in long dates
|
|
(re.compile(r'(?<=\d)\.(?=(Januar|Februar|M\u00E4rz|April|Mai|Juni|Juli|August|September|Oktober|November|Dezember))'), lambda match: '. '),
|
|
# filtering for other missing spaces
|
|
(re.compile(r'Stuttgart21'), lambda match: 'Stuttgart 21'), # the obvious
|
|
(re.compile(r'(?<=\d)(?=\u20AC)'),
|
|
lambda match: u'\u2013'), # Zahl[no space]Euro
|
|
# missing space after colon
|
|
(re.compile(r':(?=[^\d\s</])'), lambda match: ': '),
|
|
# missing space after closing quotation
|
|
(re.compile(r'\u00AB(?=[^\-\.:;,\?!<\)\s])'),
|
|
lambda match: u'\u00AB '),
|
|
# missing space before opening quotation
|
|
(re.compile(r'(?<=[^\s\(>])\u00BB'), lambda match: u' \u00BB'),
|
|
# missing space before Roman numeral
|
|
(re.compile(
|
|
r'(?<=[a-z])(?=(I|II|III|IV|V|VI|VII|VIII|IX|X|XI|XII|XIII|XIV|XV|XVI|XVII|XVIII|XIX|XX)\.)'), lambda match: ' '),
|
|
# missing space after Roman numeral
|
|
(re.compile(r'(?<=(I|V|X)\.)(?=[\w])'), lambda match: ' '),
|
|
# missing space after Roman numeral
|
|
(re.compile(
|
|
r'(?<=(II|IV|VI|IX|XI|XV|XX)\.)(?=[\w])'), lambda match: ' '),
|
|
# missing space after Roman numeral
|
|
(re.compile(
|
|
r'(?<=(III|VII|XII|XIV|XVI|XIX)\.)(?=[\w])'), lambda match: ' '),
|
|
# missing space after Roman numeral
|
|
(re.compile(r'(?<=(VIII|XIII|XVII)\.)(?=[\w])'), lambda match: ' '),
|
|
# missing space after Roman numeral
|
|
(re.compile(r'(?<=(XVIII)\.)(?=[\w])'), lambda match: ' '),
|
|
(re.compile(r'(?<=[A-Za-zÄÖÜäöü]),(?=[A-Za-zÄÖÜäöü])'),
|
|
lambda match: ', '), # missing space after comma
|
|
# missing space after full-stop
|
|
(re.compile(
|
|
r'(?<=[a-zäöü])\.(?=[A-ZÄÖÜ][A-Za-zÄÖÜäöü])'), lambda match: '. '),
|
|
# fix abbreviation that was potentially broken previously
|
|
(re.compile(r'(?<=[uU]\.) (?=a\.)'), lambda match: u'\u2008'),
|
|
# fix abbreviation that was potentially broken previously
|
|
(re.compile(r'(?<=[iI]\.) (?=A\.)'), lambda match: u'\u2008'),
|
|
# fix abbreviation that was potentially broken previously
|
|
(re.compile(r'(?<=[zZ]\.) (?=B\.)'), lambda match: u'\u2008'),
|
|
# fix e-mail address that was potentially broken previously
|
|
(re.compile(r'(?<=\w\.) (?=[A-Z][a-z]*@)'), lambda match: ''),
|
|
(re.compile(r'(?<=\d)[Pp]rozent'), lambda match: ' Prozent'),
|
|
(re.compile(r'\.\.\.\.+'), lambda match: '...'), # too many dots (....)
|
|
# spaces before ...
|
|
(re.compile(r'(?<=[^\s])\.\.\.'), lambda match: ' ...'),
|
|
# spaces after ...
|
|
(re.compile(r'\.\.\.(?=[^\s])'), lambda match: '... '),
|
|
# fix special cases of ... in brackets
|
|
(re.compile(r'(?<=[\[\(]) \.\.\. (?=[\]\)])'), lambda match: '...'),
|
|
# fix special cases of ... after a quotation mark
|
|
(re.compile(r'(?<=[\u00BB\u203A]) \.\.\.'), lambda match: '...'),
|
|
# fix special cases of ... before a quotation mark or comma
|
|
(re.compile(r'\.\.\. (?=[\u00AB\u2039,])'), lambda match: '...'),
|
|
# fix missing spaces between numbers and any sort of units, possibly
|
|
# with dot
|
|
(re.compile(r'(?<=\d)(?=(Femto|Piko|Nano|Mikro|Milli|Zenti|Dezi|Hekto|Kilo|Mega|Giga|Tera|Peta|Tausend|Trilli|Kubik|Quadrat|Meter|Uhr|Jahr|Schuljahr|Seite))'), lambda match: ' '), # noqa
|
|
(re.compile(r'(?<=\d\.)(?=(Femto|Piko|Nano|Mikro|Milli|Zenti|Dezi|Hekto|Kilo|Mega|Giga|Tera|Peta|Tausend|Trilli|Kubik|Quadrat|Meter|Uhr|Jahr|Schuljahr|Seite))'), lambda match: ' '), # noqa
|
|
# fix wrong spaces
|
|
# at beginning of paragraphs
|
|
(re.compile(
|
|
r'(?<=<p class="absatz">[A-ZÄÖÜ]) (?=[a-zäöü\-])'), lambda match: ''),
|
|
# before closing quotation
|
|
(re.compile(u' \u00AB'), lambda match: u'\u00AB '),
|
|
# after opening quotation
|
|
(re.compile(u'\u00BB '), lambda match: u' \u00BB'),
|
|
# filtering for spaces in large numbers for better readability
|
|
# end of the number with some character following
|
|
(re.compile(r'(?<=\d\d)(?=\d\d\d[ ,;\)<\?!-])'),
|
|
lambda match: u'\u2008'),
|
|
# end of the number with full-stop following, then space is necessary
|
|
# (avoid file names)
|
|
(re.compile(r'(?<=\d\d)(?=\d\d\d. )'), lambda match: u'\u2008'),
|
|
(re.compile(r'(?<=\d)(?=\d\d\d\u2008)'),
|
|
lambda match: u'\u2008'), # next level
|
|
(re.compile(r'(?<=\d)(?=\d\d\d\u2008)'),
|
|
lambda match: u'\u2008'), # next level
|
|
(re.compile(r'(?<=\d)(?=\d\d\d\u2008)'),
|
|
lambda match: u'\u2008'), # next level
|
|
(re.compile(r'(?<=\d)(?=\d\d\d\u2008)'),
|
|
lambda match: u'\u2008'), # next level
|
|
# filtering for unicode characters that are missing on the Kindle,
|
|
# try to replace them with meaningful work-arounds
|
|
# subscript-0
|
|
(re.compile(u'\u2080'), lambda match: '<span style="font-size: 40%;">0</span>'),
|
|
# subscript-1
|
|
(re.compile(u'\u2081'), lambda match: '<span style="font-size: 40%;">1</span>'),
|
|
# subscript-2
|
|
(re.compile(u'\u2082'), lambda match: '<span style="font-size: 40%;">2</span>'),
|
|
# subscript-3
|
|
(re.compile(u'\u2083'), lambda match: '<span style="font-size: 40%;">3</span>'),
|
|
# subscript-4
|
|
(re.compile(u'\u2084'), lambda match: '<span style="font-size: 40%;">4</span>'),
|
|
# subscript-5
|
|
(re.compile(u'\u2085'), lambda match: '<span style="font-size: 40%;">5</span>'),
|
|
# subscript-6
|
|
(re.compile(u'\u2086'), lambda match: '<span style="font-size: 40%;">6</span>'),
|
|
# subscript-7
|
|
(re.compile(u'\u2087'), lambda match: '<span style="font-size: 40%;">7</span>'),
|
|
# subscript-8
|
|
(re.compile(u'\u2088'), lambda match: '<span style="font-size: 40%;">8</span>'),
|
|
# subscript-9
|
|
(re.compile(u'\u2089'), lambda match: '<span style="font-size: 40%;">9</span>'),
|
|
# always chance CO2
|
|
(re.compile(r'CO2'), lambda match: 'CO<span style="font-size: 40%;">2</span>'), # CO2
|
|
# remove *** paragraphs
|
|
(re.compile(r'<p class="absatz">\*\*\*</p>'), lambda match: ''),
|
|
# better layout for the top line of each article
|
|
(re.compile(r'(?<=DIE ZEIT N\u00B0 \d /) (?=\d\d)'),
|
|
lambda match: ' 20'), # proper year in edition number
|
|
(re.compile(r'(?<=DIE ZEIT N\u00B0 \d\d /) (?=\d\d)'),
|
|
lambda match: ' 20'), # proper year in edition number
|
|
(re.compile(r'(?<=>)(?=DIE ZEIT N\u00B0 \d\d / 20\d\d)'),
|
|
lambda match: u' \u2014 '), # m-dash between category and DIE ZEIT
|
|
]
|
|
|
|
def build_index(self):
|
|
url = "https://meine.zeit.de/anmelden?url=https%3A//epaper.zeit.de/abo/diezeit"
|
|
browser = self.get_browser()
|
|
|
|
# new login process
|
|
browser.open(url)
|
|
browser.select_form(nr=0)
|
|
browser.form['email'] = self.username
|
|
browser.form['pass'] = self.password
|
|
browser.submit()
|
|
# change into abo section - not needed between late 2016 and May 2017, and again starting from March 2020
|
|
# browser.open(url)
|
|
# abolink = browser.find_link(text_regex=re.compile(
|
|
# '.*E-Paper.*')) # used to be '.*Abo-Bereich.*'
|
|
# browser.follow_link(abolink)
|
|
# find page for latest issue
|
|
latestlink = browser.find_link(text_regex=re.compile(
|
|
'.*ZUR AKTUELLEN AUSGABE.*'))
|
|
browser.follow_link(latestlink)
|
|
# now find the correct file, we will still use the ePub file
|
|
epublink = browser.find_link(text_regex=re.compile(
|
|
'.*EPUB F.*R E-READER LADEN.*')) # change from '.*EPUB FÜR E-READER LADEN.*' in May 2017
|
|
response = browser.follow_link(epublink)
|
|
self.report_progress(1, _('next step'))
|
|
|
|
tmp = PersistentTemporaryFile(suffix='.epub')
|
|
self.report_progress(0, _('downloading epub'))
|
|
tmp.write(response.read())
|
|
tmp.close()
|
|
|
|
zfile = zipfile.ZipFile(tmp.name, 'r')
|
|
self.report_progress(0, _('extracting epub'))
|
|
|
|
zfile.extractall(self.output_dir)
|
|
|
|
tmp.close()
|
|
|
|
index = os.path.join(self.output_dir, 'content.opf')
|
|
|
|
self.report_progress(1, _('epub downloaded and extracted'))
|
|
|
|
# doing regular expression filtering
|
|
for path in walk('.'):
|
|
(shortname, extension) = os.path.splitext(path)
|
|
if extension.lower() in ('.html', '.htm', '.xhtml'):
|
|
with open(path, 'r+b') as f:
|
|
raw = f.read()
|
|
raw = raw.decode('utf-8')
|
|
for pat, func in self.preprocess_regexps:
|
|
raw = pat.sub(func, raw)
|
|
f.seek(0)
|
|
f.truncate()
|
|
f.write(raw.encode('utf-8'))
|
|
|
|
# adding real cover
|
|
self.report_progress(
|
|
0, _('trying to download cover image (titlepage)'))
|
|
self.download_cover()
|
|
self.conversion_options["cover"] = self.cover_path
|
|
|
|
return index
|
|
|
|
# getting url of the cover
|
|
def get_cover_url(self):
|
|
self.log.warning('Downloading cover')
|
|
try:
|
|
self.log.warning('Trying PDF-based cover')
|
|
url = "https://meine.zeit.de/anmelden?url=https%3A//epaper.zeit.de/abo/diezeit"
|
|
browser = self.get_browser()
|
|
|
|
# new login process
|
|
browser.open(url)
|
|
browser.select_form(nr=0)
|
|
browser.form['email'] = self.username
|
|
browser.form['pass'] = self.password
|
|
browser.submit()
|
|
# change into abo section - not needed at the moment
|
|
# browser.open(url)
|
|
# abolink = browser.find_link(text_regex=re.compile(
|
|
# '.*Abo-Bereich.*'))
|
|
# browser.follow_link(abolink)
|
|
# find page for latest issue
|
|
latestlink = browser.find_link(text_regex=re.compile(
|
|
'.*ZUR AKTUELLEN AUSGABE.*'))
|
|
browser.follow_link(latestlink)
|
|
# actual cover search
|
|
pdflink = browser.find_link(text_regex=re.compile(
|
|
'.*GESAMT-PDF LADEN.*'))
|
|
cover_url = urlparse(pdflink.base_url)[0] + '://' + urlparse(pdflink.base_url)[1] + '' + (
|
|
urlparse(pdflink.url)[2]).replace('ePaper_', '').replace('.pdf', '_001.pdf')
|
|
self.log.warning('PDF link found:')
|
|
self.log.warning(cover_url)
|
|
# download the cover (has to be here due to new login process)
|
|
with closing(browser.open(cover_url)) as r:
|
|
cdata = r.read()
|
|
from calibre.ebooks.metadata.pdf import get_metadata
|
|
stream = io.BytesIO(cdata)
|
|
cdata = None
|
|
mi = get_metadata(stream)
|
|
if mi.cover_data and mi.cover_data[1]:
|
|
cdata = mi.cover_data[1]
|
|
|
|
cpath = os.path.join(self.output_dir, 'cover.jpg')
|
|
save_cover_data_to(cdata, cpath)
|
|
cover_url = cpath
|
|
|
|
except:
|
|
self.log.warning('Trying low-res cover')
|
|
try:
|
|
inhalt = self.index_to_soup('http://www.zeit.de/inhalt')
|
|
cover_url = inhalt.find('div', attrs={'class': 'singlearchive clearfix'}).img[
|
|
'src'].replace('icon_', '')
|
|
except:
|
|
self.log.warning('Using static old low-res cover')
|
|
cover_url = 'http://images.zeit.de/bilder/titelseiten_zeit/1946/001_001.jpg'
|
|
return cover_url
|