GwR wip, KG updates

This commit is contained in:
GRiker 2010-10-06 11:10:45 -07:00
commit e018b6eabb
18 changed files with 768 additions and 442 deletions

View File

@ -8,10 +8,16 @@ www.guardian.co.uk
''' '''
from calibre import strftime from calibre import strftime
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
from datetime import date
class Guardian(BasicNewsRecipe): class Guardian(BasicNewsRecipe):
title = u'The Guardian' title = u'The Guardian / The Observer'
if date.today().weekday() == 6:
base_url = "http://www.guardian.co.uk/theobserver"
else:
base_url = "http://www.guardian.co.uk/theguardian"
__author__ = 'Seabound and Sujata Raman' __author__ = 'Seabound and Sujata Raman'
language = 'en_GB' language = 'en_GB'
@ -19,6 +25,10 @@ class Guardian(BasicNewsRecipe):
max_articles_per_feed = 100 max_articles_per_feed = 100
remove_javascript = True remove_javascript = True
# List of section titles to ignore
# For example: ['Sport']
ignore_sections = []
timefmt = ' [%a, %d %b %Y]' timefmt = ' [%a, %d %b %Y]'
keep_only_tags = [ keep_only_tags = [
dict(name='div', attrs={'id':["content","article_header","main-article-info",]}), dict(name='div', attrs={'id':["content","article_header","main-article-info",]}),
@ -28,6 +38,7 @@ class Guardian(BasicNewsRecipe):
dict(name='div', attrs={'id':["article-toolbox","subscribe-feeds",]}), dict(name='div', attrs={'id':["article-toolbox","subscribe-feeds",]}),
dict(name='ul', attrs={'class':["pagination"]}), dict(name='ul', attrs={'class':["pagination"]}),
dict(name='ul', attrs={'id':["content-actions"]}), dict(name='ul', attrs={'id':["content-actions"]}),
dict(name='img'),
] ]
use_embedded_content = False use_embedded_content = False
@ -43,18 +54,6 @@ class Guardian(BasicNewsRecipe):
#match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} #match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
''' '''
feeds = [
('Front Page', 'http://www.guardian.co.uk/rss'),
('Business', 'http://www.guardian.co.uk/business/rss'),
('Sport', 'http://www.guardian.co.uk/sport/rss'),
('Culture', 'http://www.guardian.co.uk/culture/rss'),
('Money', 'http://www.guardian.co.uk/money/rss'),
('Life & Style', 'http://www.guardian.co.uk/lifeandstyle/rss'),
('Travel', 'http://www.guardian.co.uk/travel/rss'),
('Environment', 'http://www.guardian.co.uk/environment/rss'),
('Comment','http://www.guardian.co.uk/commentisfree/rss'),
]
def get_article_url(self, article): def get_article_url(self, article):
url = article.get('guid', None) url = article.get('guid', None)
if '/video/' in url or '/flyer/' in url or '/quiz/' in url or \ if '/video/' in url or '/flyer/' in url or '/quiz/' in url or \
@ -76,7 +75,8 @@ class Guardian(BasicNewsRecipe):
return soup return soup
def find_sections(self): def find_sections(self):
soup = self.index_to_soup('http://www.guardian.co.uk/theguardian') # soup = self.index_to_soup("http://www.guardian.co.uk/theobserver")
soup = self.index_to_soup(self.base_url)
# find cover pic # find cover pic
img = soup.find( 'img',attrs ={'alt':'Guardian digital edition'}) img = soup.find( 'img',attrs ={'alt':'Guardian digital edition'})
if img is not None: if img is not None:
@ -113,13 +113,10 @@ class Guardian(BasicNewsRecipe):
try: try:
feeds = [] feeds = []
for title, href in self.find_sections(): for title, href in self.find_sections():
feeds.append((title, list(self.find_articles(href)))) if not title in self.ignore_sections:
feeds.append((title, list(self.find_articles(href))))
return feeds return feeds
except: except:
raise NotImplementedError raise NotImplementedError
def postprocess_html(self,soup,first):
return soup.findAll('html')[0]

View File

@ -1,3 +1,4 @@
from calibre.web.feeds.news import re
from calibre.web.feeds.recipes import BasicNewsRecipe from calibre.web.feeds.recipes import BasicNewsRecipe
from BeautifulSoup import Tag from BeautifulSoup import Tag
@ -10,26 +11,31 @@ class RevistaMuyInteresante(BasicNewsRecipe):
language = 'es' language = 'es'
no_stylesheets = True no_stylesheets = True
remove_attributes = ['style', 'font'] remove_javascript = True
extra_css = ' .txt_articulo{ font-family: sans-serif; font-size: medium; text-align: justify } .contentheading{font-family: serif; font-size: large; font-weight: bold; color: #000000; text-align: center}'
#then we add our own style(s) like this:
extra_css = '''
.contentheading{font-weight: bold}
p {font-size: 4px;font-family: Times New Roman;}
'''
def preprocess_html(self, soup): def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
for img_tag in soup.findAll('img'): for img_tag in soup.findAll('img'):
parent_tag = img_tag.parent imagen = img_tag
if parent_tag.name == 'td': new_tag = Tag(soup,'p')
if not parent_tag.get('class') == 'txt_articulo': break img_tag.replaceWith(new_tag)
imagen = img_tag div = soup.find(attrs={'class':'article_category'})
new_tag = Tag(soup,'p') div.insert(0,imagen)
img_tag.replaceWith(new_tag) break
div = soup.find(attrs={'class':'article_category'})
div.insert(0,imagen)
return soup return soup
preprocess_regexps = [
(re.compile(r'<td class="contentheading" width="100%">.*?</td>', re.DOTALL|re.IGNORECASE), lambda match: '<td class="contentheading">' + match.group().replace('<td class="contentheading" width="100%">','').strip().replace('</td>','').strip() + '</td>'),
]
keep_only_tags = [dict(name='div', attrs={'class':['article']}),dict(name='td', attrs={'class':['txt_articulo']})] keep_only_tags = [dict(name='div', attrs={'class':['article']}),dict(name='td', attrs={'class':['txt_articulo']})]
remove_tags = [ remove_tags = [
@ -37,6 +43,7 @@ class RevistaMuyInteresante(BasicNewsRecipe):
,dict(name='div', attrs={'id':['comment']}) ,dict(name='div', attrs={'id':['comment']})
,dict(name='td', attrs={'class':['buttonheading']}) ,dict(name='td', attrs={'class':['buttonheading']})
,dict(name='div', attrs={'class':['tags_articles']}) ,dict(name='div', attrs={'class':['tags_articles']})
,dict(name='table', attrs={'class':['pagenav']})
] ]
remove_tags_after = dict(name='div', attrs={'class':'tags_articles'}) remove_tags_after = dict(name='div', attrs={'class':'tags_articles'})
@ -71,8 +78,33 @@ class RevistaMuyInteresante(BasicNewsRecipe):
for title, url in [ for title, url in [
('Historia', ('Historia',
'http://www.muyinteresante.es/historia-articulos'), 'http://www.muyinteresante.es/historia-articulos'),
('Ciencia',
'http://www.muyinteresante.es/ciencia-articulos'),
('Naturaleza',
'http://www.muyinteresante.es/naturaleza-articulos'),
('Tecnología',
'http://www.muyinteresante.es/tecnologia-articulos'),
('Salud',
'http://www.muyinteresante.es/salud-articulos'),
('Más Muy',
'http://www.muyinteresante.es/muy'),
('Innova - Automoción',
'http://www.muyinteresante.es/articulos-innovacion-autos'),
('Innova - Salud',
'http://www.muyinteresante.es/articulos-innovacion-salud'),
('Innova - Medio Ambiente',
'http://www.muyinteresante.es/articulos-innovacion-medio-ambiente'),
('Innova - Alimentación',
'http://www.muyinteresante.es/articulos-innovacion-alimentacion'),
('Innova - Sociedad',
'http://www.muyinteresante.es/articulos-innovacion-sociedad'),
('Innova - Tecnología',
'http://www.muyinteresante.es/articulos-innovacion-tecnologia'),
('Innova - Ocio',
'http://www.muyinteresante.es/articulos-innovacion-ocio'),
]: ]:
articles = self.nz_parse_section(url) articles = self.nz_parse_section(url)
if articles: if articles:
feeds.append((title, articles)) feeds.append((title, articles))
return feeds return feeds

View File

@ -110,6 +110,9 @@ class ITUNES(DriverBase):
PRODUCT_ID = [0x1292,0x1293,0x1294,0x1297,0x1299,0x129a] PRODUCT_ID = [0x1292,0x1293,0x1294,0x1297,0x1299,0x129a]
BCD = [0x01] BCD = [0x01]
# Plugboard ID
DEVICE_PLUGBOARD_NAME = 'APPLE'
# iTunes enumerations # iTunes enumerations
Audiobooks = [ Audiobooks = [
'Audible file', 'Audible file',
@ -178,7 +181,8 @@ class ITUNES(DriverBase):
log = Log() log = Log()
manual_sync_mode = False manual_sync_mode = False
path_template = 'iTunes/%s - %s.%s' path_template = 'iTunes/%s - %s.%s'
plugboard = None plugboards = None
plugboard_func = None
problem_titles = [] problem_titles = []
problem_msg = None problem_msg = None
report_progress = None report_progress = None
@ -820,14 +824,14 @@ class ITUNES(DriverBase):
''' '''
self.report_progress = report_progress self.report_progress = report_progress
def set_plugboard(self, pb): def set_plugboards(self, plugboards, pb_func):
# This method is called with the plugboard that matches the format # This method is called with the plugboard that matches the format
# declared in use_plugboard_ext and a device name of ITUNES # declared in use_plugboard_ext and a device name of ITUNES
if DEBUG: if DEBUG:
self.log.info("ITUNES.set_plugboard()") self.log.info("ITUNES.set_plugboard()")
self.log.info(' using plugboard %s' % pb) #self.log.info(' using plugboard %s' % plugboards)
if pb is not None: self.plugboards = plugboards
self.plugboard = pb self.plugboard_func = pb_func
def sync_booklists(self, booklists, end_session=True): def sync_booklists(self, booklists, end_session=True):
''' '''
@ -992,14 +996,6 @@ class ITUNES(DriverBase):
self._dump_cached_books(header="after upload_books()",indent=2) self._dump_cached_books(header="after upload_books()",indent=2)
return (new_booklist, [], []) return (new_booklist, [], [])
def use_plugboard_ext(self):
''' Declare which plugboard extension we care about '''
ext = 'epub'
if DEBUG:
self.log.info("ITUNES.use_plugboard_ext(): declaring %s" % ext)
return ext
# Private methods # Private methods
def _add_device_book(self,fpath, metadata): def _add_device_book(self,fpath, metadata):
''' '''
@ -1347,8 +1343,8 @@ class ITUNES(DriverBase):
plist = None plist = None
if plist: if plist:
if DEBUG: if DEBUG:
self.log.info(" _delete_iTunesMetadata_plist():") self.log.info(" _delete_iTunesMetadata_plist():")
self.log.info(" deleting '%s'\n from '%s'" % (pl_name,fpath)) self.log.info(" deleting '%s'\n from '%s'" % (pl_name,fpath))
zf.delete(pl_name) zf.delete(pl_name)
zf.close() zf.close()
@ -2484,11 +2480,17 @@ class ITUNES(DriverBase):
if DEBUG: if DEBUG:
self.log.info(" unable to remove '%s' from iTunes" % cached_book['title']) self.log.info(" unable to remove '%s' from iTunes" % cached_book['title'])
def title_sorter(self, title):
return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip()
def _update_epub_metadata(self, fpath, metadata): def _update_epub_metadata(self, fpath, metadata):
''' '''
''' '''
self.log.info(" ITUNES._update_epub_metadata()") self.log.info(" ITUNES._update_epub_metadata()")
# Fetch plugboard updates
metadata_x = self._xform_metadata_via_plugboard(metadata, 'epub')
# Refresh epub metadata # Refresh epub metadata
with open(fpath,'r+b') as zfo: with open(fpath,'r+b') as zfo:
# Touch the OPF timestamp # Touch the OPF timestamp
@ -2520,9 +2522,14 @@ class ITUNES(DriverBase):
self.log.info(" add timestamp: %s" % metadata.timestamp) self.log.info(" add timestamp: %s" % metadata.timestamp)
# Force the language declaration for iBooks 1.1 # Force the language declaration for iBooks 1.1
metadata.language = get_lang().replace('_', '-') #metadata.language = get_lang().replace('_', '-')
# Updates from metadata plugboard (ignoring publisher)
metadata.language = metadata_x.language
if DEBUG: if DEBUG:
self.log.info(" rewriting language: <dc:language>%s</dc:language>" % metadata.language) if metadata.language != metadata_x.language:
self.log.info(" rewriting language: <dc:language>%s</dc:language>" % metadata.language)
zf_opf.close() zf_opf.close()
@ -2604,35 +2611,29 @@ class ITUNES(DriverBase):
# Update metadata from plugboard # Update metadata from plugboard
# If self.plugboard is None (no transforms), original metadata is returned intact # If self.plugboard is None (no transforms), original metadata is returned intact
metadata_x = self._xform_metadata_via_plugboard(metadata) metadata_x = self._xform_metadata_via_plugboard(metadata, this_book.format)
if isosx: if isosx:
if lb_added: if lb_added:
lb_added.name.set(metadata_x.title)
lb_added.album.set(metadata_x.title) lb_added.album.set(metadata_x.title)
lb_added.artist.set(authors_to_string(metadata_x.authors)) lb_added.artist.set(authors_to_string(metadata_x.authors))
lb_added.composer.set(metadata_x.uuid) lb_added.composer.set(metadata_x.uuid)
lb_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) lb_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
lb_added.enabled.set(True) lb_added.enabled.set(True)
lb_added.sort_artist.set(metadata_x.author_sort.title()) lb_added.sort_artist.set(metadata_x.author_sort.title())
lb_added.sort_name.set(this_book.title_sorter) lb_added.sort_name.set(metadata.title_sort)
if this_book.format == 'pdf':
lb_added.name.set(metadata.title)
elif this_book.format == 'epub':
lb_added.name.set(metadata_x.title)
if db_added: if db_added:
db_added.name.set(metadata_x.title)
db_added.album.set(metadata_x.title) db_added.album.set(metadata_x.title)
db_added.artist.set(authors_to_string(metadata_x.authors)) db_added.artist.set(authors_to_string(metadata_x.authors))
db_added.composer.set(metadata_x.uuid) db_added.composer.set(metadata_x.uuid)
db_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) db_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
db_added.enabled.set(True) db_added.enabled.set(True)
db_added.sort_artist.set(metadata_x.author_sort.title()) db_added.sort_artist.set(metadata_x.author_sort.title())
db_added.sort_name.set(this_book.title_sorter) db_added.sort_name.set(metadata.title_sort)
if this_book.format == 'pdf':
db_added.name.set(metadata.title)
elif this_book.format == 'epub':
db_added.name.set(metadata_x.title)
if metadata_x.comments: if metadata_x.comments:
if lb_added: if lb_added:
@ -2652,8 +2653,10 @@ class ITUNES(DriverBase):
# Set genre from series if available, else first alpha tag # Set genre from series if available, else first alpha tag
# Otherwise iTunes grabs the first dc:subject from the opf metadata # Otherwise iTunes grabs the first dc:subject from the opf metadata
# self.settings().read_metadata is used as a surrogate for "Use Series name as Genre"
if metadata_x.series and self.settings().read_metadata: if metadata_x.series and self.settings().read_metadata:
if DEBUG: if DEBUG:
self.log.info(" ITUNES._update_iTunes_metadata()")
self.log.info(" using Series name as Genre") self.log.info(" using Series name as Genre")
# Format the index as a sort key # Format the index as a sort key
@ -2662,18 +2665,35 @@ class ITUNES(DriverBase):
fraction = index-integer fraction = index-integer
series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0'))
if lb_added: if lb_added:
lb_added.sort_name.set("%s %s" % (metadata_x.series, series_index)) lb_added.sort_name.set("%s %s" % (self.title_sorter(metadata_x.series), series_index))
lb_added.genre.set(metadata_x.series)
lb_added.episode_ID.set(metadata_x.series) lb_added.episode_ID.set(metadata_x.series)
lb_added.episode_number.set(metadata_x.series_index) lb_added.episode_number.set(metadata_x.series_index)
# If no plugboard transform applied to tags, change the Genre/Category to Series
if metadata.tags == metadata_x.tags:
lb_added.genre.set(self.title_sorter(metadata_x.series))
else:
for tag in metadata_x.tags:
if self._is_alpha(tag[0]):
lb_added.genre.set(tag)
break
if db_added: if db_added:
db_added.sort_name.set("%s %s" % (metadata_x.series, series_index)) db_added.sort_name.set("%s %s" % (self.title_sorter(metadata_x.series), series_index))
db_added.genre.set(metadata_x.series)
db_added.episode_ID.set(metadata_x.series) db_added.episode_ID.set(metadata_x.series)
db_added.episode_number.set(metadata_x.series_index) db_added.episode_number.set(metadata_x.series_index)
elif metadata_x.tags: # If no plugboard transform applied to tags, change the Genre/Category to Series
if metadata.tags == metadata_x.tags:
db_added.genre.set(self.title_sorter(metadata_x.series))
else:
for tag in metadata_x.tags:
if self._is_alpha(tag[0]):
db_added.genre.set(tag)
break
elif metadata_x.tags is not None:
if DEBUG: if DEBUG:
self.log.info(" %susing Tag as Genre" % self.log.info(" %susing Tag as Genre" %
"no Series name available, " if self.settings().read_metadata else '') "no Series name available, " if self.settings().read_metadata else '')
@ -2687,30 +2707,24 @@ class ITUNES(DriverBase):
elif iswindows: elif iswindows:
if lb_added: if lb_added:
lb_added.Name = metadata_x.title
lb_added.Album = metadata_x.title lb_added.Album = metadata_x.title
lb_added.Artist = authors_to_string(metadata_x.authors) lb_added.Artist = authors_to_string(metadata_x.authors)
lb_added.Composer = metadata_x.uuid lb_added.Composer = metadata_x.uuid
lb_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) lb_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
lb_added.Enabled = True lb_added.Enabled = True
lb_added.SortArtist = (metadata_x.author_sort.title()) lb_added.SortArtist = metadata_x.author_sort.title()
lb_added.SortName = (this_book.title_sorter) lb_added.SortName = metadata.title_sort
if this_book.format == 'pdf':
lb_added.Name = metadata.title
elif this_book.format == 'epub':
lb_added.Name = metadata_x.title
if db_added: if db_added:
db_added.Name = metadata_x.title
db_added.Album = metadata_x.title db_added.Album = metadata_x.title
db_added.Artist = authors_to_string(metadata_x.authors) db_added.Artist = authors_to_string(metadata_x.authors)
db_added.Composer = metadata_x.uuid db_added.Composer = metadata_x.uuid
db_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) db_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
db_added.Enabled = True db_added.Enabled = True
db_added.SortArtist = (metadata_x.author_sort.title()) db_added.SortArtist = metadata_x.author_sort.title()
db_added.SortName = (this_book.title_sorter) db_added.SortName = metadata.title_sort
if this_book.format == 'pdf':
db_added.Name = metadata.title
elif this_book.format == 'epub':
db_added.Name = metadata_x.title
if metadata_x.comments: if metadata_x.comments:
if lb_added: if lb_added:
@ -2743,16 +2757,24 @@ class ITUNES(DriverBase):
fraction = index-integer fraction = index-integer
series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0'))
if lb_added: if lb_added:
lb_added.SortName = "%s %s" % (metadata_x.series, series_index) lb_added.SortName = "%s %s" % (self.title_sorter(metadata_x.series), series_index)
lb_added.Genre = metadata_x.series
lb_added.EpisodeID = metadata_x.series lb_added.EpisodeID = metadata_x.series
try: try:
lb_added.EpisodeNumber = metadata_x.series_index lb_added.EpisodeNumber = metadata_x.series_index
except: except:
pass pass
# If no plugboard transform applied to tags, change the Genre/Category to Series
if metadata.tags == metadata_x.tags:
lb_added.Genre = self.title_sorter(metadata_x.series)
else:
for tag in metadata_x.tags:
if self._is_alpha(tag[0]):
lb_added.Genre = tag
break
if db_added: if db_added:
db_added.SortName = "%s %s" % (metadata_x.series, series_index) db_added.SortName = "%s %s" % (self.title_sorter(metadata_x.series), series_index)
db_added.Genre = metadata_x.series
db_added.EpisodeID = metadata_x.series db_added.EpisodeID = metadata_x.series
try: try:
db_added.EpisodeNumber = metadata_x.series_index db_added.EpisodeNumber = metadata_x.series_index
@ -2760,7 +2782,17 @@ class ITUNES(DriverBase):
if DEBUG: if DEBUG:
self.log.warning(" iTunes automation interface reported an error" self.log.warning(" iTunes automation interface reported an error"
" setting EpisodeNumber on iDevice") " setting EpisodeNumber on iDevice")
elif metadata_x.tags:
# If no plugboard transform applied to tags, change the Genre/Category to Series
if metadata.tags == metadata_x.tags:
db_added.Genre = self.title_sorter(metadata_x.series)
else:
for tag in metadata_x.tags:
if self._is_alpha(tag[0]):
db_added.Genre = tag
break
elif metadata_x.tags is not None:
if DEBUG: if DEBUG:
self.log.info(" using Tag as Genre") self.log.info(" using Tag as Genre")
for tag in metadata_x.tags: for tag in metadata_x.tags:
@ -2771,20 +2803,31 @@ class ITUNES(DriverBase):
db_added.Genre = tag db_added.Genre = tag
break break
def _xform_metadata_via_plugboard(self, book): def _xform_metadata_via_plugboard(self, book, format):
''' Transform book metadata from plugboard templates ''' ''' Transform book metadata from plugboard templates '''
if DEBUG: if DEBUG:
self.log.info("ITUNES._update_metadata_from_plugboard()") self.log.info(" ITUNES._update_metadata_from_plugboard()")
if self.plugboard is not None: if self.plugboard_func:
pb = self.plugboard_func(self.DEVICE_PLUGBOARD_NAME, format, self.plugboards)
newmi = book.deepcopy_metadata() newmi = book.deepcopy_metadata()
newmi.template_to_attribute(book, self.plugboard) newmi.template_to_attribute(book, pb)
if DEBUG: if DEBUG:
if book.title != newmi.title: self.log.info(" transforming %s using %s:" % (format, pb))
self.log.info(" .title (original): %s" % book.title) self.log.info(" title: %s %s" % (book.title, ">>> %s" %
self.log.info(" .title (templated): %s" % newmi.title) newmi.title if book.title != newmi.title else ''))
else: self.log.info(" title_sort: %s %s" % (book.title_sort, ">>> %s" %
self.log.info(" .title (no change): %s" % book.title) newmi.title_sort if book.title_sort != newmi.title_sort else ''))
self.log.info(" authors: %s %s" % (book.authors, ">>> %s" %
newmi.authors if book.authors != newmi.authors else ''))
self.log.info(" author_sort: %s %s" % (book.author_sort, ">>> %s" %
newmi.author_sort if book.author_sort != newmi.author_sort else ''))
self.log.info(" language: %s %s" % (book.language, ">>> %s" %
newmi.language if book.language != newmi.language else ''))
self.log.info(" publisher: %s %s" % (book.publisher, ">>> %s" %
newmi.publisher if book.publisher != newmi.publisher else ''))
self.log.info(" tags: %s %s" % (book.tags, ">>> %s" %
newmi.tags if book.tags != newmi.tags else ''))
else: else:
newmi = book newmi = book
return newmi return newmi
@ -2800,6 +2843,9 @@ class ITUNES_ASYNC(ITUNES):
icon = I('devices/itunes.png') icon = I('devices/itunes.png')
description = _('Communicate with iTunes.') description = _('Communicate with iTunes.')
# Plugboard ID
DEVICE_PLUGBOARD_NAME = 'APPLE'
connected = False connected = False
def __init__(self,path): def __init__(self,path):
@ -3080,9 +3126,3 @@ class Book(Metadata):
Metadata.__init__(self, title, authors=[author]) Metadata.__init__(self, title, authors=[author])
@dynamic_property
def title_sorter(self):
doc = '''String to sort the title. If absent, title is returned'''
def fget(self):
return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', self.title).rstrip()
return property(doc=doc, fget=fget)

View File

@ -20,7 +20,7 @@ class IREXDR1000(USBMS):
# Ordered list of supported formats # Ordered list of supported formats
# Be sure these have an entry in calibre.devices.mime # Be sure these have an entry in calibre.devices.mime
FORMATS = ['epub', 'mobi', 'prc', 'html', 'pdf', 'txt'] FORMATS = ['epub', 'mobi', 'prc', 'html', 'pdf', 'djvu', 'txt']
VENDOR_ID = [0x1e6b] VENDOR_ID = [0x1e6b]
PRODUCT_ID = [0x001] PRODUCT_ID = [0x001]

View File

@ -151,7 +151,8 @@ class CHMReader(CHMFile):
continue continue
raise raise
self._extracted = True self._extracted = True
files = os.listdir(output_dir) files = [x for x in os.listdir(output_dir) if
os.path.isfile(os.path.join(output_dir, x))]
if self.hhc_path not in files: if self.hhc_path not in files:
for f in files: for f in files:
if f.lower() == self.hhc_path.lower(): if f.lower() == self.hhc_path.lower():

View File

@ -404,14 +404,16 @@ class MetadataUpdater(object):
if self.cover_record is not None: if self.cover_record is not None:
size = len(self.cover_record) size = len(self.cover_record)
cover = rescale_image(data, size) cover = rescale_image(data, size)
cover += '\0' * (size - len(cover)) if len(cover) <= size:
self.cover_record[:] = cover cover += '\0' * (size - len(cover))
self.cover_record[:] = cover
if self.thumbnail_record is not None: if self.thumbnail_record is not None:
size = len(self.thumbnail_record) size = len(self.thumbnail_record)
thumbnail = rescale_image(data, size, dimen=MAX_THUMB_DIMEN) thumbnail = rescale_image(data, size, dimen=MAX_THUMB_DIMEN)
thumbnail += '\0' * (size - len(thumbnail)) if len(thumbnail) <= size:
self.thumbnail_record[:] = thumbnail thumbnail += '\0' * (size - len(thumbnail))
return self.thumbnail_record[:] = thumbnail
return
def set_metadata(stream, mi): def set_metadata(stream, mi):
mu = MetadataUpdater(stream) mu = MetadataUpdater(stream)

View File

@ -112,15 +112,34 @@ def align_block(raw, multiple=4, pad='\0'):
def rescale_image(data, maxsizeb, dimen=None): def rescale_image(data, maxsizeb, dimen=None):
if dimen is not None: if dimen is not None:
return thumbnail(data, width=dimen, height=dimen)[-1] data = thumbnail(data, width=dimen, height=dimen)[-1]
# Replace transparent pixels with white pixels and convert to JPEG else:
data = save_cover_data_to(data, 'img.jpg', return_data=True) # Replace transparent pixels with white pixels and convert to JPEG
data = save_cover_data_to(data, 'img.jpg', return_data=True)
if len(data) <= maxsizeb:
return data
orig_data = data
img = Image()
quality = 95
if hasattr(img, 'set_compression_quality'):
img.load(data)
while len(data) >= maxsizeb and quality >= 10:
quality -= 5
img.set_compression_quality(quality)
data = img.export('jpg')
if len(data) <= maxsizeb:
return data
orig_data = data
scale = 0.9 scale = 0.9
while len(data) >= maxsizeb and scale >= 0.05: while len(data) >= maxsizeb and scale >= 0.05:
img = Image() img = Image()
img.load(data) img.load(orig_data)
w, h = img.size w, h = img.size
img.size = (int(scale*w), int(scale*h)) img.size = (int(scale*w), int(scale*h))
if hasattr(img, 'set_compression_quality'):
img.set_compression_quality(quality)
data = img.export('jpg') data = img.export('jpg')
scale -= 0.05 scale -= 0.05
return data return data

View File

@ -31,12 +31,14 @@ class CoverManager(object):
</style> </style>
</head> </head>
<body> <body>
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" <div>
xmlns:xlink="http://www.w3.org/1999/xlink" <svg version="1.1" xmlns="http://www.w3.org/2000/svg"
width="100%%" height="100%%" viewBox="__viewbox__" xmlns:xlink="http://www.w3.org/1999/xlink"
preserveAspectRatio="__ar__"> width="100%%" height="100%%" viewBox="__viewbox__"
<image width="__width__" height="__height__" xlink:href="%s"/> preserveAspectRatio="__ar__">
</svg> <image width="__width__" height="__height__" xlink:href="%s"/>
</svg>
</div>
</body> </body>
</html> </html>
''') ''')

View File

@ -104,6 +104,28 @@ class DeviceJob(BaseJob): # {{{
# }}} # }}}
def find_plugboard(device_name, format, plugboards):
cpb = None
if format in plugboards:
cpb = plugboards[format]
elif plugboard_any_format_value in plugboards:
cpb = plugboards[plugboard_any_format_value]
if cpb is not None:
if device_name in cpb:
cpb = cpb[device_name]
elif plugboard_any_device_value in cpb:
cpb = cpb[plugboard_any_device_value]
else:
cpb = None
if DEBUG:
prints('Device using plugboard', format, device_name, cpb)
return cpb
def device_name_for_plugboards(device_class):
if hasattr(device_class, 'DEVICE_PLUGBOARD_NAME'):
return device_class.DEVICE_PLUGBOARD_NAME
return device_class.__class__.__name__
class DeviceManager(Thread): # {{{ class DeviceManager(Thread): # {{{
def __init__(self, connected_slot, job_manager, open_feedback_slot, sleep_time=2): def __init__(self, connected_slot, job_manager, open_feedback_slot, sleep_time=2):
@ -311,12 +333,9 @@ class DeviceManager(Thread): # {{{
return self.device.card_prefix(end_session=False), self.device.free_space() return self.device.card_prefix(end_session=False), self.device.free_space()
def sync_booklists(self, done, booklists, plugboards): def sync_booklists(self, done, booklists, plugboards):
if hasattr(self.connected_device, 'use_plugboard_ext') and \ if hasattr(self.connected_device, 'set_plugboards') and \
callable(self.connected_device.use_plugboard_ext): callable(self.connected_device.set_plugboards):
ext = self.connected_device.use_plugboard_ext() self.connected_device.set_plugboards(plugboards, find_plugboard)
if ext is not None:
self.connected_device.set_plugboard(
self.find_plugboard(ext, plugboards))
return self.create_job(self._sync_booklists, done, args=[booklists], return self.create_job(self._sync_booklists, done, args=[booklists],
description=_('Send metadata to device')) description=_('Send metadata to device'))
@ -325,36 +344,18 @@ class DeviceManager(Thread): # {{{
args=[booklist, on_card], args=[booklist, on_card],
description=_('Send collections to device')) description=_('Send collections to device'))
def find_plugboard(self, ext, plugboards):
dev_name = self.connected_device.__class__.__name__
cpb = None
if ext in plugboards:
cpb = plugboards[ext]
elif plugboard_any_format_value in plugboards:
cpb = plugboards[plugboard_any_format_value]
if cpb is not None:
if dev_name in cpb:
cpb = cpb[dev_name]
elif plugboard_any_device_value in cpb:
cpb = cpb[plugboard_any_device_value]
else:
cpb = None
if DEBUG:
prints('Device using plugboard', ext, dev_name, cpb)
return cpb
def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None): def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None):
'''Upload books to device: ''' '''Upload books to device: '''
if hasattr(self.connected_device, 'use_plugboard_ext') and \ if hasattr(self.connected_device, 'set_plugboards') and \
callable(self.connected_device.use_plugboard_ext): callable(self.connected_device.set_plugboards):
ext = self.connected_device.use_plugboard_ext() self.connected_device.set_plugboards(plugboards, find_plugboard)
if ext is not None:
self.connected_device.set_plugboard(self.find_plugboard(ext, plugboards))
if metadata and files and len(metadata) == len(files): if metadata and files and len(metadata) == len(files):
for f, mi in zip(files, metadata): for f, mi in zip(files, metadata):
if isinstance(f, unicode): if isinstance(f, unicode):
ext = f.rpartition('.')[-1].lower() ext = f.rpartition('.')[-1].lower()
cpb = self.find_plugboard(ext, plugboards) cpb = find_plugboard(
device_name_for_plugboards(self.connected_device),
ext, plugboards)
if ext: if ext:
try: try:
if DEBUG: if DEBUG:
@ -362,7 +363,7 @@ class DeviceManager(Thread): # {{{
f, file=sys.__stdout__) f, file=sys.__stdout__)
with open(f, 'r+b') as stream: with open(f, 'r+b') as stream:
if cpb: if cpb:
newmi = mi.deepcopy() newmi = mi.deepcopy_metadata()
newmi.template_to_attribute(mi, cpb) newmi.template_to_attribute(mi, cpb)
else: else:
newmi = mi newmi = mi

View File

@ -308,7 +308,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
im = Image() im = Image()
im.load(cdata) im.load(cdata)
im.trim(10) im.trim(10)
cdata = im.export('jpg') cdata = im.export('png')
pix = QPixmap() pix = QPixmap()
pix.loadFromData(cdata) pix.loadFromData(cdata)
self.cover.setPixmap(pix) self.cover.setPixmap(pix)

View File

@ -490,26 +490,39 @@ class BooksView(QTableView): # {{{
drag.setMimeData(md) drag.setMimeData(md)
cover = self.drag_icon(m.cover(self.currentIndex().row()), cover = self.drag_icon(m.cover(self.currentIndex().row()),
len(selected) > 1) len(selected) > 1)
drag.setHotSpot(QPoint(cover.width()//3, cover.height()//3)) drag.setHotSpot(QPoint(-15, -15))
drag.setPixmap(cover) drag.setPixmap(cover)
return drag return drag
def event_has_mods(self, event=None):
mods = event.modifiers() if event is not None else \
QApplication.keyboardModifiers()
return mods & Qt.ControlModifier or mods & Qt.ShiftModifier
def mousePressEvent(self, event): def mousePressEvent(self, event):
if event.button() == Qt.LeftButton: if event.button() == Qt.LeftButton and not self.event_has_mods():
self.drag_start_pos = event.pos() self.drag_start_pos = event.pos()
return QTableView.mousePressEvent(self, event) return QTableView.mousePressEvent(self, event)
def mouseMoveEvent(self, event): def mouseMoveEvent(self, event):
if not (event.buttons() & Qt.LeftButton) or self.drag_start_pos is None: if self.drag_start_pos is None:
return QTableView.mouseMoveEvent(self, event)
if self.event_has_mods():
self.drag_start_pos = None
return return
if (event.pos() - self.drag_start_pos).manhattanLength() \
< QApplication.startDragDistance(): if not (event.buttons() & Qt.LeftButton) or \
(event.pos() - self.drag_start_pos).manhattanLength() \
< QApplication.startDragDistance():
return return
index = self.indexAt(event.pos()) index = self.indexAt(event.pos())
if not index.isValid(): if not index.isValid():
return return
drag = self.drag_data() drag = self.drag_data()
drag.exec_(Qt.CopyAction) drag.exec_(Qt.CopyAction)
self.drag_start_pos = None
def dragEnterEvent(self, event): def dragEnterEvent(self, event):
if int(event.possibleActions() & Qt.CopyAction) + \ if int(event.possibleActions() & Qt.CopyAction) + \
@ -643,7 +656,7 @@ class DeviceBooksView(BooksView): # {{{
drag.setMimeData(md) drag.setMimeData(md)
cover = self.drag_icon(m.cover(self.currentIndex().row()), len(paths) > cover = self.drag_icon(m.cover(self.currentIndex().row()), len(paths) >
1) 1)
drag.setHotSpot(QPoint(cover.width()//3, cover.height()//3)) drag.setHotSpot(QPoint(-15, -15))
drag.setPixmap(cover) drag.setPixmap(cover)
return drag return drag

View File

@ -9,6 +9,7 @@ from PyQt4 import QtGui
from PyQt4.Qt import Qt from PyQt4.Qt import Qt
from calibre.gui2 import error_dialog from calibre.gui2 import error_dialog
from calibre.gui2.device import device_name_for_plugboards
from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences import ConfigWidgetBase, test_widget
from calibre.gui2.preferences.plugboard_ui import Ui_Form from calibre.gui2.preferences.plugboard_ui import Ui_Form
from calibre.customize.ui import metadata_writers, device_plugins from calibre.customize.ui import metadata_writers, device_plugins
@ -45,12 +46,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
else: else:
self.device_label.setText(_('Device currently connected: None')) self.device_label.setText(_('Device currently connected: None'))
self.devices = [''] self.devices = ['', 'APPLE', 'FOLDER_DEVICE']
for device in device_plugins(): for device in device_plugins():
n = device.__class__.__name__ n = device_name_for_plugboards(device)
if n.startswith('FOLDER_DEVICE'): if n not in self.devices:
n = 'FOLDER_DEVICE' self.devices.append(n)
self.devices.append(n)
self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
self.devices.insert(1, plugboard_save_to_disk_value) self.devices.insert(1, plugboard_save_to_disk_value)
self.devices.insert(2, plugboard_any_device_value) self.devices.insert(2, plugboard_any_device_value)

View File

@ -80,6 +80,7 @@ class TagsView(QTreeView): # {{{
self.setItemDelegate(TagDelegate(self)) self.setItemDelegate(TagDelegate(self))
self.made_connections = False self.made_connections = False
self.setAcceptDrops(True) self.setAcceptDrops(True)
self.setDragDropMode(self.DropOnly)
self.setDropIndicatorShown(True) self.setDropIndicatorShown(True)
def set_database(self, db, tag_match, sort_by): def set_database(self, db, tag_match, sort_by):

View File

@ -158,7 +158,7 @@ class Image(_magick.Image): # {{{
format = ext[1:] format = ext[1:]
format = format.upper() format = format.upper()
with open(path, 'wb') as f: with lopen(path, 'wb') as f:
f.write(self.export(format)) f.write(self.export(format))
def compose(self, img, left=0, top=0, operation='OverCompositeOp'): def compose(self, img, left=0, top=0, operation='OverCompositeOp'):

View File

@ -11,22 +11,57 @@ from calibre.utils.magick import Image, DrawingWand, create_canvas
from calibre.constants import __appname__, __version__ from calibre.constants import __appname__, __version__
from calibre import fit_image from calibre import fit_image
def normalize_format_name(fmt):
fmt = fmt.lower()
if fmt == 'jpeg':
fmt = 'jpg'
return fmt
def save_cover_data_to(data, path, bgcolor='#ffffff', resize_to=None, def save_cover_data_to(data, path, bgcolor='#ffffff', resize_to=None,
return_data=False): return_data=False, compression_quality=90):
''' '''
Saves image in data to path, in the format specified by the path Saves image in data to path, in the format specified by the path
extension. Composes the image onto a blank canvas so as to extension. Removes any transparency. If there is no transparency and no
properly convert transparent images. resize and the input and output image formats are the same, no changes are
made.
:param compression_quality: The quality of the image after compression.
Number between 1 and 100. 1 means highest compression, 100 means no
compression (lossless).
:param bgcolor: The color for transparent pixels. Must be specified in hex.
:param resize_to: A tuple (width, height) or None for no resizing
''' '''
changed = False
img = Image() img = Image()
img.load(data) img.load(data)
orig_fmt = normalize_format_name(img.format)
fmt = os.path.splitext(path)[1]
fmt = normalize_format_name(fmt[1:])
if resize_to is not None: if resize_to is not None:
img.size = (resize_to[0], resize_to[1]) img.size = (resize_to[0], resize_to[1])
canvas = create_canvas(img.size[0], img.size[1], bgcolor) changed = True
canvas.compose(img) if not hasattr(img, 'has_transparent_pixels') or img.has_transparent_pixels():
canvas = create_canvas(img.size[0], img.size[1], bgcolor)
canvas.compose(img)
img = canvas
changed = True
if not changed:
changed = fmt != orig_fmt
if return_data: if return_data:
return canvas.export(os.path.splitext(path)[1][1:]) if changed:
canvas.save(path) if hasattr(img, 'set_compression_quality') and fmt == 'jpg':
img.set_compression_quality(compression_quality)
return img.export(fmt)
return data
if changed:
if hasattr(img, 'set_compression_quality') and fmt == 'jpg':
img.set_compression_quality(compression_quality)
img.save(path)
else:
with lopen(path, 'wb') as f:
f.write(data)
def thumbnail(data, width=120, height=120, bgcolor='#ffffff', fmt='jpg'): def thumbnail(data, width=120, height=120, bgcolor='#ffffff', fmt='jpg'):
img = Image() img = Image()
@ -37,6 +72,8 @@ def thumbnail(data, width=120, height=120, bgcolor='#ffffff', fmt='jpg'):
img.size = (nwidth, nheight) img.size = (nwidth, nheight)
canvas = create_canvas(img.size[0], img.size[1], bgcolor) canvas = create_canvas(img.size[0], img.size[1], bgcolor)
canvas.compose(img) canvas.compose(img)
if fmt == 'jpg' and hasattr(canvas, 'set_compression_quality'):
canvas.set_compression_quality(70)
return (canvas.size[0], canvas.size[1], canvas.export(fmt)) return (canvas.size[0], canvas.size[1], canvas.export(fmt))
def identify_data(data): def identify_data(data):

View File

@ -725,6 +725,49 @@ magick_Image_set_page(magick_Image *self, PyObject *args, PyObject *kwargs) {
} }
// }}} // }}}
// Image.set_compression_quality {{{
static PyObject *
magick_Image_set_compression_quality(magick_Image *self, PyObject *args, PyObject *kwargs) {
Py_ssize_t quality;
if (!PyArg_ParseTuple(args, "n", &quality)) return NULL;
if (!MagickSetImageCompressionQuality(self->wand, quality)) return magick_set_exception(self->wand);
Py_RETURN_NONE;
}
// }}}
// Image.has_transparent_pixels {{{
static PyObject *
magick_Image_has_transparent_pixels(magick_Image *self, PyObject *args, PyObject *kwargs) {
PixelIterator *pi = NULL;
PixelWand **pixels = NULL;
int found = 0;
size_t r, c, width, height;
double alpha;
height = MagickGetImageHeight(self->wand);
pi = NewPixelIterator(self->wand);
for (r = 0; r < height; r++) {
pixels = PixelGetNextIteratorRow(pi, &width);
for (c = 0; c < width; c++) {
alpha = PixelGetAlpha(pixels[c]);
if (alpha < 1.00) {
found = 1;
c = width; r = height;
}
}
}
pi = DestroyPixelIterator(pi);
if (found) Py_RETURN_TRUE;
Py_RETURN_FALSE;
}
// }}}
// Image.normalize {{{ // Image.normalize {{{
static PyObject * static PyObject *
@ -872,6 +915,14 @@ static PyMethodDef magick_Image_methods[] = {
"set_page(width, height, x, y) \n\n Sets the page geometry of the image." "set_page(width, height, x, y) \n\n Sets the page geometry of the image."
}, },
{"set_compression_quality", (PyCFunction)magick_Image_set_compression_quality, METH_VARARGS,
"set_compression_quality(quality) \n\n Sets the compression quality when exporting the image."
},
{"has_transparent_pixels", (PyCFunction)magick_Image_has_transparent_pixels, METH_VARARGS,
"has_transparent_pixels() \n\n Returns True iff image has a (semi-) transparent pixel"
},
{"thumbnail", (PyCFunction)magick_Image_thumbnail, METH_VARARGS, {"thumbnail", (PyCFunction)magick_Image_thumbnail, METH_VARARGS,
"thumbnail(width, height) \n\n Convert to a thumbnail of specified size." "thumbnail(width, height) \n\n Convert to a thumbnail of specified size."
}, },

File diff suppressed because it is too large Load Diff

View File

@ -18,8 +18,9 @@ If this module is run, it will perform a series of unit tests.
import sys, string, operator import sys, string, operator
from calibre.utils.pyparsing import CaselessKeyword, Group, Forward, CharsNotIn, Suppress, \ from calibre.utils.pyparsing import CaselessKeyword, Group, Forward, \
OneOrMore, MatchFirst, CaselessLiteral, Optional, NoMatch, ParseException CharsNotIn, Suppress, OneOrMore, MatchFirst, CaselessLiteral, \
Optional, NoMatch, ParseException, QuotedString
from calibre.constants import preferred_encoding from calibre.constants import preferred_encoding
@ -127,18 +128,21 @@ class SearchQueryParser(object):
location |= l location |= l
location = Optional(location, default='all') location = Optional(location, default='all')
word_query = CharsNotIn(string.whitespace + '()') word_query = CharsNotIn(string.whitespace + '()')
quoted_query = Suppress('"')+CharsNotIn('"')+Suppress('"') #quoted_query = Suppress('"')+CharsNotIn('"')+Suppress('"')
quoted_query = QuotedString('"', escChar='\\')
query = quoted_query | word_query query = quoted_query | word_query
Token = Group(location + query).setResultsName('token') Token = Group(location + query).setResultsName('token')
if test: if test:
print 'Testing Token parser:' print 'Testing Token parser:'
Token.validate()
failed = SearchQueryParser.run_tests(Token, 'token', failed = SearchQueryParser.run_tests(Token, 'token',
( (
('tag:asd', ['tag', 'asd']), ('tag:asd', ['tag', 'asd']),
('ddsä', ['all', 'ddsä']), (u'ddsä', ['all', u'ddsä']),
('"one two"', ['all', 'one two']), ('"one \\"two"', ['all', 'one "two']),
('title:"one two"', ['title', 'one two']), ('title:"one \\"1.5\\" two"', ['title', 'one "1.5" two']),
('title:abc"def', ['title', 'abc"def']),
) )
) )
@ -167,7 +171,7 @@ class SearchQueryParser(object):
).setResultsName("or") | And) ).setResultsName("or") | And)
if test: if test:
Or.validate() #Or.validate()
self._tests_failed = bool(failed) self._tests_failed = bool(failed)
self._parser = Or self._parser = Or
@ -240,6 +244,8 @@ class SearchQueryParser(object):
''' '''
return set([]) return set([])
# Testing {{{
class Tester(SearchQueryParser): class Tester(SearchQueryParser):
texts = { texts = {
@ -599,3 +605,6 @@ def main(args=sys.argv):
if __name__ == '__main__': if __name__ == '__main__':
sys.exit(main()) sys.exit(main())
# }}}