diff --git a/src/calibre/ebooks/metadata/opf.xml b/src/calibre/ebooks/metadata/opf.xml index c79ac0f09f..4e0f8d94be 100644 --- a/src/calibre/ebooks/metadata/opf.xml +++ b/src/calibre/ebooks/metadata/opf.xml @@ -15,6 +15,7 @@ ${mi.comments} ${mi.publisher} ${mi.isbn} + ${mi.rights} diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index ed074f813e..b98e345366 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -439,7 +439,7 @@ class OPF(object): publisher = MetadataField('publisher') language = MetadataField('language') comments = MetadataField('description') - category = MetadataField('category') + category = MetadataField('type') rights = MetadataField('rights') series = MetadataField('series', is_dc=False) series_index = MetadataField('series_index', is_dc=False, formatter=float, none_is=1) @@ -967,6 +967,130 @@ class OPFCreator(MetaInformation): ncx_stream.flush() +def metadata_to_opf(mi, as_string=True): + from lxml import etree + import textwrap + from calibre.ebooks.oeb.base import OPF, DC + + if not mi.application_id: + mi.application_id = str(uuid.uuid4()) + + if not mi.book_producer: + mi.book_producer = __appname__ + ' (%s) '%__version__ + \ + '[http://calibre-ebook.com]' + + if not mi.language: + mi.language = 'UND' + + root = etree.fromstring(textwrap.dedent( + ''' + + + %(id)s + + + + '''%dict(a=__appname__, id=mi.application_id))) + metadata = root[0] + guide = root[1] + metadata[0].tail = '\n'+(' '*8) + def factory(tag, text=None, sort=None, role=None, scheme=None, name=None, + content=None): + attrib = {} + if sort: + attrib[OPF('file-as')] = sort + if role: + attrib[OPF('role')] = role + if scheme: + attrib[OPF('scheme')] = scheme + if name: + attrib['name'] = name + if content: + attrib['content'] = content + elem = metadata.makeelement(tag, attrib=attrib) + elem.tail = '\n'+(' '*8) + if text: + elem.text = text.strip() + metadata.append(elem) + + factory(DC('title'), mi.title, mi.title_sort) + for au in mi.authors: + factory(DC('creator'), au, mi.author_sort, 'aut') + factory(DC('contributor'), mi.book_producer, __appname__, 'bkp') + if hasattr(mi.pubdate, 'isoformat'): + factory(DC('date'), mi.pubdate.isoformat()) + factory(DC('language'), mi.language) + if mi.category: + factory(DC('type'), mi.category) + if mi.comments: + factory(DC('description'), mi.comments) + if mi.publisher: + factory(DC('publisher'), mi.publisher) + if mi.isbn: + factory(DC('identifier'), mi.isbn, scheme='ISBN') + if mi.rights: + factory(DC('rights'), mi.rights) + if mi.tags: + for tag in mi.tags: + factory(DC('subject'), tag) + meta = lambda n, c: factory('meta', name='calibre:'+n, content=c) + if mi.series: + meta('series', mi.series) + if mi.series_index is not None: + meta('series_index', mi.format_series_index()) + if mi.rating is not None: + meta('rating', str(mi.rating)) + if hasattr(mi.timestamp, 'isoformat'): + meta('timestamp', mi.timestamp.isoformat()) + if mi.publication_type: + meta('publication_type', mi.publication_type) + + metadata[-1].tail = '\n' +(' '*4) + + if mi.cover: + guide.text = '\n'+(' '*8) + r = guide.makeelement(OPF('reference'), + attrib={'type':'cover', 'title':_('Cover'), 'href':mi.cover}) + r.tail = '\n' +(' '*4) + guide.append(r) + return etree.tostring(root, pretty_print=True, encoding='utf-8', + xml_declaration=True) if as_string else root + + +def test_m2o(): + from datetime import datetime + from cStringIO import StringIO + mi = MetaInformation('test & title', ['a"1', "a'2"]) + mi.title_sort = 'a\'"b' + mi.author_sort = 'author sort' + mi.pubdate = datetime.now() + mi.language = 'en' + mi.category = 'test' + mi.comments = 'what a fun book\n\n' + mi.publisher = 'publisher' + mi.isbn = 'boooo' + mi.tags = ['a', 'b'] + mi.series = 's"c\'l&<>' + mi.series_index = 3.34 + mi.rating = 3 + mi.timestamp = datetime.now() + mi.publication_type = 'ooooo' + mi.rights = 'yes' + mi.cover = 'asd.jpg' + opf = metadata_to_opf(mi) + print opf + newmi = MetaInformation(OPF(StringIO(opf))) + for attr in ('author_sort', 'title_sort', 'comments', 'category', + 'publisher', 'series', 'series_index', 'rating', + 'isbn', 'tags', 'cover_data', 'application_id', + 'language', 'cover', + 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', + 'pubdate', 'rights', 'publication_type'): + o, n = getattr(mi, attr), getattr(newmi, attr) + if o != n and o.strip() != n.strip(): + print 'FAILED:', attr, getattr(mi, attr), '!=', getattr(newmi, attr) + + class OPFTest(unittest.TestCase): def setUp(self): diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index e0e97ad61f..071ec77c23 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -22,7 +22,7 @@ def debug(*args): def read_metadata_(task, tdir, notification=lambda x,y:x): from calibre.ebooks.metadata.meta import metadata_from_formats - from calibre.ebooks.metadata.opf2 import OPFCreator + from calibre.ebooks.metadata.opf2 import metadata_to_opf for x in task: try: id, formats = x @@ -33,9 +33,8 @@ def read_metadata_(task, tdir, notification=lambda x,y:x): if mi.cover_data: cdata = mi.cover_data[-1] mi.cover_data = None - opf = OPFCreator(tdir, mi) with open(os.path.join(tdir, '%s.opf'%id), 'wb') as f: - opf.render(f) + f.write(metadata_to_opf(mi)) if cdata: with open(os.path.join(tdir, str(id)), 'wb') as f: f.write(cdata) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 3e218e84cc..d7344b5681 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -29,7 +29,7 @@ from calibre.ebooks.metadata import string_to_authors, authors_to_string, \ MetaInformation, authors_to_sort_string from calibre.ebooks.metadata.meta import get_metadata, set_metadata, \ metadata_from_formats -from calibre.ebooks.metadata.opf2 import OPFCreator +from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding from calibre.ptempfile import PersistentTemporaryFile from calibre.customize.ui import run_plugins_on_import @@ -1611,13 +1611,12 @@ books_series_link feeds id = idx if index_is_id else self.id(idx) id = str(id) if not single_dir and not os.path.exists(tpath): - os.mkdir(tpath) + os.makedirs(tpath) name = au + ' - ' + title if byauthor else title + ' - ' + au name += '_'+id base = dir if single_dir else tpath mi = self.get_metadata(idx, index_is_id=index_is_id, get_cover=True) - f = open(os.path.join(base, sanitize_file_name(name)+'.opf'), 'wb') if not mi.authors: mi.authors = [_('Unknown')] cdata = self.cover(int(id), index_is_id=True) @@ -1625,9 +1624,9 @@ books_series_link feeds cname = sanitize_file_name(name)+'.jpg' open(os.path.join(base, cname), 'wb').write(cdata) mi.cover = cname - opf = OPFCreator(base, mi) - opf.render(f) - f.close() + with open(os.path.join(base, sanitize_file_name(name)+'.opf'), + 'wb') as f: + f.write(metadata_to_opf(mi)) fmts = self.formats(idx, index_is_id=index_is_id) if not fmts: diff --git a/src/calibre/trac/donations/server.py b/src/calibre/trac/donations/server.py deleted file mode 100644 index d0fd24af0c..0000000000 --- a/src/calibre/trac/donations/server.py +++ /dev/null @@ -1,589 +0,0 @@ -#!/usr/bin/env python -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Keep track of donations to calibre. -''' -import sys, cStringIO, textwrap, traceback, re, os, time, calendar -from datetime import date, timedelta -from math import sqrt -os.environ['HOME'] = '/tmp' -import matplotlib -matplotlib.use('Agg') -import matplotlib.pyplot as plt -import matplotlib.dates as mdates - -import cherrypy -from lxml import etree - -def range_for_month(year, month): - ty, tm = date.today().year, date.today().month - min = max = date(year=year, month=month, day=1) - x = date.today().day if ty == year and tm == month else 31 - while x > 1: - try: - max = min.replace(day=x) - break - except ValueError: - x -= 1 - return min, max - -def range_for_year(year): - return date(year=year, month=1, day=1), date(year=year, month=12, day=31) - -def days_in_month(year, month): - c = calendar.Calendar() - ans = 0 - for x in c.itermonthdays(year, month): - if x != 0: ans += 1 - return ans - -def rationalize_country(country): - if not country: - return 'Unknown' - if re.match('(?i)(US|USA|America)', country): - country = 'USA' - elif re.match('(?i)(UK|Britain|england)', country): - country = 'UK' - elif re.match('(?i)italy', country): - country = 'Italy' - elif re.match('(?i)germany', country): - country = 'Germany' - elif re.match('(?i)france', country): - country = 'France' - elif re.match('(?i)ireland', country): - country = 'Ireland' - elif re.match('(?i)norway', country): - country = 'Norway' - elif re.match('(?i)canada', country): - country = 'Canada' - elif re.match(r'(?i)new\s*zealand', country): - country = 'New Zealand' - elif re.match('(?i)jamaica', country): - country = 'Jamaica' - elif re.match('(?i)australia', country): - country = 'Australia' - elif re.match('(?i)Netherlands', country): - country = 'Netherlands' - elif re.match('(?i)spain', country): - country = 'Spain' - elif re.match('(?i)colombia', country): - country = 'Colombia' - return country - -class Record(object): - - def __init__(self, email, country, amount, date, name): - self.email = email - self.country = country - self.amount = amount - self.date = date - self.name = name - - def __str__(self): - return ''%\ - (self.email, self.country, self.amount, self.date.isoformat(), 'name="%s"'%self.name if self.name else '') - -class Country(list): - - def __init__(self, name): - list.__init__(self) - self.name = name - self.total = 0. - self.percent = 0. - - def append(self, r): - self.total += r.amount - list.append(self, r) - - def __str__(self): - return self.name + ': %.2f%%'%self.percent - - def __cmp__(self, other): - return cmp(self.total, other.total) - - -class Stats: - - def get_deviation(self, amounts): - l = float(len(amounts)) - if l == 0: - return 0 - mean = sum(amounts)/l - return sqrt( sum([i**2 for i in amounts])/l - mean**2 ) - - def __init__(self, records, start, end): - self.total = sum([r.amount for r in records]) - self.days = {} - l, rg = date.max, date.min - self.totals = [] - for r in records: - self.totals.append(r.amount) - l, rg = min(l, r.date), max(rg, r.date) - if r.date not in self.days.keys(): - self.days[r.date] = [] - self.days[r.date].append(r) - - self.min, self.max = start, end - self.period = (self.max - self.min) + timedelta(days=1) - daily_totals = [] - day = self.min - while day <= self.max: - x = self.days.get(day, []) - daily_totals.append(sum([y.amount for y in x])) - day += timedelta(days=1) - self.daily_average = self.total/self.period.days - self.daily_deviation = self.get_deviation(daily_totals) - self.average = self.total/len(records) if len(records) else 0. - self.average_deviation = self.get_deviation(self.totals) - self.countries = {} - self.daily_totals = daily_totals - for r in records: - if r.country not in self.countries.keys(): - self.countries[r.country] = Country(r.country) - self.countries[r.country].append(r) - for country in self.countries.values(): - country.percent = (100 * country.total/self.total) if self.total else 0. - - def get_daily_averages(self): - month_buckets, month_order = {}, [] - x = self.min - for t in self.daily_totals: - month = (x.year, x.month) - if month not in month_buckets: - month_buckets[month] = 0. - month_order.append(month) - month_buckets[month] += t - x += timedelta(days=1) - c = calendar.Calendar() - month_days = [days_in_month(*x) for x in month_order] - month_averages = [month_buckets[x]/float(y) for x, y in zip(month_order, month_days)] - return month_order, month_averages - - - - def __str__(self): - buf = cStringIO.StringIO() - print >>buf, '\tTotal: %.2f'%self.total - print >>buf, '\tDaily Average: %.2f'%self.daily_average - print >>buf, '\tAverage contribution: %.2f'%self.average - print >>buf, '\tCountry breakup:' - for c in self.countries.values(): - print >>buf, '\t\t', c - return buf.getvalue() - - def to_html(self, num_of_countries=sys.maxint): - countries = sorted(self.countries.values(), cmp=cmp, reverse=True)[:num_of_countries] - crows = ['%s%.2f %%'%(c.name, c.percent) for c in countries] - ctable = '\n\n%s
CountryContribution
'%('\n'.join(crows)) - if num_of_countries < sys.maxint: - ctable = '

Top %d countries

'%num_of_countries + ctable - return textwrap.dedent(''' -
-

Donations in %(period)d days [%(min)s — %(max)s]:

- - - - - -
Total$%(total).2f (%(num)d)
Daily average$%(da).2f ± %(dd).2f
Average contribution$%(ac).2f ± %(ad).2f
Donors per day%(dpd).2f
-
- %(ctable)s -
- ''')%dict(total=self.total, da=self.daily_average, ac=self.average, - ctable=ctable, period=self.period.days, num=len(self.totals), - dd=self.daily_deviation, ad=self.average_deviation, - dpd=len(self.totals)/float(self.period.days), - min=self.min.isoformat(), max=self.max.isoformat()) - - -def expose(func): - - def do(self, *args, **kwargs): - dict.update(cherrypy.response.headers, {'Server':'Donations_server/1.0'}) - return func(self, *args, **kwargs) - - return cherrypy.expose(do) - -class Server(object): - - TRENDS = '/tmp/donations_trend.png' - MONTH_TRENDS = '/tmp/donations_month_trend.png' - AVERAGES = '/tmp/donations_averages.png' - - def __init__(self, apache=False, root='/', data_file='/tmp/donations.xml'): - self.apache = apache - self.document_root = root - self.data_file = data_file - self.read_records() - - def calculate_daily_averages(self): - stats = self.get_slice(self.earliest, self.latest) - fig = plt.figure(2, (10, 4), 96)#, facecolor, edgecolor, frameon, FigureClass) - fig.clear() - ax = fig.add_subplot(111) - month_order, month_averages = stats.get_daily_averages() - x = [date(y, m, 1) for y, m in month_order[:-1]] - ax.plot(x, month_averages[:-1]) - ax.set_xlabel('Month') - ax.set_ylabel('Daily average ($)') - ax.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) - ax.xaxis.set_major_formatter(mdates.DateFormatter('%m/%y')) - fig.savefig(self.AVERAGES) - - - def calculate_month_trend(self, days=31): - stats = self.get_slice(date.today()-timedelta(days=days-1), date.today()) - fig = plt.figure(2, (10, 4), 96)#, facecolor, edgecolor, frameon, FigureClass) - fig.clear() - ax = fig.add_subplot(111) - x = list(range(days-1, -1, -1)) - y = stats.daily_totals - ax.plot(x, y)#, align='center', width=20, color='g') - ax.set_xlabel('Days ago') - ax.set_ylabel('Income ($)') - ax.hlines([stats.daily_average], 0, days-1) - ax.hlines([stats.daily_average+stats.daily_deviation, - stats.daily_average-stats.daily_deviation], 0, days-1, - linestyle=':',color='r') - ax.set_xlim([0, days-1]) - text = u'''\ -Total: $%(total).2f -Daily average: $%(da).2f \u00b1 %(dd).2f -Average contribution: $%(ac).2f \u00b1 %(ad).2f -Donors per day: %(dpd).2f - '''%dict(total=stats.total, da=stats.daily_average, - dd=stats.daily_deviation, ac=stats.average, - ad=stats.average_deviation, - dpd=len(stats.totals)/float(stats.period.days), - ) - text = ax.annotate(text, (0.5, 0.65), textcoords='axes fraction') - fig.savefig(self.MONTH_TRENDS) - - def calculate_trend(self): - def months(start, end): - pos = range_for_month(start.year, start.month)[0] - while pos <= end: - yield (pos.year, pos.month) - if pos.month == 12: - pos = pos.replace(year = pos.year+1) - pos = pos.replace(month = 1) - else: - pos = pos.replace(month = pos.month + 1) - _months = list(months(self.earliest, self.latest))[:-1][-12:] - _months = [range_for_month(*m) for m in _months] - _months = [self.get_slice(*m) for m in _months] - x = [m.min for m in _months] - y = [m.total for m in _months] - ml = mdates.MonthLocator() # every month - fig = plt.figure(1, (8, 4), 96)#, facecolor, edgecolor, frameon, FigureClass) - fig.clear() - ax = fig.add_subplot(111) - average = sum(y)/len(y) - ax.bar(x, y, align='center', width=20, color='g') - ax.hlines([average], x[0], x[-1]) - ax.xaxis.set_major_locator(ml) - ax.xaxis.set_major_formatter(mdates.DateFormatter('%b %y')) - ax.set_xlim(_months[0].min-timedelta(days=15), _months[-1].min+timedelta(days=15)) - ax.set_xlabel('Month') - ax.set_ylabel('Income ($)') - fig.autofmt_xdate() - fig.savefig(self.TRENDS) - #plt.show() - - - def read_records(self): - self.tree = etree.parse(self.data_file) - self.last_read_time = time.time() - self.root = self.tree.getroot() - self.records = [] - min_date, max_date = date.today(), date.fromordinal(1) - for x in self.root.xpath('//donation'): - d = list(map(int, x.get('date').split('-'))) - d = date(*d) - self.records.append(Record(x.get('email'), x.get('country'), float(x.get('amount')), d, x.get('name'))) - min_date = min(min_date, d) - max_date = max(max_date, d) - self.earliest, self.latest = min_date, max_date - self.calculate_trend() - self.calculate_month_trend() - self.calculate_daily_averages() - - def get_slice(self, start_date, end_date): - stats = Stats([r for r in self.records if r.date >= start_date and r.date <= end_date], - start_date, end_date) - return stats - - def month(self, year, month): - return self.get_slice(*range_for_month(year, month)) - - def year(self, year): - return self.get_slice(*range_for_year(year)) - - def range_to_date(self, raw): - return date(*map(int, raw.split('-'))) - - def build_page(self, period_type, data): - if os.stat(self.data_file).st_mtime >= self.last_read_time: - self.read_records() - month = date.today().month - year = date.today().year - mm = data[1] if period_type == 'month' else month - my = data[0] if period_type == 'month' else year - yy = data if period_type == 'year' else year - rl = data[0] if period_type == 'range' else '' - rr = data[1] if period_type == 'range' else '' - - def build_month_list(current): - months = [] - for i in range(1, 13): - month = date(2000, i, 1).strftime('%b') - sel = 'selected="selected"' if i == current else '' - months.append(''%(i, sel, month)) - return months - - def build_year_list(current): - all_years = sorted(range(self.earliest.year, self.latest.year+1, 1)) - if current not in all_years: - current = all_years[0] - years = [] - for year in all_years: - sel = 'selected="selected"' if year == current else '' - years.append(''%(year, sel, year)) - return years - - mmlist = ''%('\n'.join(build_month_list(mm))) - mylist = ''%('\n'.join(build_year_list(my))) - yylist = ''%('\n'.join(build_year_list(yy))) - - if period_type == 'month': - range_stats = range_for_month(my, mm) - elif period_type == 'year': - range_stats = range_for_year(yy) - else: - try: - range_stats = list(map(self.range_to_date, (rl, rr))) - err = None - except: - range_stats = None - err = traceback.format_exc() - if range_stats is None: - range_stats = '
Invalid input:\n%s
'%err - else: - range_stats = self.get_slice(*range_stats).to_html(num_of_countries=10) - - today = self.get_slice(date.today(), date.today()) - - return textwrap.dedent('''\ - - - - - Calibre donations - - - - - - - - - - - -
- - - - - - -
-

Donations to date

- %(todate)s -
-
-
- Income trends -

Income trends for the last year

- Month income trend -

Income trends for the last 31 days

- Daily average - income trend -

Income trends since records started

-
- - - ''')%dict( - todate=self.get_slice(self.earliest, self.latest).to_html(), - mc = 'checked="checked"' if period_type=="month" else '', - yc = 'checked="checked"' if period_type=="year" else '', - rc = 'checked="checked"' if period_type=="range" else '', - month_month=mmlist, month_year=mylist, year_year=yylist, - rl=rl, rr=rr, range_stats=range_stats, root=self.document_root, - today=today.total - ) - - @expose - def index(self): - month = date.today().month - year = date.today().year - cherrypy.response.headers['Content-Type'] = 'application/xhtml+xml' - return self.build_page('month', (year, month)) - - @expose - def trend_png(self): - cherrypy.response.headers['Content-Type'] = 'image/png' - return open(self.TRENDS, 'rb').read() - - @expose - def month_trend_png(self): - cherrypy.response.headers['Content-Type'] = 'image/png' - return open(self.MONTH_TRENDS, 'rb').read() - - @expose - def average_trend_png(self): - cherrypy.response.headers['Content-Type'] = 'image/png' - return open(self.AVERAGES, 'rb').read() - - @expose - def show(self, period_type='month', month_month='', month_year='', - year_year='', range_left='', range_right=''): - if period_type == 'month': - mm = int(month_month) if month_month else date.today().month - my = int(month_year) if month_year else date.today().year - data = (my, mm) - elif period_type == 'year': - data = int(year_year) if year_year else date.today().year - else: - data = (range_left, range_right) - cherrypy.response.headers['Content-Type'] = 'application/xhtml+xml' - return self.build_page(period_type, data) - -def config(): - config = { - 'global': { - 'tools.gzip.on' : True, - 'tools.gzip.mime_types': ['text/html', 'text/plain', 'text/xml', 'text/javascript', 'text/css', 'application/xhtml+xml'], - } - } - return config - -def apache_start(): - cherrypy.config.update({ - 'log.screen' : False, - #'log.error_file' : '/tmp/donations.log', - 'environment' : 'production', - 'show_tracebacks' : False, - }) - cherrypy.tree.mount(Server(apache=True, root='/donations/', data_file='/var/www/calibre.kovidgoyal.net/donations.xml'), - '/donations', config=config()) - - -def main(args=sys.argv): - server = Server() - cherrypy.quickstart(server, config=config()) - return 0 - -if __name__ == '__main__': - sys.exit(main())