Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Make calibre timezone aware

Browse Source
This commit is contained in:
Kovid Goyal 2010-02-14 23:15:23 -07:00
parent 895bd5db70
commit 20504d9f17
14 changed files with 138 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/calibre/ebooks/conversion/plumber.py
View File

@ -12,6 +12,7 @@ from calibre.customize.ui import input_profiles, output_profiles, \
run_plugins_on_preprocess, run_plugins_on_postprocess
from calibre.ebooks.conversion.preprocess import HTMLPreProcessor
from calibre.ptempfile import PersistentTemporaryDirectory
from calibre.utils.date import parse_date
from calibre import extract, walk
DEBUG_README=u'''
@ -65,7 +66,7 @@ class Plumber(object):
metadata_option_names = [
'title', 'authors', 'title_sort', 'author_sort', 'cover', 'comments',
'publisher', 'series', 'series_index', 'rating', 'isbn',
'tags', 'book_producer', 'language'
'tags', 'book_producer', 'language', 'pubdate', 'timestamp'
]
def __init__(self, input, output, log, report_progress=DummyReporter(),
@ -461,6 +462,14 @@ OptionRecommendation(name='language',
recommended_value=None, level=OptionRecommendation.LOW,
help=_('Set the language.')),
OptionRecommendation(name='pubdate',
recommended_value=None, level=OptionRecommendation.LOW,
help=_('Set the publication date.')),
OptionRecommendation(name='timestamp',
recommended_value=None, level=OptionRecommendation.LOW,
help=_('Set the book timestamp (used by the date column in calibre).')),
]
input_fmt = os.path.splitext(self.input)[1]
@ -619,6 +628,14 @@ OptionRecommendation(name='language',
except ValueError:
self.log.warn(_('Values of series index and rating must'
' be numbers. Ignoring'), val)
continue
elif x in ('timestamp', 'pubdate'):
try:
val = parse_date(val, assume_utc=x=='pubdate')
except:
self.log.exception(_('Failed to parse date/time') + ' ' +
unicode(val))
continue
setattr(mi, x, val)

3
src/calibre/ebooks/lrf/pylrs/pylrs.py
View File

@ -50,6 +50,7 @@ from pylrf import (LrfWriter, LrfObject, LrfTag, LrfToc,
STREAM_COMPRESSED, LrfTagStream, LrfStreamBase, IMAGE_TYPE_ENCODING,
BINDING_DIRECTION_ENCODING, LINE_TYPE_ENCODING, LrfFileStream,
STREAM_FORCE_COMPRESSED)
from calibre.utils.date import isoformat
DEFAULT_SOURCE_ENCODING = "cp1252" # defualt is us-windows character set
DEFAULT_GENREADING = "fs" # default is yes to both lrf and lrs
@ -852,7 +853,7 @@ class DocInfo(object):
self.thumbnail = None
self.language = "en"
self.creator = None
self.creationdate = date.today().isoformat()
self.creationdate = str(isoformat(date.today()))
self.producer = "%s v%s"%(__appname__, __version__)
self.numberofpages = "0"

5
src/calibre/ebooks/metadata/__init__.py
View File

@ -13,6 +13,7 @@ from urlparse import urlparse
from calibre import relpath
from calibre.utils.config import tweaks
from calibre.utils.date import isoformat
_author_pat = re.compile(',?\s+(and|with)\s+', re.IGNORECASE)
def string_to_authors(raw):
@ -344,9 +345,9 @@ class MetaInformation(object):
if self.rating is not None:
fmt('Rating', self.rating)
if self.timestamp is not None:
fmt('Timestamp', self.timestamp.isoformat(' '))
fmt('Timestamp', isoformat(self.timestamp))
if self.pubdate is not None:
fmt('Published', self.pubdate.isoformat(' '))
fmt('Published', isoformat(self.pubdate))
if self.rights is not None:
fmt('Rights', unicode(self.rights))
if self.lccn:

7
src/calibre/ebooks/metadata/amazon.py
View File

@ -10,9 +10,9 @@ import sys, re
from datetime import datetime
from lxml import etree
from dateutil import parser
from calibre import browser
from calibre.utils.date import parse_date
from calibre.ebooks.metadata import MetaInformation, string_to_authors
AWS_NS = 'http://webservices.amazon.com/AWSECommerceService/2005-10-05'
@ -44,9 +44,8 @@ def get_social_metadata(title, authors, publisher, isbn):
try:
d = root.findtext('.//'+AWS('PublicationDate'))
if d:
default = datetime.utcnow()
default = datetime(default.year, default.month, 15)
d = parser.parse(d[0].text, default=default)
default = datetime.utcnow().replace(day=15)
d = parse_date(d[0].text, assume_utc=True, default=default)
mi.pubdate = d
except:
pass

7
src/calibre/ebooks/metadata/google_books.py
View File

@ -9,11 +9,11 @@ from functools import partial
from datetime import datetime
from lxml import etree
from dateutil import parser
from calibre import browser, preferred_encoding
from calibre.ebooks.metadata import MetaInformation
from calibre.utils.config import OptionParser
from calibre.utils.date import parse_date
NAMESPACES = {
'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
@ -156,9 +156,8 @@ class ResultList(list):
try:
d = date(entry)
if d:
default = datetime.utcnow()
default = datetime(default.year, default.month, 15)
d = parser.parse(d[0].text, default=default)
default = datetime.utcnow().replace(day=15)
d = parse_date(d[0].text, assume_utc=True, default=default)
else:
d = None
except:

11
src/calibre/ebooks/metadata/opf2.py
View File

@ -12,12 +12,12 @@ from urllib import unquote
from urlparse import urlparse
from lxml import etree
from dateutil import parser
from calibre.ebooks.chardet import xml_to_unicode
from calibre.constants import __appname__, __version__, filesystem_encoding
from calibre.ebooks.metadata.toc import TOC
from calibre.ebooks.metadata import MetaInformation, string_to_authors
from calibre.utils.date import parse_date, isoformat
class Resource(object):
@ -449,9 +449,10 @@ class OPF(object):
series = MetadataField('series', is_dc=False)
series_index = MetadataField('series_index', is_dc=False, formatter=float, none_is=1)
rating = MetadataField('rating', is_dc=False, formatter=int)
pubdate = MetadataField('date', formatter=parser.parse)
pubdate = MetadataField('date', formatter=parse_date)
publication_type = MetadataField('publication_type', is_dc=False)
timestamp = MetadataField('timestamp', is_dc=False, formatter=parser.parse)
timestamp = MetadataField('timestamp', is_dc=False,
formatter=parse_date)
def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True):
@ -1046,7 +1047,7 @@ def metadata_to_opf(mi, as_string=True):
factory(DC('creator'), au, mi.author_sort, 'aut')
factory(DC('contributor'), mi.book_producer, __appname__, 'bkp')
if hasattr(mi.pubdate, 'isoformat'):
factory(DC('date'), mi.pubdate.isoformat())
factory(DC('date'), isoformat(mi.pubdate))
factory(DC('language'), mi.language)
if mi.category:
factory(DC('type'), mi.category)
@ -1069,7 +1070,7 @@ def metadata_to_opf(mi, as_string=True):
if mi.rating is not None:
meta('rating', str(mi.rating))
if hasattr(mi.timestamp, 'isoformat'):
meta('timestamp', mi.timestamp.isoformat())
meta('timestamp', isoformat(mi.timestamp))
if mi.publication_type:
meta('publication_type', mi.publication_type)

8
src/calibre/ebooks/oeb/transforms/metadata.py
View File

@ -7,7 +7,7 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os
from datetime import datetime
from calibre.utils.date import isoformat, now
def meta_info_to_oeb_metadata(mi, m, log):
from calibre.ebooks.oeb.base import OPF
@ -60,10 +60,10 @@ def meta_info_to_oeb_metadata(mi, m, log):
m.add('subject', t)
if mi.pubdate is not None:
m.clear('date')
m.add('date', mi.pubdate.isoformat())
m.add('date', isoformat(mi.pubdate))
if mi.timestamp is not None:
m.clear('timestamp')
m.add('timestamp', mi.timestamp.isoformat())
m.add('timestamp', isoformat(mi.timestamp))
if mi.rights is not None:
m.clear('rights')
m.add('rights', mi.rights)
@ -71,7 +71,7 @@ def meta_info_to_oeb_metadata(mi, m, log):
m.clear('publication_type')
m.add('publication_type', mi.publication_type)
if not m.timestamp:
m.add('timestamp', datetime.now().isoformat())
m.add('timestamp', isoformat(now()))
class MergeMetadata(object):

5
src/calibre/gui2/dialogs/scheduler.py
View File

@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
Scheduler for automated recipe downloads
'''
from datetime import datetime, timedelta
from datetime import timedelta
from PyQt4.Qt import QDialog, SIGNAL, Qt, QTime, QObject, QMenu, \
QAction, QIcon, QMutex, QTimer
@ -17,6 +17,7 @@ from calibre.gui2.search_box import SearchBox2
from calibre.gui2 import config as gconf, error_dialog
from calibre.web.feeds.recipes.model import RecipeModel
from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.date import utcnow
class SchedulerDialog(QDialog, Ui_Dialog):
@ -185,7 +186,7 @@ class SchedulerDialog(QDialog, Ui_Dialog):
self.day.setCurrentIndex(day+1)
self.time.setTime(QTime(hour, minute))
d = datetime.utcnow() - last_downloaded
d = utcnow() - last_downloaded
def hm(x): return (x-x%3600)//3600, (x%3600 - (x%3600)%60)//60
hours, minutes = hm(d.seconds)
tm = _('%d days, %d hours and %d minutes ago')%(d.days, hours, minutes)

9
src/calibre/library/catalog.py
View File

@ -10,6 +10,7 @@ from calibre.customize.conversion import OptionRecommendation, DummyReporter
from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString
from calibre.ptempfile import PersistentTemporaryDirectory
from calibre.utils.logging import Log
from calibre.utils.date import isoformat
FIELDS = ['all', 'author_sort', 'authors', 'comments',
'cover', 'formats', 'id', 'isbn', 'pubdate', 'publisher', 'rating',
@ -102,7 +103,9 @@ class CSV_XML(CatalogPlugin):
item = ', '.join(item)
elif field == 'isbn':
# Could be 9, 10 or 13 digits
field = u'%s' % re.sub(r'[\D]','',field)
item = u'%s' % re.sub(r'[\D]', '', item)
elif field in ['pubdate', 'timestamp']:
item = isoformat(item)
if x < len(fields) - 1:
if item is not None:
@ -163,12 +166,12 @@ class CSV_XML(CatalogPlugin):
if 'date' in fields:
record_child = etree.SubElement(record, 'date')
record_child.set(PY + "if", "record['date']")
record_child.text = "${record['date']}"
record_child.text = "${record['date'].isoformat()}"
if 'pubdate' in fields:
record_child = etree.SubElement(record, 'pubdate')
record_child.set(PY + "if", "record['pubdate']")
record_child.text = "${record['pubdate']}"
record_child.text = "${record['pubdate'].isoformat()}"
if 'size' in fields:
record_child = etree.SubElement(record, 'size')

14
src/calibre/library/cli.py
View File

@ -17,6 +17,7 @@ from calibre.ebooks.metadata.meta import get_metadata
from calibre.library.database2 import LibraryDatabase2
from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
from calibre.utils.genshi.template import MarkupTemplate
from calibre.utils.date import isoformat
FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
@ -37,8 +38,8 @@ XML_TEMPLATE = '''\
</authors>
<publisher>${record['publisher']}</publisher>
<rating>${record['rating']}</rating>
<date>${record['timestamp']}</date>
<pubdate>${record['pubdate']}</pubdate>
<date>${record['timestamp'].isoformat()}</date>
<pubdate>${record['pubdate'].isoformat()}</pubdate>
<size>${record['size']}</size>
<tags py:if="record['tags']">
<py:for each="tag in record['tags']">
@ -68,7 +69,7 @@ STANZA_TEMPLATE='''\
<uri>http://calibre-ebook.com</uri>
</author>
<id>$id</id>
<updated>${updated.strftime('%Y-%m-%dT%H:%M:%SZ')}</updated>
<updated>${updated.isoformat()}</updated>
<subtitle>
${subtitle}
</subtitle>
@ -77,7 +78,7 @@ STANZA_TEMPLATE='''\
<title>${record['title']}</title>
<id>urn:calibre:${record['uuid']}</id>
<author><name>${record['author_sort']}</name></author>
<updated>${record['timestamp'].strftime('%Y-%m-%dT%H:%M:%SZ')}</updated>
<updated>${record['timestamp'].isoformat()}</updated>
<link type="application/epub+zip" href="${quote(record['fmt_epub'].replace(sep, '/'))}"/>
<link py:if="record['cover']" rel="x-stanza-cover-image" type="image/png" href="${quote(record['cover'].replace(sep, '/'))}"/>
<link py:if="record['cover']" rel="x-stanza-cover-image-thumbnail" type="image/png" href="${quote(record['cover'].replace(sep, '/'))}"/>
@ -144,7 +145,10 @@ def do_list(db, fields, sort_by, ascending, search_text, line_width, separator,
widths = list(map(lambda x : 0, fields))
for record in data:
for f in record.keys():
record[f] = unicode(record[f])
if hasattr(record[f], 'isoformat'):
record[f] = isoformat(record[f], as_utc=False)
else:
record[f] = unicode(record[f])
record[f] = record[f].replace('\n', ' ')
for i in data:
for j, field in enumerate(fields):

10
src/calibre/library/database2.py
View File

@ -9,7 +9,6 @@ The database used to store ebook metadata
import os, re, sys, shutil, cStringIO, glob, collections, textwrap, \
itertools, functools, traceback
from itertools import repeat
from datetime import datetime
from math import floor
from PyQt4.QtCore import QThread, QReadWriteLock
@ -34,6 +33,7 @@ from calibre.ptempfile import PersistentTemporaryFile
from calibre.customize.ui import run_plugins_on_import
from calibre.utils.filenames import ascii_filename
from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
from calibre.ebooks import BOOK_EXTENSIONS
if iswindows:
@ -715,12 +715,12 @@ class LibraryDatabase2(LibraryDatabase):
def last_modified(self):
''' Return last modified time as a UTC datetime object'''
return datetime.utcfromtimestamp(os.stat(self.dbpath).st_mtime)
return utcfromtimestamp(os.stat(self.dbpath).st_mtime)
def check_if_modified(self):
if self.last_modified() > self.last_update_check:
self.refresh()
self.last_update_check = datetime.utcnow()
self.last_update_check = utcnow()
def path(self, index, index_is_id=False):
'Return the relative path to the directory containing this books files as a unicode string.'
@ -1123,7 +1123,7 @@ class LibraryDatabase2(LibraryDatabase):
def tags_older_than(self, tag, delta):
tag = tag.lower().strip()
now = datetime.now()
now = nowf()
for r in self.data._data:
if r is not None:
if (now - r[FIELD_MAP['timestamp']]) > delta:
@ -1484,7 +1484,7 @@ class LibraryDatabase2(LibraryDatabase):
stream.close()
self.conn.commit()
if existing:
t = datetime.utcnow()
t = utcnow()
self.set_timestamp(db_id, t, notify=False)
self.set_pubdate(db_id, t, notify=False)
self.data.refresh_ids(self, [db_id]) # Needed to update format list and size

36
src/calibre/library/sqlite.py
View File

@ -12,46 +12,18 @@ from sqlite3 import IntegrityError, OperationalError
from threading import Thread
from Queue import Queue
from threading import RLock
from datetime import tzinfo, datetime, timedelta
from datetime import datetime
from calibre.ebooks.metadata import title_sort
from calibre.utils.date import parse_date, isoformat
global_lock = RLock()
def convert_timestamp(val):
datepart, timepart = val.split(' ')
tz, mult = None, 1
x = timepart.split('+')
if len(x) > 1:
timepart, tz = x
else:
x = timepart.split('-')
if len(x) > 1:
timepart, tz = x
mult = -1
year, month, day = map(int, datepart.split("-"))
timepart_full = timepart.split(".")
hours, minutes, seconds = map(int, timepart_full[0].split(":"))
if len(timepart_full) == 2:
microseconds = int(timepart_full[1])
else:
microseconds = 0
if tz is not None:
h, m = map(int, tz.split(':'))
delta = timedelta(minutes=mult*(60*h + m))
tz = type('CustomTZ', (tzinfo,), {'utcoffset':lambda self, dt:delta,
'dst':lambda self,dt:timedelta(0)})()
val = datetime(year, month, day, hours, minutes, seconds, microseconds,
tzinfo=tz)
if tz is not None:
val = datetime(*(val.utctimetuple()[:6]))
return val
return parse_date(val, as_utc=False)
def adapt_datetime(dt):
dt = datetime(*(dt.utctimetuple()[:6]))
return dt.isoformat(' ')
return isoformat(dt)
sqlite.register_adapter(datetime, adapt_datetime)
sqlite.register_converter('timestamp', convert_timestamp)

56
src/calibre/utils/date.py Normal file
View File

@ -0,0 +1,56 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import with_statement
__license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
from datetime import datetime
from dateutil.parser import parse
from dateutil.tz import tzlocal, tzutc
_utc_tz = tzutc()
_local_tz = tzlocal()
def parse_date(date_string, assume_utc=False, as_utc=True, default=None):
'''
Parse a date/time string into a timezone aware datetime object. The timezone
is always either UTC or the local timezone.
:param assume_utc: If True and date_string does not specify a timezone,
assume UTC, otherwise assume local timezone.
:param as_utc: If True, return a UTC datetime
:param default: Missing fields are filled in from default. If None, the
current date is used.
'''
if default is None:
func = datetime.utcnow if assume_utc else datetime.now
default = func().replace(hour=0, minute=0, second=0, microsecond=0,
tzinfo=_utc_tz if assume_utc else _local_tz)
dt = parse(date_string, default=default)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=_utc_tz if assume_utc else _local_tz)
dt = dt.astimezone(_utc_tz if as_utc else _local_tz)
return dt
def isoformat(date_time, assume_utc=False, as_utc=True):
if not hasattr(date_time, 'tzinfo'):
return unicode(date_time.isoformat())
if date_time.tzinfo is None:
date_time = date_time.replace(tzinfo=_utc_tz if assume_utc else
_local_tz)
date_time = date_time.astimezone(_utc_tz if as_utc else _local_tz)
return unicode(date_time.isoformat())
def now():
return datetime.now().replace(tzinfo=_local_tz)
def utcnow():
return datetime.utcnow().replace(tzinfo=_utc_tz)
def utcfromtimestamp(stamp):
return datetime.utcfromtimestamp(stamp).replace(tzinfo=_utc_tz)

28
src/calibre/web/feeds/recipes/collection.py
View File

@ -12,9 +12,10 @@ from datetime import datetime, timedelta
from lxml import etree
from lxml.builder import ElementMaker
from dateutil import parser
from calibre import browser
from calibre.utils.date import parse_date, now as nowf, utcnow, tzlocal, \
isoformat
NS = 'http://calibre-ebook.com/recipe_collection'
E = ElementMaker(namespace=NS, nsmap={None:NS})
@ -125,7 +126,12 @@ class SchedulerConfig(object):
self.lock = RLock()
if os.access(self.conf_path, os.R_OK):
with ExclusiveFile(self.conf_path) as f:
self.root = etree.fromstring(f.read())
try:
self.root = etree.fromstring(f.read())
except:
print 'Failed to read recipe scheduler config'
import traceback
traceback.print_exc()
elif os.path.exists(old_conf_path):
self.migrate_old_conf(old_conf_path)
@ -151,7 +157,7 @@ class SchedulerConfig(object):
ld = x.get('last_downloaded', None)
if ld and last_downloaded is None:
try:
last_downloaded = parser.parse(ld)
last_downloaded = parse_date(ld)
except:
pass
self.root.remove(x)
@ -161,7 +167,7 @@ class SchedulerConfig(object):
sr = E.scheduled_recipe({
'id' : recipe.get('id'),
'title': recipe.get('title'),
'last_downloaded':last_downloaded.isoformat(),
'last_downloaded':isoformat(last_downloaded),
}, self.serialize_schedule(schedule_type, schedule))
self.root.append(sr)
self.write_scheduler_file()
@ -189,7 +195,7 @@ class SchedulerConfig(object):
def update_last_downloaded(self, recipe_id):
with self.lock:
now = datetime.utcnow()
now = utcnow()
for x in self.iter_recipes():
if x.get('id', False) == recipe_id:
typ, sch, last_downloaded = self.un_serialize_schedule(x)
@ -199,7 +205,7 @@ class SchedulerConfig(object):
if abs(actual_interval - nominal_interval) < \
timedelta(hours=1):
now = last_downloaded + nominal_interval
x.set('last_downloaded', now.isoformat())
x.set('last_downloaded', isoformat(now))
break
self.write_scheduler_file()
@ -243,20 +249,18 @@ class SchedulerConfig(object):
sch = float(sch)
elif typ == 'day/time':
sch = list(map(int, sch.split(':')))
return typ, sch, parser.parse(recipe.get('last_downloaded'))
return typ, sch, parse_date(recipe.get('last_downloaded'))
def recipe_needs_to_be_downloaded(self, recipe):
try:
typ, sch, ld = self.un_serialize_schedule(recipe)
except:
return False
utcnow = datetime.utcnow()
if typ == 'interval':
return utcnow - ld > timedelta(sch)
return utcnow() - ld > timedelta(sch)
elif typ == 'day/time':
now = datetime.now()
offset = now - utcnow
ld_local = ld + offset
now = nowf()
ld_local = ld.astimezone(tzlocal())
day, hour, minute = sch
is_today = day < 0 or day > 6 or \